Ci/build once reuse binaries

.github/workflows/docker.yml

@@ -19,6 +19,14 @@ jobs:
     name: Cargo and npm build
     runs-on: ubuntu-latest
     #runs-on: [self-hosted, linux]
+    # Build inside the same image the Dockerfile builds in, so the binaries
+    # produced here can be copied into the runtime image instead of being
+    # compiled a second time. They are dynamically linked against glibc, and
+    # glibc is forward- but not backward-compatible: a binary built on the
+    # runner (Ubuntu 24.04, glibc 2.39) would not start on debian:bookworm-slim
+    # (glibc 2.36). Building in rust:bookworm makes the two match exactly.
+    container:
+      image: rust:bookworm
     services:
       postgres:
         image: postgres:16
@@ -42,16 +50,18 @@ jobs:
           ref: ${{ github.ref }}
       - name: Export PostgreSQL connection env
         run: |
-          echo "PGHOST=127.0.0.1" >> "$GITHUB_ENV"
+          # Inside a job container, services resolve by name on the shared
+          # network — 127.0.0.1 is the container itself.
+          echo "PGHOST=postgres" >> "$GITHUB_ENV"
           echo "PGPORT=5432" >> "$GITHUB_ENV"
           echo "PGUSER=postgres" >> "$GITHUB_ENV"
           echo "PGPASSWORD=postgres" >> "$GITHUB_ENV"
 
-      - name: Install OpenSSL and protoc build deps
-        if: runner.os == 'Linux'
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y pkg-config libssl-dev protobuf-compiler
+      # No apt step here: `rust:bookworm` already carries pkg-config, libssl-dev
+      # and a C toolchain, and protoc never comes from the system — `build.rs`
+      # points PROTOC at a vendored binary unless one is already set. The step
+      # that used to be here called `sudo`, which the image does not have, and
+      # does not need: the job runs as root.
 
       - name: Verify .sqlx cache exists
         run: |
@@ -66,23 +76,20 @@ jobs:
           override: true
           components: rustfmt, clippy
 
-      - name: Cache cargo registry
-        uses: actions/cache@v4
-        with:
-          path: ~/.cargo/registry
-          key: docker-registry-${{ hashFiles('**/Cargo.lock') }}
-          restore-keys: |
-            docker-registry-
-            docker-
-
-      - name: Cache cargo index
-        uses: actions/cache@v4
+      # Registry and git only — not `target`. Caching the target directory here
+      # filled the runner's disk: `restore-keys: docker-` pulled in an older,
+      # already-bloated cache, the build added to it, and the post-job step
+      # saved a larger one still, so every run grew the next one's starting
+      # point. A 33-minute job that passed every check then died in cleanup
+      # with "No space left on device".
+      #
+      # rust.yml reached the same conclusion for the same reason; this keeps
+      # both workflows on one rule.
+      - name: Cache Cargo registry/git (no target — it grows without bound here)
+        uses: Swatinem/rust-cache@v2
         with:
-          path: ~/.cargo/git
-          key: docker-index-${{ hashFiles('**/Cargo.lock') }}
-          restore-keys: |
-            docker-index-
-            docker-
+          cache-targets: "false"
+          key: docker-cicd
 
       - name: Generate Secret Key
         run: |
@@ -99,15 +106,6 @@ jobs:
           echo "PostgreSQL did not become ready in time" >&2
           exit 1
 
-      - name: Cache cargo build
-        uses: actions/cache@v4
-        with:
-          path: target
-          key: docker-build-${{ hashFiles('**/Cargo.lock') }}
-          restore-keys: |
-            docker-build-
-            docker-
-
       - name: Cargo check
         uses: actions-rs/cargo@v1
         with:
@@ -120,12 +118,16 @@ jobs:
       # env vars no longer race and the suite runs in parallel (no more
       # RUST_TEST_THREADS=1 serialization, no 25-minute timeout). The `bdd`
       # target uses a custom harness nextest cannot run, so it runs separately.
+      # Both suites run even if one of them fails, so a single broken test does
+      # not hide the state of the other. Not `always()`: that also runs them
+      # after an earlier *setup* step fails, and then reports something
+      # unrelated — a missing apt package once surfaced as "no such command:
+      # nextest", because the install step had been skipped.
       - name: Cargo test
-        if: ${{ always() }}
         run: cargo nextest run --tests -E 'not binary(bdd)'
 
       - name: Cargo test (bdd suite)
-        if: ${{ always() }}
+        if: success() || failure()
         run: cargo test --test bdd
 
       - name: Rustfmt
@@ -148,17 +150,21 @@ jobs:
           command: clippy
           args: -- -D warnings
 
-      - name: Build server (release)
-        uses: actions-rs/cargo@v1
-        with:
-          command: build
-          args: --release --bin server
-
-      - name: Build cleanup-notify (release)
-        uses: actions-rs/cargo@v1
-        with:
-          command: build
-          args: --release --bin cleanup-notify
+      # One invocation, so the four binaries share a single compilation of the
+      # workspace instead of four sequential ones. These are the binaries the
+      # runtime image needs; the Docker job copies them rather than rebuilding.
+      # Two invocations, not one: `--features` applies to the whole command, not
+      # to the `--bin` it follows. Listing them together builds `server` with
+      # `explain` too — a differently configured binary from the one the image
+      # has always shipped. The second call is nearly free; only `console` and
+      # the re-featured casbin dependency recompile.
+      - name: Build release binaries
+        run: |
+          cargo build --release \
+            --bin server \
+            --bin cleanup-notify \
+            --bin backfill_field_policy
+          cargo build --release --bin console --features explain
 
       - name: Set up Node.js
         if: ${{ hashFiles('web/package.json') != '' }}
@@ -193,7 +199,9 @@ jobs:
         run: |
           mkdir -p app/stacker/dist
           cp target/release/server app/stacker/server
+          cp target/release/console app/stacker/console
           cp target/release/cleanup-notify app/stacker/cleanup-notify
+          cp target/release/backfill_field_policy app/stacker/backfill_field_policy
           if [ -d web/dist ]; then cp -a web/dist/. app/stacker; fi
           cp Dockerfile app/Dockerfile
           cp access_control.conf.dist app/access_control.conf.dist
@@ -222,6 +230,28 @@ jobs:
         run: |
           test -d "${GITHUB_WORKSPACE}/tests/fixtures/pipe-contract"
 
+      # The test job already compiled these, in the same rust:bookworm image the
+      # runtime stage is based on. Without this the Dockerfile compiles the whole
+      # workspace a second time — around fourteen minutes of the run.
+      - name: Download binaries built by the test job
+        uses: actions/download-artifact@v4
+        with:
+          name: artifact-linux-docker
+
+      # Unpacked outside the workspace: `.dockerignore` is empty, so anything
+      # left here is sent to buildkit as part of `context: .` — hundreds of
+      # megabytes of release binaries, twice, eating back the time this change
+      # exists to save.
+      - name: Unpack binaries
+        run: |
+          mkdir -p "${{ runner.temp }}/prebuilt"
+          tar -xzf app.tar.gz -C "${{ runner.temp }}/prebuilt"
+          cd "${{ runner.temp }}/prebuilt"
+          mv stacker/server stacker/console stacker/cleanup-notify \
+             stacker/backfill_field_policy .
+          chmod +x server console cleanup-notify backfill_field_policy
+          rm -f "${GITHUB_WORKSPACE}/app.tar.gz"
+
       -
         name: Set up QEMU
         uses: docker/setup-qemu-action@v3
@@ -247,8 +277,11 @@ jobs:
         uses: docker/build-push-action@v6
         with:
           context: .
+          build-args: |
+            BINARIES=prebuilt
           build-contexts: |
             shared_fixtures=${{ github.workspace }}/tests/fixtures
+            prebuilt_binaries=${{ runner.temp }}/prebuilt
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.docker_tags.outputs.tags }}
 

.pre-commit-config.yaml

@@ -7,6 +7,12 @@ repos:
         stages: [commit]
   - repo: local
     hooks:
+      - id: check-staged-secrets
+        name: no concrete secret values in the commit
+        entry: scripts/check-staged-secrets.sh
+        language: script
+        pass_filenames: false
+        stages: [commit]
       - id: cargo-fmt
         name: cargo fmt --all
         entry: cargo fmt --all

CHANGELOG.md

@@ -4,7 +4,35 @@ All notable changes to this project will be documented in this file.
 
 ## [Unreleased]
 
-### Added — Developer project synchronization
+## [0.3.3] — 2026-09-18
+
+### Added — Chat session management
+
+- Added chat session endpoints with archive and per-message encryption.
+- Added Casbin RBAC rules for `/api/chat` routes.
+
+### Added — Agent hardening & ownership
+
+- Enforced per-tenant ownership on `/v1/agent` routes.
+- Agent tokens now verified against a stored digest instead of round-tripping Vault.
+- Agent authentication fails closed on Vault errors.
+- Agent registration requires the service key; marketplace registration returns 501.
+- Accept aggregate `all_health` report from agents for container status.
+- Return `project_id` in agent snapshots and one-click clone endpoint.
+- Container scope classified from Docker label, not container name.
+- Moved `rotate-token` from the console binary to the CLI (`stacker agent rotate-token`).
+
+### Added — Marketplace field policy & secret federation
+
+- Added `config_contract` field-policy support (`fixed`/`editable`/`generated` + types: `hex`, `alphanumeric`, `uuid`, `derived_jwt`).
+- At publish, `generated`-field values are stripped from the stored `stack_definition` (fail-closed).
+- `derived_jwt` fields signed on cloned boxes via HMAC.
+- Config contract federated to the User Service.
+- Added `backfill_field_policy` one-shot tool to re-gate the existing catalog.
+- Policy-driven `generate-secrets.sh` reads `mutability:generated` field policy instead of hardcoding `openssl rand`.
+- Added `DisplayType` enum and `display` field to `FieldPolicy`.
+
+### Added — Project synchronization & one-click deploy
 
 - Added `stacker sync` to synchronize declarative project and app configuration
   with Stacker without creating a deployment, contacting a target server, or
@@ -14,6 +42,57 @@ All notable changes to this project will be documented in this file.
   synchronization.
 - Added centralized sensitive environment-name redaction and validation for
   marketplace asset and seed-job metadata.
+- One-click clone now registers the server in inventory and creates a cloud firewall.
+- One-click clone seeds `project_app` records for the Applications panel.
+
+### Added — Deployment lifecycle & cleanup
+
+- Added `deployment_container` table to track containers per deployment (replaces name inference).
+- Added deployment container sweeper for stale containers.
+- Added stale project and server cleanup with notification (`cleanup-notify` binary).
+- Added scheduled audit-log cleanup cron job.
+- Added env size validator.
+
+### Added — Security & infrastructure
+
+- mTLS for Vault access; Vault client reports missing CA.
+- `yaml_quote` now escapes control characters (`\n`, `\r`, `\t`).
+- Docker preflight check (`docker info`) before deploy.
+- New `W003` warning: `deploy.server.ssh_key` silently ignored on cloud deploys.
+
+### Fixed — SSH key authorization
+
+- Cloud deploy now fails when SSH key cannot be stored (was a silent warning).
+- Cloud deploy fails when no SSH access is verified after provisioning.
+- SSH key authorization retried while the VM boots; both Vault-managed and user keys authorized independently.
+- User's configured SSH key from `deploy.cloud.ssh_key` authorized through the correct endpoint.
+
+### Fixed — CLI & config
+
+- Fixed `server --dry-run` no longer runs a real Docker deploy (#238).
+- Fixed 500 on `PUT /cloud/{id}`: owner set before conversion.
+- Fixed #251: escape env/label values so multiline config survives YAML.
+- Ports validated by range, not by digit count.
+- Port values in `stacker.yml` handled correctly when unquoted.
+- Healthcheck `test` field emitted in the form docker compose expects (CMD list vs CMD-SHELL).
+
+### Fixed — Database & migrations
+
+- Fixed migration version collisions breaking CI.
+- Guard optional cron job lookup to prevent panics.
+- Reconcile audit-log cleanup cron after extension install.
+- Restored `sqlx prepare` with missing `config_contract` field.
+
+### Fixed — Auth & Casbin
+
+- Added missing Casbin rules for admin `detect-secrets` endpoint.
+- Agent auth accepts both Vault response shapes for the token.
+- Credentials tests no longer read the developer's own config.
+
+### Fixed — Marketplace BDD
+
+- BDD marketplace analytics fixtures: cast `template_id` to UUID.
+- BDD marketplace scenarios: seed `source_project_id` + deployment.
 
 ## [0.3.2] — 2026-08-26
 

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "stacker"
-version = "0.3.2"
+version = "0.3.3"
 edition = "2021"
 default-run= "server"
 

Dockerfile

@@ -1,10 +1,22 @@
 # syntax=docker/dockerfile:1.4
+#
+# Two ways in, selected by the `binaries` build context:
+#
+#   prebuilt — the CI job already compiled the release binaries and passes them
+#              in. It builds inside this same `rust:bookworm` image, so the
+#              glibc the binaries link against matches the runtime stage. That
+#              skips a second full compile of the workspace.
+#
+#   builder  — nothing was passed in (a local `docker build`, or CI without the
+#              artifact). Compiles from source, as before.
+#
+# Select with `--build-arg BINARIES=prebuilt`. Default is a self-contained build.
+ARG BINARIES=builder
+
 FROM rust:bookworm AS builder
 
 RUN apt-get update && apt-get install --no-install-recommends -y protobuf-compiler libprotobuf-dev && rm -rf /var/lib/apt/lists/*
 
-RUN cargo install sqlx-cli
-
 WORKDIR /app
 COPY --from=shared_fixtures / /shared-fixtures
 # copy manifests
@@ -45,6 +57,30 @@ RUN apt-get update && apt-get install --no-install-recommends -y libssl-dev; \
 
 #RUN ls -la /app/target/release/ >&2
 
+# Config files and the sqlx CLI, needed by both paths. Separate from `builder`
+# so the prebuilt path does not drag in a compile of the workspace just to get
+# two YAML files.
+FROM rust:bookworm AS config
+RUN cargo install sqlx-cli --no-default-features --features rustls,postgres
+WORKDIR /app
+COPY ./docker/local/.env .
+COPY ./docker/local/configuration.yaml .
+
+# The two sources of binaries, each putting them at the image root so the
+# production stage copies from one place regardless of which was used.
+
+# Handed in by CI, already compiled in this same rust:bookworm image.
+FROM scratch AS prebuilt-source
+COPY --from=prebuilt_binaries / /
+
+FROM scratch AS builder-source
+COPY --from=builder /app/target/release/server /server
+COPY --from=builder /app/target/release/console /console
+COPY --from=builder /app/target/release/cleanup-notify /cleanup-notify
+COPY --from=builder /app/target/release/backfill_field_policy /backfill_field_policy
+
+FROM ${BINARIES}-source AS binaries
+
 # deploy production
 FROM debian:bookworm-slim AS production
 
@@ -54,13 +90,13 @@ WORKDIR /app
 RUN mkdir ./files && chmod 0777 ./files
 
 # copy binary and configuration files
-COPY --from=builder /app/target/release/server .
-COPY --from=builder /app/target/release/console .
-COPY --from=builder /app/target/release/cleanup-notify .
-COPY --from=builder /app/target/release/backfill_field_policy .
-COPY --from=builder /app/.env .
-COPY --from=builder /app/configuration.yaml .
-COPY --from=builder /usr/local/cargo/bin/sqlx /usr/local/bin/sqlx
+COPY --from=binaries /server .
+COPY --from=binaries /console .
+COPY --from=binaries /cleanup-notify .
+COPY --from=binaries /backfill_field_policy .
+COPY --from=config /app/.env .
+COPY --from=config /app/configuration.yaml .
+COPY --from=config /usr/local/cargo/bin/sqlx /usr/local/bin/sqlx
 COPY ./access_control.conf.dist ./access_control.conf
 
 EXPOSE 8000