diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 8ddc86d1..8b97853b 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -19,6 +19,14 @@ jobs: name: Cargo and npm build runs-on: ubuntu-latest #runs-on: [self-hosted, linux] + # Build inside the same image the Dockerfile builds in, so the binaries + # produced here can be copied into the runtime image instead of being + # compiled a second time. They are dynamically linked against glibc, and + # glibc is forward- but not backward-compatible: a binary built on the + # runner (Ubuntu 24.04, glibc 2.39) would not start on debian:bookworm-slim + # (glibc 2.36). Building in rust:bookworm makes the two match exactly. + container: + image: rust:bookworm services: postgres: image: postgres:16 @@ -42,16 +50,18 @@ jobs: ref: ${{ github.ref }} - name: Export PostgreSQL connection env run: | - echo "PGHOST=127.0.0.1" >> "$GITHUB_ENV" + # Inside a job container, services resolve by name on the shared + # network — 127.0.0.1 is the container itself. + echo "PGHOST=postgres" >> "$GITHUB_ENV" echo "PGPORT=5432" >> "$GITHUB_ENV" echo "PGUSER=postgres" >> "$GITHUB_ENV" echo "PGPASSWORD=postgres" >> "$GITHUB_ENV" - - name: Install OpenSSL and protoc build deps - if: runner.os == 'Linux' - run: | - sudo apt-get update - sudo apt-get install -y pkg-config libssl-dev protobuf-compiler + # No apt step here: `rust:bookworm` already carries pkg-config, libssl-dev + # and a C toolchain, and protoc never comes from the system — `build.rs` + # points PROTOC at a vendored binary unless one is already set. The step + # that used to be here called `sudo`, which the image does not have, and + # does not need: the job runs as root. - name: Verify .sqlx cache exists run: | @@ -108,12 +118,16 @@ jobs: # env vars no longer race and the suite runs in parallel (no more # RUST_TEST_THREADS=1 serialization, no 25-minute timeout). The `bdd` # target uses a custom harness nextest cannot run, so it runs separately. + # Both suites run even if one of them fails, so a single broken test does + # not hide the state of the other. Not `always()`: that also runs them + # after an earlier *setup* step fails, and then reports something + # unrelated — a missing apt package once surfaced as "no such command: + # nextest", because the install step had been skipped. - name: Cargo test - if: ${{ always() }} run: cargo nextest run --tests -E 'not binary(bdd)' - name: Cargo test (bdd suite) - if: ${{ always() }} + if: success() || failure() run: cargo test --test bdd - name: Rustfmt @@ -136,17 +150,21 @@ jobs: command: clippy args: -- -D warnings - - name: Build server (release) - uses: actions-rs/cargo@v1 - with: - command: build - args: --release --bin server - - - name: Build cleanup-notify (release) - uses: actions-rs/cargo@v1 - with: - command: build - args: --release --bin cleanup-notify + # One invocation, so the four binaries share a single compilation of the + # workspace instead of four sequential ones. These are the binaries the + # runtime image needs; the Docker job copies them rather than rebuilding. + # Two invocations, not one: `--features` applies to the whole command, not + # to the `--bin` it follows. Listing them together builds `server` with + # `explain` too — a differently configured binary from the one the image + # has always shipped. The second call is nearly free; only `console` and + # the re-featured casbin dependency recompile. + - name: Build release binaries + run: | + cargo build --release \ + --bin server \ + --bin cleanup-notify \ + --bin backfill_field_policy + cargo build --release --bin console --features explain - name: Set up Node.js if: ${{ hashFiles('web/package.json') != '' }} @@ -181,7 +199,9 @@ jobs: run: | mkdir -p app/stacker/dist cp target/release/server app/stacker/server + cp target/release/console app/stacker/console cp target/release/cleanup-notify app/stacker/cleanup-notify + cp target/release/backfill_field_policy app/stacker/backfill_field_policy if [ -d web/dist ]; then cp -a web/dist/. app/stacker; fi cp Dockerfile app/Dockerfile cp access_control.conf.dist app/access_control.conf.dist @@ -210,6 +230,28 @@ jobs: run: | test -d "${GITHUB_WORKSPACE}/tests/fixtures/pipe-contract" + # The test job already compiled these, in the same rust:bookworm image the + # runtime stage is based on. Without this the Dockerfile compiles the whole + # workspace a second time — around fourteen minutes of the run. + - name: Download binaries built by the test job + uses: actions/download-artifact@v4 + with: + name: artifact-linux-docker + + # Unpacked outside the workspace: `.dockerignore` is empty, so anything + # left here is sent to buildkit as part of `context: .` — hundreds of + # megabytes of release binaries, twice, eating back the time this change + # exists to save. + - name: Unpack binaries + run: | + mkdir -p "${{ runner.temp }}/prebuilt" + tar -xzf app.tar.gz -C "${{ runner.temp }}/prebuilt" + cd "${{ runner.temp }}/prebuilt" + mv stacker/server stacker/console stacker/cleanup-notify \ + stacker/backfill_field_policy . + chmod +x server console cleanup-notify backfill_field_policy + rm -f "${GITHUB_WORKSPACE}/app.tar.gz" + - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -235,8 +277,11 @@ jobs: uses: docker/build-push-action@v6 with: context: . + build-args: | + BINARIES=prebuilt build-contexts: | shared_fixtures=${{ github.workspace }}/tests/fixtures + prebuilt_binaries=${{ runner.temp }}/prebuilt push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.docker_tags.outputs.tags }} diff --git a/Dockerfile b/Dockerfile index a04089a7..8ca9f0c3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,22 @@ # syntax=docker/dockerfile:1.4 +# +# Two ways in, selected by the `binaries` build context: +# +# prebuilt — the CI job already compiled the release binaries and passes them +# in. It builds inside this same `rust:bookworm` image, so the +# glibc the binaries link against matches the runtime stage. That +# skips a second full compile of the workspace. +# +# builder — nothing was passed in (a local `docker build`, or CI without the +# artifact). Compiles from source, as before. +# +# Select with `--build-arg BINARIES=prebuilt`. Default is a self-contained build. +ARG BINARIES=builder + FROM rust:bookworm AS builder RUN apt-get update && apt-get install --no-install-recommends -y protobuf-compiler libprotobuf-dev && rm -rf /var/lib/apt/lists/* -RUN cargo install sqlx-cli - WORKDIR /app COPY --from=shared_fixtures / /shared-fixtures # copy manifests @@ -45,6 +57,30 @@ RUN apt-get update && apt-get install --no-install-recommends -y libssl-dev; \ #RUN ls -la /app/target/release/ >&2 +# Config files and the sqlx CLI, needed by both paths. Separate from `builder` +# so the prebuilt path does not drag in a compile of the workspace just to get +# two YAML files. +FROM rust:bookworm AS config +RUN cargo install sqlx-cli --no-default-features --features rustls,postgres +WORKDIR /app +COPY ./docker/local/.env . +COPY ./docker/local/configuration.yaml . + +# The two sources of binaries, each putting them at the image root so the +# production stage copies from one place regardless of which was used. + +# Handed in by CI, already compiled in this same rust:bookworm image. +FROM scratch AS prebuilt-source +COPY --from=prebuilt_binaries / / + +FROM scratch AS builder-source +COPY --from=builder /app/target/release/server /server +COPY --from=builder /app/target/release/console /console +COPY --from=builder /app/target/release/cleanup-notify /cleanup-notify +COPY --from=builder /app/target/release/backfill_field_policy /backfill_field_policy + +FROM ${BINARIES}-source AS binaries + # deploy production FROM debian:bookworm-slim AS production @@ -54,13 +90,13 @@ WORKDIR /app RUN mkdir ./files && chmod 0777 ./files # copy binary and configuration files -COPY --from=builder /app/target/release/server . -COPY --from=builder /app/target/release/console . -COPY --from=builder /app/target/release/cleanup-notify . -COPY --from=builder /app/target/release/backfill_field_policy . -COPY --from=builder /app/.env . -COPY --from=builder /app/configuration.yaml . -COPY --from=builder /usr/local/cargo/bin/sqlx /usr/local/bin/sqlx +COPY --from=binaries /server . +COPY --from=binaries /console . +COPY --from=binaries /cleanup-notify . +COPY --from=binaries /backfill_field_policy . +COPY --from=config /app/.env . +COPY --from=config /app/configuration.yaml . +COPY --from=config /usr/local/cargo/bin/sqlx /usr/local/bin/sqlx COPY ./access_control.conf.dist ./access_control.conf EXPOSE 8000 diff --git a/docs/STACKER_YML_REFERENCE.md b/docs/STACKER_YML_REFERENCE.md index ac7212b1..da7658fe 100644 --- a/docs/STACKER_YML_REFERENCE.md +++ b/docs/STACKER_YML_REFERENCE.md @@ -23,7 +23,6 @@ - [install — Marketplace Install Inputs](#install) - [environments — Named Environments](#environments) - [volumes — Named Volumes](#volumes) -- [config_contract — Service Config Contracts](#config_contract) - [ai — AI Assistant](#ai) - [monitoring — Health & Metrics](#monitoring) - [status_panel](#monitoringstatus_panel) · [healthcheck](#monitoringhealthcheck) · [metrics](#monitoringmetrics) · [alerts](#monitoringalerts) @@ -847,6 +846,62 @@ environments: --- +## `config_contract` + +Declares who controls each of a service's inputs when somebody else installs the +stack. Read at publish time and on the marketplace install path; ignored by a +plain local deploy. + +Without it, the literal values that are correct for *your* deployment — a +`JWT_SECRET`, a database password — are copied verbatim into every buyer's +install, so every buyer and you share one set of credentials. + +```yaml +config_contract: + services: + my-service: # must match a service name, or `app` + fields: + DATABASE_URL: + mutability: fixed # your value ships as-is + LOG_LEVEL: + mutability: editable # your value is a default the buyer may override + LICENSE_KEY: + mutability: provided # the buyer must supply it; yours is never shipped + SECRET_KEY: + mutability: generated # a fresh value per install; the buyer never types it + type: alphanumeric + length: 32 + display: password +``` + +| Key | Applies to | Meaning | +|---|---|---| +| `mutability` | every field | `fixed`, `editable`, `provided` or `generated` — see above | +| `required` | every field | whether a value must resolve at all. Default `true` | +| `type` | `generated` | `hex`, `base64`, `alphanumeric`, `uuid`, `enum`, `derived_jwt` | +| `length` / `min_length` | `generated` | exact or minimum length | +| `values` | `enum` | the allowed set | +| `signing_key`, `claims`, `alg` | `derived_jwt` | `"service.FIELD"` to sign with, the claims, and one of `HS256`/`HS384`/`HS512` | +| `display` | any field | UI hint — `boolean`, `string`, `number`, `password`. Independent of `type` | + +Publishing to the marketplace is refused until every secret-shaped field carries +a `generated` or `provided` policy. + +**Shorthand.** Three plain lists are still accepted and mean +`fixed`+required, `fixed`+optional, and `generated` respectively: + +```yaml +config_contract: + services: + my-service: + required: [DATABASE_URL] + optional: [LOG_LEVEL] + secret: [SECRET_KEY] +``` + +Mixing is fine; an explicit `fields:` entry wins over a list mentioning the same +name. + ### Volume policy in `config_contract` A baked marketplace image is cloned for every buyer, and a volume that travels @@ -905,25 +960,6 @@ Named volumes referenced in `app.volumes` or `services[].volumes` but not listed --- -## `config_contract` - -*Optional* · `object` · Default: none - -Declares service-level configuration contracts — metadata consumed by the TryDirect Install Service and marketplace pipeline to validate and pre-populate service inputs. Not used during local deploys. - -```yaml -config_contract: - services: - my-service: - required_env: - - DATABASE_URL - - SECRET_KEY -``` - -> This section is primarily written by `stacker install` and the marketplace generator. You rarely need to set it by hand. - ---- - ## `ai` *Optional* · `object` · Default: `enabled: false`