test: add proxy burn-in tooling

.github/workflows/test.yml

@@ -60,3 +60,48 @@ jobs:
         with:
           channel: engineering
           webhook_url: ${{ secrets.SLACK_NOTIFICATION_WEBHOOK_URL }}
+
+  burn-in:
+    name: Encrypted burn-in (PostgreSQL 17)
+    runs-on: blacksmith-16vcpu-ubuntu-2204
+    timeout-minutes: 15
+    env:
+      PG_VERSION: 17
+      CS_ZEROKMS_HOST: https://us-east-1.aws.zerokms.cipherstashmanaged.net
+      CS_CTS_HOST: https://ap-southeast-2.aws.cts.cipherstashmanaged.net
+      RUST_BACKTRACE: "1"
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup-test
+
+      - name: Decrypt secrets
+        uses: cipherstash/secrets-action@main
+        with:
+          secrets-file: .github/secrets.env.encrypted
+        env:
+          CS_CLIENT_ID: ${{ secrets.CS_VAULT_CLIENT_ID }}
+          CS_CLIENT_KEY: ${{ secrets.CS_VAULT_CLIENT_KEY }}
+          CS_CLIENT_ACCESS_KEY: ${{ secrets.CS_VAULT_CLIENT_ACCESS_KEY }}
+          CS_WORKSPACE_CRN: ${{ secrets.CS_VAULT_WORKSPACE_CRN }}
+
+      - name: Start PostgreSQL and install EQL
+        run: |
+          mise run postgres:up --extra-args "--detach --wait"
+          mise run postgres:setup
+
+      - name: Run encrypted burn-in
+        run: mise run test:burn-in
+
+      - name: Upload burn-in RSS report
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: proxy-burn-in-pg17
+          path: target/burn-in/soak-report.json
+          if-no-files-found: warn
+
+      - uses: ./.github/actions/send-slack-notification
+        with:
+          channel: engineering
+          webhook_url: ${{ secrets.SLACK_NOTIFICATION_WEBHOOK_URL }}

mise.toml

@@ -173,6 +173,22 @@ run = """
 cargo nextest run --no-fail-fast --nocapture -p cipherstash-proxy-integration
 """
 
+[tasks."test:burn-in"]
+description = "Run a bounded encrypted CRUD soak against a release Proxy"
+run = """
+set -e
+duration="${BURN_IN_DURATION_SECONDS:-30}"
+concurrency="${BURN_IN_CONCURRENCY:-4}"
+
+# The burn-in owns its Proxy process. Disable the optional metrics listener so
+# shared developer and CI environments cannot collide on its separate port.
+CS_PROMETHEUS__ENABLED=false cargo run --locked -p cipherstash-proxy-burn-in -- \
+  soak \
+  --duration-seconds "${duration}" \
+  --concurrency "${concurrency}" \
+  --output target/burn-in/soak-report.json
+"""
+
 [tasks."test:integration:setup:tls"]
 description = "Setup for TLS integration tests: preflight, postgres, proxy"
 run = """

packages/cipherstash-proxy-burn-in/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "cipherstash-proxy-burn-in"
+version = "0.1.0"
+edition.workspace = true
+publish = false
+
+[dependencies]
+anyhow = "1"
+clap = { version = "4.5", features = ["derive", "env"] }
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+tokio = { workspace = true }
+tokio-postgres = { version = "0.7", features = ["with-serde_json-1"] }

packages/cipherstash-proxy-burn-in/README.md

@@ -0,0 +1,41 @@
+# CipherStash Proxy burn-in
+
+This package drives deterministic conformance checks and a timed mixed CRUD workload through a
+real Proxy into PostgreSQL. The fixture schema and seed migration are adapted from pg-proto's
+burn-in package so results use the same type-lab and commerce model while exercising EQL domains.
+
+Start the test PostgreSQL service and configure the CipherStash credentials used by Proxy in
+`mise.local.toml`:
+
+```toml
+[env]
+CS_WORKSPACE_CRN = "crn:region:workspace-id"
+CS_CLIENT_ACCESS_KEY = "your-access-key"
+CS_DEFAULT_KEYSET_ID = "your-keyset-id"
+CS_CLIENT_ID = "your-client-id"
+CS_CLIENT_KEY = "your-client-key"
+```
+
+The commands inherit these values from the environment when they launch Proxy.
+The target database must also have EQL installed. If it does not, the burn-in installs
+`cipherstash-encrypt.sql` automatically; run `mise run eql:download` first or provide a different
+file with `--eql-path` / `BURN_IN_EQL_PATH`.
+
+```bash
+cargo run -p cipherstash-proxy-burn-in -- conformance
+cargo run -p cipherstash-proxy-burn-in -- soak --duration-seconds 300
+```
+
+`soak` always runs `cargo build --locked --release --package cipherstash-proxy` and starts that
+exact release binary. It samples the Proxy process RSS once per second and writes the full series
+to `target/burn-in/soak-report.json`. Use `--max-rss-growth-mib` to turn retained growth into a
+hard failure, and `--concurrency` to adjust load. The workload creates public fixture tables with
+EQL domain columns and verifies CRUD through Proxy, so its memory measurements include encryption
+and decryption work.
+
+Override connection URLs with `--proxy-database-url` / `--direct-database-url` or the
+`BURN_IN_PROXY_DATABASE_URL` / `BURN_IN_DIRECT_DATABASE_URL` environment variables.
+
+CI runs a bounded PostgreSQL 17 soak with `mise run test:burn-in` and uploads
+`target/burn-in/soak-report.json`. Override its defaults locally with
+`BURN_IN_DURATION_SECONDS` and `BURN_IN_CONCURRENCY`.

packages/cipherstash-proxy-burn-in/migrations/0001_schema.sql

@@ -0,0 +1,54 @@
+-- EQL is installed from cipherstash-encrypt.sql before this migration runs.
+-- Keep these tables in public: Proxy loads only schemas on its search path and
+-- EQL Mapper resolves tables in a single, unqualified namespace.
+DROP SCHEMA IF EXISTS burnin_type_lab CASCADE;
+DROP SCHEMA IF EXISTS burnin_commerce CASCADE;
+
+DROP TABLE IF EXISTS public.burnin_commerce_order_lines;
+DROP TABLE IF EXISTS public.burnin_commerce_orders;
+DROP TABLE IF EXISTS public.burnin_commerce_products;
+DROP TABLE IF EXISTS public.burnin_commerce_customers;
+DROP TABLE IF EXISTS public.burnin_type_lab_bulk_values;
+DROP TABLE IF EXISTS public.burnin_type_lab_samples;
+
+CREATE TABLE public.burnin_type_lab_samples (
+    id integer PRIMARY KEY,
+    scalar eql_v3_integer_ord NOT NULL,
+    nullable_text eql_v3_text,
+    binary_value bytea NOT NULL,
+    tags text[] NOT NULL,
+    document eql_v3_json NOT NULL,
+    wide_text eql_v3_text NOT NULL
+);
+
+CREATE TABLE public.burnin_type_lab_bulk_values (
+    id integer PRIMARY KEY,
+    nullable_text eql_v3_text,
+    binary_value bytea NOT NULL,
+    wide_text eql_v3_text NOT NULL
+);
+
+CREATE TABLE public.burnin_commerce_customers (
+    id integer PRIMARY KEY,
+    name eql_v3_text NOT NULL
+);
+
+CREATE TABLE public.burnin_commerce_products (
+    id integer PRIMARY KEY,
+    sku eql_v3_text NOT NULL,
+    price_cents integer NOT NULL CHECK (price_cents > 0)
+);
+
+CREATE TABLE public.burnin_commerce_orders (
+    id integer PRIMARY KEY,
+    customer_id integer NOT NULL REFERENCES public.burnin_commerce_customers(id),
+    status eql_v3_text NOT NULL
+);
+
+CREATE TABLE public.burnin_commerce_order_lines (
+    order_id integer NOT NULL REFERENCES public.burnin_commerce_orders(id),
+    line_number integer NOT NULL,
+    product_id integer NOT NULL REFERENCES public.burnin_commerce_products(id),
+    quantity integer NOT NULL CHECK (quantity > 0),
+    PRIMARY KEY (order_id, line_number)
+);

packages/cipherstash-proxy-burn-in/migrations/0002_seed.sql

@@ -0,0 +1,5 @@
+-- This migration must run through Proxy so values assigned to EQL domains are
+-- encrypted before PostgreSQL stores them.
+TRUNCATE burnin_commerce_order_lines, burnin_commerce_orders,
+    burnin_commerce_products, burnin_commerce_customers,
+    burnin_type_lab_bulk_values, burnin_type_lab_samples;