From 94738143606d220d1acae9df16ea61ecb2faa7f4 Mon Sep 17 00:00:00 2001 From: mnoah1 Date: Thu, 22 Oct 2026 19:07:44 +0000 Subject: [PATCH 1/3] feat(stovepipe): record partial validation and advance build baselines early Summary: ## Why? Successful builds should supply the next incremental baseline while project resolution is pending. Failed repositories with project results need a partial degree instead of the fully broken degree. ## What? - Add DegreePartial = 0.5 for failed Requests with project results, preserving each project's resolved degree. - Resolve projects before recording facts; persist facts and history before promoting a green fact that still owns the baseline. - Share request-owned slot Claim and Complete operations. Save the winning Request outcome first, then atomically advance a successful baseline and release its Queue slot. Replays and DLQ recovery cannot release another Request's capacity. - Append the MySQL slot-owner column and document the required drained rollout. ## Issue None; this implements the agreed Stovepipe validation behavior. Test Plan: Run ./tool/bazel test //stovepipe/... //service/stovepipe/server/... //test/integration/stovepipe/extension/storage/mysql:go_default_test //test/e2e/stovepipe:go_default_test --test_output=errors. All 29 targets pass, including replay, admission-baseline timing, project-resolution ordering, partial-degree, MySQL, and end-to-end coverage. Formatting, lint, module-tidy, and Gazelle checks pass. Before deployment, drain old work and stop all old Queue writers, append the nullable in_flight_request_ids JSON column, then deploy all Queue readers and writers together. See stovepipe/extension/storage/mysql/MIGRATIONS.md. Revert Plan: Drain and stop the new workers before reverting the application. Retain the nullable column; old binaries can ignore it once all owned reservations have completed. Do not run mixed old and new writers. --- doc/rfc/stovepipe/steps/buildsignal.md | 40 +-- doc/rfc/stovepipe/steps/process.md | 43 ++- doc/rfc/stovepipe/steps/record.md | 123 ++------ doc/rfc/stovepipe/workflow.md | 10 +- .../controller/build/build_analysis_test.go | 2 +- stovepipe/controller/buildsignal/BUILD.bazel | 1 + .../controller/buildsignal/buildsignal.go | 84 +---- .../buildsignal/buildsignal_test.go | 59 +++- stovepipe/controller/dlq/BUILD.bazel | 1 + stovepipe/controller/dlq/build_test.go | 2 +- stovepipe/controller/dlq/buildsignal.go | 4 - stovepipe/controller/dlq/buildsignal_test.go | 12 +- stovepipe/controller/dlq/dlq.go | 147 +++------ stovepipe/controller/dlq/dlq_test.go | 50 ++- stovepipe/controller/process/BUILD.bazel | 2 + stovepipe/controller/process/process.go | 101 ++---- stovepipe/controller/process/process_test.go | 159 +++++++--- stovepipe/controller/project_status_test.go | 1 + stovepipe/controller/record/record.go | 134 ++------ stovepipe/controller/record/record_test.go | 290 +++++++++--------- stovepipe/core/buildslot/BUILD.bazel | 29 ++ stovepipe/core/buildslot/README.md | 9 + stovepipe/core/buildslot/claim.go | 52 ++++ stovepipe/core/buildslot/complete.go | 80 +++++ stovepipe/core/buildslot/complete_test.go | 140 +++++++++ stovepipe/entity/queue.go | 36 ++- stovepipe/entity/validation_fact.go | 8 +- stovepipe/entity/validation_fact_test.go | 3 +- .../extension/storage/mysql/MIGRATIONS.md | 9 + .../extension/storage/mysql/queue_store.go | 28 +- .../storage/mysql/queue_store_test.go | 32 +- .../extension/storage/mysql/schema/queue.sql | 1 + .../stovepipe/extension/storage/suite.go | 7 +- 33 files changed, 933 insertions(+), 766 deletions(-) create mode 100644 stovepipe/core/buildslot/BUILD.bazel create mode 100644 stovepipe/core/buildslot/README.md create mode 100644 stovepipe/core/buildslot/claim.go create mode 100644 stovepipe/core/buildslot/complete.go create mode 100644 stovepipe/core/buildslot/complete_test.go create mode 100644 stovepipe/extension/storage/mysql/MIGRATIONS.md diff --git a/doc/rfc/stovepipe/steps/buildsignal.md b/doc/rfc/stovepipe/steps/buildsignal.md index e1095acf5..7fc0812f4 100644 --- a/doc/rfc/stovepipe/steps/buildsignal.md +++ b/doc/rfc/stovepipe/steps/buildsignal.md @@ -1,6 +1,6 @@ # Buildsignal stage -`buildsignal` polls the build-runner until a build reaches a terminal state, records the status, and — once the build is terminal — releases the queue's build slot, projects the outcome onto the `Request`, and publishes the **request id** to `record`. See [workflow.md](../workflow.md) for where it sits in the pipeline and [build.md](build.md) for how the build it polls was triggered. +`buildsignal` polls the build-runner until a build reaches a terminal state, records the status, and — once the build is terminal — persists the winning `Request` outcome, completes its build slot and successful baseline in one Queue update, and publishes the **request id** to `record`. See [workflow.md](../workflow.md) for where it sits in the pipeline and [build.md](build.md) for how the build it polls was triggered. It handles only the poll loop: it does not decide build strategy, write greenness, or map targets to projects — those are `process`'s, `record`'s, and `analyze`'s jobs. It reports the *fact* (this request's build ended succeeded, failed, or cancelled); `record` derives what that fact *means* for greenness. @@ -16,7 +16,7 @@ Its logic does not branch on phase: it loads the `Build`, polls it toward termin Its early-exit guard is deliberately narrower than `State.IsTerminal()`: it proceeds when the request is `processing` **or** already carries a build outcome. The second case matters because a redelivery after the outcome was stamped but before the `record` publish landed must re-publish rather than drop the signal; everything it re-runs is a no-op (the status is unchanged, the outcome is already recorded, the slot is not released twice) and the `record` publish is idempotent. -The states the guard actually rejects are terminal-without-an-outcome — today `superseded` alone, which `process` sets only from `accepted`, so it cannot reach the poll loop at all. The guard is kept rather than dropped because the slot release in step 7a is conditioned on *not* already having an outcome, so a superseded request arriving here would decrement a slot that superseding never claimed. +A superseded Request has no build outcome and is ignored. Slot release is keyed by Request ownership, so duplicate deliveries cannot release another Request’s capacity. ## Algorithm @@ -35,9 +35,7 @@ For a delivery carrying build id `B`: - a request reaches the poll loop only after process admitted it, so it is processing — or already carries this build's outcome on a redelivery, which falls through so the signal is re-published to record instead of dropped. - - the only other terminal-without-an-outcome state is superseded, and process supersedes - solely from accepted, so it cannot occur here. Guarded anyway: step 7a would otherwise - release a build slot the request never claimed, since superseding consumes none. + - superseded requests have no build outcome and do not enter the poll loop. - mirrors submitqueue's halted-batch short-circuit in buildsignal.go. 4. Resolve the build-runner: buildRunner = Factory.For(Config{QueueName: R.Queue}). @@ -60,19 +58,21 @@ For a delivery carrying build id `B`: - with the write-once rule, accepted -> running -> {succeeded|failed|cancelled} is monotonic by mechanism, not by assumption about the backend. -7. If the stored status is terminal, and R does not already carry an outcome: - a. Release the queue's build slot: CAS-decrement Queue.in_flight_count, clamped at zero. - - failure here aborts the step: R must not go terminal while still holding a slot. - b. CAS R from processing to the outcome the stored status projects onto it and persist B as R's terminal build id: - succeeded -> succeeded, failed -> failed, cancelled -> cancelled. First writer wins. - Then publish R.ID to the record topic, partitioned by request id; ack, return. - No re-publish to buildsignal. - - record loads the Request directly by this key, derives greenness from its outcome, and - supplies its terminal build id to project-result resolution; it never reaches a Build. - - the message id is the request id, so a redelivery republishing the same terminal signal - dedups into the original message; record is idempotent regardless. - - publish failure -> return raw (non-retryable); the outcome is persisted, operational - republish recovers. +7. If the stored status is terminal: + a. Retain the build-finished history event. + b. CAS R from processing to the projected outcome, persisting B as TerminalBuildID: + succeeded -> succeeded, failed -> failed, cancelled -> cancelled. First terminal writer wins. + Adopt a concurrent terminal winner rather than replacing its State or TerminalBuildID. + c. Complete R's owned slot with a Queue CAS loop: + - if R.ID is absent from in_flight_request_ids, completion is already done; no write. + - remove only R.ID and recompute in_flight_count from the remaining owners. + - if R succeeded and is newer than LastGreenRequestID, advance LastGreenURI and + LastGreenRequestID in the SAME Queue write, before capacity becomes available. + - on version conflict, reload and retry. Other errors fail the delivery. + d. Retain outcome history, then publish R.ID to record and a terminal build-analysis nudge. + Both publications are attempted independently and errors are combined with errs.Group. + The record message id and partition key are R.ID; record reloads the durable outcome. + e. Ack; no re-publish to buildsignal. 8. Else hold the delivery for delayMs (postpone: the same message redelivers after the delay): - delayMs = pollDelay(status): shorter while running, longer while accepted. @@ -82,9 +82,9 @@ For a delivery carrying build id `B`: exactly one poll chain; no message ids are minted and no new rows are written per tick. ``` -**Why the slot is released before the outcome write, and why a failed release aborts it**: `Queue` and `Request` are separate entities with no cross-entity transaction, so the ordering picks which crash failure mode we accept. Both rules serve one invariant — *the request must not go terminal while still holding a slot* — because a terminal request is skipped by redelivery and by the DLQ reconciler alike, so nothing would ever decrement it. Failing this way leaves the request non-terminal: redelivery re-runs both steps and decrements again, transiently over-admitting by one slot until the zero clamp reconverges. Over-admission is the failure mode this pipeline already prefers, for the same reason and in the same words as the DLQ reconciler (see [process.md](process.md#in_flight_count-integrity)). +**Why the outcome is persisted first:** Queue and Request are separate entities. A durable outcome lets retries and DLQ reconciliation finish the same completion after a crash without guessing which build won. Slot ownership makes removal idempotent, including when a Queue write commits but its response is lost. Completion must run for terminal Requests as well as newly finished ones. -**Why `buildsignal` releases the slot rather than `record`**: the gate `process` claims is a *build* slot — it exists to bound concurrent builds per Queue — and once the build is terminal the build is over. Releasing here also keeps the invariant *a terminal `Request` has already released its slot*, which is what makes the DLQ reconciler's early-return on terminal requests safe. +**Why baseline advancement and slot release share a Queue write:** the next admitted build must see the successful baseline as soon as it sees freed capacity. Project resolution, immutable validation facts, history, and Git promotion belong to `record` and may finish later. A baseline is evidence of build success; deployments continue to gate on validation facts. **Why `record` hears only terminal signals**: `record` has no non-terminal work — by its own contract a non-terminal signal would be a pure no-op — and step 7 already branches on terminality to decide whether to keep polling, so gating the publish costs nothing and spares `record` a no-op delivery on every poll tick of every running build. Crash-safety is unaffected: a crash between the terminal `Update` and the publish redelivers the message; step 5 re-polls (the runner reports the same terminal status), step 6 no-ops, step 7 publishes. This is a deliberate divergence from SubmitQueue, whose buildsignal republishes to `speculate` on every tick — sound there because speculate is a state machine that may act on any signal; stovepipe has no such consumer. diff --git a/doc/rfc/stovepipe/steps/process.md b/doc/rfc/stovepipe/steps/process.md index 5ca099e6f..8490099d0 100644 --- a/doc/rfc/stovepipe/steps/process.md +++ b/doc/rfc/stovepipe/steps/process.md @@ -9,7 +9,7 @@ It handles everything *before* the first build is triggered: it does not run bui Ingest publishes a `ProcessRequest` (request id only — producer and consumer share the store) to the process topic, **partitioned by Queue name**. Two consequences the design relies on: 1. **One partition per Queue**, and `BatchSize = 1` (strict serialization; see [sql-queue-rfc.md](../../sql-queue-rfc.md#strict-serialization-opt-in)) — so at most one `process` invocation runs per Queue at a time. The read-modify-write on the Queue row (gate check, count increment) is therefore race-free **without** a transaction, the same property SubmitQueue's `validate` gets from per-queue partition leasing. -2. **`process` is not the only Queue-row writer.** `ingest` stamps `latest_request_id` (the request id of the newest accepted head, not a URI; see [Backlog coalescing](#backlog-coalescing)) and `buildsignal` releases slots when a build goes terminal, and `record` advances the `last_green_uri` bookmark. They touch different fields under optimistic-locking CAS, so concurrent writes converge instead of clobbering. +2. **`process` is not the only Queue-row writer.** `ingest` stamps `latest_request_id` (the request id of the newest accepted head, not a URI; see [Backlog coalescing](#backlog-coalescing)) and `buildsignal` releases slots when a build goes terminal, and advances the successful `last_green_uri` baseline in that same write. They touch different fields under optimistic-locking CAS, so concurrent writes converge instead of clobbering. ## Process algorithm @@ -19,17 +19,19 @@ For a delivery carrying request id `R`: 1. Load Request R from the request store. - not found -> non-retryable (storage is read-after-write consistent; see [storage README](../../../../stovepipe/extension/storage/README.md)). 2. If R.State is terminal (superseded / succeeded / failed / cancelled): - - ack and return (idempotent no-op). + - complete any owned reservation using the durable outcome, retain supersession history if applicable, then ack. 3. If R.State is processing (strategy already recorded): - re-announce validation start and re-publish R to build (either prior publish may have failed), ack, return. 4. R.State is accepted. Load the Queue row Q. 5. Coalesce: if CompareRequestID(R.Queue, R.ID, Q.latest_request_id) < 0: - a newer head exists -> mark R superseded, ack, return. (No slot consumed.) -6. R is the latest head. Gate: if Q.in_flight_count >= max_concurrent (from queue config; see below): +6. R is the latest head. Gate: if R does not already own a slot and Q.in_flight_count >= max_concurrent (from queue config; see below): - defer (hold the delivery) -> re-check on redelivery until the slot frees (admit) or a newer head supersedes it. See [Waiting for a slot](#waiting-for-a-slot). 7. Admit R: a. Derive build strategy + baseline (see "Build-strategy decision"). - b. CAS the Queue row: in_flight_count += 1. + b. CAS the Queue row: add R.ID to in_flight_request_ids once and recompute in_flight_count. + A preexisting reservation is reused even at capacity. A conflict reloads Q and + reruns coalescing, gate, and baseline derivation. c. CAS the Request: accepted -> processing, persist build_strategy + base_uri. d. Announce validation start on the hook topic (see "Hooks"). e. Publish R to build. @@ -59,15 +61,15 @@ Validation is expensive and shares a baseline, so heads arriving while an earlie | Source | Field | Meaning | |---|---|---| -| Queue row | `last_green_uri` | Bookmark `record` advances on whole-repo green; empty until first green. | -| Queue row | `in_flight_count` | Requests past `process` and not yet terminal. `process` increments on admit; `record` (or DLQ reconciliation) decrements on terminal. | +| Queue row | `last_green_uri` | Successful-build baseline advanced by buildsignal; empty until first successful build. | +| Queue row | `in_flight_count` | Count of owned build reservations; always equal to the length of `in_flight_request_ids`. | | Queue config | `max_concurrent` | Cap on concurrent in-flight validations. **Default 1** (global wiring default for MVP; per-queue override when a Stovepipe `queueconfig` extension lands). | A slot is held from admit until the build goes terminal (`process → build → buildsignal`), not just while `process` runs. It is released when the Request reaches **any** terminal state and `in_flight_count` is decremented — `buildsignal` recording the build's outcome, success *or* failure, or the DLQ reconciler forcing a terminal `failed` (see [integrity](#in_flight_count-integrity)). A build *failure* frees the slot just like a success; only a Request that never terminates keeps its slot. **Liveness — a stuck Request wedges the whole Queue.** The slot is shared per-Queue, so a Request admitted but never driven terminal holds the only slot and stalls the whole Queue. The fail-closed path in [workflow.md](../workflow.md#fail-closed-on-unprocessable-work) prevents this: every admitted Request terminates — `build`/`buildsignal` errors retry to `MaxAttempts`, then dead-letter, and the DLQ reconciler forces a conservative terminal `failed`, freeing the slot. Residual risk is operational: a **poison DLQ message** the reconciler can't process loops forever (it runs always-retryable), wedging that Queue until an operator removes it. The gate makes the blast radius the whole Queue, not one Request — monitor and alert on the DLQ. -**Future alternative — time-bounded leases.** To self-heal instead of relying on operator cleanup, `in_flight_count` could become a set of `{owner, expires_at}` leases: the gate counts only unexpired leases, terminal transitions drop the owner's lease, and a leaked lease is reclaimed on expiry — bounding any stall to a `max_validation_ms`. A *list* of leases would also generalize to `max_concurrent > 1`. Deferred; `in_flight_count` plus the fail-closed path is enough for MVP. +**Future alternative — time-bounded leases.** Reservations could gain expiry to bound recovery time for abandoned work. This needs a policy for still-running external builds and remains deferred; ownership and DLQ completion provide replay safety today. `max_concurrent` is a scalar, and `= 1` is an MVP simplicity choice, **not** a correctness requirement — see [Raising `max_concurrent`](#raising-max_concurrent-speculative-validation). @@ -111,11 +113,11 @@ Superseded Requests reach an explicit terminal `superseded` state, so "not yet v ## Concurrency lifecycle -The gate is **not** tied to `process` returning; a slot taken at admit is held until Phase 1 terminates at `record` (or DLQ reconciliation). +The gate is **not** tied to `process` returning; a slot taken at admit is held until the build completes at `buildsignal` (or DLQ reconciliation). **Rules** -1. **One slot per in-flight validation** (MVP: one per Queue). `process` increments `in_flight_count` on admit; `record` decrements on terminal. +1. **One slot per in-flight validation** (MVP: one per Queue). `process` increments `in_flight_count` on admit; `buildsignal` completes its owned reservation on terminal. 2. **No skip-ahead while in-flight.** The latest head waits for a slot until the running validation completes; it never preempts. 3. **Intermediates are superseded on sight**, gate open or closed — no slot consumed (step 5). 4. **Coalesce-to-latest on gate open.** When a slot frees, the waiting latest head is admitted. @@ -126,7 +128,7 @@ The gate is **not** tied to `process` returning; a slot taken at admit is held u 1. **A** admitted (`in_flight_count = 1`), published to `build`. 2. While A runs, **B**, **C**, **D** are ingested (`latest_request_id = D.id`). 3. B: older than D → **superseded** (acked), though A is still in flight. Same for **C**. D is latest but the gate is closed → **waits for slot** (held). -4. A's build finishes → `record` records A's greenness, `in_flight_count → 0`. +4. A's build finishes → `buildsignal` persists A's outcome and atomically advances its successful baseline and frees its slot; `record` can finish later. 5. D's re-check → gate open, D still latest → **D admitted**, published to `build`. 6. While D runs, **E**, **F** ingested (`latest_request_id = F.id`). E superseded on sight; F waits for slot. 7. D completes → slot frees → **F admitted**. @@ -135,7 +137,7 @@ A, D, F each get a full cycle; B, C, E end `superseded`. No intermediate is vali **What does not happen** -- `process` returning does **not** free a slot — only `record` (or DLQ reconciliation) does. +- `process` returning does **not** free a slot — only build completion (or DLQ reconciliation) does. - A newer head does **not** preempt an in-flight validation. - Deferred messages are **not** failed or dead-lettered — they wait for the gate (see [Waiting for a slot](#waiting-for-a-slot)). @@ -159,21 +161,16 @@ Only an admit announces. A Request that coalescing supersedes never reaches step Every branch is safe under redelivery: -- **accepted, no strategy** → full admit path. On a crash after incrementing `in_flight_count` but before persisting `processing`, redelivery re-reads `accepted` and re-runs; the increment re-applies only if the count CAS hasn't already moved (see integrity below). +- **accepted, no strategy** → full admit path. On a crash after incrementing `in_flight_count` but before persisting `processing`, redelivery reuses the reservation by Request id without incrementing the count again. - **processing** → re-announce the start event, re-publish to `build`, ack. The `build` consumer is keyed on the request id and idempotent, so a duplicate publish is harmless, and the start event's id is derived from the transition rather than the clock, so a re-announce carries the id the first attempt would have and consumers dedupe on it. Re-announcing here is what makes the event at-least-once rather than at-most-once: this is the only branch a redelivery takes once `processing` is durable, so an admit that failed after the state write would otherwise lose the event for good. -- **terminal** (superseded / recorded) → ack, no-op. +- **terminal** → finish any owned reservation from the durable outcome; never remove another owner. - **deferred (waiting for slot)** → no state or count change; pure deferral (re-enters when the held delivery comes due). -The window to handle is "count incremented, state not yet `processing`". Admit does the increment and the state transition as two ordered CAS writes, and the decrement is tied to the state transition, not a side counter (see integrity below). - ## `in_flight_count` integrity -`in_flight_count` is a cache; the source of truth is **the set of non-terminal Request rows for the Queue**. Two rules keep it from drifting: - -1. **Decrement is bound to the terminal transition.** The single CAS that moves a Request non-terminal → terminal (in `record` or the DLQ reconciler) also decrements. Being CAS-guarded, it fires exactly once per Request even under redelivery. -2. **Increment is bound to the admit transition.** `process` increments only on the `accepted → processing` CAS; a redelivery of an already-`processing` Request takes step 3 and does not increment again. +The Queue owns a bounded list of `in_flight_request_ids`; `in_flight_count` must equal its length. Admission validates this invariant and reserves by Request id under CAS before writing `processing`. A duplicate delivery reuses its existing reservation, even at capacity. A failed or ambiguous Request write keeps the reservation for retry or DLQ recovery; a losing writer must not release the winner's shared reservation. -On a crash between admit and `record`, the Request stays non-terminal; visibility-timeout redelivery drives it forward, and the fail-closed DLQ path eventually forces it terminal, decrementing as it does. The count can drift high only transiently and self-heals as stuck Requests terminate. A reconciler that recomputes the count from non-terminal rows can be added later if drift proves real, but isn't required for MVP. +Completion first persists the winning terminal Request outcome. A single Queue CAS removes only that Request's reservation and, for a newer successful Request, advances its baseline. Replays with no reservation do nothing. A crash between the two entity writes temporarily holds capacity; buildsignal redelivery, a terminal process delivery, or DLQ reconciliation completes it. No cross-entity transaction or zero clamp is needed. See [buildslot](../../../../stovepipe/core/buildslot/README.md). ## Edge cases @@ -191,8 +188,8 @@ Runtime coordination only — fields the pipeline writes under CAS: | Field | Role | Written by | |---|---|---| | `name` | Stable logical id (`monorepo/main`); the string ingest accepts | ingest (create) | -| `last_green_uri` | Bookmark; empty until first green | record | -| `in_flight_count` | Active Phase 1 validations | process (+1), record/DLQ (−1) | +| `last_green_uri`, `last_green_request_id` | Successful-build baseline; may precede recorded facts | buildsignal / terminal completion | +| `in_flight_count`, `in_flight_request_ids` | Count and owners of build reservations | process (reserve), completion (release) | | `latest_request_id` | Request id of the newest head ingest accepted | ingest | | `version` | Optimistic-locking version | all writers | @@ -222,7 +219,7 @@ Transitions use the repo's optimistic-locking pattern: compute `newVersion = old New key/value-shaped operations (single-key reads/writes, no server-side filtering or aggregation): -- **`QueueStore`** (new): `Create(ctx, queue)`, `Get(ctx, name)`, and `Update(ctx, queue, oldVersion, newVersion)` (CAS). Callers orchestrate get-or-create; ingest CASes `latest_request_id`; `process` CASes `in_flight_count`; `record` CASes `last_green_uri` + `in_flight_count`. +- **`QueueStore`** (new): `Create(ctx, queue)`, `Get(ctx, name)`, and `Update(ctx, queue, oldVersion, newVersion)` (CAS). Callers orchestrate get-or-create; ingest CASes `latest_request_id`; `process` CASes slot ownership and count; build completion CASes the baseline and slot ownership together. - **`RequestStore`**: no new methods — the added `Request` fields ride the existing `Create`/`Update` CAS. No "list requests by queue/state" query is introduced; coalescing uses the single-row `latest_request_id` pointer instead, keeping the contract satisfiable by a plain KV backend. diff --git a/doc/rfc/stovepipe/steps/record.md b/doc/rfc/stovepipe/steps/record.md index 5d0140027..e28e56e68 100644 --- a/doc/rfc/stovepipe/steps/record.md +++ b/doc/rfc/stovepipe/steps/record.md @@ -2,83 +2,30 @@ `record` turns a terminal build outcome into a durable validation fact. -- Phase 1 records whole-repository greenness. On green it also advances the Queue's last-green bookmark and promotes the commit onto the Queue's promotion ref, then announces the outcome as a hook event. This is implemented: see [stovepipe/controller/record/record.go](../../../../stovepipe/controller/record/record.go). -- Phase 2 records greenness per project instead of per repository. Sketched here, to be expanded before implementation. +`record` resolves optional project results before selecting the repository degree. It records the repository fact and project facts, then promotes a green repository fact while its Request still owns the successful-build baseline, and finally publishes the repository hook. See [record.go](../../../../stovepipe/controller/record/record.go). Notifying downstream systems rides the cross-domain hook framework rather than a Stovepipe-specific extension; see [Hooks](#hooks). See [workflow.md](../workflow.md) for the whole pipeline, [build.md](build.md) for how builds are created, and [buildsignal.md](buildsignal.md) for the terminal-only handoff into this stage. -`record` owns the validation facts and the two caches derived from them, the last-green bookmark and the promotion ref. It does not decide build scope, poll a build, release the Queue's build slot, interpret a target graph, or map targets to projects. Those belong to `process`, `buildsignal`, and `analyze`. +`record` owns validation facts and Git promotion. `buildsignal` owns the incremental-build baseline and build-slot completion. A successful build becomes available to the next build before project resolution finishes; external validation facts and promotion wait for resolution. -## Phase 1 algorithm +## Record algorithm -For a delivery carrying a `Record` payload: +1. Decode the request id and queue name, resolve storage, and load the Request. Reject a mismatched queue identity. +2. A succeeded or failed Request proceeds. A cancelled Request publishes `validation.repository.cancelled` without a fact. A superseded Request is acknowledged without effects; a nonterminal Request is an invariant error. +3. Resolve project results using the durable Request and its `TerminalBuildID`. Validate the entire result set before writing facts. Resolver errors leave facts and promotion untouched. +4. Select the repository degree: succeeded → `DegreeGreen`; failed with nonempty project results → `DegreePartial`; failed without project results → `DegreeBroken`. +5. Create the immutable repository fact and its history event. On `ErrAlreadyExists`, adopt the stored fact if it belongs to this Request; a different owner is an invariant error. +6. Create each project fact with its resolver-provided degree, then retain the project-facts-recorded history event. A partial write is completed on replay; existing facts remain authoritative. +7. For a green repository fact, read the Queue and promote only if both `LastGreenRequestID` and `LastGreenURI` still match the Request. Report the last-green timestamp when promotion is eligible. A non-green fact never promotes; the creator reports failure detection latency. +8. Publish `validation.repository.recorded` and acknowledge. A failed operation returns its raw error for backend classification. -``` -1. Deserialize the payload into (request id R, queue name Q). - - malformed -> return raw; non-retryable. - -2. Resolve the queue's storage from Q. - - unresolvable -> return raw; non-retryable. A queue with no storage is a - malformed message or a config error, not a transient condition. - -3. Load Request R. - - ErrNotFound -> return raw; non-retryable. - - other store error -> return raw; the classifier decides. - -4. If Q is non-empty and disagrees with R.Queue, return a non-retryable error. - The Request is the authoritative binding; Q is a routing hint, empty on - payloads written before the field existed. - -5. Inspect R.State. - - succeeded / failed -> continue. buildsignal stamps the outcome before it - publishes here, and both values are verdicts about the code. - - cancelled -> publish a validation.repository.cancelled HookEvent and ack, no - fact: the build decided nothing about the commit (see "When to record an outcome"). - - superseded -> ack, no fact. Unreachable in practice. - - accepted / processing / anything else -> return a non-retryable invariant error. - -6. Map R.State to a whole-repository degree and create the ValidationFact keyed by - (R.URI, empty project) in the queue-bound fact store. - - ErrAlreadyExists -> load and reconcile the existing immutable fact. - - other store error -> return raw. - -7. If the persisted fact is not green, skip to step 10: a break moves neither the - bookmark nor the ref, but is still announced. Report how long it went undetected - first, and only if step 6 is the write that created the fact (see "Observability"). - -8. Advance the bookmark to (R.URI, R.ID) in a CAS retry loop, which also reports - whether R holds the bookmark afterwards: - - stored id empty or older than R.ID -> write under the version guard; R holds it. - - stored id equals R.ID -> R set it on an earlier delivery; no write, - and R still holds it. - - stored id newer -> no write, and R does not hold it. - - ErrVersionMismatch -> reload and re-evaluate. - -9. If R holds the bookmark, ask SourceControl to point the promotion ref at R.URI. - Otherwise skip: whichever commit holds the bookmark owns the ref. - - ErrNotFound -> count and skip. A rewritten history dropped the commit from the - ref, and no retry can promote it. - - other error -> return raw. - -10. Resolve and record optional project results. - - failure -> return raw; the repository fact, bookmark, and promotion are not undone. - -11. Publish a validation.repository.recorded HookEvent naming the request whose fact - was persisted. - - publish failure -> return raw; non-retryable, so it dead-letters. - -12. ack. -``` - -Every decision after step 6 uses the persisted fact, not the outcome read from this delivery's Request. The first immutable fact controls the bookmark, the ref, and the hook event. - -Project-status processing does not delay baseline advancement or promotion. It still completes before the recorded notification. Bookmark advancement and ref promotion remain separate operations; this ordering does not make them atomic. +The repository fact can become visible before the final project fact is durable; recording is not a transaction across facts. Promotion and the recorded hook occur after all project writes and history complete. Project resolution does not delay the stored baseline, which was already advanced by build completion. ## Validation facts -A fact answers "how broken was this scope at this Queue URI?" Its identity is `(queue, uri, project)`, where `project` is empty for whole-repository greenness and a stable project id in Phase 2. +A fact answers "how broken was this scope at this Queue URI?" Its identity is `(queue, uri, project)`, where `project` is empty for whole-repository greenness and a stable project id. `queue` is the binding of the store the fact lives in, not a field on the entity. Storage is resolved per queue through `storage.Factory.For`, so a fact only ever exists inside one queue's store and does not need to name it. The column still leads the primary key, `PRIMARY KEY (queue, uri, project)`, because every domain table here has to be shardable by queue: the primary key leads with the queue column and no secondary index leads with anything else. That keeps one queue's rows unreachable through another queue's binding and makes every read a primary-key-prefix scan inside a single queue. It is an invariant rather than a per-table choice, and `make lint-queue-shard` enforces it. It is not MySQL table partitioning; no schema here uses `PARTITION BY`. @@ -88,8 +35,8 @@ So `entity.ValidationFact` carries only the rest of the identity: | Field | Meaning | | ----------- | ----------------------------------------------------------------------------------------------------------------- | | `URI` | Commit URI under validation | -| `Project` | Empty for the whole repository; stable project id for Phase 2 | -| `Degree` | Health degree in the closed interval `[entity.DegreeGreen, entity.DegreeBroken]` — `0` is green, `1` fully broken | +| `Project` | Empty for the whole repository; stable project id | +| `Degree` | Health degree in the closed interval `[entity.DegreeGreen, entity.DegreeBroken]` — `0` is green, `0.5` is partial, `1` fully broken | | `RequestID` | Request that established the fact | | `CreatedAt` | Millisecond timestamp at which the fact was first recorded | @@ -119,37 +66,28 @@ Whole-repository builds use only the endpoints, mapped from the outcome `buildsi | Request outcome | Result | | --------------- | ----------------------------- | | `succeeded` | fact at `entity.DegreeGreen` | -| `failed` | fact at `entity.DegreeBroken` | +| `failed`, project results available | fact at `entity.DegreePartial` (`0.5`) | +| `failed`, no project results | fact at `entity.DegreeBroken` (`1`) | | `cancelled` | no fact | -Intermediate degrees are reserved for project analysis and deferred with the project mapping contract. Phase 1 does not manufacture fractional values. +`DegreePartial` is hard coded to `0.5`; it records that project results are available for a failed repository, not the fraction of projects failing. Project degrees are preserved exactly as returned by the resolver. ## Last-green advancement (Queue bookmark) -The bookmark only moves forward. `entity.Queue` carries `LastGreenURI` alongside `LastGreenRequestID`, the request id that owns the current URI, plus `InFlightCount` and `LatestRequestID`. The guard compares ids with `entity.CompareRequestID(R.Queue, …)`, the same ingest-order comparison `ingest` and `process` use for coalescing. It errors on an id that does not match the queue's format, which is non-retryable: re-parsing the same ids cannot start succeeding. - -The write goes through `QueueStore.Update(ctx, queue, oldVersion, newVersion)`, so version arithmetic stays in the controller: it computes `newVersion = oldVersion + 1` and the store does a pure conditional write. `ErrVersionMismatch` is absorbed by the loop rather than returned, since a concurrent writer moved the row and reloading converges. - -**Why the bookmark moves only after the fact is durable.** The bookmark is a cache of "newest green URI" derived from the facts, so it must never lead them. Losing the advance to a crash is recoverable, since the redelivery reloads the same fact and retries. A bookmark with no fact behind it would point at greenness nothing recorded, and `process` would pick a baseline URI that no validation supports. - -A failed or cancelled build never moves the bookmark, and so never promotes either. +`buildsignal` advances `LastGreenURI` and `LastGreenRequestID` from the winning durable successful Request outcome, atomically with removal of its slot ownership. The bookmark moves only to a newer ingest id. It is a build baseline and may precede project results, validation facts, or Git promotion; absence of a fact remains fail-closed for deployment gating. Failed and cancelled outcomes never advance it. See [buildsignal.md](buildsignal.md). ## Promotion ref -A green commit is also promoted: `record` asks `SourceControl` to point the Queue's promotion ref, a stable branch name like `verified-main`, at that commit. This is the pull-shaped counterpart to the hook event's push, so a deploy gate or cache warmer can resolve a name and know nothing about Stovepipe, URIs, or degrees. See [Promotion ref](../workflow.md#promotion-ref--the-last-green-commit-by-name) in the pipeline overview. The stage names only the commit; which ref a Queue promotes to, and whether it has one at all, is integrator configuration injected into the `SourceControl` implementation alongside the endpoint and credentials, so a backend with no promotion target makes `Promote` a no-op. - -**Promotion is gated on holding the bookmark**, and only the holder promotes. Without that gate an older green commit recording late would drag the ref backwards: the ref has no version guard of its own, and `Promote` lands the URI it is given rather than refusing a non-descendant, so the caller has to enforce monotonicity. Deciding it from the bookmark instead of a fresh comparison means both caches move on one already-serialized decision. - -The equal case matters as much as the newer one. A redelivery that finds its own id on the bookmark reports that it *holds* it rather than skipping, so the promotion behind it is retried. That is what makes a crash between the bookmark write and the promotion recoverable, and it is safe because `Promote` is idempotent. Promotion follows the durable fact for the same reason the bookmark does: the ref is a cache of the facts and must not lead them. +Promotion requires a finalized green repository fact, all resolved project facts and history, and ownership of the current build baseline. Equal Request id and URI permit retrying an idempotent promotion; a newer baseline causes an older record delivery to skip promotion. This read guard and the external Git write are separate operations, not an atomic transaction. -`ErrNotFound` from `Promote` means a rewritten history dropped the commit from the ref. It is counted and acked rather than retried, since no retry will put that commit back and the next green commit corrects the ref. +`SourceControl.Promote` returning `ErrNotFound` means a history rewrite removed the commit from the ref; count and skip it. Other failures return to the classifier. ## Observability Two things are measured here, because this is where greenness becomes known. Both need a `SourceControl` lookup of a commit's creation time. -- **Last-green age.** A gauge carrying the creation timestamp of the commit the bookmark now points at, emitted once the bookmark write is durable. It is a timestamp rather than an elapsed time so that subtracting it at scrape time gives the current age, and a queue that stops going green keeps reporting a staler value without the stage re-emitting anything. +- **Last-green age.** A gauge carrying the creation timestamp of the commit the bookmark now points at, emitted after facts are durable while the Request owns the baseline. It is a timestamp rather than an elapsed time so that subtracting it at scrape time gives the current age, and a queue that stops going green keeps reporting a staler value without the stage re-emitting anything. - **Failure-detection latency.** A histogram of how long the break a build failed on went undetected, measured from the creation time of the base URI it validated against. A distribution rather than a gauge, because what matters is how long breaks typically survive, not how long the last one did. There is no later moment to sample it from, since an elapsed time only means something against the failure that just became known, so this lookup cannot move off the delivery path onto a clock. Only the delivery that *wrote* the fact reports the latency. That is why `recordFact` returns whether it created the fact and not just the fact itself: a redelivery adopts the stored fact, and a second sample would count one break twice. A full build pins no base URI, so its failures are counted as unmeasurable instead of timed; absent is the ordinary case there, not a fault. @@ -158,7 +96,7 @@ Both are best-effort. Every way the lookup can fail is counted, logged, and swal ## Build slot release -`Queue.in_flight_count` is released by `buildsignal` before it stamps the outcome and publishes to the record step. The DLQ reconciler releases the slot on the fail-closed path for the same reason. `record`'s only Queue write is the bookmark. +`buildsignal` first persists the winning Request outcome, then completes its slot and successful baseline in one Queue update, before publishing to record. The DLQ reconciler repairs the same completion even for an already-terminal Request. `record` never writes the Queue. ## Hooks @@ -173,10 +111,10 @@ The shared `HookEvent` contract at `api/base/hook/`, the hook extension at `plat Last thing before the ack, after the fact write and after both caches derived from it have moved: ``` -create fact → advance bookmark (green only) → promote (bookmark holder only) → publish HookEvent → [Phase 2: publish to analyze] → ack +resolve projects → create repository fact and history → create project facts and history → promote (green baseline holder only) → publish HookEvent → ack ``` -Last because the payload names the Request rather than snapshotting it, and hooks resolve entities from stores: a hook reacting to "URI is green" by reading `LastGreenURI`, or by fetching the promotion ref, must not find either still pointing at the previous commit. Inside the delivery rather than after it, because that is what makes the event lossless without an outbox — the state writes are recognize-and-skip on redelivery, so a crash before the ack replays the whole chain. +Last because hooks resolve entities from stores and must find the facts already durable and any eligible promotion complete. A newer successful build may already own the baseline while its record work is pending. Inside the delivery rather than after it, because that is what makes the event lossless without an outbox — the state writes are recognize-and-skip on redelivery, so a crash before the ack replays the whole chain. ### Event shape @@ -220,7 +158,7 @@ Phase 2 broadens "complete" to "all planned facts recorded", which needs a marke `ValidationFactStore` is key/value-shaped, and queue-bound rather than queue-parameterized: - `Create(ctx, fact)` writes one immutable fact for the bound queue. It returns `ErrAlreadyExists` when the composite identity is already taken, leaving the stored fact untouched. -- `Get(ctx, uri, project)` reads one fact by the rest of its identity, and returns `ErrNotFound` when none exists. The `project` argument is reserved for Phase 2. +- `Get(ctx, uri, project)` reads one fact by the rest of its identity, and returns `ErrNotFound` when none exists. The `project` argument is empty for the repository and otherwise names a resolved project. There is no `Update`. The first fact written for an identity is the permanent answer, and a caller that needs to know whether it won the race reads `ErrAlreadyExists` and then loads the winner. @@ -248,7 +186,6 @@ Partitioning by request id keeps completion bookkeeping single-writer per Reques | Request carrying no build outcome | non-retryable | producer/state-machine invariant violation | | Existing fact owned by a different Request | non-retryable | ingest dedup invariant violated; the stored fact is immutable | | Malformed request id at bookmark comparison | non-retryable | re-parsing the same ids cannot succeed | -| Queue CAS version mismatch | absorbed, not raised | `storage.ErrVersionMismatch` is handled by the bookmark loop: reload and re-apply | | Promotion target unknown to the ref | absorbed, not raised | `sourcecontrol.ErrNotFound` means a rewritten history dropped the commit | | SourceControl resolution or `Promote` failure | raw error | backend classifier has the required failure knowledge | | Commit-timestamp lookup for a metric | swallowed | counted and logged; observability must not retry a recorded outcome | @@ -268,9 +205,9 @@ Every effect is recognize-and-skip, so a redelivery after a complete run re-runs - **Green fact recorded out of order across Requests.** An older green commit can reach this stage after a newer one. Its fact is written as usual, since facts are per-URI and independent, but the bookmark guard skips it, and because it does not hold the bookmark it does not promote either. Neither cache moves backwards. - **Cancelled build.** Stovepipe never initiates cancellation, but a backend may still report it. No fact is written; the slot was already released and the Request already stamped `cancelled`. The fact identity stays unclaimed, and nothing can claim it today, since `cancelled` is terminal and re-validation does not exist. Recovery in practice is the next commit; the unclaimed identity only matters to a future re-run mechanism (see [Supporting re-run of the same URI](#supporting-re-run-of-the-same-uri)). - **History rewrite while a build runs.** The Request keeps the strategy and URI pinned at admission, and record stores the fact about that immutable URI. A later head is handled independently by `process`. If the rewrite dropped the commit from the ref, the fact and the bookmark still stand, since they describe a commit and not a ref, and only the promotion is skipped. -- **A fail-closed terminal outranks a build that passed.** The degree derives from `R.State`, so a Request forced to `failed` by DLQ reconciliation records `DegreeBroken` even when one of its builds reports success afterwards. Reachable today, and permanent once written; see [DLQ and fail-closed behavior](#dlq-and-fail-closed-behavior). -- **Crash between the fact write and the bookmark advance.** Redelivery reloads the existing fact and re-applies the idempotent guard. -- **Crash between the bookmark advance and the promotion.** Redelivery finds its own id on the bookmark, reports that it holds it, and retries the idempotent promotion. +- **A fail-closed terminal outranks a build that passed.** The degree derives from `R.State`, so a Request forced to `failed` by DLQ reconciliation records a non-green degree even when one of its builds reports success afterwards. Reachable today, and permanent once written; see [DLQ and fail-closed behavior](#dlq-and-fail-closed-behavior). +- **Crash during fact persistence.** Redelivery resolves projects again, adopts existing immutable facts, and completes missing facts and history before promotion. +- **Crash before promotion.** Redelivery retries promotion if the Request still owns the baseline. @@ -280,7 +217,7 @@ Every primary subscription comes from `DefaultSubscriptionConfig`, which enables Two different things put a message there, and only one is a poison payload. A delivery that fails with its retry budget spent is dead-lettered by the nack itself, carrying the reason it actually failed. A delivery that never reaches a nack, because it crashed or because its **ack failed** and the visibility timeout redelivered it, is dead-lettered by the poll loop once `retry_count` reaches `MaxAttempts` (3 by default), without the controller running on that final attempt and with only a generic reason recorded. So a missing reconciler exposes more than malformed messages: a fact can be lost to a storage failure that would have succeeded on a later retry, or to an ack that never landed even though the write did. -Gating stays safe, because everything this stage can lose reads as not-green: a Request with no fact is indistinguishable from one not yet validated. What is lost is the *fact*. A green build whose fact write permanently failed leaves the URI looking unvalidated, which costs the queue an incremental baseline and forces a full build at the next head. A lost notification joins that list, and unlike the fact it gets no second chance from a later commit. +Gating stays safe, because everything this stage can lose reads as not-green: a Request with no fact is indistinguishable from one not yet validated. What is lost is the *fact*. A green build whose fact write permanently failed leaves the URI looking unvalidated, while its successful-build baseline remains available to the next head. A lost notification joins that list, and unlike the fact it gets no second chance from a later commit. The reconciler does not re-run the stage. It loads the Request only to identify existing durable state and retain a stable `record_abandoned` event. The DLQ consumer uses `errs.AlwaysRetryableProcessor`, so a transient Request read or history persistence failure keeps retrying until that observable abandonment is durable. A malformed payload, invalid or unresolvable queue identity, or missing Request cannot be repaired by redelivery; those cases are logged, counted, and acknowledged so poison cannot occupy the DLQ indefinitely. No DLQ path creates a fact, advances a bookmark, invokes source control, or publishes a hook. diff --git a/doc/rfc/stovepipe/workflow.md b/doc/rfc/stovepipe/workflow.md index 4383a0175..684e53a0e 100644 --- a/doc/rfc/stovepipe/workflow.md +++ b/doc/rfc/stovepipe/workflow.md @@ -117,12 +117,12 @@ What runs is one pass per Request. It establishes whole-repository greenness, an 1. **ingest** — invoked by the external poller with a **Queue name**. It asks `SourceControl` for that Queue's current head URI, mints a Request namespaced by the Queue, persists it with no recorded greenness yet, and dedups on `(Queue, head URI)` so a re-reported head is processed once. It publishes the RequestID onward. 2. **process** — decides build strategy (incremental since last-green vs full monorepo), gates concurrent work per Queue, coalesces backlog to the latest head, publishes a **hook event** announcing that validation of the commit has begun, and publishes to `build`. See [process.md](steps/process.md). 3. **build** — runs the build-runner for the chosen scope. A flag derived from `process` decides whether to build relative to the last-green **baseline URI** (incremental) or from scratch (full). It records a build and publishes the BuildID. -4. **buildsignal** — polls until the build is terminal, records that status, releases the Queue's `in_flight_count` slot, projects the terminal status and build id onto the Request (`succeeded` / `failed` / `cancelled`), and publishes the RequestID to `record`. -5. **record** — for a succeeded or failed Request, writes the whole-repo greenness for the head URI (`0` green / `1` broken to start), derived from the Request's build outcome. On green it advances the Queue's **last-green URI** so the next `process` can build incrementally from here, and asks `SourceControl` to advance the Queue's **promotion ref** to the same commit (see [Promotion ref](#promotion-ref--the-last-green-commit-by-name)). It then gives the Request's terminal build id to `projectstatus.Resolver` and writes one validation fact per returned result. The example server uses the noop resolver, so that list is empty unless a deployment supplies another. It publishes `validation.repository.recorded` for that fact. The Queue's `in_flight_count` was already released by `buildsignal` when the build went terminal. A cancelled Request writes no fact and publishes `validation.repository.cancelled`, so a consumer can stop waiting on the commit. +4. **buildsignal** — polls until terminal, persists the winning Request outcome and terminal build id, then atomically completes its owned Queue slot and advances a newer successful-build baseline. It publishes the RequestID to record and a terminal build-analysis nudge. +5. **record** — resolves project results, then records repository degree `0` for success, `0.5` for failure with project results, or `1` for failure without results. It persists project facts with their resolved degrees. After facts and history are durable, a green fact owning the current baseline promotes in Git; the repository recorded hook follows. A cancelled Request writes no fact and publishes `validation.repository.cancelled`. ### Designed, not built: project analysis -An `analyze` stage is not part of the pipeline. `stovepipe/core/messagequeue` has no analyze topic, and `stovepipe/controller` has no analyze package. The design, still only a design, is a stage that would take a build's target graph, map broken or at-risk targets to projects, and publish project-scoped builds back through `build` → `buildsignal` → `record`. That second pass, a per-project hook event, and intermediate greenness degrees are not what the controllers do. Open questions for that design stay at the bottom of this doc. +An `analyze` stage is not part of the pipeline. `stovepipe/core/messagequeue` has no analyze topic, and `stovepipe/controller` has no analyze package. The design, still only a design, is a stage that would take a build's target graph, map broken or at-risk targets to projects, and publish project-scoped builds back through `build` → `buildsignal` → `record`. That second pass and a per-project hook event are not what the controllers do. Record currently uses the fixed DegreePartial value when project results are available for a failed repository. Open questions for that design stay at the bottom of this doc. ## Per-controller summary @@ -131,8 +131,8 @@ An `analyze` stage is not part of the pipeline. `stovepipe/core/messagequeue` ha | **ingest** | Queue name (from poller) | process | Resolve head URI via SourceControl, mint Request, persist (no greenness), dedup on `(Queue, head URI)` | | **process** | RequestID | build, hook topic | Build strategy, concurrency gate, backlog coalescing; announce validation start on admit → [process.md](steps/process.md) | | **build** | RequestID | buildsignal | Run the build-runner for the chosen scope; baseline = last-green URI iff incremental | -| **buildsignal** | BuildID | record | Record terminal build status; release `in_flight_count`; project the outcome onto the Request; publish the request id | -| **record** | RequestID | hook topic | Write whole-repo greenness; resolve project results inline; on green advance last-green URI and the promotion ref; publish the recorded or cancelled repository hook event | +| **buildsignal** | BuildID | record, buildanalysis | Persist the winning outcome; complete slot ownership and successful baseline together; publish downstream work | +| **record** | RequestID | hook topic | Resolve projects, record repository and project facts; promote a finalized green baseline owner; publish the repository hook | | **hook** | HookEvent | — | Invoke the hooks the resolver returns. The example server resolves every event to noop | ## Step RFCs diff --git a/stovepipe/controller/build/build_analysis_test.go b/stovepipe/controller/build/build_analysis_test.go index e6b2cfcfd..e1b63d8d7 100644 --- a/stovepipe/controller/build/build_analysis_test.go +++ b/stovepipe/controller/build/build_analysis_test.go @@ -43,7 +43,7 @@ func TestBuildAnalysisPipeline(t *testing.T) { var savedBuild entity.Build request := processingRequest(entity.BuildStrategyFull, "") - queue := entity.Queue{Name: testQueue, InFlightCount: 1, Version: 1} + queue := entity.Queue{Name: testQueue, InFlightCount: 1, InFlightRequestIDs: []string{testID}, Version: 1} queueStore := storagemock.NewMockQueueStore(ctrl) issueStore := storagemock.NewMockBuildIssueStore(ctrl) diff --git a/stovepipe/controller/buildsignal/BUILD.bazel b/stovepipe/controller/buildsignal/BUILD.bazel index 77ac7b005..a327d9c31 100644 --- a/stovepipe/controller/buildsignal/BUILD.bazel +++ b/stovepipe/controller/buildsignal/BUILD.bazel @@ -11,6 +11,7 @@ go_library( "//platform/errs:go_default_library", "//platform/metrics:go_default_library", "//platform/publish:go_default_library", + "//stovepipe/core/buildslot:go_default_library", "//stovepipe/core/loader:go_default_library", "//stovepipe/core/messagequeue:go_default_library", "//stovepipe/core/requestlog:go_default_library", diff --git a/stovepipe/controller/buildsignal/buildsignal.go b/stovepipe/controller/buildsignal/buildsignal.go index 32ceb7be4..c518e5e86 100644 --- a/stovepipe/controller/buildsignal/buildsignal.go +++ b/stovepipe/controller/buildsignal/buildsignal.go @@ -15,7 +15,7 @@ // Package buildsignal holds the buildsignal-stage queue controller. It consumes // BuildSignal messages (a build id), polls the build-runner until the build // reaches a terminal status, persists that status, and — once terminal — -// releases the queue's build slot, projects the outcome onto the request, and +// persists the request outcome, completes its build slot and baseline, and // publishes directly to record plus terminal analysis. See // doc/rfc/stovepipe/steps/buildsignal.md. package buildsignal @@ -31,6 +31,7 @@ import ( "github.com/uber/submitqueue/platform/errs" "github.com/uber/submitqueue/platform/metrics" "github.com/uber/submitqueue/platform/publish" + "github.com/uber/submitqueue/stovepipe/core/buildslot" "github.com/uber/submitqueue/stovepipe/core/loader" stovepipemq "github.com/uber/submitqueue/stovepipe/core/messagequeue" "github.com/uber/submitqueue/stovepipe/core/requestlog" @@ -54,11 +55,7 @@ var ( PollDelayRunningMs int64 = 2000 ) -// Controller consumes BuildSignal messages, polls the build-runner toward a -// terminal status, persists the result, and either holds the delivery until -// the next poll or releases the queue's build slot, projects the outcome onto -// the request, and publishes the request id to record. Implements -// consumer.Controller. +// Controller polls builds and completes request outcomes before publishing to record. type Controller struct { logger *zap.SugaredLogger metricsScope tally.Scope @@ -96,12 +93,7 @@ func NewController( } } -// Process reloads the build referenced by the delivery, polls its runner for -// the latest status, persists a real transition, and either holds the delivery -// until the next poll or, once terminal, releases the queue's build slot, -// projects the outcome onto the request, and publishes the request id to -// record. Returns nil to ack (success) or an error to nack (retry) / reject -// (DLQ). +// Process polls a build, completes its terminal outcome and baseline, and publishes downstream work. func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) error { msg := delivery.Message() @@ -140,17 +132,6 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er return fmt.Errorf("payload queue %q does not match queue %q of request %s", sig.GetQueueName(), request.Queue, request.ID) } - // A request only reaches the poll loop after process admitted it, so it is - // `processing` — or it already carries this build's outcome, on a redelivery - // after the outcome was stamped but before record was published. Both cases - // proceed: the redelivery re-publishes to record rather than dropping the - // signal, and every write below is a no-op the second time. - // - // Anything else is terminal without a build outcome, which today means only - // `superseded` — and process supersedes solely from `accepted` (see - // supersedeRequest), so a superseded request can never reach this point. It - // is guarded rather than assumed away because finishRequest would otherwise - // release a build slot the request never claimed: superseding consumes none. if request.State != entity.RequestStateProcessing && !request.State.HasBuildOutcome() { return nil } @@ -225,34 +206,14 @@ func (c *Controller) persistBuildFinishedLog(ctx context.Context, store storage. return nil } -// finishRequest releases the queue's build slot and projects the build's -// terminal status and ID onto the request, leaving it terminal. It is a no-op -// when the request already carries an outcome, so a redelivery neither -// double-releases the slot nor rewrites the state. -// -// The slot is released *before* the terminal write, and a failed release aborts -// the write, matching the DLQ reconciler (see stovepipe/controller/dlq/dlq.go). -// Both rules exist for the same reason: the request must not go terminal while -// still holding a slot, because a terminal request is skipped on redelivery and -// by the reconciler, so nothing would ever decrement it. Failing this way leaves -// the request non-terminal, so redelivery re-runs both steps and decrements again -// — transiently over-admitting by one until releaseBuildSlot's zero clamp -// reconverges, which is the failure mode this pipeline prefers. +// finishRequest preserves the winning outcome before atomically releasing its slot +// and advancing a successful baseline. Recovery can repeat both operations. func (c *Controller) finishRequest(ctx context.Context, store storage.Storage, request *entity.Request, buildID string, status entity.BuildStatus) error { - if request.State.HasBuildOutcome() { - return nil - } - - if err := c.releaseBuildSlot(ctx, store, request.Queue); err != nil { - metrics.NamedCounter(c.metricsScope, _opName, "storage_errors", 1, metrics.TagsFromContext(ctx)...) - return err - } - if err := c.markOutcome(ctx, store, request, buildID, outcomeState(status)); err != nil { metrics.NamedCounter(c.metricsScope, _opName, "storage_errors", 1, metrics.TagsFromContext(ctx)...) return err } - return nil + return buildslot.Complete(ctx, store, *request) } func (c *Controller) persistOutcomeLog(ctx context.Context, store storage.Storage, request entity.Request) error { @@ -329,37 +290,6 @@ func (c *Controller) markOutcome(ctx context.Context, store storage.Storage, req } } -// releaseBuildSlot CAS-decrements the queue's in_flight_count, reopening the process -// concurrency gate now that this request's build is over. It decrements relatively -// (preserving concurrent updates), clamps at zero, and retries on version conflicts. -// Unlike process's unwind-path release this is not best-effort: the caller must not -// mark the request terminal if the slot was not freed, so a hard failure is returned. -func (c *Controller) releaseBuildSlot(ctx context.Context, store storage.Storage, queueName string) error { - queueStore := store.GetQueueStore() - - for { - queueRow, err := queueStore.Get(ctx, queueName) - if err != nil { - return fmt.Errorf("failed to load queue %s to release build slot: %w", queueName, err) - } - if queueRow.InFlightCount <= 0 { - return nil - } - - updated := queueRow - updated.InFlightCount = queueRow.InFlightCount - 1 - newVersion := queueRow.Version + 1 - if err := queueStore.Update(ctx, updated, queueRow.Version, newVersion); err != nil { - if errors.Is(err, storage.ErrVersionMismatch) { - continue - } - return fmt.Errorf("failed to release build slot for queue %s: %w", queueName, err) - } - metrics.NamedCounter(c.metricsScope, _opName, "slot_released", 1, metrics.TagsFromContext(ctx)...) - return nil - } -} - // reconcile persists a real status transition and returns the status that // should drive the rest of Process: the polled status when persisted (or // already unchanged), or the stored status when a stored terminal status is diff --git a/stovepipe/controller/buildsignal/buildsignal_test.go b/stovepipe/controller/buildsignal/buildsignal_test.go index d789f839e..97a99aab5 100644 --- a/stovepipe/controller/buildsignal/buildsignal_test.go +++ b/stovepipe/controller/buildsignal/buildsignal_test.go @@ -156,6 +156,7 @@ func requestWithState(state entity.RequestState) entity.Request { request := entity.Request{ ID: testID, Queue: testQueue, + URI: "git://repo/main/green", State: state, Version: 1, } @@ -177,18 +178,28 @@ func build(status entity.BuildStatus, version int32) entity.Build { // queueRow returns the testQueue's row holding inFlight admitted validations. func queueRow(inFlight, version int32) entity.Queue { + var owners []string + if inFlight > 0 { + owners = []string{testID} + } return entity.Queue{ - Name: testQueue, - InFlightCount: inFlight, - Version: version, + Name: testQueue, + InFlightCount: inFlight, + InFlightRequestIDs: owners, + Version: version, } } func expectFinishWrites(m buildsignalMocks, state entity.RequestState) *gomock.Call { eventCall := expectBuildFinished(m) - m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(queueRow(1, 4), nil).After(eventCall) - m.queueStore.EXPECT().Update(gomock.Any(), queueRow(0, 4), int32(4), int32(5)).Return(nil) - return m.reqStore.EXPECT().Update(gomock.Any(), requestWithState(state), int32(1), int32(2)).Return(nil) + outcomeCall := m.reqStore.EXPECT().Update(gomock.Any(), requestWithState(state), int32(1), int32(2)).Return(nil).After(eventCall) + getCall := m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(queueRow(1, 4), nil).After(outcomeCall) + updated := queueRow(0, 4) + if state == entity.RequestStateSucceeded { + updated.LastGreenURI = requestWithState(state).URI + updated.LastGreenRequestID = testID + } + return m.queueStore.EXPECT().Update(gomock.Any(), updated, int32(4), int32(5)).Return(nil).After(getCall) } func expectBuildFinished(m buildsignalMocks) *gomock.Call { @@ -382,6 +393,7 @@ func TestProcess(t *testing.T) { // No build/queue/request write: the status is unchanged and the outcome // is already recorded, so the slot must not be released a second time. eventCall := expectBuildFinished(m) + m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(queueRow(0, 5), nil).After(eventCall) logCall := expectOutcomeLog(m, entity.RequestStateSucceeded, 2).After(eventCall) m.publisher.EXPECT().Publish(gomock.Any(), "record", gomock.Any()).Return(nil).After(logCall) m.publisher.EXPECT().Publish(gomock.Any(), "buildanalysis", gomock.Any()).Return(nil).After(logCall) @@ -419,6 +431,7 @@ func TestProcess(t *testing.T) { m.runnerFactory.EXPECT().For(buildrunner.Config{QueueName: testQueue}).Return(m.runner, nil) m.runner.EXPECT().Status(gomock.Any(), entity.BuildID{ID: testBuildID}).Return(entity.BuildStatusSucceeded, nil, nil) eventCall := expectBuildFinished(m) + m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(queueRow(0, 5), nil).After(eventCall) log := requestlog.NewRequestStateLog(request, entity.RequestOutcomeReasonBuildSucceeded) log.Metadata[requestlog.MetadataKeyBuildID] = request.TerminalBuildID m.materializer.EXPECT().PersistLog( @@ -476,13 +489,11 @@ func TestProcess(t *testing.T) { m.runnerFactory.EXPECT().For(buildrunner.Config{QueueName: testQueue}).Return(m.runner, nil) m.runner.EXPECT().Status(gomock.Any(), entity.BuildID{ID: testBuildID}).Return(entity.BuildStatusSucceeded, nil, nil) eventCall := expectBuildFinished(m) - m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(queueRow(1, 4), nil).After(eventCall) - m.queueStore.EXPECT().Update(gomock.Any(), queueRow(0, 4), int32(4), int32(5)).Return(nil) - m.reqStore.EXPECT().Update(gomock.Any(), gomock.Any(), int32(1), int32(2)).Return(errors.New("db down")) + m.reqStore.EXPECT().Update(gomock.Any(), gomock.Any(), int32(1), int32(2)).Return(errors.New("db down")).After(eventCall) }, }, { - name: "slot release failure leaves the request non-terminal", + name: "slot release failure preserves the terminal outcome for recovery", wantErr: true, wantRetry: false, setup: func(m buildsignalMocks) { @@ -491,10 +502,8 @@ func TestProcess(t *testing.T) { m.runnerFactory.EXPECT().For(buildrunner.Config{QueueName: testQueue}).Return(m.runner, nil) m.runner.EXPECT().Status(gomock.Any(), entity.BuildID{ID: testBuildID}).Return(entity.BuildStatusSucceeded, nil, nil) eventCall := expectBuildFinished(m) + m.reqStore.EXPECT().Update(gomock.Any(), requestWithState(entity.RequestStateSucceeded), int32(1), int32(2)).Return(nil).After(eventCall) m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{}, errors.New("db down")).After(eventCall) - // No request Update and no record publish: marking the request terminal - // while it still holds a slot would strand the slot, since a terminal - // request is skipped by both redelivery and the DLQ reconciler. }, }, { @@ -645,6 +654,7 @@ func TestTerminalAnalysisPublicationDoesNotGateRecordPublication(t *testing.T) { m.runnerFactory.EXPECT().For(buildrunner.Config{QueueName: testQueue}).Return(m.runner, nil) m.runner.EXPECT().Status(gomock.Any(), entity.BuildID{ID: testBuildID}).Return(entity.BuildStatusFailed, nil, nil) + m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(queueRow(0, 5), nil) m.materializer.EXPECT().PersistLog(gomock.Any(), m.store, gomock.Any()).Return(nil).Times(2) m.publisher.EXPECT().Publish(gomock.Any(), "record", gomock.Any()).Return(tt.recordErr) @@ -697,3 +707,26 @@ func TestNonterminalPollDoesNotPublishTerminalAnalysis(t *testing.T) { require.NoError(t, c.Process(queueContext(), d)) } + +func TestFinishRequestRecoversDurableOutcomeBeforeQueueCompletion(t *testing.T) { + ctrl := gomock.NewController(t) + c, m := newController(t, ctrl) + request := requestWithState(entity.RequestStateProcessing) + durable := requestWithState(entity.RequestStateSucceeded) + durable.Version = 2 + queue := queueRow(1, 4) + + outcome := m.reqStore.EXPECT().Update(gomock.Any(), requestWithState(entity.RequestStateSucceeded), int32(1), int32(2)).Return(nil) + m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{}, assert.AnError).After(outcome) + require.ErrorIs(t, c.finishRequest(queueContext(), m.store, &request, testBuildID, entity.BuildStatusSucceeded), assert.AnError) + assert.Equal(t, durable, request) + + m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(queue, nil) + updated := queueRow(0, 4) + updated.LastGreenURI = durable.URI + updated.LastGreenRequestID = durable.ID + m.queueStore.EXPECT().Update(gomock.Any(), updated, int32(4), int32(5)).Return(nil) + // A different delivery's failure cannot replace the durable successful outcome. + require.NoError(t, c.finishRequest(queueContext(), m.store, &request, "other-build", entity.BuildStatusFailed)) + assert.Equal(t, durable, request) +} diff --git a/stovepipe/controller/dlq/BUILD.bazel b/stovepipe/controller/dlq/BUILD.bazel index da7c418f1..906825438 100644 --- a/stovepipe/controller/dlq/BUILD.bazel +++ b/stovepipe/controller/dlq/BUILD.bazel @@ -14,6 +14,7 @@ go_library( "//platform/base/messagequeue:go_default_library", "//platform/consumer:go_default_library", "//platform/metrics:go_default_library", + "//stovepipe/core/buildslot:go_default_library", "//stovepipe/core/messagequeue:go_default_library", "//stovepipe/core/requestlog:go_default_library", "//stovepipe/entity:go_default_library", diff --git a/stovepipe/controller/dlq/build_test.go b/stovepipe/controller/dlq/build_test.go index c97686202..3cac9a138 100644 --- a/stovepipe/controller/dlq/build_test.go +++ b/stovepipe/controller/dlq/build_test.go @@ -67,7 +67,7 @@ func TestBuildProcess(t *testing.T) { name: "processing request is failed", setup: func(m dlqMocks) { m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(requestWithState(entity.RequestStateProcessing), nil) - m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{Name: testQueue, InFlightCount: 1, Version: 5}, nil) + m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{Name: testQueue, InFlightCount: 1, InFlightRequestIDs: []string{testID}, Version: 5}, nil) m.queueStore.EXPECT().Update(gomock.Any(), entity.Queue{Name: testQueue, Version: 5}, int32(5), int32(6)).Return(nil) updated := requestWithState(entity.RequestStateProcessing) updated.State = entity.RequestStateFailed diff --git a/stovepipe/controller/dlq/buildsignal.go b/stovepipe/controller/dlq/buildsignal.go index 69b2b9399..1b5b6bb43 100644 --- a/stovepipe/controller/dlq/buildsignal.go +++ b/stovepipe/controller/dlq/buildsignal.go @@ -149,10 +149,6 @@ func (c *buildSignalController) Process(ctx context.Context, delivery consumer.D return nil } - // Every request reachable from a build row is either still processing, and holding - // the slot failRequest releases, or already terminal, and past releasing it: build - // triggers only once process has written the strategy, which lands in the same CAS - // as accepted→processing, and processing exits only to a terminal outcome. if err := failRequest(ctx, store, c.materializer, c.logger, build.RequestID, entity.RequestOutcomeReasonBuildPollingExhausted); err != nil { metrics.NamedCounter(c.metricsScope, _buildSignalOpName, "reconcile_errors", 1, metrics.TagsFromContext(ctx)...) return err diff --git a/stovepipe/controller/dlq/buildsignal_test.go b/stovepipe/controller/dlq/buildsignal_test.go index 08c0f4aab..9069b2706 100644 --- a/stovepipe/controller/dlq/buildsignal_test.go +++ b/stovepipe/controller/dlq/buildsignal_test.go @@ -98,12 +98,12 @@ func TestBuildSignalProcess(t *testing.T) { { // The case this reconciler exists for: a poll message that // dead-lettered while its request was holding a build slot. - name: "processing request releases the queue slot before marking failed", + name: "processing request marks failed before releasing its queue slot", setup: func(m buildSignalDLQMocks) { m.buildStore.EXPECT().Get(gomock.Any(), testBuildID).Return(build(), nil) m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(requestWithState(entity.RequestStateProcessing), nil) m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{ - Name: testQueue, InFlightCount: 1, Version: 5, + Name: testQueue, InFlightCount: 1, InFlightRequestIDs: []string{testID}, Version: 5, }, nil) m.queueStore.EXPECT().Update(gomock.Any(), entity.Queue{ Name: testQueue, InFlightCount: 0, Version: 5, @@ -121,10 +121,12 @@ func TestBuildSignalProcess(t *testing.T) { }, }, { - name: "already terminal request is a no-op", + name: "already terminal request repairs its outcome log", setup: func(m buildSignalDLQMocks) { m.buildStore.EXPECT().Get(gomock.Any(), testBuildID).Return(build(), nil) m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(requestWithState(entity.RequestStateSucceeded), nil) + m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{Name: testQueue}, nil) + m.materializer.EXPECT().PersistLog(gomock.Any(), m.store, requestlog.NewRequestStateLog(requestWithState(entity.RequestStateSucceeded), entity.RequestOutcomeReasonBuildSucceeded)).Return(nil) }, }, { @@ -149,10 +151,12 @@ func TestBuildSignalProcess(t *testing.T) { }, }, { - name: "slot release failure aborts the terminal write", + name: "slot release failure retains terminal outcome for retry", setup: func(m buildSignalDLQMocks) { m.buildStore.EXPECT().Get(gomock.Any(), testBuildID).Return(build(), nil) m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(requestWithState(entity.RequestStateProcessing), nil) + updated := requestWithState(entity.RequestStateFailed) + m.reqStore.EXPECT().Update(gomock.Any(), updated, int32(2), int32(3)).Return(nil) m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{}, assert.AnError) }, wantErr: true, diff --git a/stovepipe/controller/dlq/dlq.go b/stovepipe/controller/dlq/dlq.go index cbde8e207..7be209c31 100644 --- a/stovepipe/controller/dlq/dlq.go +++ b/stovepipe/controller/dlq/dlq.go @@ -22,16 +22,8 @@ // on greenness would see that indistinguishably from "not yet validated" (see // doc/rfc/stovepipe/workflow.md#fail-closed-on-unprocessable-work). // -// Reconciliation strategy. Each DLQ topic carries the same payload as its originating -// topic (the queue framework preserves the bytes verbatim under a new `{topic}_dlq` -// name). The DLQ controller decodes that payload to recover the affected request, then -// transitions it to RequestStateFailed — the conservative unsuccessful outcome, which -// whatever records greenness treats as not-green for gating (see entity.RequestState) — -// with an idempotent optimistic-locking write so -// concurrent activity (a late successful pipeline transition) wins cleanly. If the request had -// already been admitted (processing) and was holding a concurrency slot, the -// reconciler also releases it by CAS-decrementing the queue's in_flight_count, per -// doc/rfc/stovepipe/steps/process.md#in_flight_count-integrity. +// Reconciliation preserves the first terminal outcome, then completes the request-owned +// build slot and successful baseline. Terminal requests remain repairable on replay. package dlq import ( @@ -40,6 +32,7 @@ import ( "fmt" "github.com/uber/submitqueue/platform/consumer" + "github.com/uber/submitqueue/stovepipe/core/buildslot" "github.com/uber/submitqueue/stovepipe/core/requestlog" "github.com/uber/submitqueue/stovepipe/entity" "github.com/uber/submitqueue/stovepipe/extension/storage" @@ -59,32 +52,7 @@ func TopicKey(main consumer.TopicKey) consumer.TopicKey { return consumer.TopicKey(string(main) + topicSuffix) } -// failRequest transitions request to RequestStateFailed if it is not already -// in a terminal state. If the request had reached RequestStateProcessing — meaning process's -// admit step already CAS-incremented the queue's in_flight_count for it and no terminal -// outcome has released it yet — the queue's -// slot is released first. -// -// Processing is the only non-terminal state that can own a slot, so the condition is not a -// narrowing of some broader set: process claims the slot and CAS-marks accepted→processing, -// releasing its own claim if that CAS never lands, and the exits from processing are the -// terminal outcomes, which release the slot themselves. Widening the release to accepted -// would decrement for the far more common request that never claimed a slot, over-admitting -// against MaxConcurrent. The one case that escapes both this reconciler and process's -// compensation is a hard crash between the two admit writes, which leaves an accepted -// request holding a slot that nothing here can tell apart from a request that never -// claimed one; distinguishing them needs per-request slot ownership on the row. -// -// Queue and Request are separate entities with no cross-entity -// transaction, so the two writes cannot be atomic and the ordering picks which crash -// failure mode we accept: a crash between the writes leaves the request non-terminal, -// redelivery re-runs reconciliation, and releaseSlot (which tracks no per-request slot -// ownership) decrements again — transiently over-admitting by one slot until the -// under-count re-converges at releaseSlot's zero clamp. The reverse order would leak -// the slot instead: redelivery skips terminal requests, permanently shrinking the -// queue's capacity toward a wedge. Over-admission is the failure mode we prefer. See -// doc/rfc/stovepipe/steps/process.md#in_flight_count-integrity for the broader -// counter-drift story. +// failRequest preserves a concurrent terminal outcome and completes its owned slot. func failRequest( ctx context.Context, store storage.Storage, @@ -104,43 +72,42 @@ func failRequest( return fmt.Errorf("failed to get request %s: %w", requestID, err) } - if request.State.IsTerminal() { - if request.State == entity.RequestStateFailed { - // The originating DLQ remains the retry trigger after the state write. Its stage-specific - // reason repairs that write's missing log; PersistLog rejects an already-retained conflict. - return persistFailureLog(ctx, store, materializer, request, reason) - } - logger.Infow("dlq reconcile: request already terminal, skipping", - "request_id", requestID, - "state", string(request.State), - ) - return nil - } - - if request.State == entity.RequestStateProcessing { - if err := releaseSlot(ctx, store, logger, request.Queue); err != nil { - return fmt.Errorf("failed to release queue slot for request %s: %w", requestID, err) + for !request.State.IsTerminal() { + updated := request + updated.State = entity.RequestStateFailed + newVersion := request.Version + 1 + if err := store.GetRequestStore().Update(ctx, updated, request.Version, newVersion); err != nil { + if errors.Is(err, storage.ErrVersionMismatch) { + request, err = store.GetRequestStore().Get(ctx, requestID) + if err != nil { + return fmt.Errorf("reload request %s during dlq completion: %w", requestID, err) + } + continue + } + return fmt.Errorf("failed to update request %s state to failed: %w", requestID, err) } + updated.Version = newVersion + request = updated } - - updated := request - updated.State = entity.RequestStateFailed - newVersion := request.Version + 1 - if err := store.GetRequestStore().Update(ctx, updated, request.Version, newVersion); err != nil { - return fmt.Errorf("failed to update request %s state to failed: %w", requestID, err) - } - updated.Version = newVersion - if err := persistFailureLog(ctx, store, materializer, updated, reason); err != nil { + if err := buildslot.Complete(ctx, store, request); err != nil { return err } - logger.Infow("dlq reconcile: request forced terminal failed", - "request_id", requestID, - "previous_state", string(request.State), - ) - return nil + switch request.State { + case entity.RequestStateSucceeded: + reason = entity.RequestOutcomeReasonBuildSucceeded + case entity.RequestStateCancelled: + reason = entity.RequestOutcomeReasonBuildCancelled + case entity.RequestStateSuperseded: + reason = entity.RequestOutcomeReasonSupersededByNewerHead + case entity.RequestStateFailed: + if request.TerminalBuildID != "" { + reason = entity.RequestOutcomeReasonBuildFailed + } + } + return persistOutcomeLog(ctx, store, materializer, request, reason) } -func persistFailureLog( +func persistOutcomeLog( ctx context.Context, store storage.Storage, materializer requestlog.Materializer, @@ -148,49 +115,11 @@ func persistFailureLog( reason entity.RequestOutcomeReason, ) error { log := requestlog.NewRequestStateLog(request, reason) + if request.TerminalBuildID != "" { + log.Metadata[requestlog.MetadataKeyBuildID] = request.TerminalBuildID + } if err := materializer.PersistLog(ctx, store, log); err != nil { - return fmt.Errorf("failed to record failed state for request %s: %w", request.ID, err) + return fmt.Errorf("failed to record terminal state for request %s: %w", request.ID, err) } return nil } - -// releaseSlot CAS-decrements the queue's in_flight_count, retrying on version -// conflicts, mirroring process.Controller's own CAS-retry loop for queue updates. -func releaseSlot(ctx context.Context, store storage.Storage, logger *zap.SugaredLogger, queueName string) error { - queueStore := store.GetQueueStore() - - for { - queueRow, err := queueStore.Get(ctx, queueName) - if err != nil { - if errors.Is(err, storage.ErrNotFound) { - logger.Warnw("dlq reconcile: queue not found, skipping slot release", - "queue", queueName, - ) - return nil - } - return fmt.Errorf("failed to get queue %s: %w", queueName, err) - } - - if queueRow.InFlightCount <= 0 { - logger.Warnw("dlq reconcile: queue in_flight_count already at zero, skipping slot release", - "queue", queueName, - ) - return nil - } - - updated := queueRow - updated.InFlightCount-- - newVersion := queueRow.Version + 1 - if err := queueStore.Update(ctx, updated, queueRow.Version, newVersion); err != nil { - if errors.Is(err, storage.ErrVersionMismatch) { - continue - } - return fmt.Errorf("failed to release slot for queue %s: %w", queueName, err) - } - logger.Infow("dlq reconcile: released queue slot", - "queue", queueName, - "in_flight_count", updated.InFlightCount, - ) - return nil - } -} diff --git a/stovepipe/controller/dlq/dlq_test.go b/stovepipe/controller/dlq/dlq_test.go index eeb7c838c..89120c122 100644 --- a/stovepipe/controller/dlq/dlq_test.go +++ b/stovepipe/controller/dlq/dlq_test.go @@ -134,8 +134,6 @@ func TestProcess(t *testing.T) { wantMetric string }{ { - // No queue expectations: an accepted request never claimed a slot, - // so releasing one here would over-admit against MaxConcurrent. name: "accepted request is marked failed without releasing a slot", setup: func(m dlqMocks) { m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(requestWithState(entity.RequestStateAccepted), nil) @@ -143,14 +141,15 @@ func TestProcess(t *testing.T) { updated.State = entity.RequestStateFailed updateCall := m.reqStore.EXPECT().Update(gomock.Any(), updated, int32(2), int32(3)).Return(nil) expectFailureLog(m, entity.RequestOutcomeReasonProcessingFailed, 3).After(updateCall) + m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{Name: testQueue}, nil) }, }, { - name: "processing request releases the queue slot before marking failed", + name: "processing request marks failed before releasing its queue slot", setup: func(m dlqMocks) { m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(requestWithState(entity.RequestStateProcessing), nil) m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{ - Name: testQueue, InFlightCount: 1, Version: 5, + Name: testQueue, InFlightCount: 1, InFlightRequestIDs: []string{testID}, Version: 5, }, nil) m.queueStore.EXPECT().Update(gomock.Any(), entity.Queue{ Name: testQueue, InFlightCount: 0, Version: 5, @@ -162,9 +161,11 @@ func TestProcess(t *testing.T) { }, }, { - name: "already superseded is a no-op", + name: "already superseded repairs its state log", setup: func(m dlqMocks) { m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(requestWithState(entity.RequestStateSuperseded), nil) + m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{Name: testQueue}, nil) + m.materializer.EXPECT().PersistLog(gomock.Any(), m.store, requestlog.NewRequestStateLog(requestWithState(entity.RequestStateSuperseded), entity.RequestOutcomeReasonSupersededByNewerHead)).Return(nil) }, }, { @@ -172,6 +173,7 @@ func TestProcess(t *testing.T) { setup: func(m dlqMocks) { m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(requestWithState(entity.RequestStateFailed), nil) expectFailureLog(m, entity.RequestOutcomeReasonProcessingFailed, 2) + m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{Name: testQueue}, nil) }, }, { @@ -187,21 +189,23 @@ func TestProcess(t *testing.T) { updated := requestWithState(entity.RequestStateAccepted) updated.State = entity.RequestStateFailed m.reqStore.EXPECT().Update(gomock.Any(), updated, int32(2), int32(3)).Return(storage.ErrVersionMismatch) + m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(requestWithState(entity.RequestStateFailed), nil) + m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{Name: testQueue}, nil) + expectFailureLog(m, entity.RequestOutcomeReasonProcessingFailed, 2) }, - wantErr: true, }, { name: "queue update retries on version mismatch then succeeds", setup: func(m dlqMocks) { m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(requestWithState(entity.RequestStateProcessing), nil) m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{ - Name: testQueue, InFlightCount: 1, Version: 5, + Name: testQueue, InFlightCount: 1, InFlightRequestIDs: []string{testID}, Version: 5, }, nil) m.queueStore.EXPECT().Update(gomock.Any(), entity.Queue{ Name: testQueue, InFlightCount: 0, Version: 5, }, int32(5), int32(6)).Return(storage.ErrVersionMismatch) m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{ - Name: testQueue, InFlightCount: 1, Version: 6, + Name: testQueue, InFlightCount: 1, InFlightRequestIDs: []string{testID}, Version: 6, }, nil) m.queueStore.EXPECT().Update(gomock.Any(), entity.Queue{ Name: testQueue, InFlightCount: 0, Version: 6, @@ -240,6 +244,7 @@ func TestProcess(t *testing.T) { m.store, requestlog.NewRequestStateLog(request, entity.RequestOutcomeReasonProcessingFailed), ).Return(errors.New("db down")).After(updateCall) + m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{Name: testQueue}, nil) }, }, { @@ -294,3 +299,32 @@ func TestProcess(t *testing.T) { func TestTopicKey(t *testing.T) { assert.Equal(t, consumer.TopicKey("process_dlq"), TopicKey(stovepipemq.TopicKeyProcess)) } + +func TestDLQCompletesPersistedSuccessWithoutReleasingAnotherRequestsSlot(t *testing.T) { + ctrl := gomock.NewController(t) + c, m := newController(t, ctrl) + request := requestWithState(entity.RequestStateSucceeded) + request.URI = "git://repo/main/green" + request.TerminalBuildID = "build-7" + queue := entity.Queue{Name: testQueue, Version: 5, InFlightCount: 2, InFlightRequestIDs: []string{testID, "8"}} + m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(request, nil).Times(2) + m.queueStore.EXPECT().Get(gomock.Any(), testQueue).DoAndReturn(func(context.Context, string) (entity.Queue, error) { + return queue, nil + }).Times(2) + m.queueStore.EXPECT().Update(gomock.Any(), gomock.Any(), int32(5), int32(6)).DoAndReturn( + func(_ context.Context, updated entity.Queue, _, next int32) error { + queue = updated + queue.Version = next + return nil + }) + log := requestlog.NewRequestStateLog(request, entity.RequestOutcomeReasonBuildSucceeded) + log.Metadata[requestlog.MetadataKeyBuildID] = request.TerminalBuildID + m.materializer.EXPECT().PersistLog(gomock.Any(), m.store, log).Return(nil).Times(2) + for range 2 { + require.NoError(t, c.Process(queueContext(), delivery(t, ctrl, processPayload(t, testID)))) + } + assert.Equal(t, request.URI, queue.LastGreenURI) + assert.Equal(t, request.ID, queue.LastGreenRequestID) + assert.Equal(t, []string{"8"}, queue.InFlightRequestIDs) + assert.EqualValues(t, 1, queue.InFlightCount) +} diff --git a/stovepipe/controller/process/BUILD.bazel b/stovepipe/controller/process/BUILD.bazel index 27fbe8d7d..33d60c9ea 100644 --- a/stovepipe/controller/process/BUILD.bazel +++ b/stovepipe/controller/process/BUILD.bazel @@ -13,6 +13,7 @@ go_library( "//platform/hook:go_default_library", "//platform/metrics:go_default_library", "//platform/publish:go_default_library", + "//stovepipe/core/buildslot:go_default_library", "//stovepipe/core/hookevent:go_default_library", "//stovepipe/core/loader:go_default_library", "//stovepipe/core/messagequeue:go_default_library", @@ -38,6 +39,7 @@ go_test( "//platform/errs:go_default_library", "//platform/extension/messagequeue/mock:go_default_library", "//platform/metrics:go_default_library", + "//stovepipe/core/buildslot:go_default_library", "//stovepipe/core/hookevent:go_default_library", "//stovepipe/core/messagequeue:go_default_library", "//stovepipe/core/requestlog:go_default_library", diff --git a/stovepipe/controller/process/process.go b/stovepipe/controller/process/process.go index f334f70bd..ffdffa403 100644 --- a/stovepipe/controller/process/process.go +++ b/stovepipe/controller/process/process.go @@ -14,8 +14,7 @@ // Package process holds the process-stage queue controller. It consumes request // ids from ingest, reloads the Request from storage, coalesces older heads, and -// admits the latest head when a build slot is open. Build queue publish lands in -// a follow-up PR. +// admits the latest head when a build slot is open. package process import ( @@ -31,6 +30,7 @@ import ( platformhook "github.com/uber/submitqueue/platform/hook" "github.com/uber/submitqueue/platform/metrics" "github.com/uber/submitqueue/platform/publish" + "github.com/uber/submitqueue/stovepipe/core/buildslot" "github.com/uber/submitqueue/stovepipe/core/hookevent" "github.com/uber/submitqueue/stovepipe/core/loader" stovepipemq "github.com/uber/submitqueue/stovepipe/core/messagequeue" @@ -141,11 +141,12 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er } return nil case entity.RequestStateSuperseded: + if err := buildslot.Complete(ctx, store, request); err != nil { + return err + } return c.persistStateLog(ctx, store, request, entity.RequestOutcomeReasonSupersededByNewerHead) case entity.RequestStateSucceeded, entity.RequestStateFailed, entity.RequestStateCancelled: - // Terminal: a newer head preempted this request, or its build already finished. - // A stale redelivery has nothing left to do. - return nil + return buildslot.Complete(ctx, store, request) case entity.RequestStateAccepted: return c.processAccepted(ctx, store, delivery, request) default: @@ -196,7 +197,7 @@ func (c *Controller) processAccepted(ctx context.Context, store storage.Storage, // coalesce supersedes request when a newer head exists (RFC process step 5), returning // true so the caller acks. It returns false when request is still the latest head and -// should proceed to the gate. Superseding consumes no build slot. +// should proceed to the gate. Superseding also completes any prior reservation. func (c *Controller) coalesce(ctx context.Context, store storage.Storage, request entity.Request, latestRequestID string) (bool, error) { cmp, err := entity.CompareRequestID(request.ID, latestRequestID) if err != nil { @@ -213,6 +214,9 @@ func (c *Controller) coalesce(ctx context.Context, store storage.Storage, reques if request.State != entity.RequestStateSuperseded { return true, nil } + if err := buildslot.Complete(ctx, store, request); err != nil { + return false, err + } if err := c.persistStateLog(ctx, store, request, entity.RequestOutcomeReasonSupersededByNewerHead); err != nil { return false, err } @@ -236,7 +240,10 @@ func (c *Controller) admitLatestHead(ctx context.Context, store storage.Storage, var err error for { - if queueRow.InFlightCount >= cfg.MaxConcurrent { + if err := queueRow.ValidateBuildSlots(); err != nil { + return err + } + if !queueRow.HasBuildSlot(request.ID) && queueRow.InFlightCount >= cfg.MaxConcurrent { return c.holdForBuildSlot(ctx, delivery, request, queueRow.InFlightCount, cfg.GateWaitDelayMs) } @@ -255,14 +262,14 @@ func (c *Controller) admitLatestHead(ctx context.Context, store storage.Storage, return err } - err = c.claimBuildSlot(ctx, store, &queueRow) + queueRow, err = buildslot.Claim(ctx, store, queueRow, request.ID) if err == nil { break } if !errors.Is(err, storage.ErrVersionMismatch) { return err } - // claimBuildSlot reloaded queueRow. Re-coalesce: supersede if a newer head arrived, + // Claim reloaded queueRow. Re-coalesce: supersede if a newer head arrived, // otherwise loop to re-check the gate. superseded, err := c.coalesce(ctx, store, request, queueRow.LatestRequestID) if err != nil || superseded { @@ -272,14 +279,13 @@ func (c *Controller) admitLatestHead(ctx context.Context, store storage.Storage, transitioned, err := c.markProcessing(ctx, store, &request, strategy, baseURI) if err != nil { - // Slot claimed but never admitted: release best-effort so the slot isn't leaked - // (a redelivery would find the gate closed by its own claim and nothing decrements it). - c.releaseBuildSlot(ctx, store, request.Queue) + // Keep the reservation: a retry recognizes this request's ownership even at capacity. return err } if !transitioned { - // Lost the admit race: another delivery advanced this request. Release and skip. - c.releaseBuildSlot(ctx, store, request.Queue) + if request.State.IsTerminal() { + return buildslot.Complete(ctx, store, request) + } return nil } @@ -349,35 +355,7 @@ func (c *Controller) deriveBuildStrategy(ctx context.Context, sc sourcecontrol.S return entity.BuildStrategyFull, "", nil } -// claimBuildSlot CAS-increments queue.in_flight_count by one. On version mismatch it -// reloads queueRow and returns ErrVersionMismatch so the caller can retry. -func (c *Controller) claimBuildSlot(ctx context.Context, store storage.Storage, queueRow *entity.Queue) error { - queueStore := store.GetQueueStore() - - updated := *queueRow - updated.InFlightCount = queueRow.InFlightCount + 1 - newVersion := queueRow.Version + 1 - if err := queueStore.Update(ctx, updated, queueRow.Version, newVersion); err != nil { - if errors.Is(err, storage.ErrVersionMismatch) { - got, getErr := queueStore.Get(ctx, queueRow.Name) - if getErr != nil { - return fmt.Errorf("failed to reload queue %s after version mismatch: %w", queueRow.Name, getErr) - } - *queueRow = got - return storage.ErrVersionMismatch - } - return fmt.Errorf("failed to claim build slot for queue %s: %w", queueRow.Name, err) - } - updated.Version = newVersion - *queueRow = updated - return nil -} - -// markProcessing CAS-marks request accepted→processing and persists the strategy chosen after the -// build slot was claimed. It reapplies the values after a request reload so an accepted concurrent -// update cannot discard them. transitioned is true only when this call performed the CAS; false -// means a concurrent writer already advanced the request past accepted, so the caller must release -// its claimed slot. +// markProcessing preserves a concurrent transition and its pinned build strategy. func (c *Controller) markProcessing(ctx context.Context, store storage.Storage, request *entity.Request, strategy entity.BuildStrategy, baseURI string) (transitioned bool, err error) { reqStore := store.GetRequestStore() @@ -408,43 +386,6 @@ func (c *Controller) markProcessing(ctx context.Context, store storage.Storage, } } -// releaseBuildSlot CAS-decrements queue.in_flight_count to compensate a slot claimed but never -// admitted. It decrements relatively (preserving a concurrent record decrement) and retries on -// version conflicts. Best-effort: it only logs on a hard failure, since the caller is unwinding. -func (c *Controller) releaseBuildSlot(ctx context.Context, store storage.Storage, queueName string) { - queueStore := store.GetQueueStore() - - for { - queueRow, err := queueStore.Get(ctx, queueName) - if err != nil { - c.logger.Errorw("failed to release claimed build slot", - "queue", queueName, - "error", err, - ) - return - } - if queueRow.InFlightCount <= 0 { - return - } - - updated := queueRow - updated.InFlightCount = queueRow.InFlightCount - 1 - newVersion := queueRow.Version + 1 - if err := queueStore.Update(ctx, updated, queueRow.Version, newVersion); err != nil { - if errors.Is(err, storage.ErrVersionMismatch) { - continue - } - c.logger.Errorw("failed to release claimed build slot", - "queue", queueName, - "error", err, - ) - return - } - metrics.NamedCounter(c.metricsScope, _opName, "slot_released", 1, metrics.TagsFromContext(ctx)...) - return - } -} - // supersedeRequest returns the canonical row so the log uses the version that won any CAS race. func (c *Controller) supersedeRequest(ctx context.Context, store storage.Storage, request entity.Request) (entity.Request, error) { reqStore := store.GetRequestStore() diff --git a/stovepipe/controller/process/process_test.go b/stovepipe/controller/process/process_test.go index cd8ef4c61..c4c0cadeb 100644 --- a/stovepipe/controller/process/process_test.go +++ b/stovepipe/controller/process/process_test.go @@ -29,6 +29,7 @@ import ( "github.com/uber/submitqueue/platform/errs" mqmock "github.com/uber/submitqueue/platform/extension/messagequeue/mock" "github.com/uber/submitqueue/platform/metrics" + "github.com/uber/submitqueue/stovepipe/core/buildslot" "github.com/uber/submitqueue/stovepipe/core/hookevent" stovepipemq "github.com/uber/submitqueue/stovepipe/core/messagequeue" "github.com/uber/submitqueue/stovepipe/core/requestlog" @@ -176,10 +177,11 @@ func expectAdmit(t *testing.T, m processMocks, id string) { t.Helper() updatedQueue := entity.Queue{ - Name: testQueue, - LatestRequestID: id, - InFlightCount: 1, - Version: 1, + Name: testQueue, + LatestRequestID: id, + InFlightCount: 1, + InFlightRequestIDs: []string{id}, + Version: 1, } m.queueStore.EXPECT().Update(gomock.Any(), updatedQueue, int32(1), int32(2)).Return(nil) @@ -467,8 +469,10 @@ func TestProcessRederivesStrategyAfterQueueReload(t *testing.T) { } initialClaim := initialQueue initialClaim.InFlightCount = 1 + initialClaim.InFlightRequestIDs = []string{testID} claimedQueue := reloadedQueue claimedQueue.InFlightCount = 1 + claimedQueue.InFlightRequestIDs = []string{testID} updatedRequest := request updatedRequest.State = entity.RequestStateProcessing @@ -515,6 +519,7 @@ func TestProcess(t *testing.T) { } m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(request, nil) expectStateLog(t, m, request, entity.RequestOutcomeReasonSupersededByNewerHead) + m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{Name: testQueue}, nil) }, }, { @@ -530,6 +535,7 @@ func TestProcess(t *testing.T) { m.store, requestlog.NewRequestStateLog(request, entity.RequestOutcomeReasonSupersededByNewerHead), ).Return(errors.New("db down")) + m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{Name: testQueue}, nil) }, }, { @@ -538,6 +544,7 @@ func TestProcess(t *testing.T) { m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(entity.Request{ ID: testID, Queue: testQueue, State: entity.RequestStateSucceeded, Version: 2, }, nil) + m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{Name: testQueue}, nil) }, }, { @@ -546,6 +553,7 @@ func TestProcess(t *testing.T) { m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(entity.Request{ ID: testID, Queue: testQueue, State: entity.RequestStateFailed, Version: 2, }, nil) + m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{Name: testQueue}, nil) }, }, { @@ -554,6 +562,7 @@ func TestProcess(t *testing.T) { m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(entity.Request{ ID: testID, Queue: testQueue, State: entity.RequestStateCancelled, Version: 2, }, nil) + m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{Name: testQueue}, nil) }, }, { @@ -612,10 +621,11 @@ func TestProcess(t *testing.T) { Version: 1, }, nil) updatedQueue := entity.Queue{ - Name: testQueue, - LatestRequestID: testID, - InFlightCount: 1, - Version: 1, + Name: testQueue, + LatestRequestID: testID, + InFlightCount: 1, + InFlightRequestIDs: []string{testID}, + Version: 1, } m.queueStore.EXPECT().Update(gomock.Any(), updatedQueue, int32(1), int32(2)).Return(nil) updatedRequest := acceptedRequest(testID) @@ -642,10 +652,11 @@ func TestProcess(t *testing.T) { Version: 1, }, nil) updatedQueue := entity.Queue{ - Name: testQueue, - LatestRequestID: testID, - InFlightCount: 1, - Version: 1, + Name: testQueue, + LatestRequestID: testID, + InFlightCount: 1, + InFlightRequestIDs: []string{testID}, + Version: 1, } m.queueStore.EXPECT().Update(gomock.Any(), updatedQueue, int32(1), int32(2)).Return(nil) updatedRequest := acceptedRequest(testID) @@ -674,10 +685,11 @@ func TestProcess(t *testing.T) { Version: 1, }, nil) updatedQueue := entity.Queue{ - Name: testQueue, - LatestRequestID: testID, - InFlightCount: 1, - Version: 1, + Name: testQueue, + LatestRequestID: testID, + InFlightCount: 1, + InFlightRequestIDs: []string{testID}, + Version: 1, } m.queueStore.EXPECT().Update(gomock.Any(), updatedQueue, int32(1), int32(2)).Return(nil) updatedRequest := acceptedRequest(testID) @@ -725,11 +737,12 @@ func TestProcess(t *testing.T) { setup: func(m processMocks) { m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(acceptedRequest(testID), nil) m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{ - Name: testQueue, - LatestRequestID: testID, - InFlightCount: 1, - LastGreenURI: "git://repo/monorepo/main/green", - Version: 1, + Name: testQueue, + LatestRequestID: testID, + InFlightCount: 1, + InFlightRequestIDs: []string{"other"}, + LastGreenURI: "git://repo/monorepo/main/green", + Version: 1, }, nil) }, }, @@ -744,16 +757,18 @@ func TestProcess(t *testing.T) { Version: 1, }, nil) m.queueStore.EXPECT().Update(gomock.Any(), entity.Queue{ - Name: testQueue, - LatestRequestID: testID, - InFlightCount: 1, - Version: 1, + Name: testQueue, + LatestRequestID: testID, + InFlightCount: 1, + InFlightRequestIDs: []string{testID}, + Version: 1, }, int32(1), int32(2)).Return(storage.ErrVersionMismatch) m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{ - Name: testQueue, - LatestRequestID: testID, - InFlightCount: 1, - Version: 2, + Name: testQueue, + LatestRequestID: testID, + InFlightCount: 1, + InFlightRequestIDs: []string{"other"}, + Version: 2, }, nil) }, }, @@ -766,7 +781,7 @@ func TestProcess(t *testing.T) { }, nil) // First claim CAS loses to a concurrent writer. m.queueStore.EXPECT().Update(gomock.Any(), entity.Queue{ - Name: testQueue, LatestRequestID: testID, InFlightCount: 1, Version: 1, + Name: testQueue, LatestRequestID: testID, InFlightCount: 1, InFlightRequestIDs: []string{testID}, Version: 1, }, int32(1), int32(2)).Return(storage.ErrVersionMismatch) // Reload: still latest, slot still free (version advanced by an unrelated field). m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{ @@ -774,7 +789,7 @@ func TestProcess(t *testing.T) { }, nil) // Retry claim succeeds, then admit. m.queueStore.EXPECT().Update(gomock.Any(), entity.Queue{ - Name: testQueue, LatestRequestID: testID, InFlightCount: 1, Version: 2, + Name: testQueue, LatestRequestID: testID, InFlightCount: 1, InFlightRequestIDs: []string{testID}, Version: 2, }, int32(2), int32(3)).Return(nil) updatedReq := acceptedRequest(testID) updatedReq.State = entity.RequestStateProcessing @@ -793,7 +808,7 @@ func TestProcess(t *testing.T) { Name: testQueue, LatestRequestID: testID, Version: 1, }, nil) m.queueStore.EXPECT().Update(gomock.Any(), entity.Queue{ - Name: testQueue, LatestRequestID: testID, InFlightCount: 1, Version: 1, + Name: testQueue, LatestRequestID: testID, InFlightCount: 1, InFlightRequestIDs: []string{testID}, Version: 1, }, int32(1), int32(2)).Return(storage.ErrVersionMismatch) // Reload: ingest stamped a newer head — our head is no longer latest. m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{ @@ -805,6 +820,7 @@ func TestProcess(t *testing.T) { loggedRequest := superseded loggedRequest.Version = 2 expectStateLog(t, m, loggedRequest, entity.RequestOutcomeReasonSupersededByNewerHead).After(updateCall) + m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{Name: testQueue}, nil) }, }, { @@ -819,7 +835,7 @@ func TestProcess(t *testing.T) { m.sourceFactory.EXPECT().For(sourcecontrol.Config{QueueName: testQueue}).Return(m.sourceControl, nil) m.sourceControl.EXPECT().IsAncestor(gomock.Any(), lastGreenURI, testURI).Return(true, nil) m.queueStore.EXPECT().Update(gomock.Any(), entity.Queue{ - Name: testQueue, LatestRequestID: testID, InFlightCount: 1, LastGreenURI: lastGreenURI, Version: 1, + Name: testQueue, LatestRequestID: testID, InFlightCount: 1, InFlightRequestIDs: []string{testID}, LastGreenURI: lastGreenURI, Version: 1, }, int32(1), int32(2)).Return(nil) firstAttempt := acceptedRequest(testID) @@ -843,7 +859,7 @@ func TestProcess(t *testing.T) { }, }, { - name: "mark processing lost race releases slot and skips admit", + name: "mark processing duplicate preserves the winning reservation", setup: func(m processMocks) { m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(acceptedRequest(testID), nil) m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{ @@ -851,7 +867,7 @@ func TestProcess(t *testing.T) { }, nil) // Claim succeeds. m.queueStore.EXPECT().Update(gomock.Any(), entity.Queue{ - Name: testQueue, LatestRequestID: testID, InFlightCount: 1, Version: 1, + Name: testQueue, LatestRequestID: testID, InFlightCount: 1, InFlightRequestIDs: []string{testID}, Version: 1, }, int32(1), int32(2)).Return(nil) // markProcessing CAS loses, reload shows a concurrent writer already advanced it. updatedReq := acceptedRequest(testID) @@ -861,17 +877,10 @@ func TestProcess(t *testing.T) { m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(entity.Request{ ID: testID, Queue: testQueue, State: entity.RequestStateProcessing, Version: 2, }, nil) - // Compensating decrement of the spurious slot. - m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{ - Name: testQueue, LatestRequestID: testID, InFlightCount: 1, Version: 2, - }, nil) - m.queueStore.EXPECT().Update(gomock.Any(), entity.Queue{ - Name: testQueue, LatestRequestID: testID, InFlightCount: 0, Version: 2, - }, int32(2), int32(3)).Return(nil) }, }, { - name: "mark processing error releases slot and returns error", + name: "mark processing error retains reservation for retry", wantErr: true, setup: func(m processMocks) { m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(acceptedRequest(testID), nil) @@ -879,19 +888,12 @@ func TestProcess(t *testing.T) { Name: testQueue, LatestRequestID: testID, Version: 1, }, nil) m.queueStore.EXPECT().Update(gomock.Any(), entity.Queue{ - Name: testQueue, LatestRequestID: testID, InFlightCount: 1, Version: 1, + Name: testQueue, LatestRequestID: testID, InFlightCount: 1, InFlightRequestIDs: []string{testID}, Version: 1, }, int32(1), int32(2)).Return(nil) updatedReq := acceptedRequest(testID) updatedReq.State = entity.RequestStateProcessing updatedReq.BuildStrategy = entity.BuildStrategyFull m.reqStore.EXPECT().Update(gomock.Any(), updatedReq, int32(1), int32(2)).Return(errors.New("db down")) - // Best-effort compensating decrement before the error propagates. - m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{ - Name: testQueue, LatestRequestID: testID, InFlightCount: 1, Version: 2, - }, nil) - m.queueStore.EXPECT().Update(gomock.Any(), entity.Queue{ - Name: testQueue, LatestRequestID: testID, InFlightCount: 0, Version: 2, - }, int32(2), int32(3)).Return(nil) }, }, { @@ -910,6 +912,7 @@ func TestProcess(t *testing.T) { loggedRequest := updated loggedRequest.Version = 2 expectStateLog(t, m, loggedRequest, entity.RequestOutcomeReasonSupersededByNewerHead).After(updateCall) + m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{Name: testQueue}, nil) }, }, { @@ -934,6 +937,7 @@ func TestProcess(t *testing.T) { m.store, requestlog.NewRequestStateLog(request, entity.RequestOutcomeReasonSupersededByNewerHead), ).Return(errors.New("db down")).After(updateCall) + m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{Name: testQueue}, nil) }, }, { @@ -954,6 +958,7 @@ func TestProcess(t *testing.T) { } reloadCall := m.reqStore.EXPECT().Get(gomock.Any(), testOlderID).Return(reloaded, nil) expectStateLog(t, m, reloaded, entity.RequestOutcomeReasonSupersededByNewerHead).After(reloadCall) + m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{Name: testQueue}, nil) }, }, { @@ -1052,3 +1057,55 @@ func TestHoldForBuildSlotRequiresPositiveDelay(t *testing.T) { require.Error(t, err) assert.False(t, errs.IsRetryable(err)) } + +func TestAdmissionUsesCompletedBuildBaselineBeforeFactsAreRecorded(t *testing.T) { + ctrl := gomock.NewController(t) + c, m := newController(t, ctrl) + previous := acceptedRequest("6") + previous.State = entity.RequestStateSucceeded + queue := entity.Queue{Name: testQueue, LatestRequestID: testID, Version: 1, + InFlightCount: 1, InFlightRequestIDs: []string{previous.ID}} + m.queueStore.EXPECT().Get(gomock.Any(), testQueue).DoAndReturn(func(context.Context, string) (entity.Queue, error) { + return queue, nil + }).Times(2) + m.queueStore.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, updated entity.Queue, old, next int32) error { + require.Equal(t, queue.Version, old) + queue = updated + queue.Version = next + return nil + }).Times(2) + require.NoError(t, buildslot.Complete(queueContext(testQueue), m.store, previous)) + assert.Zero(t, queue.InFlightCount) + assert.Equal(t, previous.URI, queue.LastGreenURI) + + m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(acceptedRequest(testID), nil) + m.sourceFactory.EXPECT().For(sourcecontrol.Config{QueueName: testQueue}).Return(m.sourceControl, nil) + m.sourceControl.EXPECT().IsAncestor(gomock.Any(), previous.URI, testURI).Return(true, nil) + processing := acceptedRequest(testID) + processing.State = entity.RequestStateProcessing + processing.BuildStrategy = entity.BuildStrategyIncrementalSinceGreen + processing.BaseURI = previous.URI + m.reqStore.EXPECT().Update(gomock.Any(), processing, int32(1), int32(2)).Return(nil) + processing.Version = 2 + expectStartAnnounceAndBuildPublish(t, m, processing) + + require.NoError(t, c.Process(queueContext(testQueue), delivery(t, ctrl, processPayload(t, testID)))) + assert.Equal(t, []string{testID}, queue.InFlightRequestIDs) +} + +func TestAdmissionReusesReservationAfterRequestWriteFailure(t *testing.T) { + ctrl := gomock.NewController(t) + c, m := newController(t, ctrl) + queue := entity.Queue{Name: testQueue, LatestRequestID: testID, Version: 2, + InFlightCount: 1, InFlightRequestIDs: []string{testID}} + m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(acceptedRequest(testID), nil) + m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(queue, nil) + processing := acceptedRequest(testID) + processing.State = entity.RequestStateProcessing + processing.BuildStrategy = entity.BuildStrategyFull + m.reqStore.EXPECT().Update(gomock.Any(), processing, int32(1), int32(2)).Return(nil) + processing.Version = 2 + expectStartAnnounceAndBuildPublish(t, m, processing) + require.NoError(t, c.Process(queueContext(testQueue), delivery(t, ctrl, processPayload(t, testID)))) +} diff --git a/stovepipe/controller/project_status_test.go b/stovepipe/controller/project_status_test.go index 6832c7152..97d8017a7 100644 --- a/stovepipe/controller/project_status_test.go +++ b/stovepipe/controller/project_status_test.go @@ -70,6 +70,7 @@ func TestGetProjectStatus(t *testing.T) { wantConsistency bool }{ {name: "in progress without fact", request: entity.GetProjectStatusRequest{Queue: projectStatusQueue, ChangeURI: projectStatusURI, Projects: []string{"project-a"}}, factErr: storage.ErrNotFound}, + {name: "recorded partial repository fact", request: entity.GetProjectStatusRequest{Queue: projectStatusQueue, ChangeURI: projectStatusURI}, fact: entity.ValidationFact{URI: projectStatusURI, RequestID: projectStatusID, Degree: entity.DegreePartial}, wantFact: true}, {name: "recorded green fact", request: entity.GetProjectStatusRequest{Queue: projectStatusQueue, ChangeURI: projectStatusURI, Projects: []string{"project-a"}}, fact: entity.ValidationFact{URI: projectStatusURI, RequestID: projectStatusID, Degree: entity.DegreeGreen}, wantFact: true}, {name: "missing uri", request: entity.GetProjectStatusRequest{Queue: projectStatusQueue, ChangeURI: projectStatusURI, Projects: []string{"project-a"}}, uriErr: storage.ErrNotFound, wantNotFound: true}, {name: "request summary visibility gap", request: entity.GetProjectStatusRequest{Queue: projectStatusQueue, ChangeURI: projectStatusURI, Projects: []string{"project-a"}}, summaryErr: storage.ErrNotFound, wantRetryable: true}, diff --git a/stovepipe/controller/record/record.go b/stovepipe/controller/record/record.go index 2b6ef37df..1401e64e3 100644 --- a/stovepipe/controller/record/record.go +++ b/stovepipe/controller/record/record.go @@ -16,10 +16,8 @@ // messages (a request id) published by buildsignal once a build reaches a // terminal status, and turns that outcome into durable validation state. // -// The durable state is a ValidationFact per validated commit, plus the queue's -// last-green bookmark, which process reads to choose an incremental build -// baseline. A green commit is also promoted, moving the queue's promotion ref so -// downstream systems can pull the latest green commit by name. +// Repository and project facts are persisted before promotion. Build completion +// owns the stored incremental baseline, which can precede these facts. // // The outcome is then announced as a hook event, so integrations outside the // pipeline learn that a commit is green or broken. See @@ -147,17 +145,21 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er switch request.State { case entity.RequestStateSucceeded, entity.RequestStateFailed: - fact, created, err := c.recordFact(ctx, store, request) + results, err := c.resolveProjectResults(ctx, request) + if err != nil { + return err + } + fact, created, err := c.recordFact(ctx, store, request, results) if err != nil { return err } if err := c.persistValidationFactRecordedLog(ctx, store, request, fact); err != nil { return err } - if err := c.applyFactToDerivedCaches(ctx, store, request, fact, created); err != nil { + if err := c.persistProjectFacts(ctx, store, request, results); err != nil { return err } - if err := c.recordProjectFacts(ctx, store, request); err != nil { + if err := c.applyFactToDerivedCaches(ctx, store, request, fact, created); err != nil { return err } return c.publishHookEvent(ctx, request, hookevent.NewValidationRepositoryRecorded(request)) @@ -183,19 +185,23 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er } } -func (c *Controller) recordProjectFacts(ctx context.Context, store storage.Storage, request entity.Request) error { +func (c *Controller) resolveProjectResults(ctx context.Context, request entity.Request) ([]projectstatus.Result, error) { resolver, err := c.projectStatusResolverFactory.For(projectstatus.ResolverConfig{QueueName: request.Queue}) if err != nil { - return fmt.Errorf("resolve project status resolver for queue %q: %w", request.Queue, err) + return nil, fmt.Errorf("resolve project status resolver for queue %q: %w", request.Queue, err) } results, err := resolver.Resolve(ctx, request, request.TerminalBuildID) if err != nil { - return fmt.Errorf("resolve project status results for request %q: %w", request.ID, err) + return nil, fmt.Errorf("resolve project status results for request %q: %w", request.ID, err) } if err := validateProjectStatusResults(request, results); err != nil { - return err + return nil, err } + return results, nil +} + +func (c *Controller) persistProjectFacts(ctx context.Context, store storage.Storage, request entity.Request, results []projectstatus.Result) error { for _, result := range results { _, _, err := c.recordValidationFact(ctx, store.GetValidationFactStore(), entity.ValidationFact{ URI: request.URI, @@ -270,10 +276,8 @@ func (c *Controller) persistValidationFactRecordedLog( return nil } -// applyFactToDerivedCaches moves the two caches that follow the persisted fact: a -// green fact advances the queue's bookmark and, when this request ends up holding -// it, promotes the commit. A broken fact moves neither, and instead reports how -// long the break it names went undetected. +// applyFactToDerivedCaches promotes a finalized green fact only while its request +// owns the build baseline. Build completion owns the baseline itself. func (c *Controller) applyFactToDerivedCaches( ctx context.Context, store storage.Storage, @@ -292,32 +296,29 @@ func (c *Controller) applyFactToDerivedCaches( return nil } - holdsBookmark, err := c.advanceLastGreen(ctx, store, request) + queue, err := store.GetQueueStore().Get(ctx, request.Queue) if err != nil { metrics.NamedCounter(c.metricsScope, _opName, "storage_errors", 1, metrics.TagsFromContext(ctx)...) return err } - if !holdsBookmark { - // A later green commit already holds the bookmark, so it also owns the - // promotion ref: promoting this older commit would move the ref backwards. + if queue.LastGreenRequestID != "" { + if _, err := entity.CompareRequestID(request.ID, queue.LastGreenRequestID); err != nil { + return err + } + } + if queue.LastGreenRequestID != request.ID || queue.LastGreenURI != request.URI { return nil } + c.emitLastGreenTimestamp(ctx, request) return c.promote(ctx, request) } -// recordFact writes the request's outcome as an immutable whole-repository fact and -// returns the fact that is actually stored, which is not always the one just built: -// facts are first-writer-wins, so an identity already claimed by this same request — -// a redelivery after the write but before the bookmark advanced — yields the stored -// fact instead. Every decision downstream reads that stored fact rather than the -// request, so a redelivery cannot reach a different verdict than the original. The -// second return reports whether this call is the one that wrote the fact, which is -// how a caller tells the original delivery from a redelivery. -func (c *Controller) recordFact(ctx context.Context, store storage.Storage, request entity.Request) (entity.ValidationFact, bool, error) { +// recordFact adopts the stored fact on replay; immutable facts remain authoritative. +func (c *Controller) recordFact(ctx context.Context, store storage.Storage, request entity.Request, results []projectstatus.Result) (entity.ValidationFact, bool, error) { return c.recordValidationFact(ctx, store.GetValidationFactStore(), entity.ValidationFact{ URI: request.URI, Project: wholeRepositoryProject, - Degree: degreeFor(request.State), + Degree: degreeFor(request.State, results), RequestID: request.ID, CreatedAt: time.Now().UnixMilli(), }) @@ -429,67 +430,14 @@ func (c *Controller) failureDetectionUnobserved(ctx context.Context, request ent ) } -// degreeFor maps a request's build outcome onto a whole-repository degree. Only the -// endpoints are produced: a whole-repository build is either clean or it is not, and -// intermediate degrees need per-project attribution that this stage does not do. -func degreeFor(state entity.RequestState) float64 { +func degreeFor(state entity.RequestState, results []projectstatus.Result) float64 { if state == entity.RequestStateSucceeded { return entity.DegreeGreen } - return entity.DegreeBroken -} - -// advanceLastGreen points the queue's bookmark at request, retrying on version -// conflicts, and reports whether request holds the bookmark afterwards. The -// bookmark only moves forward: an older candidate is skipped without a write and -// does not hold it, while a redelivery of the request that already set it holds it -// without a write, so the promotion that follows is retried. -// -// The bookmark is a cache of "newest green URI" derived from the facts, so it is -// advanced only after the green fact is durable. Losing the advance to a crash is -// recoverable — the redelivery reloads the same fact and retries — whereas a -// bookmark with no fact behind it would point at greenness nothing recorded. -func (c *Controller) advanceLastGreen(ctx context.Context, store storage.Storage, request entity.Request) (bool, error) { - queueStore := store.GetQueueStore() - - for { - queueRow, err := queueStore.Get(ctx, request.Queue) - if err != nil { - return false, fmt.Errorf("failed to load queue %s to advance last green: %w", request.Queue, err) - } - - cmp, err := compareToBookmark(request.Queue, request.ID, queueRow.LastGreenRequestID) - if err != nil { - // Non-retryable: re-parsing the same ids cannot start succeeding. - return false, err - } - if cmp < 0 { - return false, nil - } - if cmp == 0 { - return true, nil - } - - updated := queueRow - updated.LastGreenURI = request.URI - updated.LastGreenRequestID = request.ID - newVersion := queueRow.Version + 1 - if err := queueStore.Update(ctx, updated, queueRow.Version, newVersion); err != nil { - if errors.Is(err, storage.ErrVersionMismatch) { - continue - } - return false, fmt.Errorf("failed to advance last green for queue %s: %w", request.Queue, err) - } - - metrics.NamedCounter(c.metricsScope, _opName, "last_green_advanced", 1, metrics.TagsFromContext(ctx)...) - c.logger.Infow("advanced last green bookmark", - "queue", request.Queue, - "request_id", request.ID, - "last_green_uri", request.URI, - ) - c.emitLastGreenTimestamp(ctx, request) - return true, nil + if len(results) > 0 { + return entity.DegreePartial } + return entity.DegreeBroken } // emitLastGreenTimestamp emits the creation time of the change the bookmark now @@ -616,20 +564,6 @@ func (c *Controller) publishHookEvent(ctx context.Context, request entity.Reques return nil } -// compareToBookmark orders candidate against the request id currently holding the -// bookmark, by ingest order, using the sign convention of entity.CompareRequestID. -// An empty current means the bookmark has never been set, so any candidate is newer. -func compareToBookmark(queue, candidate, current string) (int, error) { - if current == "" { - return 1, nil - } - cmp, err := entity.CompareRequestID(candidate, current) - if err != nil { - return 0, fmt.Errorf("failed to compare request ids for queue %s: %w", queue, err) - } - return cmp, nil -} - // loadRequest loads the request by id. func (c *Controller) loadRequest(ctx context.Context, store storage.Storage, id string) (entity.Request, error) { return loader.ByID(ctx, id, store.GetRequestStore().Get, "request") diff --git a/stovepipe/controller/record/record_test.go b/stovepipe/controller/record/record_test.go index 0743ef77a..d773551d4 100644 --- a/stovepipe/controller/record/record_test.go +++ b/stovepipe/controller/record/record_test.go @@ -275,88 +275,31 @@ func queueRow(lastGreenURI, lastGreenRequestID string, version int32) entity.Que } } -func TestProcess_AdvancesBookmarkOnSuccess(t *testing.T) { - tests := []struct { - name string - stored entity.Queue - wantURI string - }{ - { - name: "bookmark unset", - stored: queueRow("", "", 1), - wantURI: testURI, - }, - { - name: "stored bookmark is older", - stored: queueRow("git://remote/monorepo/main/old", "3", 4), - wantURI: testURI, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ctrl := gomock.NewController(t) - c, m := newController(t, ctrl) - - m.reqStore.EXPECT().Get(gomock.Any(), testID). - Return(requestWithState(entity.RequestStateSucceeded), nil) - var fact entity.ValidationFact - m.expectFactCreated(&fact) - m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(tt.stored, nil) - - var written entity.Queue - m.queueStore.EXPECT(). - Update(gomock.Any(), gomock.Any(), tt.stored.Version, tt.stored.Version+1). - DoAndReturn(func(_ context.Context, q entity.Queue, _, _ int32) error { - written = q - return nil - }) - m.sourceControl.EXPECT().ChangeInfo(gomock.Any(), testURI). - Return(sourcecontrol.ChangeInfo{CreatedAt: testChangeTime.UnixMilli()}, nil) - m.sourceControl.EXPECT().Promote(gomock.Any(), testURI).Return(nil) - - require.NoError(t, c.Process(queueContext(), delivery(t, ctrl, recordPayload(t, testID)))) - assert.Equal(t, testQueue, m.hooks.tenant) - assert.Equal(t, tt.wantURI, written.LastGreenURI) - assert.Equal(t, testID, written.LastGreenRequestID) - - gauge, ok := m.metricsScope.Snapshot().Gauges()["record_controller.record.last_green_timestamp_seconds+queue=monorepo/main"] - require.True(t, ok) - assert.Equal(t, float64(testChangeTime.Unix()), gauge.Value()) - - // The green fact is what authorises the advance. - assert.Equal(t, entity.DegreeGreen, fact.Degree) - assert.Equal(t, testURI, fact.URI) - assert.Equal(t, testID, fact.RequestID) - assert.Equal(t, wholeRepositoryProject, fact.Project) - assert.Positive(t, fact.CreatedAt) - }) - } +func TestProcess_PromotesFinalizedGreenAtCurrentBaseline(t *testing.T) { + ctrl := gomock.NewController(t) + c, m := newController(t, ctrl) + m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(requestWithState(entity.RequestStateSucceeded), nil) + var fact entity.ValidationFact + m.expectFactCreated(&fact) + m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(queueRow(testURI, testID, 1), nil) + m.sourceControl.EXPECT().ChangeInfo(gomock.Any(), testURI).Return(sourcecontrol.ChangeInfo{CreatedAt: testChangeTime.UnixMilli()}, nil) + m.sourceControl.EXPECT().Promote(gomock.Any(), testURI).Return(nil) + require.NoError(t, c.Process(queueContext(), delivery(t, ctrl, recordPayload(t, testID)))) + assert.Equal(t, entity.DegreeGreen, fact.Degree) + assert.Len(t, m.hooks.events, 1) } -func TestProcess_PromotesGreenBeforeProjectResolution(t *testing.T) { +func TestProcess_WaitsForProjectResolutionBeforeRecordingOrPromotion(t *testing.T) { ctrl := gomock.NewController(t) c, m := newController(t, ctrl) request := requestWithState(entity.RequestStateSucceeded) m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(request, nil) - m.factStore.EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil) - m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(queueRow("", "", 1), nil) - m.queueStore.EXPECT().Update(gomock.Any(), gomock.Any(), int32(1), int32(2)).Return(nil) - m.sourceControl.EXPECT().ChangeInfo(gomock.Any(), testURI). - Return(sourcecontrol.ChangeInfo{CreatedAt: testChangeTime.UnixMilli()}, nil) - promote := m.sourceControl.EXPECT().Promote(gomock.Any(), testURI).Return(nil) - - resolverFactory := projectstatusmock.NewMockResolverFactory(ctrl) + factory := projectstatusmock.NewMockResolverFactory(ctrl) resolver := projectstatusmock.NewMockResolver(ctrl) - c.projectStatusResolverFactory = resolverFactory - resolverFactory.EXPECT().For(projectstatus.ResolverConfig{QueueName: testQueue}). - After(promote). - Return(resolver, nil) - sentinel := errors.New("resolution unavailable") - resolver.EXPECT().Resolve(gomock.Any(), request, request.TerminalBuildID).Return(nil, sentinel) - - err := c.Process(queueContext(), delivery(t, ctrl, recordPayload(t, testID))) - require.ErrorIs(t, err, sentinel) + c.projectStatusResolverFactory = factory + factory.EXPECT().For(projectstatus.ResolverConfig{QueueName: testQueue}).Return(resolver, nil) + resolver.EXPECT().Resolve(gomock.Any(), request, request.TerminalBuildID).Return(nil, assert.AnError) + require.ErrorIs(t, c.Process(queueContext(), delivery(t, ctrl, recordPayload(t, testID))), assert.AnError) assert.Empty(t, m.hooks.events) } @@ -416,7 +359,11 @@ func TestProcess_RecordsNamedProjectStatusResults(t *testing.T) { for _, fact := range facts { assert.Equal(t, testURI, fact.URI) assert.Equal(t, testID, fact.RequestID) - assert.Equal(t, entity.DegreeBroken, fact.Degree) + if fact.Project == "" { + assert.Equal(t, entity.DegreePartial, fact.Degree) + } else { + assert.Equal(t, entity.DegreeBroken, fact.Degree) + } assert.Positive(t, fact.CreatedAt) } require.Len(t, logs, 2) @@ -427,7 +374,7 @@ func TestProcess_RecordsNamedProjectStatusResults(t *testing.T) { func TestRecordProjectFacts_PassesEmptyTerminalBuild(t *testing.T) { ctrl := gomock.NewController(t) - c, m := newController(t, ctrl) + c, _ := newController(t, ctrl) request := requestWithState(entity.RequestStateFailed) request.TerminalBuildID = "" projectStatusResolverFactory := projectstatusmock.NewMockResolverFactory(ctrl) @@ -437,7 +384,8 @@ func TestRecordProjectFacts_PassesEmptyTerminalBuild(t *testing.T) { Return(projectStatusResolver, nil) projectStatusResolver.EXPECT().Resolve(gomock.Any(), request, "").Return(nil, nil) - err := c.recordProjectFacts(queueContext(), m.store, request) + results, err := c.resolveProjectResults(queueContext(), request) + assert.Empty(t, results) require.NoError(t, err) } @@ -454,13 +402,10 @@ func TestProcess_RejectsInvalidProjectStatusResultsBeforeWritingFacts(t *testing {Project: "project-b", Degree: math.NaN()}, }, nil) m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(requestWithState(entity.RequestStateFailed), nil) - var fact entity.ValidationFact - m.expectFactCreated(&fact) err := c.Process(queueContext(), delivery(t, ctrl, recordPayload(t, testID))) require.Error(t, err) assert.ErrorContains(t, err, "invalid degree") - assert.Equal(t, wholeRepositoryProject, fact.Project) } func TestProcess_TimestampReportingFailureDoesNotFailRecord(t *testing.T) { @@ -493,8 +438,7 @@ func TestProcess_TimestampReportingFailureDoesNotFailRecord(t *testing.T) { Return(requestWithState(entity.RequestStateSucceeded), nil) var fact entity.ValidationFact m.expectFactCreated(&fact) - m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(queueRow("", "", 1), nil) - m.queueStore.EXPECT().Update(gomock.Any(), gomock.Any(), int32(1), int32(2)).Return(nil) + m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(queueRow(testURI, testID, 1), nil) m.sourceControl.EXPECT().ChangeInfo(gomock.Any(), testURI).Return(tt.info, tt.err) m.sourceControl.EXPECT().Promote(gomock.Any(), testURI).Return(nil) @@ -519,8 +463,7 @@ func TestProcess_UnresolvableSourceControlCountsTimestampFailure(t *testing.T) { Return(requestWithState(entity.RequestStateSucceeded), nil) var fact entity.ValidationFact m.expectFactCreated(&fact) - m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(queueRow("", "", 1), nil) - m.queueStore.EXPECT().Update(gomock.Any(), gomock.Any(), int32(1), int32(2)).Return(nil) + m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(queueRow(testURI, testID, 1), nil) require.Error(t, c.Process(queueContext(), delivery(t, ctrl, recordPayload(t, testID)))) assert.Empty(t, m.metricsScope.Snapshot().Gauges()) @@ -573,10 +516,9 @@ func TestProcess_RecordsFactEventBeforeDerivedWork(t *testing.T) { m.queueStore.EXPECT().Get(gomock.Any(), testQueue).DoAndReturn( func(context.Context, string) (entity.Queue, error) { require.True(t, eventPersisted) - return queueRow("", "", 1), nil + return queueRow(testURI, testID, 1), nil }, ) - m.queueStore.EXPECT().Update(gomock.Any(), gomock.Any(), int32(1), int32(2)).Return(nil) m.sourceControl.EXPECT().ChangeInfo(gomock.Any(), testURI). Return(sourcecontrol.ChangeInfo{CreatedAt: testChangeTime.UnixMilli()}, nil) m.sourceControl.EXPECT().Promote(gomock.Any(), testURI).Return(nil) @@ -722,14 +664,14 @@ func TestProcess_UnobservableDetectionLatencyDoesNotFailRecord(t *testing.T) { func TestProcess_AdoptsExistingFactFromSameRequest(t *testing.T) { tests := []struct { - name string - stored entity.ValidationFact - wantUpdate bool + name string + stored entity.ValidationFact + wantPromoteStoredFact bool }{ { - name: "green fact still advances the bookmark", - stored: entity.ValidationFact{URI: testURI, Degree: entity.DegreeGreen, RequestID: testID, CreatedAt: testChangeTime.UnixMilli()}, - wantUpdate: true, + name: "green fact still promotes the current baseline", + stored: entity.ValidationFact{URI: testURI, Degree: entity.DegreeGreen, RequestID: testID, CreatedAt: testChangeTime.UnixMilli()}, + wantPromoteStoredFact: true, }, { name: "broken fact does not", @@ -742,16 +684,14 @@ func TestProcess_AdoptsExistingFactFromSameRequest(t *testing.T) { ctrl := gomock.NewController(t) c, m := newController(t, ctrl) - // A redelivery after the fact was written but before the bookmark - // advanced: the write loses, and the stored fact decides. + // A stored fact controls promotion even if the current Request suggests another degree. m.reqStore.EXPECT().Get(gomock.Any(), testID). Return(requestWithState(entity.RequestStateSucceeded), nil) m.factStore.EXPECT().Create(gomock.Any(), gomock.Any()).Return(storage.ErrAlreadyExists) m.factStore.EXPECT().Get(gomock.Any(), testURI, wholeRepositoryProject).Return(tt.stored, nil) - if tt.wantUpdate { - m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(queueRow("", "", 1), nil) - m.queueStore.EXPECT().Update(gomock.Any(), gomock.Any(), int32(1), int32(2)).Return(nil) + if tt.wantPromoteStoredFact { + m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(queueRow(testURI, testID, 1), nil) m.sourceControl.EXPECT().ChangeInfo(gomock.Any(), testURI). Return(sourcecontrol.ChangeInfo{CreatedAt: testChangeTime.UnixMilli()}, nil) m.sourceControl.EXPECT().Promote(gomock.Any(), testURI).Return(nil) @@ -776,7 +716,7 @@ func TestProcess_ExistingFactFromDifferentRequestFails(t *testing.T) { require.Error(t, c.Process(queueContext(), delivery(t, ctrl, recordPayload(t, testID)))) } -func TestProcess_SkipsBookmarkWhenNotNewer(t *testing.T) { +func TestProcessPromotesOnlyCurrentBaselineOwner(t *testing.T) { tests := []struct { name string stored entity.Queue @@ -807,9 +747,10 @@ func TestProcess_SkipsBookmarkWhenNotNewer(t *testing.T) { var fact entity.ValidationFact m.expectFactCreated(&fact) m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(tt.stored, nil) - // No Update: the bookmark only moves forward. + // Record reads the baseline without changing it. if tt.wantPromote { + m.sourceControl.EXPECT().ChangeInfo(gomock.Any(), testURI).Return(sourcecontrol.ChangeInfo{}, nil) m.sourceControl.EXPECT().Promote(gomock.Any(), testURI).Return(nil) } @@ -831,8 +772,7 @@ func TestProcess_SkipsPromotionWhenCommitLeftTheRef(t *testing.T) { Return(requestWithState(entity.RequestStateSucceeded), nil) var fact entity.ValidationFact m.expectFactCreated(&fact) - m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(queueRow("", "", 1), nil) - m.queueStore.EXPECT().Update(gomock.Any(), gomock.Any(), int32(1), int32(2)).Return(nil) + m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(queueRow(testURI, testID, 1), nil) m.sourceControl.EXPECT().ChangeInfo(gomock.Any(), testURI). Return(sourcecontrol.ChangeInfo{CreatedAt: testChangeTime.UnixMilli()}, nil) @@ -873,8 +813,7 @@ func TestProcess_PromotionErrorsPropagate(t *testing.T) { Return(requestWithState(entity.RequestStateSucceeded), nil) var fact entity.ValidationFact m.expectFactCreated(&fact) - m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(queueRow("", "", 1), nil) - m.queueStore.EXPECT().Update(gomock.Any(), gomock.Any(), int32(1), int32(2)).Return(nil) + m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(queueRow(testURI, testID, 1), nil) tt.setup(c, m) // The bookmark already advanced, so the redelivery re-promotes the @@ -945,8 +884,7 @@ func TestProcess_AnnouncesRecordedValidation(t *testing.T) { m.expectFactCreated(&fact) if tt.wantDegree == entity.DegreeGreen { - m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(queueRow("", "", 1), nil) - m.queueStore.EXPECT().Update(gomock.Any(), gomock.Any(), int32(1), int32(2)).Return(nil) + m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(queueRow(testURI, testID, 1), nil) m.sourceControl.EXPECT().ChangeInfo(gomock.Any(), testURI). Return(sourcecontrol.ChangeInfo{CreatedAt: testChangeTime.UnixMilli()}, nil) m.sourceControl.EXPECT().Promote(gomock.Any(), testURI).Return(nil) @@ -994,12 +932,7 @@ func TestProcess_AnnouncesOnlyAfterDerivedCachesMove(t *testing.T) { Return(requestWithState(entity.RequestStateSucceeded), nil) var fact entity.ValidationFact m.expectFactCreated(&fact) - m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(queueRow("", "", 1), nil) - m.queueStore.EXPECT().Update(gomock.Any(), gomock.Any(), int32(1), int32(2)). - DoAndReturn(func(context.Context, entity.Queue, int32, int32) error { - assert.Empty(t, m.hooks.events, "the bookmark must advance before the announcement") - return nil - }) + m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(queueRow(testURI, testID, 1), nil) m.sourceControl.EXPECT().ChangeInfo(gomock.Any(), testURI). Return(sourcecontrol.ChangeInfo{CreatedAt: testChangeTime.UnixMilli()}, nil) m.sourceControl.EXPECT().Promote(gomock.Any(), testURI). @@ -1074,33 +1007,6 @@ func TestProcess_NonTerminalRequestFails(t *testing.T) { } } -func TestProcess_RetriesBookmarkOnVersionMismatch(t *testing.T) { - ctrl := gomock.NewController(t) - c, m := newController(t, ctrl) - - stale := queueRow("", "", 1) - fresh := queueRow("", "", 2) - - m.reqStore.EXPECT().Get(gomock.Any(), testID). - Return(requestWithState(entity.RequestStateSucceeded), nil) - var fact entity.ValidationFact - m.expectFactCreated(&fact) - - gomock.InOrder( - m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(stale, nil), - m.queueStore.EXPECT().Update(gomock.Any(), gomock.Any(), stale.Version, stale.Version+1). - Return(storage.ErrVersionMismatch), - m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(fresh, nil), - m.queueStore.EXPECT().Update(gomock.Any(), gomock.Any(), fresh.Version, fresh.Version+1). - Return(nil), - ) - m.sourceControl.EXPECT().ChangeInfo(gomock.Any(), testURI). - Return(sourcecontrol.ChangeInfo{CreatedAt: testChangeTime.UnixMilli()}, nil) - m.sourceControl.EXPECT().Promote(gomock.Any(), testURI).Return(nil) - - require.NoError(t, c.Process(queueContext(), delivery(t, ctrl, recordPayload(t, testID)))) -} - func TestProcess_MalformedRequestIDFails(t *testing.T) { ctrl := gomock.NewController(t) c, m := newController(t, ctrl) @@ -1156,18 +1062,6 @@ func TestProcess_StorageErrorsPropagate(t *testing.T) { Return(entity.Queue{}, errors.New("boom")) }, }, - { - name: "queue update fails", - setup: func(m recordMocks) { - m.reqStore.EXPECT().Get(gomock.Any(), testID). - Return(requestWithState(entity.RequestStateSucceeded), nil) - var fact entity.ValidationFact - m.expectFactCreated(&fact) - m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(queueRow("", "", 1), nil) - m.queueStore.EXPECT().Update(gomock.Any(), gomock.Any(), int32(1), int32(2)). - Return(errors.New("boom")) - }, - }, } for _, tt := range tests { @@ -1217,3 +1111,97 @@ func TestProcess_QueueMismatchFails(t *testing.T) { require.Error(t, c.Process(queueContext(), d)) } + +func TestProcessRecordsDegreeAfterResolvingProjects(t *testing.T) { + for _, tt := range []struct { + name string + state entity.RequestState + results []projectstatus.Result + degree float64 + }{ + {name: "successful repository", state: entity.RequestStateSucceeded, degree: entity.DegreeGreen}, + {name: "failed repository without project results", state: entity.RequestStateFailed, degree: entity.DegreeBroken}, + {name: "failed repository with project results", state: entity.RequestStateFailed, degree: entity.DegreePartial, + results: []projectstatus.Result{{Project: "a", Degree: entity.DegreeGreen}, {Project: "b", Degree: 0.25}}}, + {name: "successful repository with project results", state: entity.RequestStateSucceeded, degree: entity.DegreeGreen, + results: []projectstatus.Result{{Project: "a", Degree: entity.DegreeGreen}}}, + } { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + c, m := newController(t, ctrl) + logs := requestlogmock.NewMockMaterializer(ctrl) + logs.EXPECT().PersistLog(gomock.Any(), m.store, gomock.Any()).Return(nil).AnyTimes() + c.materializer = logs + request := requestWithState(tt.state) + m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(request, nil) + factory := projectstatusmock.NewMockResolverFactory(ctrl) + resolver := projectstatusmock.NewMockResolver(ctrl) + c.projectStatusResolverFactory = factory + factory.EXPECT().For(projectstatus.ResolverConfig{QueueName: testQueue}).Return(resolver, nil) + resolved := false + resolver.EXPECT().Resolve(gomock.Any(), request, request.TerminalBuildID).DoAndReturn( + func(context.Context, entity.Request, string) ([]projectstatus.Result, error) { + resolved = true + return tt.results, nil + }) + facts := map[string]entity.ValidationFact{} + m.factStore.EXPECT().Create(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, fact entity.ValidationFact) error { + assert.True(t, resolved) + facts[fact.Project] = fact + return nil + }).Times(len(tt.results) + 1) + if tt.degree == entity.DegreeGreen { + m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(queueRow(testURI, testID, 1), nil) + m.sourceControl.EXPECT().ChangeInfo(gomock.Any(), testURI).Return(sourcecontrol.ChangeInfo{}, nil) + m.sourceControl.EXPECT().Promote(gomock.Any(), testURI).DoAndReturn(func(context.Context, string) error { + assert.Len(t, facts, len(tt.results)+1) + assert.Empty(t, m.hooks.events) + return nil + }) + } + require.NoError(t, c.Process(queueContext(), delivery(t, ctrl, recordPayload(t, testID)))) + assert.Equal(t, tt.degree, facts[""].Degree) + for _, result := range tt.results { + assert.Equal(t, result.Degree, facts[result.Project].Degree) + } + assert.Len(t, m.hooks.events, 1) + }) + } +} + +func TestProcessRetriesProjectPersistenceWithImmutablePartialFact(t *testing.T) { + ctrl := gomock.NewController(t) + c, m := newController(t, ctrl) + logs := requestlogmock.NewMockMaterializer(ctrl) + logs.EXPECT().PersistLog(gomock.Any(), m.store, gomock.Any()).Return(nil).AnyTimes() + c.materializer = logs + request := requestWithState(entity.RequestStateFailed) + m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(request, nil).Times(2) + factory := projectstatusmock.NewMockResolverFactory(ctrl) + resolver := projectstatusmock.NewMockResolver(ctrl) + c.projectStatusResolverFactory = factory + factory.EXPECT().For(projectstatus.ResolverConfig{QueueName: testQueue}).Return(resolver, nil).Times(2) + resolver.EXPECT().Resolve(gomock.Any(), request, request.TerminalBuildID).Return( + []projectstatus.Result{{Project: "a", Degree: entity.DegreeBroken}}, nil).Times(2) + var stored entity.ValidationFact + m.factStore.EXPECT().Create(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, fact entity.ValidationFact) error { + assert.Empty(t, fact.Project) + stored = fact + return nil + }) + m.factStore.EXPECT().Create(gomock.Any(), gomock.Any()).Return(assert.AnError) + require.ErrorIs(t, c.Process(queueContext(), delivery(t, ctrl, recordPayload(t, testID))), assert.AnError) + assert.Equal(t, entity.DegreePartial, stored.Degree) + assert.Empty(t, m.hooks.events) + + m.factStore.EXPECT().Create(gomock.Any(), gomock.Any()).Return(storage.ErrAlreadyExists) + m.factStore.EXPECT().Get(gomock.Any(), testURI, "").Return(stored, nil) + m.factStore.EXPECT().Create(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, fact entity.ValidationFact) error { + assert.Equal(t, "a", fact.Project) + assert.Equal(t, entity.DegreeBroken, fact.Degree) + return nil + }) + require.NoError(t, c.Process(queueContext(), delivery(t, ctrl, recordPayload(t, testID)))) + assert.Len(t, m.hooks.events, 1) +} diff --git a/stovepipe/core/buildslot/BUILD.bazel b/stovepipe/core/buildslot/BUILD.bazel new file mode 100644 index 000000000..bbbf47cc5 --- /dev/null +++ b/stovepipe/core/buildslot/BUILD.bazel @@ -0,0 +1,29 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = [ + "claim.go", + "complete.go", + ], + importpath = "github.com/uber/submitqueue/stovepipe/core/buildslot", + visibility = ["//visibility:public"], + deps = [ + "//stovepipe/entity:go_default_library", + "//stovepipe/extension/storage:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = ["complete_test.go"], + embed = [":go_default_library"], + deps = [ + "//stovepipe/entity:go_default_library", + "//stovepipe/extension/storage:go_default_library", + "//stovepipe/extension/storage/mock:go_default_library", + "@com_github_stretchr_testify//assert:go_default_library", + "@com_github_stretchr_testify//require:go_default_library", + "@org_uber_go_mock//gomock:go_default_library", + ], +) diff --git a/stovepipe/core/buildslot/README.md b/stovepipe/core/buildslot/README.md new file mode 100644 index 000000000..cf4bb1c6a --- /dev/null +++ b/stovepipe/core/buildslot/README.md @@ -0,0 +1,9 @@ +# Build-slot ownership + +`Claim` reserves capacity once per Request in the Queue snapshot provided by process. Process owns the admission policy: coalescing, capacity, and baseline selection. A successful claim returns the persisted snapshot with its new version. A version conflict returns the current Queue and `storage.ErrVersionMismatch`, so process rechecks that policy before retrying. An existing owner reuses its reservation; neither operation mutates the supplied snapshot. + +`Complete` accepts a durable terminal Request. It removes only that Request’s reservation from `Queue.InFlightRequestIDs` and recomputes `InFlightCount`. For a newer successful outcome it advances `LastGreenURI` and `LastGreenRequestID` in the same Queue CAS, so the next admission sees the successful baseline when it sees available capacity. + +The Request outcome must be persisted first. A crash before Queue completion temporarily holds capacity; primary retries and DLQ recovery repeat completion even for terminal Requests. Absence of the Request id makes completion a no-op, including after an ambiguous committed Queue write. CAS conflicts reload the Queue to preserve concurrent owners, enablement, and newer baselines. Inputs remain immutable snapshots. + +The baseline can lead validation facts and Git promotion. Record resolves projects and persists facts and history before promotion. Buildslot neither writes facts nor calls Git. diff --git a/stovepipe/core/buildslot/claim.go b/stovepipe/core/buildslot/claim.go new file mode 100644 index 000000000..282a4f41c --- /dev/null +++ b/stovepipe/core/buildslot/claim.go @@ -0,0 +1,52 @@ +// Copyright (c) 2026 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package buildslot + +import ( + "context" + "errors" + "fmt" + "slices" + + "github.com/uber/submitqueue/stovepipe/entity" + "github.com/uber/submitqueue/stovepipe/extension/storage" +) + +// Claim reserves a request's slot in a validated queue snapshot without mutating it. +// The caller must check capacity first. A version conflict returns the current queue +// and storage.ErrVersionMismatch so admission can recheck capacity, coalescing, and baseline. +func Claim(ctx context.Context, store storage.Storage, queueRow entity.Queue, requestID string) (entity.Queue, error) { + if queueRow.HasBuildSlot(requestID) { + return queueRow, nil + } + queueStore := store.GetQueueStore() + + updated := queueRow + updated.InFlightRequestIDs = append(slices.Clone(queueRow.InFlightRequestIDs), requestID) + updated.InFlightCount = int32(len(updated.InFlightRequestIDs)) + newVersion := queueRow.Version + 1 + if err := queueStore.Update(ctx, updated, queueRow.Version, newVersion); err != nil { + if errors.Is(err, storage.ErrVersionMismatch) { + got, getErr := queueStore.Get(ctx, queueRow.Name) + if getErr != nil { + return queueRow, fmt.Errorf("failed to reload queue %s after version mismatch: %w", queueRow.Name, getErr) + } + return got, storage.ErrVersionMismatch + } + return queueRow, fmt.Errorf("failed to claim build slot for queue %s: %w", queueRow.Name, err) + } + updated.Version = newVersion + return updated, nil +} diff --git a/stovepipe/core/buildslot/complete.go b/stovepipe/core/buildslot/complete.go new file mode 100644 index 000000000..df8e14653 --- /dev/null +++ b/stovepipe/core/buildslot/complete.go @@ -0,0 +1,80 @@ +// Copyright (c) 2026 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package buildslot reserves and completes request-owned build capacity and baselines. +package buildslot + +import ( + "context" + "errors" + "fmt" + "slices" + + "github.com/uber/submitqueue/stovepipe/entity" + "github.com/uber/submitqueue/stovepipe/extension/storage" +) + +// Complete releases a terminal request's slot and advances its successful baseline +// in one queue update. The durable Request outcome must be written first. +// Repeated calls cannot release another request's capacity. +func Complete(ctx context.Context, store storage.Storage, request entity.Request) error { + if !request.State.IsTerminal() || request.ID == "" || request.Queue == "" { + return fmt.Errorf("build slot completion requires a terminal request with an ID and queue") + } + queueStore := store.GetQueueStore() + for { + queue, err := queueStore.Get(ctx, request.Queue) + if err != nil { + return fmt.Errorf("load queue %q for build slot completion: %w", request.Queue, err) + } + if err := queue.ValidateBuildSlots(); err != nil { + return err + } + index := slices.Index(queue.InFlightRequestIDs, request.ID) + if index < 0 { + return nil + } + + updated := queue + updated.InFlightRequestIDs = slices.Delete(slices.Clone(queue.InFlightRequestIDs), index, index+1) + if len(updated.InFlightRequestIDs) == 0 { + updated.InFlightRequestIDs = nil + } + updated.InFlightCount = int32(len(updated.InFlightRequestIDs)) + if request.State == entity.RequestStateSucceeded { + if request.URI == "" { + return fmt.Errorf("successful request %q has no baseline URI", request.ID) + } + newer := queue.LastGreenRequestID == "" + if !newer { + comparison, err := entity.CompareRequestID(request.ID, queue.LastGreenRequestID) + if err != nil { + return fmt.Errorf("compare build baseline for queue %q: %w", request.Queue, err) + } + newer = comparison > 0 + } + if newer { + updated.LastGreenURI = request.URI + updated.LastGreenRequestID = request.ID + } + } + if err := queueStore.Update(ctx, updated, queue.Version, queue.Version+1); err != nil { + if errors.Is(err, storage.ErrVersionMismatch) { + continue + } + return fmt.Errorf("complete build slot for request %q: %w", request.ID, err) + } + return nil + } +} diff --git a/stovepipe/core/buildslot/complete_test.go b/stovepipe/core/buildslot/complete_test.go new file mode 100644 index 000000000..947cce574 --- /dev/null +++ b/stovepipe/core/buildslot/complete_test.go @@ -0,0 +1,140 @@ +// Copyright (c) 2026 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package buildslot + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/uber/submitqueue/stovepipe/entity" + "github.com/uber/submitqueue/stovepipe/extension/storage" + storagemock "github.com/uber/submitqueue/stovepipe/extension/storage/mock" + "go.uber.org/mock/gomock" +) + +func TestCompleteReleasesOnlyOwnedCapacityAndAdvancesSuccessfulBaseline(t *testing.T) { + for _, tt := range []struct { + name string + state entity.RequestState + bookmark string + wantGreen string + }{ + {name: "successful build", state: entity.RequestStateSucceeded, bookmark: "3", wantGreen: "7"}, + {name: "newer build already succeeded", state: entity.RequestStateSucceeded, bookmark: "9", wantGreen: "9"}, + {name: "first green build", state: entity.RequestStateSucceeded, wantGreen: "7"}, + {name: "failed build", state: entity.RequestStateFailed, bookmark: "3", wantGreen: "3"}, + {name: "cancelled build", state: entity.RequestStateCancelled, bookmark: "3", wantGreen: "3"}, + {name: "superseded reservation", state: entity.RequestStateSuperseded, bookmark: "3", wantGreen: "3"}, + } { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + store := storagemock.NewMockStorage(ctrl) + queues := storagemock.NewMockQueueStore(ctrl) + store.EXPECT().GetQueueStore().Return(queues).AnyTimes() + request := entity.Request{ID: "7", Queue: "repo/main", URI: "commit-7", State: tt.state} + queue := entity.Queue{Name: request.Queue, Version: 4, InFlightCount: 2, + InFlightRequestIDs: []string{"7", "8"}, LastGreenRequestID: tt.bookmark, LastGreenURI: "old-commit"} + queues.EXPECT().Get(gomock.Any(), request.Queue).DoAndReturn(func(context.Context, string) (entity.Queue, error) { + return queue, nil + }).Times(2) + queues.EXPECT().Update(gomock.Any(), gomock.Any(), int32(4), int32(5)).DoAndReturn( + func(_ context.Context, updated entity.Queue, _, version int32) error { + assert.Equal(t, []string{"7", "8"}, queue.InFlightRequestIDs) + assert.Equal(t, int32(1), updated.InFlightCount) + assert.Equal(t, []string{"8"}, updated.InFlightRequestIDs) + assert.Equal(t, tt.wantGreen, updated.LastGreenRequestID) + if tt.wantGreen == "7" { + assert.Equal(t, request.URI, updated.LastGreenURI) + } + updated.Version = version + queue = updated + return nil + }) + require.NoError(t, Complete(context.Background(), store, request)) + require.NoError(t, Complete(context.Background(), store, request)) + }) + } +} + +func TestCompleteRecoversAfterCommittedWriteReturnsAnError(t *testing.T) { + ctrl := gomock.NewController(t) + store := storagemock.NewMockStorage(ctrl) + queues := storagemock.NewMockQueueStore(ctrl) + store.EXPECT().GetQueueStore().Return(queues).AnyTimes() + request := entity.Request{ID: "7", Queue: "repo/main", URI: "commit-7", State: entity.RequestStateSucceeded} + queue := entity.Queue{Name: request.Queue, Version: 1, InFlightCount: 2, InFlightRequestIDs: []string{"7", "8"}} + queues.EXPECT().Get(gomock.Any(), request.Queue).DoAndReturn(func(context.Context, string) (entity.Queue, error) { + return queue, nil + }).Times(2) + queues.EXPECT().Update(gomock.Any(), gomock.Any(), int32(1), int32(2)).DoAndReturn( + func(_ context.Context, updated entity.Queue, _, version int32) error { + queue = updated + queue.Version = version + return assert.AnError + }) + require.ErrorIs(t, Complete(context.Background(), store, request), assert.AnError) + require.NoError(t, Complete(context.Background(), store, request)) + assert.Equal(t, []string{"8"}, queue.InFlightRequestIDs) + assert.Equal(t, int32(1), queue.InFlightCount) + assert.Equal(t, request.URI, queue.LastGreenURI) +} + +func TestCompleteReloadsConcurrentQueueUpdates(t *testing.T) { + ctrl := gomock.NewController(t) + store := storagemock.NewMockStorage(ctrl) + queues := storagemock.NewMockQueueStore(ctrl) + store.EXPECT().GetQueueStore().Return(queues) + request := entity.Request{ID: "7", Queue: "repo/main", URI: "commit-7", State: entity.RequestStateSucceeded} + stale := entity.Queue{Name: request.Queue, Version: 1, InFlightCount: 1, InFlightRequestIDs: []string{"7"}} + fresh := stale + fresh.Version = 2 + fresh.InFlightCount = 2 + fresh.InFlightRequestIDs = []string{"7", "8"} + fresh.LastGreenRequestID = "9" + fresh.LastGreenURI = "commit-9" + gomock.InOrder( + queues.EXPECT().Get(gomock.Any(), request.Queue).Return(stale, nil), + queues.EXPECT().Update(gomock.Any(), gomock.Any(), int32(1), int32(2)).Return(storage.ErrVersionMismatch), + queues.EXPECT().Get(gomock.Any(), request.Queue).Return(fresh, nil), + queues.EXPECT().Update(gomock.Any(), gomock.Any(), int32(2), int32(3)).DoAndReturn( + func(_ context.Context, updated entity.Queue, _, _ int32) error { + assert.Equal(t, []string{"8"}, updated.InFlightRequestIDs) + assert.Equal(t, fresh.LastGreenURI, updated.LastGreenURI) + return nil + }), + ) + require.NoError(t, Complete(context.Background(), store, request)) +} + +func TestCompleteRejectsInvalidOwnership(t *testing.T) { + for _, queue := range []entity.Queue{ + {InFlightCount: 1}, + {InFlightCount: 2, InFlightRequestIDs: []string{"7", "7"}}, + {InFlightCount: 1, InFlightRequestIDs: []string{""}}, + {InFlightCount: -1}, + } { + t.Run(fmt.Sprint(queue), func(t *testing.T) { + ctrl := gomock.NewController(t) + store := storagemock.NewMockStorage(ctrl) + queues := storagemock.NewMockQueueStore(ctrl) + store.EXPECT().GetQueueStore().Return(queues) + queues.EXPECT().Get(gomock.Any(), "repo/main").Return(queue, nil) + require.Error(t, Complete(context.Background(), store, entity.Request{ID: "7", Queue: "repo/main", State: entity.RequestStateFailed})) + }) + } +} diff --git a/stovepipe/entity/queue.go b/stovepipe/entity/queue.go index 9ca1c8fcc..3b486f07e 100644 --- a/stovepipe/entity/queue.go +++ b/stovepipe/entity/queue.go @@ -14,6 +14,11 @@ package entity +import ( + "fmt" + "slices" +) + // Queue holds per-queue pipeline coordination state for a named repo+ref (e.g. "monorepo/main"). type Queue struct { // **************** @@ -27,8 +32,8 @@ type Queue struct { // Following fields could be changed throughout the lifecycle of the queue // **************** - // LastGreenURI is the queue's last-known-good commit: the most recent head at which - // whole-repo validation recorded green (health degree 0). Empty until the first such outcome. + // LastGreenURI is the newest head with a successful build outcome, usable as an + // incremental baseline before validation facts and Git promotion are complete. LastGreenURI string `json:"last_green_uri"` // LastGreenRequestID is the request that established LastGreenURI. The bookmark only @@ -36,8 +41,10 @@ type Queue struct { // than the candidate's, compared by ingest order via CompareRequestID. LastGreenRequestID string `json:"last_green_request_id"` - // InFlightCount is the number of trunk validations admitted by process but not yet terminal. + // InFlightCount is the number of reserved build slots, equal to len(InFlightRequestIDs). InFlightCount int32 `json:"in_flight_count"` + // InFlightRequestIDs identifies the requests holding reserved build slots. + InFlightRequestIDs []string `json:"in_flight_request_ids"` // LatestRequestID is the request id of the newest head ingest accepted for this queue. // Empty until the first request is created. Coalescing compares IDs via CompareRequestID. @@ -61,6 +68,29 @@ type Queue struct { Version int32 `json:"version"` } +// HasBuildSlot reports whether requestID owns a reserved build slot. +func (q Queue) HasBuildSlot(requestID string) bool { + return slices.Contains(q.InFlightRequestIDs, requestID) +} + +// ValidateBuildSlots checks that the slot count and ownership agree. +func (q Queue) ValidateBuildSlots() error { + if q.InFlightCount != int32(len(q.InFlightRequestIDs)) { + return fmt.Errorf("queue %q has %d slots but %d owners", q.Name, q.InFlightCount, len(q.InFlightRequestIDs)) + } + seen := make(map[string]struct{}, len(q.InFlightRequestIDs)) + for _, id := range q.InFlightRequestIDs { + if id == "" { + return fmt.Errorf("queue %q has an empty slot owner", q.Name) + } + if _, exists := seen[id]; exists { + return fmt.Errorf("queue %q has duplicate slot owner %q", q.Name, id) + } + seen[id] = struct{}{} + } + return nil +} + func (q Queue) EnablementState() QueueState { return QueueState{Queue: q.Name, State: q.Enablement, Version: q.EnablementVersion, EffectiveFromCommitURI: q.EffectiveFromCommitURI, ChangedAtMs: q.EnablementChangedAtMs} diff --git a/stovepipe/entity/validation_fact.go b/stovepipe/entity/validation_fact.go index 198a6d7e4..9fe9fbbfc 100644 --- a/stovepipe/entity/validation_fact.go +++ b/stovepipe/entity/validation_fact.go @@ -19,6 +19,8 @@ package entity const ( // DegreeGreen is a fully healthy scope: nothing broken. DegreeGreen = 0.0 + // DegreePartial is a broken scope with individual project results available. + DegreePartial = 0.5 // DegreeBroken is a fully broken scope. DegreeBroken = 1.0 ) @@ -41,10 +43,8 @@ type ValidationFact struct { // repository; named projects narrow the fact to part of it. Project string `json:"project"` // Degree is how broken the scope was, on the closed interval - // [DegreeGreen, DegreeBroken]. A whole-repository scope only ever takes an - // endpoint: the build either passed or it did not. Intermediate values - // belong to project-level scopes, whose analysis defines how a partial - // degree is derived. + // [DegreeGreen, DegreeBroken]. A failed repository with project results has + // DegreePartial; without project results it has DegreeBroken. Degree float64 `json:"degree"` // RequestID is the request that established the fact. RequestID string `json:"request_id"` diff --git a/stovepipe/entity/validation_fact_test.go b/stovepipe/entity/validation_fact_test.go index e5eb657f5..0341f973c 100644 --- a/stovepipe/entity/validation_fact_test.go +++ b/stovepipe/entity/validation_fact_test.go @@ -28,7 +28,8 @@ func TestValidationFact_IsGreen(t *testing.T) { }{ {name: "green degree is green", degree: DegreeGreen, expected: true}, {name: "broken degree is not green", degree: DegreeBroken, expected: false}, - {name: "partial breakage is not green", degree: 0.25, expected: false}, + {name: "partial repository is not green", degree: DegreePartial, expected: false}, + {name: "partial project breakage is not green", degree: 0.25, expected: false}, {name: "zero value is green", degree: 0, expected: true}, } diff --git a/stovepipe/extension/storage/mysql/MIGRATIONS.md b/stovepipe/extension/storage/mysql/MIGRATIONS.md new file mode 100644 index 000000000..668d127be --- /dev/null +++ b/stovepipe/extension/storage/mysql/MIGRATIONS.md @@ -0,0 +1,9 @@ +# Queue build-slot ownership rollout + +Fresh databases use `schema/queue.sql`. Existing databases need the nullable `in_flight_request_ids JSON DEFAULT NULL` column on `queue` before these binaries start. The Queue store reads and writes this column with the rest of the versioned snapshot. + +1. Pause ingestion and drain all accepted and processing Requests, pending pipeline messages, and DLQ reconciliation with the old workers. Verify every Queue has `in_flight_count = 0`. +2. Stop every old writer. Append `in_flight_request_ids JSON DEFAULT NULL` as the final column of `queue`. Always append new columns so existing column positions remain unchanged. Existing drained rows need no backfill: SQL NULL represents an empty owner set. +3. Deploy all Queue readers and writers together, then resume ingestion. Do not mix old and new writers: an old writer cannot preserve slot ownership or the atomic baseline-and-release invariant. + +An old nonzero counter cannot safely be converted into owner identities from the count alone. Admission and completion reject mismatched counts, empty owner ids, and duplicate owners rather than silently over-admitting. Resolve an undrained queue before deployment instead of resetting its counter while builds remain live. Rollback likewise requires draining and stopping the new workers first. diff --git a/stovepipe/extension/storage/mysql/queue_store.go b/stovepipe/extension/storage/mysql/queue_store.go index c4fa43ca3..76af81a41 100644 --- a/stovepipe/extension/storage/mysql/queue_store.go +++ b/stovepipe/extension/storage/mysql/queue_store.go @@ -17,6 +17,7 @@ package mysql import ( "context" "database/sql" + "encoding/json" "errors" "fmt" @@ -47,10 +48,14 @@ func (q *queueStore) Create(ctx context.Context, queue entity.Queue) (retErr err if queue.Name != q.queue { return fmt.Errorf("queue row name %q does not match the store's bound queue %q", queue.Name, q.queue) } + owners, err := json.Marshal(queue.InFlightRequestIDs) + if err != nil { + return fmt.Errorf("encode build slot owners: %w", err) + } - _, err := q.db.ExecContext(ctx, - `INSERT INTO queue (name, last_green_uri, in_flight_count, latest_request_id, version, last_green_request_id, enablement, enablement_version, effective_from_commit_uri, enablement_changed_at_ms) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + _, err = q.db.ExecContext(ctx, + `INSERT INTO queue (name, last_green_uri, in_flight_count, latest_request_id, version, last_green_request_id, enablement, enablement_version, effective_from_commit_uri, enablement_changed_at_ms, in_flight_request_ids) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, queue.Name, queue.LastGreenURI, queue.InFlightCount, @@ -58,6 +63,7 @@ func (q *queueStore) Create(ctx context.Context, queue entity.Queue) (retErr err queue.Version, queue.LastGreenRequestID, queue.Enablement, queue.EnablementVersion, queue.EffectiveFromCommitURI, queue.EnablementChangedAtMs, + owners, ) if err != nil { if isDuplicateEntry(err) { @@ -78,8 +84,9 @@ func (q *queueStore) Get(ctx context.Context, name string) (ret entity.Queue, re } var queue entity.Queue + var owners []byte err := q.db.QueryRowContext(ctx, - "SELECT name, last_green_uri, in_flight_count, latest_request_id, version, last_green_request_id, enablement, enablement_version, effective_from_commit_uri, enablement_changed_at_ms FROM queue WHERE name = ?", + "SELECT name, last_green_uri, in_flight_count, latest_request_id, version, last_green_request_id, enablement, enablement_version, effective_from_commit_uri, enablement_changed_at_ms, in_flight_request_ids FROM queue WHERE name = ?", name, ).Scan( &queue.Name, @@ -89,6 +96,7 @@ func (q *queueStore) Get(ctx context.Context, name string) (ret entity.Queue, re &queue.Version, &queue.LastGreenRequestID, &queue.Enablement, &queue.EnablementVersion, &queue.EffectiveFromCommitURI, &queue.EnablementChangedAtMs, + &owners, ) if errors.Is(err, sql.ErrNoRows) { @@ -97,6 +105,11 @@ func (q *queueStore) Get(ctx context.Context, name string) (ret entity.Queue, re if err != nil { return entity.Queue{}, fmt.Errorf("failed to get queue name=%s from the database: %w", name, err) } + if len(owners) > 0 { + if err := json.Unmarshal(owners, &queue.InFlightRequestIDs); err != nil { + return entity.Queue{}, fmt.Errorf("decode build slot owners for queue %q: %w", name, err) + } + } return queue, nil } @@ -110,10 +123,14 @@ func (q *queueStore) Update(ctx context.Context, queue entity.Queue, oldVersion, if queue.Name != q.queue { return fmt.Errorf("queue row name %q does not match the store's bound queue %q", queue.Name, q.queue) } + owners, err := json.Marshal(queue.InFlightRequestIDs) + if err != nil { + return fmt.Errorf("encode build slot owners: %w", err) + } result, err := q.db.ExecContext(ctx, `UPDATE queue - SET last_green_uri = ?, in_flight_count = ?, latest_request_id = ?, version = ?, last_green_request_id = ?, enablement = ?, enablement_version = ?, effective_from_commit_uri = ?, enablement_changed_at_ms = ? + SET last_green_uri = ?, in_flight_count = ?, latest_request_id = ?, version = ?, last_green_request_id = ?, enablement = ?, enablement_version = ?, effective_from_commit_uri = ?, enablement_changed_at_ms = ?, in_flight_request_ids = ? WHERE name = ? AND version = ?`, queue.LastGreenURI, queue.InFlightCount, @@ -121,6 +138,7 @@ func (q *queueStore) Update(ctx context.Context, queue entity.Queue, oldVersion, newVersion, queue.LastGreenRequestID, queue.Enablement, queue.EnablementVersion, queue.EffectiveFromCommitURI, queue.EnablementChangedAtMs, + owners, queue.Name, oldVersion, ) diff --git a/stovepipe/extension/storage/mysql/queue_store_test.go b/stovepipe/extension/storage/mysql/queue_store_test.go index e96365daf..d50fa31d9 100644 --- a/stovepipe/extension/storage/mysql/queue_store_test.go +++ b/stovepipe/extension/storage/mysql/queue_store_test.go @@ -17,6 +17,7 @@ package mysql import ( "context" "database/sql" + "encoding/json" "fmt" "testing" @@ -60,7 +61,7 @@ func TestQueueStore_Create(t *testing.T) { name: "success", setup: func(mock sqlmock.Sqlmock) { mock.ExpectExec("INSERT INTO queue"). - WithArgs(queue.Name, queue.LastGreenURI, queue.InFlightCount, queue.LatestRequestID, queue.Version, queue.LastGreenRequestID, queue.Enablement, queue.EnablementVersion, queue.EffectiveFromCommitURI, queue.EnablementChangedAtMs). + WithArgs(queue.Name, queue.LastGreenURI, queue.InFlightCount, queue.LatestRequestID, queue.Version, queue.LastGreenRequestID, queue.Enablement, queue.EnablementVersion, queue.EffectiveFromCommitURI, queue.EnablementChangedAtMs, queueOwnersJSON(t, queue)). WillReturnResult(sqlmock.NewResult(0, 1)) }, }, @@ -68,7 +69,7 @@ func TestQueueStore_Create(t *testing.T) { name: "duplicate name returns ErrAlreadyExists", setup: func(mock sqlmock.Sqlmock) { mock.ExpectExec("INSERT INTO queue"). - WithArgs(queue.Name, queue.LastGreenURI, queue.InFlightCount, queue.LatestRequestID, queue.Version, queue.LastGreenRequestID, queue.Enablement, queue.EnablementVersion, queue.EffectiveFromCommitURI, queue.EnablementChangedAtMs). + WithArgs(queue.Name, queue.LastGreenURI, queue.InFlightCount, queue.LatestRequestID, queue.Version, queue.LastGreenRequestID, queue.Enablement, queue.EnablementVersion, queue.EffectiveFromCommitURI, queue.EnablementChangedAtMs, queueOwnersJSON(t, queue)). WillReturnError(&mysql.MySQLError{Number: mysqlErrDuplicateEntry}) }, wantErr: true, @@ -78,7 +79,7 @@ func TestQueueStore_Create(t *testing.T) { name: "other exec error", setup: func(mock sqlmock.Sqlmock) { mock.ExpectExec("INSERT INTO queue"). - WithArgs(queue.Name, queue.LastGreenURI, queue.InFlightCount, queue.LatestRequestID, queue.Version, queue.LastGreenRequestID, queue.Enablement, queue.EnablementVersion, queue.EffectiveFromCommitURI, queue.EnablementChangedAtMs). + WithArgs(queue.Name, queue.LastGreenURI, queue.InFlightCount, queue.LatestRequestID, queue.Version, queue.LastGreenRequestID, queue.Enablement, queue.EnablementVersion, queue.EffectiveFromCommitURI, queue.EnablementChangedAtMs, queueOwnersJSON(t, queue)). WillReturnError(fmt.Errorf("connection reset")) }, wantErr: true, @@ -128,9 +129,9 @@ func TestQueueStore_Get(t *testing.T) { name: "found", queueName: want.Name, setup: func(mock sqlmock.Sqlmock) { - rows := sqlmock.NewRows([]string{"name", "last_green_uri", "in_flight_count", "latest_request_id", "version", "last_green_request_id", "enablement", "enablement_version", "effective_from_commit_uri", "enablement_changed_at_ms"}). - AddRow(want.Name, want.LastGreenURI, want.InFlightCount, want.LatestRequestID, want.Version, want.LastGreenRequestID, want.Enablement, want.EnablementVersion, want.EffectiveFromCommitURI, want.EnablementChangedAtMs) - mock.ExpectQuery("SELECT name, last_green_uri, in_flight_count, latest_request_id, version, last_green_request_id, enablement, enablement_version, effective_from_commit_uri, enablement_changed_at_ms FROM queue"). + rows := sqlmock.NewRows([]string{"name", "last_green_uri", "in_flight_count", "latest_request_id", "version", "last_green_request_id", "enablement", "enablement_version", "effective_from_commit_uri", "enablement_changed_at_ms", "in_flight_request_ids"}). + AddRow(want.Name, want.LastGreenURI, want.InFlightCount, want.LatestRequestID, want.Version, want.LastGreenRequestID, want.Enablement, want.EnablementVersion, want.EffectiveFromCommitURI, want.EnablementChangedAtMs, queueOwnersJSON(t, want)) + mock.ExpectQuery("SELECT name, last_green_uri, in_flight_count, latest_request_id, version, last_green_request_id, enablement, enablement_version, effective_from_commit_uri, enablement_changed_at_ms, in_flight_request_ids FROM queue"). WithArgs(want.Name). WillReturnRows(rows) }, @@ -140,7 +141,7 @@ func TestQueueStore_Get(t *testing.T) { name: "not found", queueName: want.Name, setup: func(mock sqlmock.Sqlmock) { - mock.ExpectQuery("SELECT name, last_green_uri, in_flight_count, latest_request_id, version, last_green_request_id, enablement, enablement_version, effective_from_commit_uri, enablement_changed_at_ms FROM queue"). + mock.ExpectQuery("SELECT name, last_green_uri, in_flight_count, latest_request_id, version, last_green_request_id, enablement, enablement_version, effective_from_commit_uri, enablement_changed_at_ms, in_flight_request_ids FROM queue"). WithArgs(want.Name). WillReturnError(sql.ErrNoRows) }, @@ -151,7 +152,7 @@ func TestQueueStore_Get(t *testing.T) { name: "query error", queueName: want.Name, setup: func(mock sqlmock.Sqlmock) { - mock.ExpectQuery("SELECT name, last_green_uri, in_flight_count, latest_request_id, version, last_green_request_id, enablement, enablement_version, effective_from_commit_uri, enablement_changed_at_ms FROM queue"). + mock.ExpectQuery("SELECT name, last_green_uri, in_flight_count, latest_request_id, version, last_green_request_id, enablement, enablement_version, effective_from_commit_uri, enablement_changed_at_ms, in_flight_request_ids FROM queue"). WithArgs(want.Name). WillReturnError(fmt.Errorf("connection reset")) }, @@ -201,7 +202,7 @@ func TestQueueStore_Update(t *testing.T) { name: "success", setup: func(mock sqlmock.Sqlmock) { mock.ExpectExec("UPDATE queue"). - WithArgs(queue.LastGreenURI, queue.InFlightCount, queue.LatestRequestID, newVersion, queue.LastGreenRequestID, queue.Enablement, queue.EnablementVersion, queue.EffectiveFromCommitURI, queue.EnablementChangedAtMs, queue.Name, oldVersion). + WithArgs(queue.LastGreenURI, queue.InFlightCount, queue.LatestRequestID, newVersion, queue.LastGreenRequestID, queue.Enablement, queue.EnablementVersion, queue.EffectiveFromCommitURI, queue.EnablementChangedAtMs, queueOwnersJSON(t, queue), queue.Name, oldVersion). WillReturnResult(sqlmock.NewResult(0, 1)) }, }, @@ -209,7 +210,7 @@ func TestQueueStore_Update(t *testing.T) { name: "version mismatch", setup: func(mock sqlmock.Sqlmock) { mock.ExpectExec("UPDATE queue"). - WithArgs(queue.LastGreenURI, queue.InFlightCount, queue.LatestRequestID, newVersion, queue.LastGreenRequestID, queue.Enablement, queue.EnablementVersion, queue.EffectiveFromCommitURI, queue.EnablementChangedAtMs, queue.Name, oldVersion). + WithArgs(queue.LastGreenURI, queue.InFlightCount, queue.LatestRequestID, newVersion, queue.LastGreenRequestID, queue.Enablement, queue.EnablementVersion, queue.EffectiveFromCommitURI, queue.EnablementChangedAtMs, queueOwnersJSON(t, queue), queue.Name, oldVersion). WillReturnResult(sqlmock.NewResult(0, 0)) }, wantErr: true, @@ -219,7 +220,7 @@ func TestQueueStore_Update(t *testing.T) { name: "exec error", setup: func(mock sqlmock.Sqlmock) { mock.ExpectExec("UPDATE queue"). - WithArgs(queue.LastGreenURI, queue.InFlightCount, queue.LatestRequestID, newVersion, queue.LastGreenRequestID, queue.Enablement, queue.EnablementVersion, queue.EffectiveFromCommitURI, queue.EnablementChangedAtMs, queue.Name, oldVersion). + WithArgs(queue.LastGreenURI, queue.InFlightCount, queue.LatestRequestID, newVersion, queue.LastGreenRequestID, queue.Enablement, queue.EnablementVersion, queue.EffectiveFromCommitURI, queue.EnablementChangedAtMs, queueOwnersJSON(t, queue), queue.Name, oldVersion). WillReturnError(fmt.Errorf("connection reset")) }, wantErr: true, @@ -228,7 +229,7 @@ func TestQueueStore_Update(t *testing.T) { name: "rows affected error", setup: func(mock sqlmock.Sqlmock) { mock.ExpectExec("UPDATE queue"). - WithArgs(queue.LastGreenURI, queue.InFlightCount, queue.LatestRequestID, newVersion, queue.LastGreenRequestID, queue.Enablement, queue.EnablementVersion, queue.EffectiveFromCommitURI, queue.EnablementChangedAtMs, queue.Name, oldVersion). + WithArgs(queue.LastGreenURI, queue.InFlightCount, queue.LatestRequestID, newVersion, queue.LastGreenRequestID, queue.Enablement, queue.EnablementVersion, queue.EffectiveFromCommitURI, queue.EnablementChangedAtMs, queueOwnersJSON(t, queue), queue.Name, oldVersion). WillReturnResult(sqlmock.NewErrorResult(fmt.Errorf("driver error"))) }, wantErr: true, @@ -255,3 +256,10 @@ func TestQueueStore_Update(t *testing.T) { }) } } + +func queueOwnersJSON(t *testing.T, queue entity.Queue) []byte { + t.Helper() + owners, err := json.Marshal(queue.InFlightRequestIDs) + require.NoError(t, err) + return owners +} diff --git a/stovepipe/extension/storage/mysql/schema/queue.sql b/stovepipe/extension/storage/mysql/schema/queue.sql index 05acbc350..eeeccc141 100644 --- a/stovepipe/extension/storage/mysql/schema/queue.sql +++ b/stovepipe/extension/storage/mysql/schema/queue.sql @@ -11,5 +11,6 @@ CREATE TABLE IF NOT EXISTS queue ( enablement_version BIGINT NOT NULL DEFAULT 0, effective_from_commit_uri VARCHAR(255) NOT NULL DEFAULT '', enablement_changed_at_ms BIGINT NOT NULL DEFAULT 0, + in_flight_request_ids JSON DEFAULT NULL, PRIMARY KEY (name) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; diff --git a/test/integration/stovepipe/extension/storage/suite.go b/test/integration/stovepipe/extension/storage/suite.go index fec3a3d8b..3ed204ea4 100644 --- a/test/integration/stovepipe/extension/storage/suite.go +++ b/test/integration/stovepipe/extension/storage/suite.go @@ -90,6 +90,8 @@ func (s *QueueStoreContractSuite) TestQueueStore_CreateWithFields() { Name: name, LastGreenURI: "git://remote/monorepo/main/green-bbbb", LastGreenRequestID: "98", + InFlightCount: 2, + InFlightRequestIDs: []string{"99", "100"}, LatestRequestID: "99", Version: 1, } @@ -142,6 +144,7 @@ func (s *QueueStoreContractSuite) TestQueueStore_UpdateCAS() { updated.LastGreenRequestID = "41" updated.LatestRequestID = "42" updated.InFlightCount = 1 + updated.InFlightRequestIDs = []string{"42"} require.NoError(t, s.storeFor(name).Update(s.ctx, updated, 1, 2)) got, err := s.storeFor(name).Get(s.ctx, name) @@ -150,6 +153,7 @@ func (s *QueueStoreContractSuite) TestQueueStore_UpdateCAS() { assert.Equal(t, "41", got.LastGreenRequestID) assert.Equal(t, "42", got.LatestRequestID) assert.Equal(t, int32(1), got.InFlightCount) + assert.Len(t, got.InFlightRequestIDs, 1) assert.Equal(t, int32(2), got.Version) err = s.storeFor(name).Update(s.ctx, updated, 1, 2) @@ -174,13 +178,14 @@ func (s *QueueStoreContractSuite) TestQueueStore_UpdateSequentialCAS() { v2 := entity.Queue{Name: name, LatestRequestID: "10", Version: 1} require.NoError(t, s.storeFor(name).Update(s.ctx, v2, 1, 2)) - v3 := entity.Queue{Name: name, LatestRequestID: "10", InFlightCount: 1, Version: 2} + v3 := entity.Queue{Name: name, LatestRequestID: "10", InFlightCount: 1, InFlightRequestIDs: []string{"10"}, Version: 2} require.NoError(t, s.storeFor(name).Update(s.ctx, v3, 2, 3)) got, err := s.storeFor(name).Get(s.ctx, name) require.NoError(t, err) assert.Equal(t, "10", got.LatestRequestID) assert.Equal(t, int32(1), got.InFlightCount) + assert.Len(t, got.InFlightRequestIDs, 1) assert.Equal(t, int32(3), got.Version) } From b549470f5e212eec8640cc01fe4528b367617b4c Mon Sep 17 00:00:00 2001 From: mnoah1 Date: Thu, 22 Oct 2026 19:14:11 +0000 Subject: [PATCH 2/3] chore(stovepipe): restore unrelated comment wording --- .../controller/buildsignal/buildsignal.go | 18 ++++++++++-- stovepipe/controller/dlq/buildsignal.go | 4 +++ stovepipe/controller/dlq/dlq.go | 11 ++++++-- stovepipe/controller/process/process.go | 9 ++++-- stovepipe/controller/record/record.go | 28 +++++++++++++++---- stovepipe/controller/record/record_test.go | 5 ++-- 6 files changed, 61 insertions(+), 14 deletions(-) diff --git a/stovepipe/controller/buildsignal/buildsignal.go b/stovepipe/controller/buildsignal/buildsignal.go index c518e5e86..4646bba8e 100644 --- a/stovepipe/controller/buildsignal/buildsignal.go +++ b/stovepipe/controller/buildsignal/buildsignal.go @@ -55,7 +55,11 @@ var ( PollDelayRunningMs int64 = 2000 ) -// Controller polls builds and completes request outcomes before publishing to record. +// Controller consumes BuildSignal messages, polls the build-runner toward a +// terminal status, persists the result, and either holds the delivery until +// the next poll or projects the outcome onto the request, completes the queue's +// build slot and baseline, and publishes the request id to record. Implements +// consumer.Controller. type Controller struct { logger *zap.SugaredLogger metricsScope tally.Scope @@ -93,7 +97,12 @@ func NewController( } } -// Process polls a build, completes its terminal outcome and baseline, and publishes downstream work. +// Process reloads the build referenced by the delivery, polls its runner for +// the latest status, persists a real transition, and either holds the delivery +// until the next poll or, once terminal, projects the outcome onto the request, +// completes the queue's build slot and baseline, and publishes the request id to +// record. Returns nil to ack (success) or an error to nack (retry) / reject +// (DLQ). func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) error { msg := delivery.Message() @@ -132,6 +141,11 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er return fmt.Errorf("payload queue %q does not match queue %q of request %s", sig.GetQueueName(), request.Queue, request.ID) } + // A request only reaches the poll loop after process admitted it, so it is + // `processing` — or it already carries this build's outcome, on a redelivery + // after the outcome was stamped but before record was published. Both cases + // proceed: the redelivery re-publishes to record rather than dropping the + // signal, and repeats any unfinished slot and baseline completion. if request.State != entity.RequestStateProcessing && !request.State.HasBuildOutcome() { return nil } diff --git a/stovepipe/controller/dlq/buildsignal.go b/stovepipe/controller/dlq/buildsignal.go index 1b5b6bb43..bec62dbdd 100644 --- a/stovepipe/controller/dlq/buildsignal.go +++ b/stovepipe/controller/dlq/buildsignal.go @@ -149,6 +149,10 @@ func (c *buildSignalController) Process(ctx context.Context, delivery consumer.D return nil } + // Every request reachable from a build row is either still processing, and holding + // the slot failRequest releases, or already terminal, with completion safe to retry: build + // triggers only once process has written the strategy, which lands in the same CAS + // as accepted→processing, and processing exits only to a terminal outcome. if err := failRequest(ctx, store, c.materializer, c.logger, build.RequestID, entity.RequestOutcomeReasonBuildPollingExhausted); err != nil { metrics.NamedCounter(c.metricsScope, _buildSignalOpName, "reconcile_errors", 1, metrics.TagsFromContext(ctx)...) return err diff --git a/stovepipe/controller/dlq/dlq.go b/stovepipe/controller/dlq/dlq.go index 7be209c31..ef203bb03 100644 --- a/stovepipe/controller/dlq/dlq.go +++ b/stovepipe/controller/dlq/dlq.go @@ -22,8 +22,15 @@ // on greenness would see that indistinguishably from "not yet validated" (see // doc/rfc/stovepipe/workflow.md#fail-closed-on-unprocessable-work). // -// Reconciliation preserves the first terminal outcome, then completes the request-owned -// build slot and successful baseline. Terminal requests remain repairable on replay. +// Reconciliation strategy. Each DLQ topic carries the same payload as its originating +// topic (the queue framework preserves the bytes verbatim under a new `{topic}_dlq` +// name). The DLQ controller decodes that payload to recover the affected request, then +// transitions it to RequestStateFailed — the conservative unsuccessful outcome, which +// whatever records greenness treats as not-green for gating (see entity.RequestState) — +// with an idempotent optimistic-locking write so +// concurrent activity (a late successful pipeline transition) wins cleanly. If the request had +// claimed a concurrency slot, the reconciler completes that reservation and any +// successful baseline in one Queue update, including on terminal redeliveries. package dlq import ( diff --git a/stovepipe/controller/process/process.go b/stovepipe/controller/process/process.go index ffdffa403..f8d2e941c 100644 --- a/stovepipe/controller/process/process.go +++ b/stovepipe/controller/process/process.go @@ -14,7 +14,8 @@ // Package process holds the process-stage queue controller. It consumes request // ids from ingest, reloads the Request from storage, coalesces older heads, and -// admits the latest head when a build slot is open. +// admits the latest head when a build slot is open. Build queue publish lands in +// a follow-up PR. package process import ( @@ -355,7 +356,11 @@ func (c *Controller) deriveBuildStrategy(ctx context.Context, sc sourcecontrol.S return entity.BuildStrategyFull, "", nil } -// markProcessing preserves a concurrent transition and its pinned build strategy. +// markProcessing CAS-marks request accepted→processing and persists the strategy chosen after the +// build slot was claimed. It reapplies the values after a request reload so an accepted concurrent +// update cannot discard them. transitioned is true only when this call performed the CAS; false +// means a concurrent writer already advanced the request past accepted, so the caller completes +// the shared reservation only if the request is terminal. func (c *Controller) markProcessing(ctx context.Context, store storage.Storage, request *entity.Request, strategy entity.BuildStrategy, baseURI string) (transitioned bool, err error) { reqStore := store.GetRequestStore() diff --git a/stovepipe/controller/record/record.go b/stovepipe/controller/record/record.go index 1401e64e3..a53489055 100644 --- a/stovepipe/controller/record/record.go +++ b/stovepipe/controller/record/record.go @@ -16,8 +16,11 @@ // messages (a request id) published by buildsignal once a build reaches a // terminal status, and turns that outcome into durable validation state. // -// Repository and project facts are persisted before promotion. Build completion -// owns the stored incremental baseline, which can precede these facts. +// The durable state is a ValidationFact per validated commit and project. Build +// completion owns the queue's last-green bookmark, which process reads to choose +// an incremental build baseline. A green commit is also promoted after its facts +// are persisted, moving the queue's promotion ref so downstream systems can pull +// the latest green commit by name. // // The outcome is then announced as a hook event, so integrations outside the // pipeline learn that a commit is green or broken. See @@ -276,8 +279,10 @@ func (c *Controller) persistValidationFactRecordedLog( return nil } -// applyFactToDerivedCaches promotes a finalized green fact only while its request -// owns the build baseline. Build completion owns the baseline itself. +// applyFactToDerivedCaches follows the persisted fact: a green fact promotes the +// commit when this request holds the queue's bookmark, set by build completion. +// A broken fact skips promotion, and instead reports how long the break it names +// went undetected. func (c *Controller) applyFactToDerivedCaches( ctx context.Context, store storage.Storage, @@ -307,13 +312,22 @@ func (c *Controller) applyFactToDerivedCaches( } } if queue.LastGreenRequestID != request.ID || queue.LastGreenURI != request.URI { + // Only the current baseline owner can promote: promoting an older commit + // would move the ref backwards. return nil } c.emitLastGreenTimestamp(ctx, request) return c.promote(ctx, request) } -// recordFact adopts the stored fact on replay; immutable facts remain authoritative. +// recordFact writes the request's outcome as an immutable whole-repository fact and +// returns the fact that is actually stored, which is not always the one just built: +// facts are first-writer-wins, so an identity already claimed by this same request — +// a redelivery after the write but before promotion — yields the stored +// fact instead. Every decision downstream reads that stored fact rather than the +// request, so a redelivery cannot reach a different verdict than the original. The +// second return reports whether this call is the one that wrote the fact, which is +// how a caller tells the original delivery from a redelivery. func (c *Controller) recordFact(ctx context.Context, store storage.Storage, request entity.Request, results []projectstatus.Result) (entity.ValidationFact, bool, error) { return c.recordValidationFact(ctx, store.GetValidationFactStore(), entity.ValidationFact{ URI: request.URI, @@ -430,6 +444,8 @@ func (c *Controller) failureDetectionUnobserved(ctx context.Context, request ent ) } +// degreeFor maps a request's build outcome onto a whole-repository degree. +// A failed build with project results has a partial degree. func degreeFor(state entity.RequestState, results []projectstatus.Result) float64 { if state == entity.RequestStateSucceeded { return entity.DegreeGreen @@ -495,7 +511,7 @@ func (c *Controller) emitLastGreenTimestamp(ctx context.Context, request entity. // the queue has one at all — is source-control configuration, so this stage names // only the commit. // -// Like the bookmark, the ref is a cache of the facts, so it moves only after the +// The ref is a cache of the facts, so it moves only after the // green fact is durable. Promotion is idempotent, so a redelivery repeats it // harmlessly. A commit that a rewritten history dropped from the ref cannot be // promoted by any retry, so that case is counted and skipped rather than failed. diff --git a/stovepipe/controller/record/record_test.go b/stovepipe/controller/record/record_test.go index d773551d4..31e2f4539 100644 --- a/stovepipe/controller/record/record_test.go +++ b/stovepipe/controller/record/record_test.go @@ -684,7 +684,8 @@ func TestProcess_AdoptsExistingFactFromSameRequest(t *testing.T) { ctrl := gomock.NewController(t) c, m := newController(t, ctrl) - // A stored fact controls promotion even if the current Request suggests another degree. + // A redelivery after the fact was written but before promotion: + // the write loses, and the stored fact decides. m.reqStore.EXPECT().Get(gomock.Any(), testID). Return(requestWithState(entity.RequestStateSucceeded), nil) m.factStore.EXPECT().Create(gomock.Any(), gomock.Any()).Return(storage.ErrAlreadyExists) @@ -747,7 +748,7 @@ func TestProcessPromotesOnlyCurrentBaselineOwner(t *testing.T) { var fact entity.ValidationFact m.expectFactCreated(&fact) m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(tt.stored, nil) - // Record reads the baseline without changing it. + // No Update: the bookmark only moves forward. if tt.wantPromote { m.sourceControl.EXPECT().ChangeInfo(gomock.Any(), testURI).Return(sourcecontrol.ChangeInfo{}, nil) From 9d4110bed6880c589346ba0cf61b59c4a18ae1dc Mon Sep 17 00:00:00 2001 From: mnoah1 Date: Thu, 22 Oct 2026 19:31:07 +0000 Subject: [PATCH 3/3] refactor(stovepipe): preserve existing slot accounting for partial validation --- doc/rfc/stovepipe/steps/buildsignal.md | 43 +++-- doc/rfc/stovepipe/steps/process.md | 45 ++--- doc/rfc/stovepipe/steps/record.md | 119 ++++++++++--- doc/rfc/stovepipe/workflow.md | 22 +-- .../controller/build/build_analysis_test.go | 2 +- stovepipe/controller/buildsignal/BUILD.bazel | 1 - .../controller/buildsignal/buildsignal.go | 141 +++++++++++++-- .../buildsignal/buildsignal_test.go | 166 +++++++++++++----- stovepipe/controller/dlq/BUILD.bazel | 1 - stovepipe/controller/dlq/build_test.go | 2 +- stovepipe/controller/dlq/buildsignal.go | 2 +- stovepipe/controller/dlq/buildsignal_test.go | 12 +- stovepipe/controller/dlq/dlq.go | 140 +++++++++++---- stovepipe/controller/dlq/dlq_test.go | 50 +----- stovepipe/controller/process/BUILD.bazel | 2 - stovepipe/controller/process/process.go | 96 +++++++--- stovepipe/controller/process/process_test.go | 159 ++++++----------- stovepipe/controller/record/record.go | 2 +- stovepipe/core/buildslot/BUILD.bazel | 29 --- stovepipe/core/buildslot/README.md | 9 - stovepipe/core/buildslot/claim.go | 52 ------ stovepipe/core/buildslot/complete.go | 80 --------- stovepipe/core/buildslot/complete_test.go | 140 --------------- stovepipe/entity/queue.go | 36 +--- .../extension/storage/mysql/MIGRATIONS.md | 9 - .../extension/storage/mysql/queue_store.go | 28 +-- .../storage/mysql/queue_store_test.go | 32 ++-- .../extension/storage/mysql/schema/queue.sql | 1 - .../stovepipe/extension/storage/suite.go | 7 +- 29 files changed, 668 insertions(+), 760 deletions(-) delete mode 100644 stovepipe/core/buildslot/BUILD.bazel delete mode 100644 stovepipe/core/buildslot/README.md delete mode 100644 stovepipe/core/buildslot/claim.go delete mode 100644 stovepipe/core/buildslot/complete.go delete mode 100644 stovepipe/core/buildslot/complete_test.go delete mode 100644 stovepipe/extension/storage/mysql/MIGRATIONS.md diff --git a/doc/rfc/stovepipe/steps/buildsignal.md b/doc/rfc/stovepipe/steps/buildsignal.md index 7fc0812f4..cf0b7f78b 100644 --- a/doc/rfc/stovepipe/steps/buildsignal.md +++ b/doc/rfc/stovepipe/steps/buildsignal.md @@ -1,6 +1,6 @@ # Buildsignal stage -`buildsignal` polls the build-runner until a build reaches a terminal state, records the status, and — once the build is terminal — persists the winning `Request` outcome, completes its build slot and successful baseline in one Queue update, and publishes the **request id** to `record`. See [workflow.md](../workflow.md) for where it sits in the pipeline and [build.md](build.md) for how the build it polls was triggered. +`buildsignal` polls the build-runner until a build reaches a terminal state, records the status, and — once the build is terminal — releases the queue's build slot, projects the outcome onto the `Request`, advances a successful Request's last-green build baseline, and publishes the **request id** to `record`. See [workflow.md](../workflow.md) for where it sits in the pipeline and [build.md](build.md) for how the build it polls was triggered. It handles only the poll loop: it does not decide build strategy, write greenness, or map targets to projects — those are `process`'s, `record`'s, and `analyze`'s jobs. It reports the *fact* (this request's build ended succeeded, failed, or cancelled); `record` derives what that fact *means* for greenness. @@ -16,7 +16,7 @@ Its logic does not branch on phase: it loads the `Build`, polls it toward termin Its early-exit guard is deliberately narrower than `State.IsTerminal()`: it proceeds when the request is `processing` **or** already carries a build outcome. The second case matters because a redelivery after the outcome was stamped but before the `record` publish landed must re-publish rather than drop the signal; everything it re-runs is a no-op (the status is unchanged, the outcome is already recorded, the slot is not released twice) and the `record` publish is idempotent. -A superseded Request has no build outcome and is ignored. Slot release is keyed by Request ownership, so duplicate deliveries cannot release another Request’s capacity. +The states the guard actually rejects are terminal-without-an-outcome — today `superseded` alone, which `process` sets only from `accepted`, so it cannot reach the poll loop at all. The guard is kept rather than dropped because the slot release in step 7a is conditioned on *not* already having an outcome, so a superseded request arriving here would decrement a slot that superseding never claimed. ## Algorithm @@ -35,7 +35,9 @@ For a delivery carrying build id `B`: - a request reaches the poll loop only after process admitted it, so it is processing — or already carries this build's outcome on a redelivery, which falls through so the signal is re-published to record instead of dropped. - - superseded requests have no build outcome and do not enter the poll loop. + - the only other terminal-without-an-outcome state is superseded, and process supersedes + solely from accepted, so it cannot occur here. Guarded anyway: step 7a would otherwise + release a build slot the request never claimed, since superseding consumes none. - mirrors submitqueue's halted-batch short-circuit in buildsignal.go. 4. Resolve the build-runner: buildRunner = Factory.For(Config{QueueName: R.Queue}). @@ -59,20 +61,21 @@ For a delivery carrying build id `B`: by mechanism, not by assumption about the backend. 7. If the stored status is terminal: - a. Retain the build-finished history event. - b. CAS R from processing to the projected outcome, persisting B as TerminalBuildID: - succeeded -> succeeded, failed -> failed, cancelled -> cancelled. First terminal writer wins. - Adopt a concurrent terminal winner rather than replacing its State or TerminalBuildID. - c. Complete R's owned slot with a Queue CAS loop: - - if R.ID is absent from in_flight_request_ids, completion is already done; no write. - - remove only R.ID and recompute in_flight_count from the remaining owners. - - if R succeeded and is newer than LastGreenRequestID, advance LastGreenURI and - LastGreenRequestID in the SAME Queue write, before capacity becomes available. - - on version conflict, reload and retry. Other errors fail the delivery. - d. Retain outcome history, then publish R.ID to record and a terminal build-analysis nudge. - Both publications are attempted independently and errors are combined with errs.Group. - The record message id and partition key are R.ID; record reloads the durable outcome. - e. Ack; no re-publish to buildsignal. + a. If R does not already carry an outcome, release the queue's build slot: + CAS-decrement Queue.in_flight_count, clamped at zero. + - failure here aborts the step: R must not go terminal while still holding a slot. + b. If R does not already carry an outcome, CAS R from processing to the outcome the stored status projects onto it and persist B as R's terminal build id: + succeeded -> succeeded, failed -> failed, cancelled -> cancelled. First writer wins. + c. If the winning durable R.State is succeeded, advance the last-green bookmark + in a CAS retry loop, even on a terminal redelivery. Preserve a newer bookmark. + Then publish R.ID to the record topic, partitioned by request id; ack, return. + No re-publish to buildsignal. + - record loads the Request directly by this key, derives greenness from its outcome, and + supplies its terminal build id to project-result resolution; it never reaches a Build. + - the message id is the request id, so a redelivery republishing the same terminal signal + dedups into the original message; record is idempotent regardless. + - publish failure -> return raw (non-retryable); the outcome is persisted, operational + republish recovers. 8. Else hold the delivery for delayMs (postpone: the same message redelivers after the delay): - delayMs = pollDelay(status): shorter while running, longer while accepted. @@ -82,9 +85,11 @@ For a delivery carrying build id `B`: exactly one poll chain; no message ids are minted and no new rows are written per tick. ``` -**Why the outcome is persisted first:** Queue and Request are separate entities. A durable outcome lets retries and DLQ reconciliation finish the same completion after a crash without guessing which build won. Slot ownership makes removal idempotent, including when a Queue write commits but its response is lost. Completion must run for terminal Requests as well as newly finished ones. +**Why the slot is released before the outcome write, and why a failed release aborts it**: `Queue` and `Request` are separate entities with no cross-entity transaction, so the ordering picks which crash failure mode we accept. Both rules serve one invariant — *the request must not go terminal while still holding a slot* — because a terminal request is skipped by redelivery and by the DLQ reconciler alike, so nothing would ever decrement it. Failing this way leaves the request non-terminal: redelivery re-runs both steps and decrements again, transiently over-admitting by one slot until the zero clamp reconverges. Over-admission is the failure mode this pipeline already prefers, for the same reason and in the same words as the DLQ reconciler (see [process.md](process.md#in_flight_count-integrity)). -**Why baseline advancement and slot release share a Queue write:** the next admitted build must see the successful baseline as soon as it sees freed capacity. Project resolution, immutable validation facts, history, and Git promotion belong to `record` and may finish later. A baseline is evidence of build success; deployments continue to gate on validation facts. +**Baseline timing.** After the winning Request outcome is durable, a successful Request advances `last_green_uri` and `last_green_request_id` in a CAS retry loop before publishing to `record`. Older candidates never regress the bookmark. A terminal redelivery retries the advance without releasing capacity again. Slot release and bookmark advancement are separate updates, so an admission between them can still select the previous baseline. Project resolution, facts, and Git promotion remain in `record`. + +**Why `buildsignal` releases the slot rather than `record`**: the gate `process` claims is a *build* slot — it exists to bound concurrent builds per Queue — and once the build is terminal the build is over. Releasing here also keeps the invariant *a terminal `Request` has already released its slot*, which is what makes the DLQ reconciler's early-return on terminal requests safe. **Why `record` hears only terminal signals**: `record` has no non-terminal work — by its own contract a non-terminal signal would be a pure no-op — and step 7 already branches on terminality to decide whether to keep polling, so gating the publish costs nothing and spares `record` a no-op delivery on every poll tick of every running build. Crash-safety is unaffected: a crash between the terminal `Update` and the publish redelivers the message; step 5 re-polls (the runner reports the same terminal status), step 6 no-ops, step 7 publishes. This is a deliberate divergence from SubmitQueue, whose buildsignal republishes to `speculate` on every tick — sound there because speculate is a state machine that may act on any signal; stovepipe has no such consumer. diff --git a/doc/rfc/stovepipe/steps/process.md b/doc/rfc/stovepipe/steps/process.md index 8490099d0..b1089fbe0 100644 --- a/doc/rfc/stovepipe/steps/process.md +++ b/doc/rfc/stovepipe/steps/process.md @@ -9,7 +9,7 @@ It handles everything *before* the first build is triggered: it does not run bui Ingest publishes a `ProcessRequest` (request id only — producer and consumer share the store) to the process topic, **partitioned by Queue name**. Two consequences the design relies on: 1. **One partition per Queue**, and `BatchSize = 1` (strict serialization; see [sql-queue-rfc.md](../../sql-queue-rfc.md#strict-serialization-opt-in)) — so at most one `process` invocation runs per Queue at a time. The read-modify-write on the Queue row (gate check, count increment) is therefore race-free **without** a transaction, the same property SubmitQueue's `validate` gets from per-queue partition leasing. -2. **`process` is not the only Queue-row writer.** `ingest` stamps `latest_request_id` (the request id of the newest accepted head, not a URI; see [Backlog coalescing](#backlog-coalescing)) and `buildsignal` releases slots when a build goes terminal, and advances the successful `last_green_uri` baseline in that same write. They touch different fields under optimistic-locking CAS, so concurrent writes converge instead of clobbering. +2. **`process` is not the only Queue-row writer.** `ingest` stamps `latest_request_id` (the request id of the newest accepted head, not a URI; see [Backlog coalescing](#backlog-coalescing)) and `buildsignal` releases slots when a build goes terminal, then advances the `last_green_uri` bookmark for a successful Request. They touch different fields under optimistic-locking CAS, so concurrent writes converge instead of clobbering. ## Process algorithm @@ -19,19 +19,17 @@ For a delivery carrying request id `R`: 1. Load Request R from the request store. - not found -> non-retryable (storage is read-after-write consistent; see [storage README](../../../../stovepipe/extension/storage/README.md)). 2. If R.State is terminal (superseded / succeeded / failed / cancelled): - - complete any owned reservation using the durable outcome, retain supersession history if applicable, then ack. + - ack and return (idempotent no-op). 3. If R.State is processing (strategy already recorded): - re-announce validation start and re-publish R to build (either prior publish may have failed), ack, return. 4. R.State is accepted. Load the Queue row Q. 5. Coalesce: if CompareRequestID(R.Queue, R.ID, Q.latest_request_id) < 0: - a newer head exists -> mark R superseded, ack, return. (No slot consumed.) -6. R is the latest head. Gate: if R does not already own a slot and Q.in_flight_count >= max_concurrent (from queue config; see below): +6. R is the latest head. Gate: if Q.in_flight_count >= max_concurrent (from queue config; see below): - defer (hold the delivery) -> re-check on redelivery until the slot frees (admit) or a newer head supersedes it. See [Waiting for a slot](#waiting-for-a-slot). 7. Admit R: a. Derive build strategy + baseline (see "Build-strategy decision"). - b. CAS the Queue row: add R.ID to in_flight_request_ids once and recompute in_flight_count. - A preexisting reservation is reused even at capacity. A conflict reloads Q and - reruns coalescing, gate, and baseline derivation. + b. CAS the Queue row: in_flight_count += 1. c. CAS the Request: accepted -> processing, persist build_strategy + base_uri. d. Announce validation start on the hook topic (see "Hooks"). e. Publish R to build. @@ -61,15 +59,15 @@ Validation is expensive and shares a baseline, so heads arriving while an earlie | Source | Field | Meaning | |---|---|---| -| Queue row | `last_green_uri` | Successful-build baseline advanced by buildsignal; empty until first successful build. | -| Queue row | `in_flight_count` | Count of owned build reservations; always equal to the length of `in_flight_request_ids`. | +| Queue row | `last_green_uri` | Bookmark `buildsignal` advances on a successful build; empty until first green. | +| Queue row | `in_flight_count` | Requests past `process` and not yet terminal. `process` increments on admit; `record` (or DLQ reconciliation) decrements on terminal. | | Queue config | `max_concurrent` | Cap on concurrent in-flight validations. **Default 1** (global wiring default for MVP; per-queue override when a Stovepipe `queueconfig` extension lands). | A slot is held from admit until the build goes terminal (`process → build → buildsignal`), not just while `process` runs. It is released when the Request reaches **any** terminal state and `in_flight_count` is decremented — `buildsignal` recording the build's outcome, success *or* failure, or the DLQ reconciler forcing a terminal `failed` (see [integrity](#in_flight_count-integrity)). A build *failure* frees the slot just like a success; only a Request that never terminates keeps its slot. **Liveness — a stuck Request wedges the whole Queue.** The slot is shared per-Queue, so a Request admitted but never driven terminal holds the only slot and stalls the whole Queue. The fail-closed path in [workflow.md](../workflow.md#fail-closed-on-unprocessable-work) prevents this: every admitted Request terminates — `build`/`buildsignal` errors retry to `MaxAttempts`, then dead-letter, and the DLQ reconciler forces a conservative terminal `failed`, freeing the slot. Residual risk is operational: a **poison DLQ message** the reconciler can't process loops forever (it runs always-retryable), wedging that Queue until an operator removes it. The gate makes the blast radius the whole Queue, not one Request — monitor and alert on the DLQ. -**Future alternative — time-bounded leases.** Reservations could gain expiry to bound recovery time for abandoned work. This needs a policy for still-running external builds and remains deferred; ownership and DLQ completion provide replay safety today. +**Future alternative — time-bounded leases.** To self-heal instead of relying on operator cleanup, `in_flight_count` could become a set of `{owner, expires_at}` leases: the gate counts only unexpired leases, terminal transitions drop the owner's lease, and a leaked lease is reclaimed on expiry — bounding any stall to a `max_validation_ms`. A *list* of leases would also generalize to `max_concurrent > 1`. Deferred; `in_flight_count` plus the fail-closed path is enough for MVP. `max_concurrent` is a scalar, and `= 1` is an MVP simplicity choice, **not** a correctness requirement — see [Raising `max_concurrent`](#raising-max_concurrent-speculative-validation). @@ -82,7 +80,7 @@ The scheme: each admit pins its baseline to last-green *at admit time*; a green Correctness rests on four rules, all with MVP primitives already in place: 1. **Pin to last-green at admit** — a known-green baseline, which `process` already reads. -2. **Advance last-green by ingest order, monotonically** — `record` adopts the highest-counter green (per `CompareRequestID`) and never regresses. (Free at N=1, since verdicts land in order; explicit at N > 1.) +2. **Advance last-green by ingest order, monotonically** — `buildsignal` adopts the highest-counter green (per `CompareRequestID`) and never regresses. (Free at N=1, since verdicts land in order; explicit at N > 1.) 3. **Coalesce to the latest N** instead of the latest 1 — otherwise intermediates are superseded on sight and nothing overlaps. 4. **Fall back to serial on a rewrite** — the linear/superset property is what makes "adopt highest green" sound; rewrites already force a full build with no baseline. @@ -113,11 +111,11 @@ Superseded Requests reach an explicit terminal `superseded` state, so "not yet v ## Concurrency lifecycle -The gate is **not** tied to `process` returning; a slot taken at admit is held until the build completes at `buildsignal` (or DLQ reconciliation). +The gate is **not** tied to `process` returning; a slot taken at admit is held until Phase 1 terminates at `record` (or DLQ reconciliation). **Rules** -1. **One slot per in-flight validation** (MVP: one per Queue). `process` increments `in_flight_count` on admit; `buildsignal` completes its owned reservation on terminal. +1. **One slot per in-flight validation** (MVP: one per Queue). `process` increments `in_flight_count` on admit; `record` decrements on terminal. 2. **No skip-ahead while in-flight.** The latest head waits for a slot until the running validation completes; it never preempts. 3. **Intermediates are superseded on sight**, gate open or closed — no slot consumed (step 5). 4. **Coalesce-to-latest on gate open.** When a slot frees, the waiting latest head is admitted. @@ -128,7 +126,7 @@ The gate is **not** tied to `process` returning; a slot taken at admit is held u 1. **A** admitted (`in_flight_count = 1`), published to `build`. 2. While A runs, **B**, **C**, **D** are ingested (`latest_request_id = D.id`). 3. B: older than D → **superseded** (acked), though A is still in flight. Same for **C**. D is latest but the gate is closed → **waits for slot** (held). -4. A's build finishes → `buildsignal` persists A's outcome and atomically advances its successful baseline and frees its slot; `record` can finish later. +4. A's build finishes → `record` records A's greenness, `in_flight_count → 0`. 5. D's re-check → gate open, D still latest → **D admitted**, published to `build`. 6. While D runs, **E**, **F** ingested (`latest_request_id = F.id`). E superseded on sight; F waits for slot. 7. D completes → slot frees → **F admitted**. @@ -137,7 +135,7 @@ A, D, F each get a full cycle; B, C, E end `superseded`. No intermediate is vali **What does not happen** -- `process` returning does **not** free a slot — only build completion (or DLQ reconciliation) does. +- `process` returning does **not** free a slot — only `record` (or DLQ reconciliation) does. - A newer head does **not** preempt an in-flight validation. - Deferred messages are **not** failed or dead-lettered — they wait for the gate (see [Waiting for a slot](#waiting-for-a-slot)). @@ -161,16 +159,21 @@ Only an admit announces. A Request that coalescing supersedes never reaches step Every branch is safe under redelivery: -- **accepted, no strategy** → full admit path. On a crash after incrementing `in_flight_count` but before persisting `processing`, redelivery reuses the reservation by Request id without incrementing the count again. +- **accepted, no strategy** → full admit path. On a crash after incrementing `in_flight_count` but before persisting `processing`, redelivery re-reads `accepted` and re-runs; the increment re-applies only if the count CAS hasn't already moved (see integrity below). - **processing** → re-announce the start event, re-publish to `build`, ack. The `build` consumer is keyed on the request id and idempotent, so a duplicate publish is harmless, and the start event's id is derived from the transition rather than the clock, so a re-announce carries the id the first attempt would have and consumers dedupe on it. Re-announcing here is what makes the event at-least-once rather than at-most-once: this is the only branch a redelivery takes once `processing` is durable, so an admit that failed after the state write would otherwise lose the event for good. -- **terminal** → finish any owned reservation from the durable outcome; never remove another owner. +- **terminal** (superseded / recorded) → ack, no-op. - **deferred (waiting for slot)** → no state or count change; pure deferral (re-enters when the held delivery comes due). +The window to handle is "count incremented, state not yet `processing`". Admit does the increment and the state transition as two ordered CAS writes, and the decrement is tied to the state transition, not a side counter (see integrity below). + ## `in_flight_count` integrity -The Queue owns a bounded list of `in_flight_request_ids`; `in_flight_count` must equal its length. Admission validates this invariant and reserves by Request id under CAS before writing `processing`. A duplicate delivery reuses its existing reservation, even at capacity. A failed or ambiguous Request write keeps the reservation for retry or DLQ recovery; a losing writer must not release the winner's shared reservation. +`in_flight_count` is a cache; the source of truth is **the set of non-terminal Request rows for the Queue**. Two rules keep it from drifting: + +1. **Decrement is bound to the terminal transition.** The single CAS that moves a Request non-terminal → terminal (in `record` or the DLQ reconciler) also decrements. Being CAS-guarded, it fires exactly once per Request even under redelivery. +2. **Increment is bound to the admit transition.** `process` increments only on the `accepted → processing` CAS; a redelivery of an already-`processing` Request takes step 3 and does not increment again. -Completion first persists the winning terminal Request outcome. A single Queue CAS removes only that Request's reservation and, for a newer successful Request, advances its baseline. Replays with no reservation do nothing. A crash between the two entity writes temporarily holds capacity; buildsignal redelivery, a terminal process delivery, or DLQ reconciliation completes it. No cross-entity transaction or zero clamp is needed. See [buildslot](../../../../stovepipe/core/buildslot/README.md). +On a crash between admit and `record`, the Request stays non-terminal; visibility-timeout redelivery drives it forward, and the fail-closed DLQ path eventually forces it terminal, decrementing as it does. The count can drift high only transiently and self-heals as stuck Requests terminate. A reconciler that recomputes the count from non-terminal rows can be added later if drift proves real, but isn't required for MVP. ## Edge cases @@ -188,8 +191,8 @@ Runtime coordination only — fields the pipeline writes under CAS: | Field | Role | Written by | |---|---|---| | `name` | Stable logical id (`monorepo/main`); the string ingest accepts | ingest (create) | -| `last_green_uri`, `last_green_request_id` | Successful-build baseline; may precede recorded facts | buildsignal / terminal completion | -| `in_flight_count`, `in_flight_request_ids` | Count and owners of build reservations | process (reserve), completion (release) | +| `last_green_uri` | Bookmark; empty until first green | buildsignal | +| `in_flight_count` | Active Phase 1 validations | process (+1), record/DLQ (−1) | | `latest_request_id` | Request id of the newest head ingest accepted | ingest | | `version` | Optimistic-locking version | all writers | @@ -219,7 +222,7 @@ Transitions use the repo's optimistic-locking pattern: compute `newVersion = old New key/value-shaped operations (single-key reads/writes, no server-side filtering or aggregation): -- **`QueueStore`** (new): `Create(ctx, queue)`, `Get(ctx, name)`, and `Update(ctx, queue, oldVersion, newVersion)` (CAS). Callers orchestrate get-or-create; ingest CASes `latest_request_id`; `process` CASes slot ownership and count; build completion CASes the baseline and slot ownership together. +- **`QueueStore`** (new): `Create(ctx, queue)`, `Get(ctx, name)`, and `Update(ctx, queue, oldVersion, newVersion)` (CAS). Callers orchestrate get-or-create; ingest CASes `latest_request_id`; `process` CASes `in_flight_count`; `record` CASes `last_green_uri` + `in_flight_count`. - **`RequestStore`**: no new methods — the added `Request` fields ride the existing `Create`/`Update` CAS. No "list requests by queue/state" query is introduced; coalescing uses the single-row `latest_request_id` pointer instead, keeping the contract satisfiable by a plain KV backend. diff --git a/doc/rfc/stovepipe/steps/record.md b/doc/rfc/stovepipe/steps/record.md index e28e56e68..1a7da8267 100644 --- a/doc/rfc/stovepipe/steps/record.md +++ b/doc/rfc/stovepipe/steps/record.md @@ -2,30 +2,82 @@ `record` turns a terminal build outcome into a durable validation fact. -`record` resolves optional project results before selecting the repository degree. It records the repository fact and project facts, then promotes a green repository fact while its Request still owns the successful-build baseline, and finally publishes the repository hook. See [record.go](../../../../stovepipe/controller/record/record.go). +- Phase 1 records whole-repository greenness and optional project facts. On green it promotes the commit onto the Queue's promotion ref if the Request still holds the last-green bookmark, then announces the outcome as a hook event. This is implemented: see [stovepipe/controller/record/record.go](../../../../stovepipe/controller/record/record.go). +- Phase 2 records greenness per project instead of per repository. Sketched here, to be expanded before implementation. Notifying downstream systems rides the cross-domain hook framework rather than a Stovepipe-specific extension; see [Hooks](#hooks). See [workflow.md](../workflow.md) for the whole pipeline, [build.md](build.md) for how builds are created, and [buildsignal.md](buildsignal.md) for the terminal-only handoff into this stage. -`record` owns validation facts and Git promotion. `buildsignal` owns the incremental-build baseline and build-slot completion. A successful build becomes available to the next build before project resolution finishes; external validation facts and promotion wait for resolution. +`record` owns the validation facts and the promotion ref. `buildsignal` advances the last-green build baseline before handing off to this stage. `record` does not decide build scope, poll a build, release the Queue's build slot, interpret a target graph, or map targets to projects. Those belong to `process`, `buildsignal`, and `analyze`. -## Record algorithm +## Phase 1 algorithm -1. Decode the request id and queue name, resolve storage, and load the Request. Reject a mismatched queue identity. -2. A succeeded or failed Request proceeds. A cancelled Request publishes `validation.repository.cancelled` without a fact. A superseded Request is acknowledged without effects; a nonterminal Request is an invariant error. -3. Resolve project results using the durable Request and its `TerminalBuildID`. Validate the entire result set before writing facts. Resolver errors leave facts and promotion untouched. -4. Select the repository degree: succeeded → `DegreeGreen`; failed with nonempty project results → `DegreePartial`; failed without project results → `DegreeBroken`. -5. Create the immutable repository fact and its history event. On `ErrAlreadyExists`, adopt the stored fact if it belongs to this Request; a different owner is an invariant error. -6. Create each project fact with its resolver-provided degree, then retain the project-facts-recorded history event. A partial write is completed on replay; existing facts remain authoritative. -7. For a green repository fact, read the Queue and promote only if both `LastGreenRequestID` and `LastGreenURI` still match the Request. Report the last-green timestamp when promotion is eligible. A non-green fact never promotes; the creator reports failure detection latency. -8. Publish `validation.repository.recorded` and acknowledge. A failed operation returns its raw error for backend classification. +For a delivery carrying a `Record` payload: -The repository fact can become visible before the final project fact is durable; recording is not a transaction across facts. Promotion and the recorded hook occur after all project writes and history complete. Project resolution does not delay the stored baseline, which was already advanced by build completion. +``` +1. Deserialize the payload into (request id R, queue name Q). + - malformed -> return raw; non-retryable. + +2. Resolve the queue's storage from Q. + - unresolvable -> return raw; non-retryable. A queue with no storage is a + malformed message or a config error, not a transient condition. + +3. Load Request R. + - ErrNotFound -> return raw; non-retryable. + - other store error -> return raw; the classifier decides. + +4. If Q is non-empty and disagrees with R.Queue, return a non-retryable error. + The Request is the authoritative binding; Q is a routing hint, empty on + payloads written before the field existed. + +5. Inspect R.State. + - succeeded / failed -> continue. buildsignal stamps the outcome before it + publishes here, and both values are verdicts about the code. + - cancelled -> publish a validation.repository.cancelled HookEvent and ack, no + fact: the build decided nothing about the commit (see "When to record an outcome"). + - superseded -> ack, no fact. Unreachable in practice. + - accepted / processing / anything else -> return a non-retryable invariant error. + +6. Resolve optional project results using R and its TerminalBuildID, and validate + the entire result set before writing facts. A resolver error returns raw. + +7. Map R.State and the project results to a whole-repository degree and create the + ValidationFact keyed by (R.URI, empty project) in the queue-bound fact store. + - succeeded -> DegreeGreen. + - failed with project results -> DegreePartial (0.5). + - failed without project results -> DegreeBroken. + - ErrAlreadyExists -> load and reconcile the existing immutable fact. + - other store error -> return raw. + +8. Record each project fact with its resolver-provided degree and persist history + for the repository fact and completed project facts. A failure returns raw; + replay completes missing writes without replacing immutable facts. + +9. If the persisted repository fact is not green, skip to step 11. Report how long + the break went undetected only if step 7 created the fact (see "Observability"). + Otherwise load the Queue and check that both LastGreenRequestID and LastGreenURI + still match R. If they do not, skip promotion and continue to step 11. + +10. Ask SourceControl to point the promotion ref at R.URI. + - ErrNotFound -> count and skip. A rewritten history dropped the commit from the + ref, and no retry can promote it. + - other error -> return raw. + +11. Publish a validation.repository.recorded HookEvent naming the request whose fact + was persisted. + - publish failure -> return raw; non-retryable, so it dead-letters. + +12. ack. +``` + +Every decision after step 7 uses the persisted fact, not the outcome read from this delivery's Request. The first immutable fact controls the ref and the hook event. + +Project-status processing delays fact creation and promotion, while the build baseline is already available from `buildsignal`. All fact writes and history complete before promotion and the recorded notification. The fact writes, bookmark advancement, and ref promotion remain separate operations. ## Validation facts -A fact answers "how broken was this scope at this Queue URI?" Its identity is `(queue, uri, project)`, where `project` is empty for whole-repository greenness and a stable project id. +A fact answers "how broken was this scope at this Queue URI?" Its identity is `(queue, uri, project)`, where `project` is empty for whole-repository greenness and a stable project id in Phase 2. `queue` is the binding of the store the fact lives in, not a field on the entity. Storage is resolved per queue through `storage.Factory.For`, so a fact only ever exists inside one queue's store and does not need to name it. The column still leads the primary key, `PRIMARY KEY (queue, uri, project)`, because every domain table here has to be shardable by queue: the primary key leads with the queue column and no secondary index leads with anything else. That keeps one queue's rows unreachable through another queue's binding and makes every read a primary-key-prefix scan inside a single queue. It is an invariant rather than a per-table choice, and `make lint-queue-shard` enforces it. It is not MySQL table partitioning; no schema here uses `PARTITION BY`. @@ -35,8 +87,8 @@ So `entity.ValidationFact` carries only the rest of the identity: | Field | Meaning | | ----------- | ----------------------------------------------------------------------------------------------------------------- | | `URI` | Commit URI under validation | -| `Project` | Empty for the whole repository; stable project id | -| `Degree` | Health degree in the closed interval `[entity.DegreeGreen, entity.DegreeBroken]` — `0` is green, `0.5` is partial, `1` fully broken | +| `Project` | Empty for the whole repository; stable project id for Phase 2 | +| `Degree` | Health degree in the closed interval `[entity.DegreeGreen, entity.DegreeBroken]` — `0` is green, `1` fully broken | | `RequestID` | Request that established the fact | | `CreatedAt` | Millisecond timestamp at which the fact was first recorded | @@ -60,7 +112,7 @@ The fail-closed path is subtler than "no fact". A DLQ reconciler forces the Requ ### Phase 1 degree mapping -Whole-repository builds use only the endpoints, mapped from the outcome `buildsignal` stamped on the Request: +Whole-repository degree depends on the outcome `buildsignal` stamped on the Request and whether project results are available: | Request outcome | Result | @@ -71,23 +123,33 @@ Whole-repository builds use only the endpoints, mapped from the outcome `buildsi | `cancelled` | no fact | -`DegreePartial` is hard coded to `0.5`; it records that project results are available for a failed repository, not the fraction of projects failing. Project degrees are preserved exactly as returned by the resolver. +`DegreePartial` is hard coded to `0.5` for now; it does not measure the fraction of projects failing. Each project fact preserves the degree returned by the resolver. ## Last-green advancement (Queue bookmark) -`buildsignal` advances `LastGreenURI` and `LastGreenRequestID` from the winning durable successful Request outcome, atomically with removal of its slot ownership. The bookmark moves only to a newer ingest id. It is a build baseline and may precede project results, validation facts, or Git promotion; absence of a fact remains fail-closed for deployment gating. Failed and cancelled outcomes never advance it. See [buildsignal.md](buildsignal.md). +`buildsignal` advances the bookmark after releasing the build slot and persisting the successful Request outcome, before publishing to `record`. The bookmark only moves forward. `entity.Queue` carries `LastGreenURI` alongside `LastGreenRequestID`, the request id that owns the current URI, plus `InFlightCount` and `LatestRequestID`. The guard compares ids with `entity.CompareRequestID(R.Queue, …)`, the same ingest-order comparison `ingest` and `process` use for coalescing. It errors on an id that does not match the queue's format, which is non-retryable: re-parsing the same ids cannot start succeeding. + +The write goes through `QueueStore.Update(ctx, queue, oldVersion, newVersion)`, so version arithmetic stays in the controller: it computes `newVersion = oldVersion + 1` and the store does a pure conditional write. `ErrVersionMismatch` is absorbed by the loop rather than returned, since a concurrent writer moved the row and reloading converges. + +The bookmark is a successful-build baseline and can precede validation facts. Project resolution and Git promotion remain in `record`. Releasing capacity and advancing the bookmark are separate Queue updates, so a new build may be admitted with the previous baseline before the advance lands. A terminal redelivery retries the bookmark update without releasing the slot again. + +A failed or cancelled build never moves the bookmark, and so never promotes either. ## Promotion ref -Promotion requires a finalized green repository fact, all resolved project facts and history, and ownership of the current build baseline. Equal Request id and URI permit retrying an idempotent promotion; a newer baseline causes an older record delivery to skip promotion. This read guard and the external Git write are separate operations, not an atomic transaction. +A green commit is also promoted: `record` asks `SourceControl` to point the Queue's promotion ref, a stable branch name like `verified-main`, at that commit. This is the pull-shaped counterpart to the hook event's push, so a deploy gate or cache warmer can resolve a name and know nothing about Stovepipe, URIs, or degrees. See [Promotion ref](../workflow.md#promotion-ref--the-last-green-commit-by-name) in the pipeline overview. The stage names only the commit; which ref a Queue promotes to, and whether it has one at all, is integrator configuration injected into the `SourceControl` implementation alongside the endpoint and credentials, so a backend with no promotion target makes `Promote` a no-op. + +**Promotion is gated on holding the bookmark**, and only the holder promotes. Without that gate an older green commit recording late would drag the ref backwards: the ref has no version guard of its own, and `Promote` lands the URI it is given rather than refusing a non-descendant, so the caller has to enforce monotonicity. The stored Request id and URI must both match before promotion. The Queue read and external Git write remain separate operations. + +The equal case matters as much as the newer one. A redelivery that finds its own id on the bookmark reports that it *holds* it rather than skipping, so the promotion behind it is retried. That is what makes a crash between the bookmark write and the promotion recoverable, and it is safe because `Promote` is idempotent. Promotion follows the durable fact for the same reason the bookmark does: the ref is a cache of the facts and must not lead them. -`SourceControl.Promote` returning `ErrNotFound` means a history rewrite removed the commit from the ref; count and skip it. Other failures return to the classifier. +`ErrNotFound` from `Promote` means a rewritten history dropped the commit from the ref. It is counted and acked rather than retried, since no retry will put that commit back and the next green commit corrects the ref. ## Observability Two things are measured here, because this is where greenness becomes known. Both need a `SourceControl` lookup of a commit's creation time. -- **Last-green age.** A gauge carrying the creation timestamp of the commit the bookmark now points at, emitted after facts are durable while the Request owns the baseline. It is a timestamp rather than an elapsed time so that subtracting it at scrape time gives the current age, and a queue that stops going green keeps reporting a staler value without the stage re-emitting anything. +- **Last-green age.** A gauge carrying the creation timestamp of the commit the bookmark now points at, emitted once the bookmark write is durable. It is a timestamp rather than an elapsed time so that subtracting it at scrape time gives the current age, and a queue that stops going green keeps reporting a staler value without the stage re-emitting anything. - **Failure-detection latency.** A histogram of how long the break a build failed on went undetected, measured from the creation time of the base URI it validated against. A distribution rather than a gauge, because what matters is how long breaks typically survive, not how long the last one did. There is no later moment to sample it from, since an elapsed time only means something against the failure that just became known, so this lookup cannot move off the delivery path onto a clock. Only the delivery that *wrote* the fact reports the latency. That is why `recordFact` returns whether it created the fact and not just the fact itself: a redelivery adopts the stored fact, and a second sample would count one break twice. A full build pins no base URI, so its failures are counted as unmeasurable instead of timed; absent is the ordinary case there, not a fault. @@ -96,7 +158,7 @@ Both are best-effort. Every way the lookup can fail is counted, logged, and swal ## Build slot release -`buildsignal` first persists the winning Request outcome, then completes its slot and successful baseline in one Queue update, before publishing to record. The DLQ reconciler repairs the same completion even for an already-terminal Request. `record` never writes the Queue. +`Queue.in_flight_count` is released by `buildsignal` before it stamps the outcome and publishes to the record step. The DLQ reconciler releases the slot on the fail-closed path for the same reason. `record`'s only Queue write is the bookmark. ## Hooks @@ -111,10 +173,10 @@ The shared `HookEvent` contract at `api/base/hook/`, the hook extension at `plat Last thing before the ack, after the fact write and after both caches derived from it have moved: ``` -resolve projects → create repository fact and history → create project facts and history → promote (green baseline holder only) → publish HookEvent → ack +create fact → advance bookmark (green only) → promote (bookmark holder only) → publish HookEvent → [Phase 2: publish to analyze] → ack ``` -Last because hooks resolve entities from stores and must find the facts already durable and any eligible promotion complete. A newer successful build may already own the baseline while its record work is pending. Inside the delivery rather than after it, because that is what makes the event lossless without an outbox — the state writes are recognize-and-skip on redelivery, so a crash before the ack replays the whole chain. +Last because the payload names the Request rather than snapshotting it, and hooks resolve entities from stores: a hook reacting to "URI is green" by reading `LastGreenURI`, or by fetching the promotion ref, must not find either still pointing at the previous commit. Inside the delivery rather than after it, because that is what makes the event lossless without an outbox — the state writes are recognize-and-skip on redelivery, so a crash before the ack replays the whole chain. ### Event shape @@ -158,7 +220,7 @@ Phase 2 broadens "complete" to "all planned facts recorded", which needs a marke `ValidationFactStore` is key/value-shaped, and queue-bound rather than queue-parameterized: - `Create(ctx, fact)` writes one immutable fact for the bound queue. It returns `ErrAlreadyExists` when the composite identity is already taken, leaving the stored fact untouched. -- `Get(ctx, uri, project)` reads one fact by the rest of its identity, and returns `ErrNotFound` when none exists. The `project` argument is empty for the repository and otherwise names a resolved project. +- `Get(ctx, uri, project)` reads one fact by the rest of its identity, and returns `ErrNotFound` when none exists. The `project` argument is reserved for Phase 2. There is no `Update`. The first fact written for an identity is the permanent answer, and a caller that needs to know whether it won the race reads `ErrAlreadyExists` and then loads the winner. @@ -186,6 +248,7 @@ Partitioning by request id keeps completion bookkeeping single-writer per Reques | Request carrying no build outcome | non-retryable | producer/state-machine invariant violation | | Existing fact owned by a different Request | non-retryable | ingest dedup invariant violated; the stored fact is immutable | | Malformed request id at bookmark comparison | non-retryable | re-parsing the same ids cannot succeed | +| Queue CAS version mismatch | absorbed, not raised | `storage.ErrVersionMismatch` is handled by the bookmark loop: reload and re-apply | | Promotion target unknown to the ref | absorbed, not raised | `sourcecontrol.ErrNotFound` means a rewritten history dropped the commit | | SourceControl resolution or `Promote` failure | raw error | backend classifier has the required failure knowledge | | Commit-timestamp lookup for a metric | swallowed | counted and logged; observability must not retry a recorded outcome | @@ -205,9 +268,9 @@ Every effect is recognize-and-skip, so a redelivery after a complete run re-runs - **Green fact recorded out of order across Requests.** An older green commit can reach this stage after a newer one. Its fact is written as usual, since facts are per-URI and independent, but the bookmark guard skips it, and because it does not hold the bookmark it does not promote either. Neither cache moves backwards. - **Cancelled build.** Stovepipe never initiates cancellation, but a backend may still report it. No fact is written; the slot was already released and the Request already stamped `cancelled`. The fact identity stays unclaimed, and nothing can claim it today, since `cancelled` is terminal and re-validation does not exist. Recovery in practice is the next commit; the unclaimed identity only matters to a future re-run mechanism (see [Supporting re-run of the same URI](#supporting-re-run-of-the-same-uri)). - **History rewrite while a build runs.** The Request keeps the strategy and URI pinned at admission, and record stores the fact about that immutable URI. A later head is handled independently by `process`. If the rewrite dropped the commit from the ref, the fact and the bookmark still stand, since they describe a commit and not a ref, and only the promotion is skipped. -- **A fail-closed terminal outranks a build that passed.** The degree derives from `R.State`, so a Request forced to `failed` by DLQ reconciliation records a non-green degree even when one of its builds reports success afterwards. Reachable today, and permanent once written; see [DLQ and fail-closed behavior](#dlq-and-fail-closed-behavior). -- **Crash during fact persistence.** Redelivery resolves projects again, adopts existing immutable facts, and completes missing facts and history before promotion. -- **Crash before promotion.** Redelivery retries promotion if the Request still owns the baseline. +- **A fail-closed terminal outranks a build that passed.** The degree derives from `R.State`, so a Request forced to `failed` by DLQ reconciliation records `DegreeBroken` even when one of its builds reports success afterwards. Reachable today, and permanent once written; see [DLQ and fail-closed behavior](#dlq-and-fail-closed-behavior). +- **Crash between the fact write and the bookmark advance.** Redelivery reloads the existing fact and re-applies the idempotent guard. +- **Crash between the bookmark advance and the promotion.** Redelivery finds its own id on the bookmark, reports that it holds it, and retries the idempotent promotion. diff --git a/doc/rfc/stovepipe/workflow.md b/doc/rfc/stovepipe/workflow.md index 684e53a0e..3d54e7c9c 100644 --- a/doc/rfc/stovepipe/workflow.md +++ b/doc/rfc/stovepipe/workflow.md @@ -100,29 +100,29 @@ What runs is one pass per Request. It establishes whole-repository greenness, an │ buildsignal │ │ Poll until terminal; record │ │ status; release the slot; │ - │ project the outcome │ + │ save outcome; advance green │ └───────────────┬──────────────┘ │ RequestID ▼ ┌──────────────────────────────┐ Hooks │ record │┄┄┄┄┄► "URI green, - │ Write whole-repo greenness; │ not green, or - │ on green advance last-green │ cancelled" - │ and the promotion ref; │ - │ resolve project results │ - │ inline │ + │ Resolve project results; │ not green, or + │ record facts; on green move │ cancelled" + │ the promotion ref; │ + │ announce recorded outcome │ + │ │ └──────────────────────────────┘ ``` 1. **ingest** — invoked by the external poller with a **Queue name**. It asks `SourceControl` for that Queue's current head URI, mints a Request namespaced by the Queue, persists it with no recorded greenness yet, and dedups on `(Queue, head URI)` so a re-reported head is processed once. It publishes the RequestID onward. 2. **process** — decides build strategy (incremental since last-green vs full monorepo), gates concurrent work per Queue, coalesces backlog to the latest head, publishes a **hook event** announcing that validation of the commit has begun, and publishes to `build`. See [process.md](steps/process.md). 3. **build** — runs the build-runner for the chosen scope. A flag derived from `process` decides whether to build relative to the last-green **baseline URI** (incremental) or from scratch (full). It records a build and publishes the BuildID. -4. **buildsignal** — polls until terminal, persists the winning Request outcome and terminal build id, then atomically completes its owned Queue slot and advances a newer successful-build baseline. It publishes the RequestID to record and a terminal build-analysis nudge. -5. **record** — resolves project results, then records repository degree `0` for success, `0.5` for failure with project results, or `1` for failure without results. It persists project facts with their resolved degrees. After facts and history are durable, a green fact owning the current baseline promotes in Git; the repository recorded hook follows. A cancelled Request writes no fact and publishes `validation.repository.cancelled`. +4. **buildsignal** — polls until terminal, records the status, releases the build slot, persists the winning Request outcome, and advances a successful-build baseline before publishing the RequestID to `record`. Slot release and baseline advancement are separate updates, so the next admission may still select the previous baseline. +5. **record** — resolves project results, then records repository degree `0` for success, `0.5` for failure with project results, or `1` for failure without results. Project facts preserve their resolved degrees. Once facts and history are durable, a green fact holding the current baseline promotes in Git; the recorded hook follows. A cancelled Request writes no fact and publishes `validation.repository.cancelled`. ### Designed, not built: project analysis -An `analyze` stage is not part of the pipeline. `stovepipe/core/messagequeue` has no analyze topic, and `stovepipe/controller` has no analyze package. The design, still only a design, is a stage that would take a build's target graph, map broken or at-risk targets to projects, and publish project-scoped builds back through `build` → `buildsignal` → `record`. That second pass and a per-project hook event are not what the controllers do. Record currently uses the fixed DegreePartial value when project results are available for a failed repository. Open questions for that design stay at the bottom of this doc. +An `analyze` stage is not part of the pipeline. `stovepipe/core/messagequeue` has no analyze topic, and `stovepipe/controller` has no analyze package. The design, still only a design, is a stage that would take a build's target graph, map broken or at-risk targets to projects, and publish project-scoped builds back through `build` → `buildsignal` → `record`. That second pass, a per-project hook event, and intermediate greenness degrees are not what the controllers do. Open questions for that design stay at the bottom of this doc. ## Per-controller summary @@ -131,8 +131,8 @@ An `analyze` stage is not part of the pipeline. `stovepipe/core/messagequeue` ha | **ingest** | Queue name (from poller) | process | Resolve head URI via SourceControl, mint Request, persist (no greenness), dedup on `(Queue, head URI)` | | **process** | RequestID | build, hook topic | Build strategy, concurrency gate, backlog coalescing; announce validation start on admit → [process.md](steps/process.md) | | **build** | RequestID | buildsignal | Run the build-runner for the chosen scope; baseline = last-green URI iff incremental | -| **buildsignal** | BuildID | record, buildanalysis | Persist the winning outcome; complete slot ownership and successful baseline together; publish downstream work | -| **record** | RequestID | hook topic | Resolve projects, record repository and project facts; promote a finalized green baseline owner; publish the repository hook | +| **buildsignal** | BuildID | record, buildanalysis | Poll until terminal, release capacity, persist the outcome, advance a successful baseline, and publish downstream work | +| **record** | RequestID | hook topic | Resolve projects, record repository and project facts, promote a green baseline holder, and publish the repository hook | | **hook** | HookEvent | — | Invoke the hooks the resolver returns. The example server resolves every event to noop | ## Step RFCs diff --git a/stovepipe/controller/build/build_analysis_test.go b/stovepipe/controller/build/build_analysis_test.go index e1b63d8d7..e6b2cfcfd 100644 --- a/stovepipe/controller/build/build_analysis_test.go +++ b/stovepipe/controller/build/build_analysis_test.go @@ -43,7 +43,7 @@ func TestBuildAnalysisPipeline(t *testing.T) { var savedBuild entity.Build request := processingRequest(entity.BuildStrategyFull, "") - queue := entity.Queue{Name: testQueue, InFlightCount: 1, InFlightRequestIDs: []string{testID}, Version: 1} + queue := entity.Queue{Name: testQueue, InFlightCount: 1, Version: 1} queueStore := storagemock.NewMockQueueStore(ctrl) issueStore := storagemock.NewMockBuildIssueStore(ctrl) diff --git a/stovepipe/controller/buildsignal/BUILD.bazel b/stovepipe/controller/buildsignal/BUILD.bazel index a327d9c31..77ac7b005 100644 --- a/stovepipe/controller/buildsignal/BUILD.bazel +++ b/stovepipe/controller/buildsignal/BUILD.bazel @@ -11,7 +11,6 @@ go_library( "//platform/errs:go_default_library", "//platform/metrics:go_default_library", "//platform/publish:go_default_library", - "//stovepipe/core/buildslot:go_default_library", "//stovepipe/core/loader:go_default_library", "//stovepipe/core/messagequeue:go_default_library", "//stovepipe/core/requestlog:go_default_library", diff --git a/stovepipe/controller/buildsignal/buildsignal.go b/stovepipe/controller/buildsignal/buildsignal.go index 4646bba8e..c6d3e0f30 100644 --- a/stovepipe/controller/buildsignal/buildsignal.go +++ b/stovepipe/controller/buildsignal/buildsignal.go @@ -15,8 +15,9 @@ // Package buildsignal holds the buildsignal-stage queue controller. It consumes // BuildSignal messages (a build id), polls the build-runner until the build // reaches a terminal status, persists that status, and — once terminal — -// persists the request outcome, completes its build slot and baseline, and -// publishes directly to record plus terminal analysis. See +// releases the queue's build slot, projects the outcome onto the request, +// advances the successful-build bookmark, and publishes directly to record plus +// terminal analysis. See // doc/rfc/stovepipe/steps/buildsignal.md. package buildsignal @@ -31,7 +32,6 @@ import ( "github.com/uber/submitqueue/platform/errs" "github.com/uber/submitqueue/platform/metrics" "github.com/uber/submitqueue/platform/publish" - "github.com/uber/submitqueue/stovepipe/core/buildslot" "github.com/uber/submitqueue/stovepipe/core/loader" stovepipemq "github.com/uber/submitqueue/stovepipe/core/messagequeue" "github.com/uber/submitqueue/stovepipe/core/requestlog" @@ -57,8 +57,8 @@ var ( // Controller consumes BuildSignal messages, polls the build-runner toward a // terminal status, persists the result, and either holds the delivery until -// the next poll or projects the outcome onto the request, completes the queue's -// build slot and baseline, and publishes the request id to record. Implements +// the next poll or releases the queue's build slot, projects the outcome onto +// the request, and publishes the request id to record. Implements // consumer.Controller. type Controller struct { logger *zap.SugaredLogger @@ -99,8 +99,8 @@ func NewController( // Process reloads the build referenced by the delivery, polls its runner for // the latest status, persists a real transition, and either holds the delivery -// until the next poll or, once terminal, projects the outcome onto the request, -// completes the queue's build slot and baseline, and publishes the request id to +// until the next poll or, once terminal, releases the queue's build slot, +// projects the outcome onto the request, and publishes the request id to // record. Returns nil to ack (success) or an error to nack (retry) / reject // (DLQ). func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) error { @@ -145,7 +145,13 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er // `processing` — or it already carries this build's outcome, on a redelivery // after the outcome was stamped but before record was published. Both cases // proceed: the redelivery re-publishes to record rather than dropping the - // signal, and repeats any unfinished slot and baseline completion. + // signal, and every write below is a no-op the second time. + // + // Anything else is terminal without a build outcome, which today means only + // `superseded` — and process supersedes solely from `accepted` (see + // supersedeRequest), so a superseded request can never reach this point. It + // is guarded rather than assumed away because finishRequest would otherwise + // release a build slot the request never claimed: superseding consumes none. if request.State != entity.RequestStateProcessing && !request.State.HasBuildOutcome() { return nil } @@ -173,6 +179,12 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er if err := c.finishRequest(ctx, store, &request, build.ID, effective); err != nil { return err } + if request.State == entity.RequestStateSucceeded { + if err := c.advanceLastGreen(ctx, store, request); err != nil { + metrics.NamedCounter(c.metricsScope, _opName, "storage_errors", 1, metrics.TagsFromContext(ctx)...) + return err + } + } if err := c.persistOutcomeLog(ctx, store, request); err != nil { return err } @@ -220,14 +232,34 @@ func (c *Controller) persistBuildFinishedLog(ctx context.Context, store storage. return nil } -// finishRequest preserves the winning outcome before atomically releasing its slot -// and advancing a successful baseline. Recovery can repeat both operations. +// finishRequest releases the queue's build slot and projects the build's +// terminal status and ID onto the request, leaving it terminal. It is a no-op +// when the request already carries an outcome, so a redelivery neither +// double-releases the slot nor rewrites the state. +// +// The slot is released *before* the terminal write, and a failed release aborts +// the write, matching the DLQ reconciler (see stovepipe/controller/dlq/dlq.go). +// Both rules exist for the same reason: the request must not go terminal while +// still holding a slot, because a terminal request is skipped on redelivery and +// by the reconciler, so nothing would ever decrement it. Failing this way leaves +// the request non-terminal, so redelivery re-runs both steps and decrements again +// — transiently over-admitting by one until releaseBuildSlot's zero clamp +// reconverges, which is the failure mode this pipeline prefers. func (c *Controller) finishRequest(ctx context.Context, store storage.Storage, request *entity.Request, buildID string, status entity.BuildStatus) error { + if request.State.HasBuildOutcome() { + return nil + } + + if err := c.releaseBuildSlot(ctx, store, request.Queue); err != nil { + metrics.NamedCounter(c.metricsScope, _opName, "storage_errors", 1, metrics.TagsFromContext(ctx)...) + return err + } + if err := c.markOutcome(ctx, store, request, buildID, outcomeState(status)); err != nil { metrics.NamedCounter(c.metricsScope, _opName, "storage_errors", 1, metrics.TagsFromContext(ctx)...) return err } - return buildslot.Complete(ctx, store, *request) + return nil } func (c *Controller) persistOutcomeLog(ctx context.Context, store storage.Storage, request entity.Request) error { @@ -304,6 +336,93 @@ func (c *Controller) markOutcome(ctx context.Context, store storage.Storage, req } } +// releaseBuildSlot CAS-decrements the queue's in_flight_count, reopening the process +// concurrency gate now that this request's build is over. It decrements relatively +// (preserving concurrent updates), clamps at zero, and retries on version conflicts. +// Unlike process's unwind-path release this is not best-effort: the caller must not +// mark the request terminal if the slot was not freed, so a hard failure is returned. +func (c *Controller) releaseBuildSlot(ctx context.Context, store storage.Storage, queueName string) error { + queueStore := store.GetQueueStore() + + for { + queueRow, err := queueStore.Get(ctx, queueName) + if err != nil { + return fmt.Errorf("failed to load queue %s to release build slot: %w", queueName, err) + } + if queueRow.InFlightCount <= 0 { + return nil + } + + updated := queueRow + updated.InFlightCount = queueRow.InFlightCount - 1 + newVersion := queueRow.Version + 1 + if err := queueStore.Update(ctx, updated, queueRow.Version, newVersion); err != nil { + if errors.Is(err, storage.ErrVersionMismatch) { + continue + } + return fmt.Errorf("failed to release build slot for queue %s: %w", queueName, err) + } + metrics.NamedCounter(c.metricsScope, _opName, "slot_released", 1, metrics.TagsFromContext(ctx)...) + return nil + } +} + +// advanceLastGreen points the queue's bookmark at the successful request, retrying +// on version conflicts. The bookmark only moves forward and can precede facts. +// Capacity is already released, so another admission may use the prior baseline. +func (c *Controller) advanceLastGreen(ctx context.Context, store storage.Storage, request entity.Request) error { + queueStore := store.GetQueueStore() + + for { + queueRow, err := queueStore.Get(ctx, request.Queue) + if err != nil { + return fmt.Errorf("failed to load queue %s to advance last green: %w", request.Queue, err) + } + + cmp, err := compareToBookmark(request.Queue, request.ID, queueRow.LastGreenRequestID) + if err != nil { + // Non-retryable: re-parsing the same ids cannot start succeeding. + return err + } + if cmp <= 0 { + return nil + } + + updated := queueRow + updated.LastGreenURI = request.URI + updated.LastGreenRequestID = request.ID + newVersion := queueRow.Version + 1 + if err := queueStore.Update(ctx, updated, queueRow.Version, newVersion); err != nil { + if errors.Is(err, storage.ErrVersionMismatch) { + continue + } + return fmt.Errorf("failed to advance last green for queue %s: %w", request.Queue, err) + } + + metrics.NamedCounter(c.metricsScope, _opName, "last_green_advanced", 1, metrics.TagsFromContext(ctx)...) + c.logger.Infow("advanced last green bookmark", + "queue", request.Queue, + "request_id", request.ID, + "last_green_uri", request.URI, + ) + return nil + } +} + +// compareToBookmark orders candidate against the request id currently holding the +// bookmark, by ingest order, using the sign convention of entity.CompareRequestID. +// An empty current means the bookmark has never been set, so any candidate is newer. +func compareToBookmark(queue, candidate, current string) (int, error) { + if current == "" { + return 1, nil + } + cmp, err := entity.CompareRequestID(candidate, current) + if err != nil { + return 0, fmt.Errorf("failed to compare request ids for queue %s: %w", queue, err) + } + return cmp, nil +} + // reconcile persists a real status transition and returns the status that // should drive the rest of Process: the polled status when persisted (or // already unchanged), or the stored status when a stored terminal status is diff --git a/stovepipe/controller/buildsignal/buildsignal_test.go b/stovepipe/controller/buildsignal/buildsignal_test.go index 97a99aab5..5731831d7 100644 --- a/stovepipe/controller/buildsignal/buildsignal_test.go +++ b/stovepipe/controller/buildsignal/buildsignal_test.go @@ -156,7 +156,6 @@ func requestWithState(state entity.RequestState) entity.Request { request := entity.Request{ ID: testID, Queue: testQueue, - URI: "git://repo/main/green", State: state, Version: 1, } @@ -178,28 +177,33 @@ func build(status entity.BuildStatus, version int32) entity.Build { // queueRow returns the testQueue's row holding inFlight admitted validations. func queueRow(inFlight, version int32) entity.Queue { - var owners []string - if inFlight > 0 { - owners = []string{testID} - } return entity.Queue{ - Name: testQueue, - InFlightCount: inFlight, - InFlightRequestIDs: owners, - Version: version, + Name: testQueue, + InFlightCount: inFlight, + Version: version, } } func expectFinishWrites(m buildsignalMocks, state entity.RequestState) *gomock.Call { eventCall := expectBuildFinished(m) - outcomeCall := m.reqStore.EXPECT().Update(gomock.Any(), requestWithState(state), int32(1), int32(2)).Return(nil).After(eventCall) - getCall := m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(queueRow(1, 4), nil).After(outcomeCall) - updated := queueRow(0, 4) + m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(queueRow(1, 4), nil).After(eventCall) + m.queueStore.EXPECT().Update(gomock.Any(), queueRow(0, 4), int32(4), int32(5)).Return(nil) + outcomeCall := m.reqStore.EXPECT().Update(gomock.Any(), requestWithState(state), int32(1), int32(2)).Return(nil) if state == entity.RequestStateSucceeded { - updated.LastGreenURI = requestWithState(state).URI - updated.LastGreenRequestID = testID + return expectAdvanceLastGreen(m, outcomeCall) + } + return outcomeCall +} + +func expectAdvanceLastGreen(m buildsignalMocks, after *gomock.Call) *gomock.Call { + getCall := m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(queueRow(0, 5), nil) + if after != nil { + getCall.After(after) } - return m.queueStore.EXPECT().Update(gomock.Any(), updated, int32(4), int32(5)).Return(nil).After(getCall) + updated := queueRow(0, 5) + updated.LastGreenURI = requestWithState(entity.RequestStateSucceeded).URI + updated.LastGreenRequestID = testID + return m.queueStore.EXPECT().Update(gomock.Any(), updated, int32(5), int32(6)).Return(nil).After(getCall) } func expectBuildFinished(m buildsignalMocks) *gomock.Call { @@ -391,10 +395,13 @@ func TestProcess(t *testing.T) { m.runnerFactory.EXPECT().For(buildrunner.Config{QueueName: testQueue}).Return(m.runner, nil) m.runner.EXPECT().Status(gomock.Any(), entity.BuildID{ID: testBuildID}).Return(entity.BuildStatusSucceeded, nil, nil) // No build/queue/request write: the status is unchanged and the outcome - // is already recorded, so the slot must not be released a second time. + // and bookmark are already recorded, so the slot must not be released a second time. eventCall := expectBuildFinished(m) - m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(queueRow(0, 5), nil).After(eventCall) - logCall := expectOutcomeLog(m, entity.RequestStateSucceeded, 2).After(eventCall) + queue := queueRow(1, 6) + queue.LastGreenRequestID = testID + queue.LastGreenURI = request.URI + getCall := m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(queue, nil).After(eventCall) + logCall := expectOutcomeLog(m, entity.RequestStateSucceeded, 2).After(getCall) m.publisher.EXPECT().Publish(gomock.Any(), "record", gomock.Any()).Return(nil).After(logCall) m.publisher.EXPECT().Publish(gomock.Any(), "buildanalysis", gomock.Any()).Return(nil).After(logCall) }, @@ -431,14 +438,14 @@ func TestProcess(t *testing.T) { m.runnerFactory.EXPECT().For(buildrunner.Config{QueueName: testQueue}).Return(m.runner, nil) m.runner.EXPECT().Status(gomock.Any(), entity.BuildID{ID: testBuildID}).Return(entity.BuildStatusSucceeded, nil, nil) eventCall := expectBuildFinished(m) - m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(queueRow(0, 5), nil).After(eventCall) + bookmarkCall := expectAdvanceLastGreen(m, eventCall) log := requestlog.NewRequestStateLog(request, entity.RequestOutcomeReasonBuildSucceeded) log.Metadata[requestlog.MetadataKeyBuildID] = request.TerminalBuildID m.materializer.EXPECT().PersistLog( gomock.Any(), m.store, log, - ).Return(errors.New("db down")).After(eventCall) + ).Return(errors.New("db down")).After(bookmarkCall) }, }, { @@ -489,11 +496,13 @@ func TestProcess(t *testing.T) { m.runnerFactory.EXPECT().For(buildrunner.Config{QueueName: testQueue}).Return(m.runner, nil) m.runner.EXPECT().Status(gomock.Any(), entity.BuildID{ID: testBuildID}).Return(entity.BuildStatusSucceeded, nil, nil) eventCall := expectBuildFinished(m) - m.reqStore.EXPECT().Update(gomock.Any(), gomock.Any(), int32(1), int32(2)).Return(errors.New("db down")).After(eventCall) + m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(queueRow(1, 4), nil).After(eventCall) + m.queueStore.EXPECT().Update(gomock.Any(), queueRow(0, 4), int32(4), int32(5)).Return(nil) + m.reqStore.EXPECT().Update(gomock.Any(), gomock.Any(), int32(1), int32(2)).Return(errors.New("db down")) }, }, { - name: "slot release failure preserves the terminal outcome for recovery", + name: "slot release failure leaves the request non-terminal", wantErr: true, wantRetry: false, setup: func(m buildsignalMocks) { @@ -502,8 +511,10 @@ func TestProcess(t *testing.T) { m.runnerFactory.EXPECT().For(buildrunner.Config{QueueName: testQueue}).Return(m.runner, nil) m.runner.EXPECT().Status(gomock.Any(), entity.BuildID{ID: testBuildID}).Return(entity.BuildStatusSucceeded, nil, nil) eventCall := expectBuildFinished(m) - m.reqStore.EXPECT().Update(gomock.Any(), requestWithState(entity.RequestStateSucceeded), int32(1), int32(2)).Return(nil).After(eventCall) m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{}, errors.New("db down")).After(eventCall) + // No request Update and no record publish: marking the request terminal + // while it still holds a slot would strand the slot, since a terminal + // request is skipped by both redelivery and the DLQ reconciler. }, }, { @@ -654,7 +665,6 @@ func TestTerminalAnalysisPublicationDoesNotGateRecordPublication(t *testing.T) { m.runnerFactory.EXPECT().For(buildrunner.Config{QueueName: testQueue}).Return(m.runner, nil) m.runner.EXPECT().Status(gomock.Any(), entity.BuildID{ID: testBuildID}).Return(entity.BuildStatusFailed, nil, nil) - m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(queueRow(0, 5), nil) m.materializer.EXPECT().PersistLog(gomock.Any(), m.store, gomock.Any()).Return(nil).Times(2) m.publisher.EXPECT().Publish(gomock.Any(), "record", gomock.Any()).Return(tt.recordErr) @@ -708,25 +718,95 @@ func TestNonterminalPollDoesNotPublishTerminalAnalysis(t *testing.T) { require.NoError(t, c.Process(queueContext(), d)) } -func TestFinishRequestRecoversDurableOutcomeBeforeQueueCompletion(t *testing.T) { +func TestProcessRetriesBookmarkWithoutReleasingSlotAgain(t *testing.T) { + for _, committed := range []bool{false, true} { + t.Run(map[bool]string{false: "write failed", true: "write committed but response lost"}[committed], func(t *testing.T) { + ctrl := gomock.NewController(t) + c, m := newController(t, ctrl) + request := requestWithState(entity.RequestStateSucceeded) + m.buildStore.EXPECT().Get(gomock.Any(), testBuildID).Return(build(entity.BuildStatusSucceeded, 3), nil).Times(2) + m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(request, nil).Times(2) + m.runnerFactory.EXPECT().For(buildrunner.Config{QueueName: testQueue}).Return(m.runner, nil).Times(2) + m.runner.EXPECT().Status(gomock.Any(), entity.BuildID{ID: testBuildID}).Return(entity.BuildStatusSucceeded, nil, nil).Times(2) + m.materializer.EXPECT().PersistLog(gomock.Any(), m.store, gomock.Any()).Return(nil).Times(3) + queue := queueRow(1, 5) + m.queueStore.EXPECT().Get(gomock.Any(), testQueue).DoAndReturn(func(context.Context, string) (entity.Queue, error) { + return queue, nil + }).Times(2) + m.queueStore.EXPECT().Update(gomock.Any(), gomock.Any(), int32(5), int32(6)).DoAndReturn( + func(_ context.Context, updated entity.Queue, _, version int32) error { + assert.Equal(t, int32(1), updated.InFlightCount) + assert.Equal(t, request.ID, updated.LastGreenRequestID) + assert.Equal(t, request.URI, updated.LastGreenURI) + if committed { + queue = updated + queue.Version = version + } + return driver.ErrBadConn + }) + require.ErrorIs(t, c.Process(queueContext(), delivery(t, ctrl, buildSignalPayload(t, testBuildID))), driver.ErrBadConn) + if !committed { + updated := queue + updated.LastGreenRequestID = request.ID + updated.LastGreenURI = request.URI + m.queueStore.EXPECT().Update(gomock.Any(), updated, int32(5), int32(6)).Return(nil) + } + m.publisher.EXPECT().Publish(gomock.Any(), "record", gomock.Any()).Return(nil) + m.publisher.EXPECT().Publish(gomock.Any(), "buildanalysis", gomock.Any()).Return(nil) + require.NoError(t, c.Process(queueContext(), delivery(t, ctrl, buildSignalPayload(t, testBuildID)))) + }) + } +} + +func TestProcessBookmarkPreservesConcurrentQueueUpdates(t *testing.T) { ctrl := gomock.NewController(t) c, m := newController(t, ctrl) - request := requestWithState(entity.RequestStateProcessing) - durable := requestWithState(entity.RequestStateSucceeded) - durable.Version = 2 - queue := queueRow(1, 4) - - outcome := m.reqStore.EXPECT().Update(gomock.Any(), requestWithState(entity.RequestStateSucceeded), int32(1), int32(2)).Return(nil) - m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{}, assert.AnError).After(outcome) - require.ErrorIs(t, c.finishRequest(queueContext(), m.store, &request, testBuildID, entity.BuildStatusSucceeded), assert.AnError) - assert.Equal(t, durable, request) - - m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(queue, nil) - updated := queueRow(0, 4) - updated.LastGreenURI = durable.URI - updated.LastGreenRequestID = durable.ID - m.queueStore.EXPECT().Update(gomock.Any(), updated, int32(4), int32(5)).Return(nil) - // A different delivery's failure cannot replace the durable successful outcome. - require.NoError(t, c.finishRequest(queueContext(), m.store, &request, "other-build", entity.BuildStatusFailed)) - assert.Equal(t, durable, request) + request := requestWithState(entity.RequestStateSucceeded) + m.buildStore.EXPECT().Get(gomock.Any(), testBuildID).Return(build(entity.BuildStatusSucceeded, 3), nil) + m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(request, nil) + m.runnerFactory.EXPECT().For(buildrunner.Config{QueueName: testQueue}).Return(m.runner, nil) + m.runner.EXPECT().Status(gomock.Any(), entity.BuildID{ID: testBuildID}).Return(entity.BuildStatusSucceeded, nil, nil) + m.materializer.EXPECT().PersistLog(gomock.Any(), m.store, gomock.Any()).Return(nil).Times(2) + stale := queueRow(0, 5) + fresh := queueRow(1, 6) + fresh.LastGreenRequestID = "9" + fresh.LastGreenURI = "newer-commit" + gomock.InOrder( + m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(stale, nil), + m.queueStore.EXPECT().Update(gomock.Any(), gomock.Any(), int32(5), int32(6)).Return(storage.ErrVersionMismatch), + m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(fresh, nil), + ) + m.publisher.EXPECT().Publish(gomock.Any(), "record", gomock.Any()).Return(nil) + m.publisher.EXPECT().Publish(gomock.Any(), "buildanalysis", gomock.Any()).Return(nil) + require.NoError(t, c.Process(queueContext(), delivery(t, ctrl, buildSignalPayload(t, testBuildID)))) +} + +func TestProcessBookmarkUsesWinningRequestOutcome(t *testing.T) { + for _, state := range []entity.RequestState{entity.RequestStateSucceeded, entity.RequestStateFailed} { + t.Run(string(state), func(t *testing.T) { + ctrl := gomock.NewController(t) + c, m := newController(t, ctrl) + status := entity.BuildStatusSucceeded + if state == entity.RequestStateSucceeded { + status = entity.BuildStatusFailed + } + m.buildStore.EXPECT().Get(gomock.Any(), testBuildID).Return(build(status, 3), nil) + m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(requestWithState(entity.RequestStateProcessing), nil) + m.runnerFactory.EXPECT().For(buildrunner.Config{QueueName: testQueue}).Return(m.runner, nil) + m.runner.EXPECT().Status(gomock.Any(), entity.BuildID{ID: testBuildID}).Return(status, nil, nil) + m.materializer.EXPECT().PersistLog(gomock.Any(), m.store, gomock.Any()).Return(nil).Times(2) + gomock.InOrder( + m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(queueRow(1, 4), nil), + m.queueStore.EXPECT().Update(gomock.Any(), queueRow(0, 4), int32(4), int32(5)).Return(nil), + m.reqStore.EXPECT().Update(gomock.Any(), gomock.Any(), int32(1), int32(2)).Return(storage.ErrVersionMismatch), + m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(requestWithState(state), nil), + ) + if state == entity.RequestStateSucceeded { + expectAdvanceLastGreen(m, nil) + } + m.publisher.EXPECT().Publish(gomock.Any(), "record", gomock.Any()).Return(nil) + m.publisher.EXPECT().Publish(gomock.Any(), "buildanalysis", gomock.Any()).Return(nil) + require.NoError(t, c.Process(queueContext(), delivery(t, ctrl, buildSignalPayload(t, testBuildID)))) + }) + } } diff --git a/stovepipe/controller/dlq/BUILD.bazel b/stovepipe/controller/dlq/BUILD.bazel index 906825438..da7c418f1 100644 --- a/stovepipe/controller/dlq/BUILD.bazel +++ b/stovepipe/controller/dlq/BUILD.bazel @@ -14,7 +14,6 @@ go_library( "//platform/base/messagequeue:go_default_library", "//platform/consumer:go_default_library", "//platform/metrics:go_default_library", - "//stovepipe/core/buildslot:go_default_library", "//stovepipe/core/messagequeue:go_default_library", "//stovepipe/core/requestlog:go_default_library", "//stovepipe/entity:go_default_library", diff --git a/stovepipe/controller/dlq/build_test.go b/stovepipe/controller/dlq/build_test.go index 3cac9a138..c97686202 100644 --- a/stovepipe/controller/dlq/build_test.go +++ b/stovepipe/controller/dlq/build_test.go @@ -67,7 +67,7 @@ func TestBuildProcess(t *testing.T) { name: "processing request is failed", setup: func(m dlqMocks) { m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(requestWithState(entity.RequestStateProcessing), nil) - m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{Name: testQueue, InFlightCount: 1, InFlightRequestIDs: []string{testID}, Version: 5}, nil) + m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{Name: testQueue, InFlightCount: 1, Version: 5}, nil) m.queueStore.EXPECT().Update(gomock.Any(), entity.Queue{Name: testQueue, Version: 5}, int32(5), int32(6)).Return(nil) updated := requestWithState(entity.RequestStateProcessing) updated.State = entity.RequestStateFailed diff --git a/stovepipe/controller/dlq/buildsignal.go b/stovepipe/controller/dlq/buildsignal.go index bec62dbdd..69b2b9399 100644 --- a/stovepipe/controller/dlq/buildsignal.go +++ b/stovepipe/controller/dlq/buildsignal.go @@ -150,7 +150,7 @@ func (c *buildSignalController) Process(ctx context.Context, delivery consumer.D } // Every request reachable from a build row is either still processing, and holding - // the slot failRequest releases, or already terminal, with completion safe to retry: build + // the slot failRequest releases, or already terminal, and past releasing it: build // triggers only once process has written the strategy, which lands in the same CAS // as accepted→processing, and processing exits only to a terminal outcome. if err := failRequest(ctx, store, c.materializer, c.logger, build.RequestID, entity.RequestOutcomeReasonBuildPollingExhausted); err != nil { diff --git a/stovepipe/controller/dlq/buildsignal_test.go b/stovepipe/controller/dlq/buildsignal_test.go index 9069b2706..08c0f4aab 100644 --- a/stovepipe/controller/dlq/buildsignal_test.go +++ b/stovepipe/controller/dlq/buildsignal_test.go @@ -98,12 +98,12 @@ func TestBuildSignalProcess(t *testing.T) { { // The case this reconciler exists for: a poll message that // dead-lettered while its request was holding a build slot. - name: "processing request marks failed before releasing its queue slot", + name: "processing request releases the queue slot before marking failed", setup: func(m buildSignalDLQMocks) { m.buildStore.EXPECT().Get(gomock.Any(), testBuildID).Return(build(), nil) m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(requestWithState(entity.RequestStateProcessing), nil) m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{ - Name: testQueue, InFlightCount: 1, InFlightRequestIDs: []string{testID}, Version: 5, + Name: testQueue, InFlightCount: 1, Version: 5, }, nil) m.queueStore.EXPECT().Update(gomock.Any(), entity.Queue{ Name: testQueue, InFlightCount: 0, Version: 5, @@ -121,12 +121,10 @@ func TestBuildSignalProcess(t *testing.T) { }, }, { - name: "already terminal request repairs its outcome log", + name: "already terminal request is a no-op", setup: func(m buildSignalDLQMocks) { m.buildStore.EXPECT().Get(gomock.Any(), testBuildID).Return(build(), nil) m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(requestWithState(entity.RequestStateSucceeded), nil) - m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{Name: testQueue}, nil) - m.materializer.EXPECT().PersistLog(gomock.Any(), m.store, requestlog.NewRequestStateLog(requestWithState(entity.RequestStateSucceeded), entity.RequestOutcomeReasonBuildSucceeded)).Return(nil) }, }, { @@ -151,12 +149,10 @@ func TestBuildSignalProcess(t *testing.T) { }, }, { - name: "slot release failure retains terminal outcome for retry", + name: "slot release failure aborts the terminal write", setup: func(m buildSignalDLQMocks) { m.buildStore.EXPECT().Get(gomock.Any(), testBuildID).Return(build(), nil) m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(requestWithState(entity.RequestStateProcessing), nil) - updated := requestWithState(entity.RequestStateFailed) - m.reqStore.EXPECT().Update(gomock.Any(), updated, int32(2), int32(3)).Return(nil) m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{}, assert.AnError) }, wantErr: true, diff --git a/stovepipe/controller/dlq/dlq.go b/stovepipe/controller/dlq/dlq.go index ef203bb03..cbde8e207 100644 --- a/stovepipe/controller/dlq/dlq.go +++ b/stovepipe/controller/dlq/dlq.go @@ -29,8 +29,9 @@ // whatever records greenness treats as not-green for gating (see entity.RequestState) — // with an idempotent optimistic-locking write so // concurrent activity (a late successful pipeline transition) wins cleanly. If the request had -// claimed a concurrency slot, the reconciler completes that reservation and any -// successful baseline in one Queue update, including on terminal redeliveries. +// already been admitted (processing) and was holding a concurrency slot, the +// reconciler also releases it by CAS-decrementing the queue's in_flight_count, per +// doc/rfc/stovepipe/steps/process.md#in_flight_count-integrity. package dlq import ( @@ -39,7 +40,6 @@ import ( "fmt" "github.com/uber/submitqueue/platform/consumer" - "github.com/uber/submitqueue/stovepipe/core/buildslot" "github.com/uber/submitqueue/stovepipe/core/requestlog" "github.com/uber/submitqueue/stovepipe/entity" "github.com/uber/submitqueue/stovepipe/extension/storage" @@ -59,7 +59,32 @@ func TopicKey(main consumer.TopicKey) consumer.TopicKey { return consumer.TopicKey(string(main) + topicSuffix) } -// failRequest preserves a concurrent terminal outcome and completes its owned slot. +// failRequest transitions request to RequestStateFailed if it is not already +// in a terminal state. If the request had reached RequestStateProcessing — meaning process's +// admit step already CAS-incremented the queue's in_flight_count for it and no terminal +// outcome has released it yet — the queue's +// slot is released first. +// +// Processing is the only non-terminal state that can own a slot, so the condition is not a +// narrowing of some broader set: process claims the slot and CAS-marks accepted→processing, +// releasing its own claim if that CAS never lands, and the exits from processing are the +// terminal outcomes, which release the slot themselves. Widening the release to accepted +// would decrement for the far more common request that never claimed a slot, over-admitting +// against MaxConcurrent. The one case that escapes both this reconciler and process's +// compensation is a hard crash between the two admit writes, which leaves an accepted +// request holding a slot that nothing here can tell apart from a request that never +// claimed one; distinguishing them needs per-request slot ownership on the row. +// +// Queue and Request are separate entities with no cross-entity +// transaction, so the two writes cannot be atomic and the ordering picks which crash +// failure mode we accept: a crash between the writes leaves the request non-terminal, +// redelivery re-runs reconciliation, and releaseSlot (which tracks no per-request slot +// ownership) decrements again — transiently over-admitting by one slot until the +// under-count re-converges at releaseSlot's zero clamp. The reverse order would leak +// the slot instead: redelivery skips terminal requests, permanently shrinking the +// queue's capacity toward a wedge. Over-admission is the failure mode we prefer. See +// doc/rfc/stovepipe/steps/process.md#in_flight_count-integrity for the broader +// counter-drift story. func failRequest( ctx context.Context, store storage.Storage, @@ -79,42 +104,43 @@ func failRequest( return fmt.Errorf("failed to get request %s: %w", requestID, err) } - for !request.State.IsTerminal() { - updated := request - updated.State = entity.RequestStateFailed - newVersion := request.Version + 1 - if err := store.GetRequestStore().Update(ctx, updated, request.Version, newVersion); err != nil { - if errors.Is(err, storage.ErrVersionMismatch) { - request, err = store.GetRequestStore().Get(ctx, requestID) - if err != nil { - return fmt.Errorf("reload request %s during dlq completion: %w", requestID, err) - } - continue - } - return fmt.Errorf("failed to update request %s state to failed: %w", requestID, err) + if request.State.IsTerminal() { + if request.State == entity.RequestStateFailed { + // The originating DLQ remains the retry trigger after the state write. Its stage-specific + // reason repairs that write's missing log; PersistLog rejects an already-retained conflict. + return persistFailureLog(ctx, store, materializer, request, reason) } - updated.Version = newVersion - request = updated - } - if err := buildslot.Complete(ctx, store, request); err != nil { - return err + logger.Infow("dlq reconcile: request already terminal, skipping", + "request_id", requestID, + "state", string(request.State), + ) + return nil } - switch request.State { - case entity.RequestStateSucceeded: - reason = entity.RequestOutcomeReasonBuildSucceeded - case entity.RequestStateCancelled: - reason = entity.RequestOutcomeReasonBuildCancelled - case entity.RequestStateSuperseded: - reason = entity.RequestOutcomeReasonSupersededByNewerHead - case entity.RequestStateFailed: - if request.TerminalBuildID != "" { - reason = entity.RequestOutcomeReasonBuildFailed + + if request.State == entity.RequestStateProcessing { + if err := releaseSlot(ctx, store, logger, request.Queue); err != nil { + return fmt.Errorf("failed to release queue slot for request %s: %w", requestID, err) } } - return persistOutcomeLog(ctx, store, materializer, request, reason) + + updated := request + updated.State = entity.RequestStateFailed + newVersion := request.Version + 1 + if err := store.GetRequestStore().Update(ctx, updated, request.Version, newVersion); err != nil { + return fmt.Errorf("failed to update request %s state to failed: %w", requestID, err) + } + updated.Version = newVersion + if err := persistFailureLog(ctx, store, materializer, updated, reason); err != nil { + return err + } + logger.Infow("dlq reconcile: request forced terminal failed", + "request_id", requestID, + "previous_state", string(request.State), + ) + return nil } -func persistOutcomeLog( +func persistFailureLog( ctx context.Context, store storage.Storage, materializer requestlog.Materializer, @@ -122,11 +148,49 @@ func persistOutcomeLog( reason entity.RequestOutcomeReason, ) error { log := requestlog.NewRequestStateLog(request, reason) - if request.TerminalBuildID != "" { - log.Metadata[requestlog.MetadataKeyBuildID] = request.TerminalBuildID - } if err := materializer.PersistLog(ctx, store, log); err != nil { - return fmt.Errorf("failed to record terminal state for request %s: %w", request.ID, err) + return fmt.Errorf("failed to record failed state for request %s: %w", request.ID, err) } return nil } + +// releaseSlot CAS-decrements the queue's in_flight_count, retrying on version +// conflicts, mirroring process.Controller's own CAS-retry loop for queue updates. +func releaseSlot(ctx context.Context, store storage.Storage, logger *zap.SugaredLogger, queueName string) error { + queueStore := store.GetQueueStore() + + for { + queueRow, err := queueStore.Get(ctx, queueName) + if err != nil { + if errors.Is(err, storage.ErrNotFound) { + logger.Warnw("dlq reconcile: queue not found, skipping slot release", + "queue", queueName, + ) + return nil + } + return fmt.Errorf("failed to get queue %s: %w", queueName, err) + } + + if queueRow.InFlightCount <= 0 { + logger.Warnw("dlq reconcile: queue in_flight_count already at zero, skipping slot release", + "queue", queueName, + ) + return nil + } + + updated := queueRow + updated.InFlightCount-- + newVersion := queueRow.Version + 1 + if err := queueStore.Update(ctx, updated, queueRow.Version, newVersion); err != nil { + if errors.Is(err, storage.ErrVersionMismatch) { + continue + } + return fmt.Errorf("failed to release slot for queue %s: %w", queueName, err) + } + logger.Infow("dlq reconcile: released queue slot", + "queue", queueName, + "in_flight_count", updated.InFlightCount, + ) + return nil + } +} diff --git a/stovepipe/controller/dlq/dlq_test.go b/stovepipe/controller/dlq/dlq_test.go index 89120c122..eeb7c838c 100644 --- a/stovepipe/controller/dlq/dlq_test.go +++ b/stovepipe/controller/dlq/dlq_test.go @@ -134,6 +134,8 @@ func TestProcess(t *testing.T) { wantMetric string }{ { + // No queue expectations: an accepted request never claimed a slot, + // so releasing one here would over-admit against MaxConcurrent. name: "accepted request is marked failed without releasing a slot", setup: func(m dlqMocks) { m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(requestWithState(entity.RequestStateAccepted), nil) @@ -141,15 +143,14 @@ func TestProcess(t *testing.T) { updated.State = entity.RequestStateFailed updateCall := m.reqStore.EXPECT().Update(gomock.Any(), updated, int32(2), int32(3)).Return(nil) expectFailureLog(m, entity.RequestOutcomeReasonProcessingFailed, 3).After(updateCall) - m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{Name: testQueue}, nil) }, }, { - name: "processing request marks failed before releasing its queue slot", + name: "processing request releases the queue slot before marking failed", setup: func(m dlqMocks) { m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(requestWithState(entity.RequestStateProcessing), nil) m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{ - Name: testQueue, InFlightCount: 1, InFlightRequestIDs: []string{testID}, Version: 5, + Name: testQueue, InFlightCount: 1, Version: 5, }, nil) m.queueStore.EXPECT().Update(gomock.Any(), entity.Queue{ Name: testQueue, InFlightCount: 0, Version: 5, @@ -161,11 +162,9 @@ func TestProcess(t *testing.T) { }, }, { - name: "already superseded repairs its state log", + name: "already superseded is a no-op", setup: func(m dlqMocks) { m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(requestWithState(entity.RequestStateSuperseded), nil) - m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{Name: testQueue}, nil) - m.materializer.EXPECT().PersistLog(gomock.Any(), m.store, requestlog.NewRequestStateLog(requestWithState(entity.RequestStateSuperseded), entity.RequestOutcomeReasonSupersededByNewerHead)).Return(nil) }, }, { @@ -173,7 +172,6 @@ func TestProcess(t *testing.T) { setup: func(m dlqMocks) { m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(requestWithState(entity.RequestStateFailed), nil) expectFailureLog(m, entity.RequestOutcomeReasonProcessingFailed, 2) - m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{Name: testQueue}, nil) }, }, { @@ -189,23 +187,21 @@ func TestProcess(t *testing.T) { updated := requestWithState(entity.RequestStateAccepted) updated.State = entity.RequestStateFailed m.reqStore.EXPECT().Update(gomock.Any(), updated, int32(2), int32(3)).Return(storage.ErrVersionMismatch) - m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(requestWithState(entity.RequestStateFailed), nil) - m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{Name: testQueue}, nil) - expectFailureLog(m, entity.RequestOutcomeReasonProcessingFailed, 2) }, + wantErr: true, }, { name: "queue update retries on version mismatch then succeeds", setup: func(m dlqMocks) { m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(requestWithState(entity.RequestStateProcessing), nil) m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{ - Name: testQueue, InFlightCount: 1, InFlightRequestIDs: []string{testID}, Version: 5, + Name: testQueue, InFlightCount: 1, Version: 5, }, nil) m.queueStore.EXPECT().Update(gomock.Any(), entity.Queue{ Name: testQueue, InFlightCount: 0, Version: 5, }, int32(5), int32(6)).Return(storage.ErrVersionMismatch) m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{ - Name: testQueue, InFlightCount: 1, InFlightRequestIDs: []string{testID}, Version: 6, + Name: testQueue, InFlightCount: 1, Version: 6, }, nil) m.queueStore.EXPECT().Update(gomock.Any(), entity.Queue{ Name: testQueue, InFlightCount: 0, Version: 6, @@ -244,7 +240,6 @@ func TestProcess(t *testing.T) { m.store, requestlog.NewRequestStateLog(request, entity.RequestOutcomeReasonProcessingFailed), ).Return(errors.New("db down")).After(updateCall) - m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{Name: testQueue}, nil) }, }, { @@ -299,32 +294,3 @@ func TestProcess(t *testing.T) { func TestTopicKey(t *testing.T) { assert.Equal(t, consumer.TopicKey("process_dlq"), TopicKey(stovepipemq.TopicKeyProcess)) } - -func TestDLQCompletesPersistedSuccessWithoutReleasingAnotherRequestsSlot(t *testing.T) { - ctrl := gomock.NewController(t) - c, m := newController(t, ctrl) - request := requestWithState(entity.RequestStateSucceeded) - request.URI = "git://repo/main/green" - request.TerminalBuildID = "build-7" - queue := entity.Queue{Name: testQueue, Version: 5, InFlightCount: 2, InFlightRequestIDs: []string{testID, "8"}} - m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(request, nil).Times(2) - m.queueStore.EXPECT().Get(gomock.Any(), testQueue).DoAndReturn(func(context.Context, string) (entity.Queue, error) { - return queue, nil - }).Times(2) - m.queueStore.EXPECT().Update(gomock.Any(), gomock.Any(), int32(5), int32(6)).DoAndReturn( - func(_ context.Context, updated entity.Queue, _, next int32) error { - queue = updated - queue.Version = next - return nil - }) - log := requestlog.NewRequestStateLog(request, entity.RequestOutcomeReasonBuildSucceeded) - log.Metadata[requestlog.MetadataKeyBuildID] = request.TerminalBuildID - m.materializer.EXPECT().PersistLog(gomock.Any(), m.store, log).Return(nil).Times(2) - for range 2 { - require.NoError(t, c.Process(queueContext(), delivery(t, ctrl, processPayload(t, testID)))) - } - assert.Equal(t, request.URI, queue.LastGreenURI) - assert.Equal(t, request.ID, queue.LastGreenRequestID) - assert.Equal(t, []string{"8"}, queue.InFlightRequestIDs) - assert.EqualValues(t, 1, queue.InFlightCount) -} diff --git a/stovepipe/controller/process/BUILD.bazel b/stovepipe/controller/process/BUILD.bazel index 33d60c9ea..27fbe8d7d 100644 --- a/stovepipe/controller/process/BUILD.bazel +++ b/stovepipe/controller/process/BUILD.bazel @@ -13,7 +13,6 @@ go_library( "//platform/hook:go_default_library", "//platform/metrics:go_default_library", "//platform/publish:go_default_library", - "//stovepipe/core/buildslot:go_default_library", "//stovepipe/core/hookevent:go_default_library", "//stovepipe/core/loader:go_default_library", "//stovepipe/core/messagequeue:go_default_library", @@ -39,7 +38,6 @@ go_test( "//platform/errs:go_default_library", "//platform/extension/messagequeue/mock:go_default_library", "//platform/metrics:go_default_library", - "//stovepipe/core/buildslot:go_default_library", "//stovepipe/core/hookevent:go_default_library", "//stovepipe/core/messagequeue:go_default_library", "//stovepipe/core/requestlog:go_default_library", diff --git a/stovepipe/controller/process/process.go b/stovepipe/controller/process/process.go index f8d2e941c..f334f70bd 100644 --- a/stovepipe/controller/process/process.go +++ b/stovepipe/controller/process/process.go @@ -31,7 +31,6 @@ import ( platformhook "github.com/uber/submitqueue/platform/hook" "github.com/uber/submitqueue/platform/metrics" "github.com/uber/submitqueue/platform/publish" - "github.com/uber/submitqueue/stovepipe/core/buildslot" "github.com/uber/submitqueue/stovepipe/core/hookevent" "github.com/uber/submitqueue/stovepipe/core/loader" stovepipemq "github.com/uber/submitqueue/stovepipe/core/messagequeue" @@ -142,12 +141,11 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er } return nil case entity.RequestStateSuperseded: - if err := buildslot.Complete(ctx, store, request); err != nil { - return err - } return c.persistStateLog(ctx, store, request, entity.RequestOutcomeReasonSupersededByNewerHead) case entity.RequestStateSucceeded, entity.RequestStateFailed, entity.RequestStateCancelled: - return buildslot.Complete(ctx, store, request) + // Terminal: a newer head preempted this request, or its build already finished. + // A stale redelivery has nothing left to do. + return nil case entity.RequestStateAccepted: return c.processAccepted(ctx, store, delivery, request) default: @@ -198,7 +196,7 @@ func (c *Controller) processAccepted(ctx context.Context, store storage.Storage, // coalesce supersedes request when a newer head exists (RFC process step 5), returning // true so the caller acks. It returns false when request is still the latest head and -// should proceed to the gate. Superseding also completes any prior reservation. +// should proceed to the gate. Superseding consumes no build slot. func (c *Controller) coalesce(ctx context.Context, store storage.Storage, request entity.Request, latestRequestID string) (bool, error) { cmp, err := entity.CompareRequestID(request.ID, latestRequestID) if err != nil { @@ -215,9 +213,6 @@ func (c *Controller) coalesce(ctx context.Context, store storage.Storage, reques if request.State != entity.RequestStateSuperseded { return true, nil } - if err := buildslot.Complete(ctx, store, request); err != nil { - return false, err - } if err := c.persistStateLog(ctx, store, request, entity.RequestOutcomeReasonSupersededByNewerHead); err != nil { return false, err } @@ -241,10 +236,7 @@ func (c *Controller) admitLatestHead(ctx context.Context, store storage.Storage, var err error for { - if err := queueRow.ValidateBuildSlots(); err != nil { - return err - } - if !queueRow.HasBuildSlot(request.ID) && queueRow.InFlightCount >= cfg.MaxConcurrent { + if queueRow.InFlightCount >= cfg.MaxConcurrent { return c.holdForBuildSlot(ctx, delivery, request, queueRow.InFlightCount, cfg.GateWaitDelayMs) } @@ -263,14 +255,14 @@ func (c *Controller) admitLatestHead(ctx context.Context, store storage.Storage, return err } - queueRow, err = buildslot.Claim(ctx, store, queueRow, request.ID) + err = c.claimBuildSlot(ctx, store, &queueRow) if err == nil { break } if !errors.Is(err, storage.ErrVersionMismatch) { return err } - // Claim reloaded queueRow. Re-coalesce: supersede if a newer head arrived, + // claimBuildSlot reloaded queueRow. Re-coalesce: supersede if a newer head arrived, // otherwise loop to re-check the gate. superseded, err := c.coalesce(ctx, store, request, queueRow.LatestRequestID) if err != nil || superseded { @@ -280,13 +272,14 @@ func (c *Controller) admitLatestHead(ctx context.Context, store storage.Storage, transitioned, err := c.markProcessing(ctx, store, &request, strategy, baseURI) if err != nil { - // Keep the reservation: a retry recognizes this request's ownership even at capacity. + // Slot claimed but never admitted: release best-effort so the slot isn't leaked + // (a redelivery would find the gate closed by its own claim and nothing decrements it). + c.releaseBuildSlot(ctx, store, request.Queue) return err } if !transitioned { - if request.State.IsTerminal() { - return buildslot.Complete(ctx, store, request) - } + // Lost the admit race: another delivery advanced this request. Release and skip. + c.releaseBuildSlot(ctx, store, request.Queue) return nil } @@ -356,11 +349,35 @@ func (c *Controller) deriveBuildStrategy(ctx context.Context, sc sourcecontrol.S return entity.BuildStrategyFull, "", nil } +// claimBuildSlot CAS-increments queue.in_flight_count by one. On version mismatch it +// reloads queueRow and returns ErrVersionMismatch so the caller can retry. +func (c *Controller) claimBuildSlot(ctx context.Context, store storage.Storage, queueRow *entity.Queue) error { + queueStore := store.GetQueueStore() + + updated := *queueRow + updated.InFlightCount = queueRow.InFlightCount + 1 + newVersion := queueRow.Version + 1 + if err := queueStore.Update(ctx, updated, queueRow.Version, newVersion); err != nil { + if errors.Is(err, storage.ErrVersionMismatch) { + got, getErr := queueStore.Get(ctx, queueRow.Name) + if getErr != nil { + return fmt.Errorf("failed to reload queue %s after version mismatch: %w", queueRow.Name, getErr) + } + *queueRow = got + return storage.ErrVersionMismatch + } + return fmt.Errorf("failed to claim build slot for queue %s: %w", queueRow.Name, err) + } + updated.Version = newVersion + *queueRow = updated + return nil +} + // markProcessing CAS-marks request accepted→processing and persists the strategy chosen after the // build slot was claimed. It reapplies the values after a request reload so an accepted concurrent // update cannot discard them. transitioned is true only when this call performed the CAS; false -// means a concurrent writer already advanced the request past accepted, so the caller completes -// the shared reservation only if the request is terminal. +// means a concurrent writer already advanced the request past accepted, so the caller must release +// its claimed slot. func (c *Controller) markProcessing(ctx context.Context, store storage.Storage, request *entity.Request, strategy entity.BuildStrategy, baseURI string) (transitioned bool, err error) { reqStore := store.GetRequestStore() @@ -391,6 +408,43 @@ func (c *Controller) markProcessing(ctx context.Context, store storage.Storage, } } +// releaseBuildSlot CAS-decrements queue.in_flight_count to compensate a slot claimed but never +// admitted. It decrements relatively (preserving a concurrent record decrement) and retries on +// version conflicts. Best-effort: it only logs on a hard failure, since the caller is unwinding. +func (c *Controller) releaseBuildSlot(ctx context.Context, store storage.Storage, queueName string) { + queueStore := store.GetQueueStore() + + for { + queueRow, err := queueStore.Get(ctx, queueName) + if err != nil { + c.logger.Errorw("failed to release claimed build slot", + "queue", queueName, + "error", err, + ) + return + } + if queueRow.InFlightCount <= 0 { + return + } + + updated := queueRow + updated.InFlightCount = queueRow.InFlightCount - 1 + newVersion := queueRow.Version + 1 + if err := queueStore.Update(ctx, updated, queueRow.Version, newVersion); err != nil { + if errors.Is(err, storage.ErrVersionMismatch) { + continue + } + c.logger.Errorw("failed to release claimed build slot", + "queue", queueName, + "error", err, + ) + return + } + metrics.NamedCounter(c.metricsScope, _opName, "slot_released", 1, metrics.TagsFromContext(ctx)...) + return + } +} + // supersedeRequest returns the canonical row so the log uses the version that won any CAS race. func (c *Controller) supersedeRequest(ctx context.Context, store storage.Storage, request entity.Request) (entity.Request, error) { reqStore := store.GetRequestStore() diff --git a/stovepipe/controller/process/process_test.go b/stovepipe/controller/process/process_test.go index c4c0cadeb..cd8ef4c61 100644 --- a/stovepipe/controller/process/process_test.go +++ b/stovepipe/controller/process/process_test.go @@ -29,7 +29,6 @@ import ( "github.com/uber/submitqueue/platform/errs" mqmock "github.com/uber/submitqueue/platform/extension/messagequeue/mock" "github.com/uber/submitqueue/platform/metrics" - "github.com/uber/submitqueue/stovepipe/core/buildslot" "github.com/uber/submitqueue/stovepipe/core/hookevent" stovepipemq "github.com/uber/submitqueue/stovepipe/core/messagequeue" "github.com/uber/submitqueue/stovepipe/core/requestlog" @@ -177,11 +176,10 @@ func expectAdmit(t *testing.T, m processMocks, id string) { t.Helper() updatedQueue := entity.Queue{ - Name: testQueue, - LatestRequestID: id, - InFlightCount: 1, - InFlightRequestIDs: []string{id}, - Version: 1, + Name: testQueue, + LatestRequestID: id, + InFlightCount: 1, + Version: 1, } m.queueStore.EXPECT().Update(gomock.Any(), updatedQueue, int32(1), int32(2)).Return(nil) @@ -469,10 +467,8 @@ func TestProcessRederivesStrategyAfterQueueReload(t *testing.T) { } initialClaim := initialQueue initialClaim.InFlightCount = 1 - initialClaim.InFlightRequestIDs = []string{testID} claimedQueue := reloadedQueue claimedQueue.InFlightCount = 1 - claimedQueue.InFlightRequestIDs = []string{testID} updatedRequest := request updatedRequest.State = entity.RequestStateProcessing @@ -519,7 +515,6 @@ func TestProcess(t *testing.T) { } m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(request, nil) expectStateLog(t, m, request, entity.RequestOutcomeReasonSupersededByNewerHead) - m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{Name: testQueue}, nil) }, }, { @@ -535,7 +530,6 @@ func TestProcess(t *testing.T) { m.store, requestlog.NewRequestStateLog(request, entity.RequestOutcomeReasonSupersededByNewerHead), ).Return(errors.New("db down")) - m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{Name: testQueue}, nil) }, }, { @@ -544,7 +538,6 @@ func TestProcess(t *testing.T) { m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(entity.Request{ ID: testID, Queue: testQueue, State: entity.RequestStateSucceeded, Version: 2, }, nil) - m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{Name: testQueue}, nil) }, }, { @@ -553,7 +546,6 @@ func TestProcess(t *testing.T) { m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(entity.Request{ ID: testID, Queue: testQueue, State: entity.RequestStateFailed, Version: 2, }, nil) - m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{Name: testQueue}, nil) }, }, { @@ -562,7 +554,6 @@ func TestProcess(t *testing.T) { m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(entity.Request{ ID: testID, Queue: testQueue, State: entity.RequestStateCancelled, Version: 2, }, nil) - m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{Name: testQueue}, nil) }, }, { @@ -621,11 +612,10 @@ func TestProcess(t *testing.T) { Version: 1, }, nil) updatedQueue := entity.Queue{ - Name: testQueue, - LatestRequestID: testID, - InFlightCount: 1, - InFlightRequestIDs: []string{testID}, - Version: 1, + Name: testQueue, + LatestRequestID: testID, + InFlightCount: 1, + Version: 1, } m.queueStore.EXPECT().Update(gomock.Any(), updatedQueue, int32(1), int32(2)).Return(nil) updatedRequest := acceptedRequest(testID) @@ -652,11 +642,10 @@ func TestProcess(t *testing.T) { Version: 1, }, nil) updatedQueue := entity.Queue{ - Name: testQueue, - LatestRequestID: testID, - InFlightCount: 1, - InFlightRequestIDs: []string{testID}, - Version: 1, + Name: testQueue, + LatestRequestID: testID, + InFlightCount: 1, + Version: 1, } m.queueStore.EXPECT().Update(gomock.Any(), updatedQueue, int32(1), int32(2)).Return(nil) updatedRequest := acceptedRequest(testID) @@ -685,11 +674,10 @@ func TestProcess(t *testing.T) { Version: 1, }, nil) updatedQueue := entity.Queue{ - Name: testQueue, - LatestRequestID: testID, - InFlightCount: 1, - InFlightRequestIDs: []string{testID}, - Version: 1, + Name: testQueue, + LatestRequestID: testID, + InFlightCount: 1, + Version: 1, } m.queueStore.EXPECT().Update(gomock.Any(), updatedQueue, int32(1), int32(2)).Return(nil) updatedRequest := acceptedRequest(testID) @@ -737,12 +725,11 @@ func TestProcess(t *testing.T) { setup: func(m processMocks) { m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(acceptedRequest(testID), nil) m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{ - Name: testQueue, - LatestRequestID: testID, - InFlightCount: 1, - InFlightRequestIDs: []string{"other"}, - LastGreenURI: "git://repo/monorepo/main/green", - Version: 1, + Name: testQueue, + LatestRequestID: testID, + InFlightCount: 1, + LastGreenURI: "git://repo/monorepo/main/green", + Version: 1, }, nil) }, }, @@ -757,18 +744,16 @@ func TestProcess(t *testing.T) { Version: 1, }, nil) m.queueStore.EXPECT().Update(gomock.Any(), entity.Queue{ - Name: testQueue, - LatestRequestID: testID, - InFlightCount: 1, - InFlightRequestIDs: []string{testID}, - Version: 1, + Name: testQueue, + LatestRequestID: testID, + InFlightCount: 1, + Version: 1, }, int32(1), int32(2)).Return(storage.ErrVersionMismatch) m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{ - Name: testQueue, - LatestRequestID: testID, - InFlightCount: 1, - InFlightRequestIDs: []string{"other"}, - Version: 2, + Name: testQueue, + LatestRequestID: testID, + InFlightCount: 1, + Version: 2, }, nil) }, }, @@ -781,7 +766,7 @@ func TestProcess(t *testing.T) { }, nil) // First claim CAS loses to a concurrent writer. m.queueStore.EXPECT().Update(gomock.Any(), entity.Queue{ - Name: testQueue, LatestRequestID: testID, InFlightCount: 1, InFlightRequestIDs: []string{testID}, Version: 1, + Name: testQueue, LatestRequestID: testID, InFlightCount: 1, Version: 1, }, int32(1), int32(2)).Return(storage.ErrVersionMismatch) // Reload: still latest, slot still free (version advanced by an unrelated field). m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{ @@ -789,7 +774,7 @@ func TestProcess(t *testing.T) { }, nil) // Retry claim succeeds, then admit. m.queueStore.EXPECT().Update(gomock.Any(), entity.Queue{ - Name: testQueue, LatestRequestID: testID, InFlightCount: 1, InFlightRequestIDs: []string{testID}, Version: 2, + Name: testQueue, LatestRequestID: testID, InFlightCount: 1, Version: 2, }, int32(2), int32(3)).Return(nil) updatedReq := acceptedRequest(testID) updatedReq.State = entity.RequestStateProcessing @@ -808,7 +793,7 @@ func TestProcess(t *testing.T) { Name: testQueue, LatestRequestID: testID, Version: 1, }, nil) m.queueStore.EXPECT().Update(gomock.Any(), entity.Queue{ - Name: testQueue, LatestRequestID: testID, InFlightCount: 1, InFlightRequestIDs: []string{testID}, Version: 1, + Name: testQueue, LatestRequestID: testID, InFlightCount: 1, Version: 1, }, int32(1), int32(2)).Return(storage.ErrVersionMismatch) // Reload: ingest stamped a newer head — our head is no longer latest. m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{ @@ -820,7 +805,6 @@ func TestProcess(t *testing.T) { loggedRequest := superseded loggedRequest.Version = 2 expectStateLog(t, m, loggedRequest, entity.RequestOutcomeReasonSupersededByNewerHead).After(updateCall) - m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{Name: testQueue}, nil) }, }, { @@ -835,7 +819,7 @@ func TestProcess(t *testing.T) { m.sourceFactory.EXPECT().For(sourcecontrol.Config{QueueName: testQueue}).Return(m.sourceControl, nil) m.sourceControl.EXPECT().IsAncestor(gomock.Any(), lastGreenURI, testURI).Return(true, nil) m.queueStore.EXPECT().Update(gomock.Any(), entity.Queue{ - Name: testQueue, LatestRequestID: testID, InFlightCount: 1, InFlightRequestIDs: []string{testID}, LastGreenURI: lastGreenURI, Version: 1, + Name: testQueue, LatestRequestID: testID, InFlightCount: 1, LastGreenURI: lastGreenURI, Version: 1, }, int32(1), int32(2)).Return(nil) firstAttempt := acceptedRequest(testID) @@ -859,7 +843,7 @@ func TestProcess(t *testing.T) { }, }, { - name: "mark processing duplicate preserves the winning reservation", + name: "mark processing lost race releases slot and skips admit", setup: func(m processMocks) { m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(acceptedRequest(testID), nil) m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{ @@ -867,7 +851,7 @@ func TestProcess(t *testing.T) { }, nil) // Claim succeeds. m.queueStore.EXPECT().Update(gomock.Any(), entity.Queue{ - Name: testQueue, LatestRequestID: testID, InFlightCount: 1, InFlightRequestIDs: []string{testID}, Version: 1, + Name: testQueue, LatestRequestID: testID, InFlightCount: 1, Version: 1, }, int32(1), int32(2)).Return(nil) // markProcessing CAS loses, reload shows a concurrent writer already advanced it. updatedReq := acceptedRequest(testID) @@ -877,10 +861,17 @@ func TestProcess(t *testing.T) { m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(entity.Request{ ID: testID, Queue: testQueue, State: entity.RequestStateProcessing, Version: 2, }, nil) + // Compensating decrement of the spurious slot. + m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{ + Name: testQueue, LatestRequestID: testID, InFlightCount: 1, Version: 2, + }, nil) + m.queueStore.EXPECT().Update(gomock.Any(), entity.Queue{ + Name: testQueue, LatestRequestID: testID, InFlightCount: 0, Version: 2, + }, int32(2), int32(3)).Return(nil) }, }, { - name: "mark processing error retains reservation for retry", + name: "mark processing error releases slot and returns error", wantErr: true, setup: func(m processMocks) { m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(acceptedRequest(testID), nil) @@ -888,12 +879,19 @@ func TestProcess(t *testing.T) { Name: testQueue, LatestRequestID: testID, Version: 1, }, nil) m.queueStore.EXPECT().Update(gomock.Any(), entity.Queue{ - Name: testQueue, LatestRequestID: testID, InFlightCount: 1, InFlightRequestIDs: []string{testID}, Version: 1, + Name: testQueue, LatestRequestID: testID, InFlightCount: 1, Version: 1, }, int32(1), int32(2)).Return(nil) updatedReq := acceptedRequest(testID) updatedReq.State = entity.RequestStateProcessing updatedReq.BuildStrategy = entity.BuildStrategyFull m.reqStore.EXPECT().Update(gomock.Any(), updatedReq, int32(1), int32(2)).Return(errors.New("db down")) + // Best-effort compensating decrement before the error propagates. + m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{ + Name: testQueue, LatestRequestID: testID, InFlightCount: 1, Version: 2, + }, nil) + m.queueStore.EXPECT().Update(gomock.Any(), entity.Queue{ + Name: testQueue, LatestRequestID: testID, InFlightCount: 0, Version: 2, + }, int32(2), int32(3)).Return(nil) }, }, { @@ -912,7 +910,6 @@ func TestProcess(t *testing.T) { loggedRequest := updated loggedRequest.Version = 2 expectStateLog(t, m, loggedRequest, entity.RequestOutcomeReasonSupersededByNewerHead).After(updateCall) - m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{Name: testQueue}, nil) }, }, { @@ -937,7 +934,6 @@ func TestProcess(t *testing.T) { m.store, requestlog.NewRequestStateLog(request, entity.RequestOutcomeReasonSupersededByNewerHead), ).Return(errors.New("db down")).After(updateCall) - m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{Name: testQueue}, nil) }, }, { @@ -958,7 +954,6 @@ func TestProcess(t *testing.T) { } reloadCall := m.reqStore.EXPECT().Get(gomock.Any(), testOlderID).Return(reloaded, nil) expectStateLog(t, m, reloaded, entity.RequestOutcomeReasonSupersededByNewerHead).After(reloadCall) - m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{Name: testQueue}, nil) }, }, { @@ -1057,55 +1052,3 @@ func TestHoldForBuildSlotRequiresPositiveDelay(t *testing.T) { require.Error(t, err) assert.False(t, errs.IsRetryable(err)) } - -func TestAdmissionUsesCompletedBuildBaselineBeforeFactsAreRecorded(t *testing.T) { - ctrl := gomock.NewController(t) - c, m := newController(t, ctrl) - previous := acceptedRequest("6") - previous.State = entity.RequestStateSucceeded - queue := entity.Queue{Name: testQueue, LatestRequestID: testID, Version: 1, - InFlightCount: 1, InFlightRequestIDs: []string{previous.ID}} - m.queueStore.EXPECT().Get(gomock.Any(), testQueue).DoAndReturn(func(context.Context, string) (entity.Queue, error) { - return queue, nil - }).Times(2) - m.queueStore.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( - func(_ context.Context, updated entity.Queue, old, next int32) error { - require.Equal(t, queue.Version, old) - queue = updated - queue.Version = next - return nil - }).Times(2) - require.NoError(t, buildslot.Complete(queueContext(testQueue), m.store, previous)) - assert.Zero(t, queue.InFlightCount) - assert.Equal(t, previous.URI, queue.LastGreenURI) - - m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(acceptedRequest(testID), nil) - m.sourceFactory.EXPECT().For(sourcecontrol.Config{QueueName: testQueue}).Return(m.sourceControl, nil) - m.sourceControl.EXPECT().IsAncestor(gomock.Any(), previous.URI, testURI).Return(true, nil) - processing := acceptedRequest(testID) - processing.State = entity.RequestStateProcessing - processing.BuildStrategy = entity.BuildStrategyIncrementalSinceGreen - processing.BaseURI = previous.URI - m.reqStore.EXPECT().Update(gomock.Any(), processing, int32(1), int32(2)).Return(nil) - processing.Version = 2 - expectStartAnnounceAndBuildPublish(t, m, processing) - - require.NoError(t, c.Process(queueContext(testQueue), delivery(t, ctrl, processPayload(t, testID)))) - assert.Equal(t, []string{testID}, queue.InFlightRequestIDs) -} - -func TestAdmissionReusesReservationAfterRequestWriteFailure(t *testing.T) { - ctrl := gomock.NewController(t) - c, m := newController(t, ctrl) - queue := entity.Queue{Name: testQueue, LatestRequestID: testID, Version: 2, - InFlightCount: 1, InFlightRequestIDs: []string{testID}} - m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(acceptedRequest(testID), nil) - m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(queue, nil) - processing := acceptedRequest(testID) - processing.State = entity.RequestStateProcessing - processing.BuildStrategy = entity.BuildStrategyFull - m.reqStore.EXPECT().Update(gomock.Any(), processing, int32(1), int32(2)).Return(nil) - processing.Version = 2 - expectStartAnnounceAndBuildPublish(t, m, processing) - require.NoError(t, c.Process(queueContext(testQueue), delivery(t, ctrl, processPayload(t, testID)))) -} diff --git a/stovepipe/controller/record/record.go b/stovepipe/controller/record/record.go index a53489055..8e9dff5f7 100644 --- a/stovepipe/controller/record/record.go +++ b/stovepipe/controller/record/record.go @@ -56,7 +56,7 @@ import ( ) // Controller consumes Record messages, records the build's validation fact, and -// when that fact is green advances the queue's last-green bookmark and promotes +// when that fact is green and holds the queue's last-green bookmark promotes // the commit. Implements consumer.Controller. type Controller struct { logger *zap.SugaredLogger diff --git a/stovepipe/core/buildslot/BUILD.bazel b/stovepipe/core/buildslot/BUILD.bazel deleted file mode 100644 index bbbf47cc5..000000000 --- a/stovepipe/core/buildslot/BUILD.bazel +++ /dev/null @@ -1,29 +0,0 @@ -load("@rules_go//go:def.bzl", "go_library", "go_test") - -go_library( - name = "go_default_library", - srcs = [ - "claim.go", - "complete.go", - ], - importpath = "github.com/uber/submitqueue/stovepipe/core/buildslot", - visibility = ["//visibility:public"], - deps = [ - "//stovepipe/entity:go_default_library", - "//stovepipe/extension/storage:go_default_library", - ], -) - -go_test( - name = "go_default_test", - srcs = ["complete_test.go"], - embed = [":go_default_library"], - deps = [ - "//stovepipe/entity:go_default_library", - "//stovepipe/extension/storage:go_default_library", - "//stovepipe/extension/storage/mock:go_default_library", - "@com_github_stretchr_testify//assert:go_default_library", - "@com_github_stretchr_testify//require:go_default_library", - "@org_uber_go_mock//gomock:go_default_library", - ], -) diff --git a/stovepipe/core/buildslot/README.md b/stovepipe/core/buildslot/README.md deleted file mode 100644 index cf4bb1c6a..000000000 --- a/stovepipe/core/buildslot/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# Build-slot ownership - -`Claim` reserves capacity once per Request in the Queue snapshot provided by process. Process owns the admission policy: coalescing, capacity, and baseline selection. A successful claim returns the persisted snapshot with its new version. A version conflict returns the current Queue and `storage.ErrVersionMismatch`, so process rechecks that policy before retrying. An existing owner reuses its reservation; neither operation mutates the supplied snapshot. - -`Complete` accepts a durable terminal Request. It removes only that Request’s reservation from `Queue.InFlightRequestIDs` and recomputes `InFlightCount`. For a newer successful outcome it advances `LastGreenURI` and `LastGreenRequestID` in the same Queue CAS, so the next admission sees the successful baseline when it sees available capacity. - -The Request outcome must be persisted first. A crash before Queue completion temporarily holds capacity; primary retries and DLQ recovery repeat completion even for terminal Requests. Absence of the Request id makes completion a no-op, including after an ambiguous committed Queue write. CAS conflicts reload the Queue to preserve concurrent owners, enablement, and newer baselines. Inputs remain immutable snapshots. - -The baseline can lead validation facts and Git promotion. Record resolves projects and persists facts and history before promotion. Buildslot neither writes facts nor calls Git. diff --git a/stovepipe/core/buildslot/claim.go b/stovepipe/core/buildslot/claim.go deleted file mode 100644 index 282a4f41c..000000000 --- a/stovepipe/core/buildslot/claim.go +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright (c) 2026 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package buildslot - -import ( - "context" - "errors" - "fmt" - "slices" - - "github.com/uber/submitqueue/stovepipe/entity" - "github.com/uber/submitqueue/stovepipe/extension/storage" -) - -// Claim reserves a request's slot in a validated queue snapshot without mutating it. -// The caller must check capacity first. A version conflict returns the current queue -// and storage.ErrVersionMismatch so admission can recheck capacity, coalescing, and baseline. -func Claim(ctx context.Context, store storage.Storage, queueRow entity.Queue, requestID string) (entity.Queue, error) { - if queueRow.HasBuildSlot(requestID) { - return queueRow, nil - } - queueStore := store.GetQueueStore() - - updated := queueRow - updated.InFlightRequestIDs = append(slices.Clone(queueRow.InFlightRequestIDs), requestID) - updated.InFlightCount = int32(len(updated.InFlightRequestIDs)) - newVersion := queueRow.Version + 1 - if err := queueStore.Update(ctx, updated, queueRow.Version, newVersion); err != nil { - if errors.Is(err, storage.ErrVersionMismatch) { - got, getErr := queueStore.Get(ctx, queueRow.Name) - if getErr != nil { - return queueRow, fmt.Errorf("failed to reload queue %s after version mismatch: %w", queueRow.Name, getErr) - } - return got, storage.ErrVersionMismatch - } - return queueRow, fmt.Errorf("failed to claim build slot for queue %s: %w", queueRow.Name, err) - } - updated.Version = newVersion - return updated, nil -} diff --git a/stovepipe/core/buildslot/complete.go b/stovepipe/core/buildslot/complete.go deleted file mode 100644 index df8e14653..000000000 --- a/stovepipe/core/buildslot/complete.go +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright (c) 2026 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package buildslot reserves and completes request-owned build capacity and baselines. -package buildslot - -import ( - "context" - "errors" - "fmt" - "slices" - - "github.com/uber/submitqueue/stovepipe/entity" - "github.com/uber/submitqueue/stovepipe/extension/storage" -) - -// Complete releases a terminal request's slot and advances its successful baseline -// in one queue update. The durable Request outcome must be written first. -// Repeated calls cannot release another request's capacity. -func Complete(ctx context.Context, store storage.Storage, request entity.Request) error { - if !request.State.IsTerminal() || request.ID == "" || request.Queue == "" { - return fmt.Errorf("build slot completion requires a terminal request with an ID and queue") - } - queueStore := store.GetQueueStore() - for { - queue, err := queueStore.Get(ctx, request.Queue) - if err != nil { - return fmt.Errorf("load queue %q for build slot completion: %w", request.Queue, err) - } - if err := queue.ValidateBuildSlots(); err != nil { - return err - } - index := slices.Index(queue.InFlightRequestIDs, request.ID) - if index < 0 { - return nil - } - - updated := queue - updated.InFlightRequestIDs = slices.Delete(slices.Clone(queue.InFlightRequestIDs), index, index+1) - if len(updated.InFlightRequestIDs) == 0 { - updated.InFlightRequestIDs = nil - } - updated.InFlightCount = int32(len(updated.InFlightRequestIDs)) - if request.State == entity.RequestStateSucceeded { - if request.URI == "" { - return fmt.Errorf("successful request %q has no baseline URI", request.ID) - } - newer := queue.LastGreenRequestID == "" - if !newer { - comparison, err := entity.CompareRequestID(request.ID, queue.LastGreenRequestID) - if err != nil { - return fmt.Errorf("compare build baseline for queue %q: %w", request.Queue, err) - } - newer = comparison > 0 - } - if newer { - updated.LastGreenURI = request.URI - updated.LastGreenRequestID = request.ID - } - } - if err := queueStore.Update(ctx, updated, queue.Version, queue.Version+1); err != nil { - if errors.Is(err, storage.ErrVersionMismatch) { - continue - } - return fmt.Errorf("complete build slot for request %q: %w", request.ID, err) - } - return nil - } -} diff --git a/stovepipe/core/buildslot/complete_test.go b/stovepipe/core/buildslot/complete_test.go deleted file mode 100644 index 947cce574..000000000 --- a/stovepipe/core/buildslot/complete_test.go +++ /dev/null @@ -1,140 +0,0 @@ -// Copyright (c) 2026 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package buildslot - -import ( - "context" - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/uber/submitqueue/stovepipe/entity" - "github.com/uber/submitqueue/stovepipe/extension/storage" - storagemock "github.com/uber/submitqueue/stovepipe/extension/storage/mock" - "go.uber.org/mock/gomock" -) - -func TestCompleteReleasesOnlyOwnedCapacityAndAdvancesSuccessfulBaseline(t *testing.T) { - for _, tt := range []struct { - name string - state entity.RequestState - bookmark string - wantGreen string - }{ - {name: "successful build", state: entity.RequestStateSucceeded, bookmark: "3", wantGreen: "7"}, - {name: "newer build already succeeded", state: entity.RequestStateSucceeded, bookmark: "9", wantGreen: "9"}, - {name: "first green build", state: entity.RequestStateSucceeded, wantGreen: "7"}, - {name: "failed build", state: entity.RequestStateFailed, bookmark: "3", wantGreen: "3"}, - {name: "cancelled build", state: entity.RequestStateCancelled, bookmark: "3", wantGreen: "3"}, - {name: "superseded reservation", state: entity.RequestStateSuperseded, bookmark: "3", wantGreen: "3"}, - } { - t.Run(tt.name, func(t *testing.T) { - ctrl := gomock.NewController(t) - store := storagemock.NewMockStorage(ctrl) - queues := storagemock.NewMockQueueStore(ctrl) - store.EXPECT().GetQueueStore().Return(queues).AnyTimes() - request := entity.Request{ID: "7", Queue: "repo/main", URI: "commit-7", State: tt.state} - queue := entity.Queue{Name: request.Queue, Version: 4, InFlightCount: 2, - InFlightRequestIDs: []string{"7", "8"}, LastGreenRequestID: tt.bookmark, LastGreenURI: "old-commit"} - queues.EXPECT().Get(gomock.Any(), request.Queue).DoAndReturn(func(context.Context, string) (entity.Queue, error) { - return queue, nil - }).Times(2) - queues.EXPECT().Update(gomock.Any(), gomock.Any(), int32(4), int32(5)).DoAndReturn( - func(_ context.Context, updated entity.Queue, _, version int32) error { - assert.Equal(t, []string{"7", "8"}, queue.InFlightRequestIDs) - assert.Equal(t, int32(1), updated.InFlightCount) - assert.Equal(t, []string{"8"}, updated.InFlightRequestIDs) - assert.Equal(t, tt.wantGreen, updated.LastGreenRequestID) - if tt.wantGreen == "7" { - assert.Equal(t, request.URI, updated.LastGreenURI) - } - updated.Version = version - queue = updated - return nil - }) - require.NoError(t, Complete(context.Background(), store, request)) - require.NoError(t, Complete(context.Background(), store, request)) - }) - } -} - -func TestCompleteRecoversAfterCommittedWriteReturnsAnError(t *testing.T) { - ctrl := gomock.NewController(t) - store := storagemock.NewMockStorage(ctrl) - queues := storagemock.NewMockQueueStore(ctrl) - store.EXPECT().GetQueueStore().Return(queues).AnyTimes() - request := entity.Request{ID: "7", Queue: "repo/main", URI: "commit-7", State: entity.RequestStateSucceeded} - queue := entity.Queue{Name: request.Queue, Version: 1, InFlightCount: 2, InFlightRequestIDs: []string{"7", "8"}} - queues.EXPECT().Get(gomock.Any(), request.Queue).DoAndReturn(func(context.Context, string) (entity.Queue, error) { - return queue, nil - }).Times(2) - queues.EXPECT().Update(gomock.Any(), gomock.Any(), int32(1), int32(2)).DoAndReturn( - func(_ context.Context, updated entity.Queue, _, version int32) error { - queue = updated - queue.Version = version - return assert.AnError - }) - require.ErrorIs(t, Complete(context.Background(), store, request), assert.AnError) - require.NoError(t, Complete(context.Background(), store, request)) - assert.Equal(t, []string{"8"}, queue.InFlightRequestIDs) - assert.Equal(t, int32(1), queue.InFlightCount) - assert.Equal(t, request.URI, queue.LastGreenURI) -} - -func TestCompleteReloadsConcurrentQueueUpdates(t *testing.T) { - ctrl := gomock.NewController(t) - store := storagemock.NewMockStorage(ctrl) - queues := storagemock.NewMockQueueStore(ctrl) - store.EXPECT().GetQueueStore().Return(queues) - request := entity.Request{ID: "7", Queue: "repo/main", URI: "commit-7", State: entity.RequestStateSucceeded} - stale := entity.Queue{Name: request.Queue, Version: 1, InFlightCount: 1, InFlightRequestIDs: []string{"7"}} - fresh := stale - fresh.Version = 2 - fresh.InFlightCount = 2 - fresh.InFlightRequestIDs = []string{"7", "8"} - fresh.LastGreenRequestID = "9" - fresh.LastGreenURI = "commit-9" - gomock.InOrder( - queues.EXPECT().Get(gomock.Any(), request.Queue).Return(stale, nil), - queues.EXPECT().Update(gomock.Any(), gomock.Any(), int32(1), int32(2)).Return(storage.ErrVersionMismatch), - queues.EXPECT().Get(gomock.Any(), request.Queue).Return(fresh, nil), - queues.EXPECT().Update(gomock.Any(), gomock.Any(), int32(2), int32(3)).DoAndReturn( - func(_ context.Context, updated entity.Queue, _, _ int32) error { - assert.Equal(t, []string{"8"}, updated.InFlightRequestIDs) - assert.Equal(t, fresh.LastGreenURI, updated.LastGreenURI) - return nil - }), - ) - require.NoError(t, Complete(context.Background(), store, request)) -} - -func TestCompleteRejectsInvalidOwnership(t *testing.T) { - for _, queue := range []entity.Queue{ - {InFlightCount: 1}, - {InFlightCount: 2, InFlightRequestIDs: []string{"7", "7"}}, - {InFlightCount: 1, InFlightRequestIDs: []string{""}}, - {InFlightCount: -1}, - } { - t.Run(fmt.Sprint(queue), func(t *testing.T) { - ctrl := gomock.NewController(t) - store := storagemock.NewMockStorage(ctrl) - queues := storagemock.NewMockQueueStore(ctrl) - store.EXPECT().GetQueueStore().Return(queues) - queues.EXPECT().Get(gomock.Any(), "repo/main").Return(queue, nil) - require.Error(t, Complete(context.Background(), store, entity.Request{ID: "7", Queue: "repo/main", State: entity.RequestStateFailed})) - }) - } -} diff --git a/stovepipe/entity/queue.go b/stovepipe/entity/queue.go index 3b486f07e..9ca1c8fcc 100644 --- a/stovepipe/entity/queue.go +++ b/stovepipe/entity/queue.go @@ -14,11 +14,6 @@ package entity -import ( - "fmt" - "slices" -) - // Queue holds per-queue pipeline coordination state for a named repo+ref (e.g. "monorepo/main"). type Queue struct { // **************** @@ -32,8 +27,8 @@ type Queue struct { // Following fields could be changed throughout the lifecycle of the queue // **************** - // LastGreenURI is the newest head with a successful build outcome, usable as an - // incremental baseline before validation facts and Git promotion are complete. + // LastGreenURI is the queue's last-known-good commit: the most recent head at which + // whole-repo validation recorded green (health degree 0). Empty until the first such outcome. LastGreenURI string `json:"last_green_uri"` // LastGreenRequestID is the request that established LastGreenURI. The bookmark only @@ -41,10 +36,8 @@ type Queue struct { // than the candidate's, compared by ingest order via CompareRequestID. LastGreenRequestID string `json:"last_green_request_id"` - // InFlightCount is the number of reserved build slots, equal to len(InFlightRequestIDs). + // InFlightCount is the number of trunk validations admitted by process but not yet terminal. InFlightCount int32 `json:"in_flight_count"` - // InFlightRequestIDs identifies the requests holding reserved build slots. - InFlightRequestIDs []string `json:"in_flight_request_ids"` // LatestRequestID is the request id of the newest head ingest accepted for this queue. // Empty until the first request is created. Coalescing compares IDs via CompareRequestID. @@ -68,29 +61,6 @@ type Queue struct { Version int32 `json:"version"` } -// HasBuildSlot reports whether requestID owns a reserved build slot. -func (q Queue) HasBuildSlot(requestID string) bool { - return slices.Contains(q.InFlightRequestIDs, requestID) -} - -// ValidateBuildSlots checks that the slot count and ownership agree. -func (q Queue) ValidateBuildSlots() error { - if q.InFlightCount != int32(len(q.InFlightRequestIDs)) { - return fmt.Errorf("queue %q has %d slots but %d owners", q.Name, q.InFlightCount, len(q.InFlightRequestIDs)) - } - seen := make(map[string]struct{}, len(q.InFlightRequestIDs)) - for _, id := range q.InFlightRequestIDs { - if id == "" { - return fmt.Errorf("queue %q has an empty slot owner", q.Name) - } - if _, exists := seen[id]; exists { - return fmt.Errorf("queue %q has duplicate slot owner %q", q.Name, id) - } - seen[id] = struct{}{} - } - return nil -} - func (q Queue) EnablementState() QueueState { return QueueState{Queue: q.Name, State: q.Enablement, Version: q.EnablementVersion, EffectiveFromCommitURI: q.EffectiveFromCommitURI, ChangedAtMs: q.EnablementChangedAtMs} diff --git a/stovepipe/extension/storage/mysql/MIGRATIONS.md b/stovepipe/extension/storage/mysql/MIGRATIONS.md deleted file mode 100644 index 668d127be..000000000 --- a/stovepipe/extension/storage/mysql/MIGRATIONS.md +++ /dev/null @@ -1,9 +0,0 @@ -# Queue build-slot ownership rollout - -Fresh databases use `schema/queue.sql`. Existing databases need the nullable `in_flight_request_ids JSON DEFAULT NULL` column on `queue` before these binaries start. The Queue store reads and writes this column with the rest of the versioned snapshot. - -1. Pause ingestion and drain all accepted and processing Requests, pending pipeline messages, and DLQ reconciliation with the old workers. Verify every Queue has `in_flight_count = 0`. -2. Stop every old writer. Append `in_flight_request_ids JSON DEFAULT NULL` as the final column of `queue`. Always append new columns so existing column positions remain unchanged. Existing drained rows need no backfill: SQL NULL represents an empty owner set. -3. Deploy all Queue readers and writers together, then resume ingestion. Do not mix old and new writers: an old writer cannot preserve slot ownership or the atomic baseline-and-release invariant. - -An old nonzero counter cannot safely be converted into owner identities from the count alone. Admission and completion reject mismatched counts, empty owner ids, and duplicate owners rather than silently over-admitting. Resolve an undrained queue before deployment instead of resetting its counter while builds remain live. Rollback likewise requires draining and stopping the new workers first. diff --git a/stovepipe/extension/storage/mysql/queue_store.go b/stovepipe/extension/storage/mysql/queue_store.go index 76af81a41..c4fa43ca3 100644 --- a/stovepipe/extension/storage/mysql/queue_store.go +++ b/stovepipe/extension/storage/mysql/queue_store.go @@ -17,7 +17,6 @@ package mysql import ( "context" "database/sql" - "encoding/json" "errors" "fmt" @@ -48,14 +47,10 @@ func (q *queueStore) Create(ctx context.Context, queue entity.Queue) (retErr err if queue.Name != q.queue { return fmt.Errorf("queue row name %q does not match the store's bound queue %q", queue.Name, q.queue) } - owners, err := json.Marshal(queue.InFlightRequestIDs) - if err != nil { - return fmt.Errorf("encode build slot owners: %w", err) - } - _, err = q.db.ExecContext(ctx, - `INSERT INTO queue (name, last_green_uri, in_flight_count, latest_request_id, version, last_green_request_id, enablement, enablement_version, effective_from_commit_uri, enablement_changed_at_ms, in_flight_request_ids) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + _, err := q.db.ExecContext(ctx, + `INSERT INTO queue (name, last_green_uri, in_flight_count, latest_request_id, version, last_green_request_id, enablement, enablement_version, effective_from_commit_uri, enablement_changed_at_ms) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, queue.Name, queue.LastGreenURI, queue.InFlightCount, @@ -63,7 +58,6 @@ func (q *queueStore) Create(ctx context.Context, queue entity.Queue) (retErr err queue.Version, queue.LastGreenRequestID, queue.Enablement, queue.EnablementVersion, queue.EffectiveFromCommitURI, queue.EnablementChangedAtMs, - owners, ) if err != nil { if isDuplicateEntry(err) { @@ -84,9 +78,8 @@ func (q *queueStore) Get(ctx context.Context, name string) (ret entity.Queue, re } var queue entity.Queue - var owners []byte err := q.db.QueryRowContext(ctx, - "SELECT name, last_green_uri, in_flight_count, latest_request_id, version, last_green_request_id, enablement, enablement_version, effective_from_commit_uri, enablement_changed_at_ms, in_flight_request_ids FROM queue WHERE name = ?", + "SELECT name, last_green_uri, in_flight_count, latest_request_id, version, last_green_request_id, enablement, enablement_version, effective_from_commit_uri, enablement_changed_at_ms FROM queue WHERE name = ?", name, ).Scan( &queue.Name, @@ -96,7 +89,6 @@ func (q *queueStore) Get(ctx context.Context, name string) (ret entity.Queue, re &queue.Version, &queue.LastGreenRequestID, &queue.Enablement, &queue.EnablementVersion, &queue.EffectiveFromCommitURI, &queue.EnablementChangedAtMs, - &owners, ) if errors.Is(err, sql.ErrNoRows) { @@ -105,11 +97,6 @@ func (q *queueStore) Get(ctx context.Context, name string) (ret entity.Queue, re if err != nil { return entity.Queue{}, fmt.Errorf("failed to get queue name=%s from the database: %w", name, err) } - if len(owners) > 0 { - if err := json.Unmarshal(owners, &queue.InFlightRequestIDs); err != nil { - return entity.Queue{}, fmt.Errorf("decode build slot owners for queue %q: %w", name, err) - } - } return queue, nil } @@ -123,14 +110,10 @@ func (q *queueStore) Update(ctx context.Context, queue entity.Queue, oldVersion, if queue.Name != q.queue { return fmt.Errorf("queue row name %q does not match the store's bound queue %q", queue.Name, q.queue) } - owners, err := json.Marshal(queue.InFlightRequestIDs) - if err != nil { - return fmt.Errorf("encode build slot owners: %w", err) - } result, err := q.db.ExecContext(ctx, `UPDATE queue - SET last_green_uri = ?, in_flight_count = ?, latest_request_id = ?, version = ?, last_green_request_id = ?, enablement = ?, enablement_version = ?, effective_from_commit_uri = ?, enablement_changed_at_ms = ?, in_flight_request_ids = ? + SET last_green_uri = ?, in_flight_count = ?, latest_request_id = ?, version = ?, last_green_request_id = ?, enablement = ?, enablement_version = ?, effective_from_commit_uri = ?, enablement_changed_at_ms = ? WHERE name = ? AND version = ?`, queue.LastGreenURI, queue.InFlightCount, @@ -138,7 +121,6 @@ func (q *queueStore) Update(ctx context.Context, queue entity.Queue, oldVersion, newVersion, queue.LastGreenRequestID, queue.Enablement, queue.EnablementVersion, queue.EffectiveFromCommitURI, queue.EnablementChangedAtMs, - owners, queue.Name, oldVersion, ) diff --git a/stovepipe/extension/storage/mysql/queue_store_test.go b/stovepipe/extension/storage/mysql/queue_store_test.go index d50fa31d9..e96365daf 100644 --- a/stovepipe/extension/storage/mysql/queue_store_test.go +++ b/stovepipe/extension/storage/mysql/queue_store_test.go @@ -17,7 +17,6 @@ package mysql import ( "context" "database/sql" - "encoding/json" "fmt" "testing" @@ -61,7 +60,7 @@ func TestQueueStore_Create(t *testing.T) { name: "success", setup: func(mock sqlmock.Sqlmock) { mock.ExpectExec("INSERT INTO queue"). - WithArgs(queue.Name, queue.LastGreenURI, queue.InFlightCount, queue.LatestRequestID, queue.Version, queue.LastGreenRequestID, queue.Enablement, queue.EnablementVersion, queue.EffectiveFromCommitURI, queue.EnablementChangedAtMs, queueOwnersJSON(t, queue)). + WithArgs(queue.Name, queue.LastGreenURI, queue.InFlightCount, queue.LatestRequestID, queue.Version, queue.LastGreenRequestID, queue.Enablement, queue.EnablementVersion, queue.EffectiveFromCommitURI, queue.EnablementChangedAtMs). WillReturnResult(sqlmock.NewResult(0, 1)) }, }, @@ -69,7 +68,7 @@ func TestQueueStore_Create(t *testing.T) { name: "duplicate name returns ErrAlreadyExists", setup: func(mock sqlmock.Sqlmock) { mock.ExpectExec("INSERT INTO queue"). - WithArgs(queue.Name, queue.LastGreenURI, queue.InFlightCount, queue.LatestRequestID, queue.Version, queue.LastGreenRequestID, queue.Enablement, queue.EnablementVersion, queue.EffectiveFromCommitURI, queue.EnablementChangedAtMs, queueOwnersJSON(t, queue)). + WithArgs(queue.Name, queue.LastGreenURI, queue.InFlightCount, queue.LatestRequestID, queue.Version, queue.LastGreenRequestID, queue.Enablement, queue.EnablementVersion, queue.EffectiveFromCommitURI, queue.EnablementChangedAtMs). WillReturnError(&mysql.MySQLError{Number: mysqlErrDuplicateEntry}) }, wantErr: true, @@ -79,7 +78,7 @@ func TestQueueStore_Create(t *testing.T) { name: "other exec error", setup: func(mock sqlmock.Sqlmock) { mock.ExpectExec("INSERT INTO queue"). - WithArgs(queue.Name, queue.LastGreenURI, queue.InFlightCount, queue.LatestRequestID, queue.Version, queue.LastGreenRequestID, queue.Enablement, queue.EnablementVersion, queue.EffectiveFromCommitURI, queue.EnablementChangedAtMs, queueOwnersJSON(t, queue)). + WithArgs(queue.Name, queue.LastGreenURI, queue.InFlightCount, queue.LatestRequestID, queue.Version, queue.LastGreenRequestID, queue.Enablement, queue.EnablementVersion, queue.EffectiveFromCommitURI, queue.EnablementChangedAtMs). WillReturnError(fmt.Errorf("connection reset")) }, wantErr: true, @@ -129,9 +128,9 @@ func TestQueueStore_Get(t *testing.T) { name: "found", queueName: want.Name, setup: func(mock sqlmock.Sqlmock) { - rows := sqlmock.NewRows([]string{"name", "last_green_uri", "in_flight_count", "latest_request_id", "version", "last_green_request_id", "enablement", "enablement_version", "effective_from_commit_uri", "enablement_changed_at_ms", "in_flight_request_ids"}). - AddRow(want.Name, want.LastGreenURI, want.InFlightCount, want.LatestRequestID, want.Version, want.LastGreenRequestID, want.Enablement, want.EnablementVersion, want.EffectiveFromCommitURI, want.EnablementChangedAtMs, queueOwnersJSON(t, want)) - mock.ExpectQuery("SELECT name, last_green_uri, in_flight_count, latest_request_id, version, last_green_request_id, enablement, enablement_version, effective_from_commit_uri, enablement_changed_at_ms, in_flight_request_ids FROM queue"). + rows := sqlmock.NewRows([]string{"name", "last_green_uri", "in_flight_count", "latest_request_id", "version", "last_green_request_id", "enablement", "enablement_version", "effective_from_commit_uri", "enablement_changed_at_ms"}). + AddRow(want.Name, want.LastGreenURI, want.InFlightCount, want.LatestRequestID, want.Version, want.LastGreenRequestID, want.Enablement, want.EnablementVersion, want.EffectiveFromCommitURI, want.EnablementChangedAtMs) + mock.ExpectQuery("SELECT name, last_green_uri, in_flight_count, latest_request_id, version, last_green_request_id, enablement, enablement_version, effective_from_commit_uri, enablement_changed_at_ms FROM queue"). WithArgs(want.Name). WillReturnRows(rows) }, @@ -141,7 +140,7 @@ func TestQueueStore_Get(t *testing.T) { name: "not found", queueName: want.Name, setup: func(mock sqlmock.Sqlmock) { - mock.ExpectQuery("SELECT name, last_green_uri, in_flight_count, latest_request_id, version, last_green_request_id, enablement, enablement_version, effective_from_commit_uri, enablement_changed_at_ms, in_flight_request_ids FROM queue"). + mock.ExpectQuery("SELECT name, last_green_uri, in_flight_count, latest_request_id, version, last_green_request_id, enablement, enablement_version, effective_from_commit_uri, enablement_changed_at_ms FROM queue"). WithArgs(want.Name). WillReturnError(sql.ErrNoRows) }, @@ -152,7 +151,7 @@ func TestQueueStore_Get(t *testing.T) { name: "query error", queueName: want.Name, setup: func(mock sqlmock.Sqlmock) { - mock.ExpectQuery("SELECT name, last_green_uri, in_flight_count, latest_request_id, version, last_green_request_id, enablement, enablement_version, effective_from_commit_uri, enablement_changed_at_ms, in_flight_request_ids FROM queue"). + mock.ExpectQuery("SELECT name, last_green_uri, in_flight_count, latest_request_id, version, last_green_request_id, enablement, enablement_version, effective_from_commit_uri, enablement_changed_at_ms FROM queue"). WithArgs(want.Name). WillReturnError(fmt.Errorf("connection reset")) }, @@ -202,7 +201,7 @@ func TestQueueStore_Update(t *testing.T) { name: "success", setup: func(mock sqlmock.Sqlmock) { mock.ExpectExec("UPDATE queue"). - WithArgs(queue.LastGreenURI, queue.InFlightCount, queue.LatestRequestID, newVersion, queue.LastGreenRequestID, queue.Enablement, queue.EnablementVersion, queue.EffectiveFromCommitURI, queue.EnablementChangedAtMs, queueOwnersJSON(t, queue), queue.Name, oldVersion). + WithArgs(queue.LastGreenURI, queue.InFlightCount, queue.LatestRequestID, newVersion, queue.LastGreenRequestID, queue.Enablement, queue.EnablementVersion, queue.EffectiveFromCommitURI, queue.EnablementChangedAtMs, queue.Name, oldVersion). WillReturnResult(sqlmock.NewResult(0, 1)) }, }, @@ -210,7 +209,7 @@ func TestQueueStore_Update(t *testing.T) { name: "version mismatch", setup: func(mock sqlmock.Sqlmock) { mock.ExpectExec("UPDATE queue"). - WithArgs(queue.LastGreenURI, queue.InFlightCount, queue.LatestRequestID, newVersion, queue.LastGreenRequestID, queue.Enablement, queue.EnablementVersion, queue.EffectiveFromCommitURI, queue.EnablementChangedAtMs, queueOwnersJSON(t, queue), queue.Name, oldVersion). + WithArgs(queue.LastGreenURI, queue.InFlightCount, queue.LatestRequestID, newVersion, queue.LastGreenRequestID, queue.Enablement, queue.EnablementVersion, queue.EffectiveFromCommitURI, queue.EnablementChangedAtMs, queue.Name, oldVersion). WillReturnResult(sqlmock.NewResult(0, 0)) }, wantErr: true, @@ -220,7 +219,7 @@ func TestQueueStore_Update(t *testing.T) { name: "exec error", setup: func(mock sqlmock.Sqlmock) { mock.ExpectExec("UPDATE queue"). - WithArgs(queue.LastGreenURI, queue.InFlightCount, queue.LatestRequestID, newVersion, queue.LastGreenRequestID, queue.Enablement, queue.EnablementVersion, queue.EffectiveFromCommitURI, queue.EnablementChangedAtMs, queueOwnersJSON(t, queue), queue.Name, oldVersion). + WithArgs(queue.LastGreenURI, queue.InFlightCount, queue.LatestRequestID, newVersion, queue.LastGreenRequestID, queue.Enablement, queue.EnablementVersion, queue.EffectiveFromCommitURI, queue.EnablementChangedAtMs, queue.Name, oldVersion). WillReturnError(fmt.Errorf("connection reset")) }, wantErr: true, @@ -229,7 +228,7 @@ func TestQueueStore_Update(t *testing.T) { name: "rows affected error", setup: func(mock sqlmock.Sqlmock) { mock.ExpectExec("UPDATE queue"). - WithArgs(queue.LastGreenURI, queue.InFlightCount, queue.LatestRequestID, newVersion, queue.LastGreenRequestID, queue.Enablement, queue.EnablementVersion, queue.EffectiveFromCommitURI, queue.EnablementChangedAtMs, queueOwnersJSON(t, queue), queue.Name, oldVersion). + WithArgs(queue.LastGreenURI, queue.InFlightCount, queue.LatestRequestID, newVersion, queue.LastGreenRequestID, queue.Enablement, queue.EnablementVersion, queue.EffectiveFromCommitURI, queue.EnablementChangedAtMs, queue.Name, oldVersion). WillReturnResult(sqlmock.NewErrorResult(fmt.Errorf("driver error"))) }, wantErr: true, @@ -256,10 +255,3 @@ func TestQueueStore_Update(t *testing.T) { }) } } - -func queueOwnersJSON(t *testing.T, queue entity.Queue) []byte { - t.Helper() - owners, err := json.Marshal(queue.InFlightRequestIDs) - require.NoError(t, err) - return owners -} diff --git a/stovepipe/extension/storage/mysql/schema/queue.sql b/stovepipe/extension/storage/mysql/schema/queue.sql index eeeccc141..05acbc350 100644 --- a/stovepipe/extension/storage/mysql/schema/queue.sql +++ b/stovepipe/extension/storage/mysql/schema/queue.sql @@ -11,6 +11,5 @@ CREATE TABLE IF NOT EXISTS queue ( enablement_version BIGINT NOT NULL DEFAULT 0, effective_from_commit_uri VARCHAR(255) NOT NULL DEFAULT '', enablement_changed_at_ms BIGINT NOT NULL DEFAULT 0, - in_flight_request_ids JSON DEFAULT NULL, PRIMARY KEY (name) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; diff --git a/test/integration/stovepipe/extension/storage/suite.go b/test/integration/stovepipe/extension/storage/suite.go index 3ed204ea4..fec3a3d8b 100644 --- a/test/integration/stovepipe/extension/storage/suite.go +++ b/test/integration/stovepipe/extension/storage/suite.go @@ -90,8 +90,6 @@ func (s *QueueStoreContractSuite) TestQueueStore_CreateWithFields() { Name: name, LastGreenURI: "git://remote/monorepo/main/green-bbbb", LastGreenRequestID: "98", - InFlightCount: 2, - InFlightRequestIDs: []string{"99", "100"}, LatestRequestID: "99", Version: 1, } @@ -144,7 +142,6 @@ func (s *QueueStoreContractSuite) TestQueueStore_UpdateCAS() { updated.LastGreenRequestID = "41" updated.LatestRequestID = "42" updated.InFlightCount = 1 - updated.InFlightRequestIDs = []string{"42"} require.NoError(t, s.storeFor(name).Update(s.ctx, updated, 1, 2)) got, err := s.storeFor(name).Get(s.ctx, name) @@ -153,7 +150,6 @@ func (s *QueueStoreContractSuite) TestQueueStore_UpdateCAS() { assert.Equal(t, "41", got.LastGreenRequestID) assert.Equal(t, "42", got.LatestRequestID) assert.Equal(t, int32(1), got.InFlightCount) - assert.Len(t, got.InFlightRequestIDs, 1) assert.Equal(t, int32(2), got.Version) err = s.storeFor(name).Update(s.ctx, updated, 1, 2) @@ -178,14 +174,13 @@ func (s *QueueStoreContractSuite) TestQueueStore_UpdateSequentialCAS() { v2 := entity.Queue{Name: name, LatestRequestID: "10", Version: 1} require.NoError(t, s.storeFor(name).Update(s.ctx, v2, 1, 2)) - v3 := entity.Queue{Name: name, LatestRequestID: "10", InFlightCount: 1, InFlightRequestIDs: []string{"10"}, Version: 2} + v3 := entity.Queue{Name: name, LatestRequestID: "10", InFlightCount: 1, Version: 2} require.NoError(t, s.storeFor(name).Update(s.ctx, v3, 2, 3)) got, err := s.storeFor(name).Get(s.ctx, name) require.NoError(t, err) assert.Equal(t, "10", got.LatestRequestID) assert.Equal(t, int32(1), got.InFlightCount) - assert.Len(t, got.InFlightRequestIDs, 1) assert.Equal(t, int32(3), got.Version) }