diff --git a/internal/analytics/collector_test.go b/internal/analytics/collector_test.go index 8d0f873b..3a61e057 100644 --- a/internal/analytics/collector_test.go +++ b/internal/analytics/collector_test.go @@ -861,6 +861,10 @@ func (m *mockConfigStore) UpsertLadderConfig(_ context.Context, cfg *config.Ladd func (m *mockConfigStore) SaveLadderRun(_ context.Context, run *config.LadderRunDB) (*config.LadderRunDB, error) { return run, nil } +func (m *mockConfigStore) PlanAndSaveLadderRun(_ context.Context, _ string, _ time.Time, build config.LadderPlanBuilder) (*config.LadderRunDB, error) { + run, _, err := build(0) + return run, err +} func (m *mockConfigStore) SaveLadderRunWithTranches(_ context.Context, run *config.LadderRunDB, _ []config.LadderTrancheDB) (*config.LadderRunDB, error) { return run, nil } diff --git a/internal/config/interfaces.go b/internal/config/interfaces.go index b5af9fd4..04aa5276 100644 --- a/internal/config/interfaces.go +++ b/internal/config/interfaces.go @@ -75,9 +75,13 @@ type StoreInterface interface { // LockPlanProvidersTx takes the transaction-scoped advisory lock of every // named provider, in sorted order, so two plan steps that resolve the same // provider's recommendations cannot both attach the same capacity. Lock - // order is advisory lock, then LockPurchasePlanTx; nothing else takes these - // locks, so it cannot deadlock. Duplicates are taken once; an empty list - // and an empty provider name are errors. + // order is advisory lock, then LockPurchasePlanTx. The ladder config lock + // (PlanAndSaveLadderRun, ClaimDueLadderTranches) is always taken BEFORE both: + // ClaimDueLadderTranches's create callback may take these locks, so they + // come after the ladder lock; code holding an execution or tranche row + // (retry linker, reconciler) must never take the ladder lock. No cycle. + // Duplicates are taken once; an empty list and an empty provider name are + // errors. LockPlanProvidersTx(ctx context.Context, tx pgx.Tx, providers []string) error // OccupiedRampStepsInRangeTx returns the steps in [from, to] of planID that // already have a fan-out unit which bought or is still working, ascending. @@ -628,6 +632,16 @@ type StoreInterface interface { // a non-nil pointer (zero when no scheduled tranches exist) so callers can // pass it directly to AllocationInput.InFlightUSDPerHour. Never returns // nil without an error. + // PlanAndSaveLadderRun is the ladder planning persist: in ONE READ COMMITTED + // transaction under the config's advisory lock it reads the in-flight + // commitment (scheduled rows within LadderFireGrace plus fired rows), calls + // build with it, then inserts the run, sweeps overdue tranches and inserts + // the new ones (append-only, L5). Overlapping runs for one config serialize: + // the second nets the first's committed rows. build must be pure compute + // (no I/O); run.StartedAt must equal asOf. The lock is also taken first by + // ClaimDueLadderTranches. SaveLadderRunWithTranches below bypasses it and + // must not be used for planning. + PlanAndSaveLadderRun(ctx context.Context, configID string, asOf time.Time, build LadderPlanBuilder) (*LadderRunDB, error) SaveLadderRun(ctx context.Context, run *LadderRunDB) (*LadderRunDB, error) SaveLadderRunWithTranches(ctx context.Context, run *LadderRunDB, tranches []LadderTrancheDB) (*LadderRunDB, error) GetInFlightLadderCommitUSDHr(ctx context.Context, configID string, asOf time.Time) (*float64, error) diff --git a/internal/config/store_postgres_ladder.go b/internal/config/store_postgres_ladder.go index 168536f9..4b1ccd66 100644 --- a/internal/config/store_postgres_ladder.go +++ b/internal/config/store_postgres_ladder.go @@ -372,18 +372,11 @@ func (s *PostgresStore) SaveLadderRunWithTranches(ctx context.Context, run *Ladd } var result LadderRunDB err := s.WithTx(ctx, func(tx pgx.Tx) error { - row := tx.QueryRow(ctx, ladderRunInsertQuery, ladderRunInsertArgs(run, ladderRunPlanJSON(run))...) - scanned, err := scanLadderRun(row) + saved, err := insertLadderRunWithTranchesTx(ctx, tx, run, tranches) if err != nil { - return fmt.Errorf("failed to insert ladder_run id=%s: %w", run.ID, err) - } - if err := cancelOverdueLadderTranchesTx(ctx, tx, run.ConfigID, run.StartedAt); err != nil { - return err - } - if err := insertLadderTranchesTx(ctx, tx, tranches); err != nil { return err } - result = scanned + result = *saved return nil }) if err != nil { @@ -392,6 +385,24 @@ func (s *PostgresStore) SaveLadderRunWithTranches(ctx context.Context, run *Ladd return &result, nil } +// insertLadderRunWithTranchesTx is the shared write half of +// SaveLadderRunWithTranches and PlanAndSaveLadderRun: run row, overdue sweep, +// then the new tranches. run.ID must be set. +func insertLadderRunWithTranchesTx(ctx context.Context, tx pgx.Tx, run *LadderRunDB, tranches []LadderTrancheDB) (*LadderRunDB, error) { + row := tx.QueryRow(ctx, ladderRunInsertQuery, ladderRunInsertArgs(run, ladderRunPlanJSON(run))...) + scanned, err := scanLadderRun(row) + if err != nil { + return nil, fmt.Errorf("failed to insert ladder_run id=%s: %w", run.ID, err) + } + if err := cancelOverdueLadderTranchesTx(ctx, tx, run.ConfigID, run.StartedAt); err != nil { + return nil, err + } + if err := insertLadderTranchesTx(ctx, tx, tranches); err != nil { + return nil, err + } + return &scanned, nil +} + // GetInFlightLadderCommitUSDHr returns the total hourly USD commitment already // in flight for the given config: the SUM of amount_usd_hr for ladder_tranches // rows where config_id=$1 and either @@ -410,15 +421,9 @@ func (s *PostgresStore) SaveLadderRunWithTranches(ctx context.Context, run *Ladd // nil without a non-nil error, so callers can pass it directly to // AllocationInput.InFlightUSDPerHour without an extra nil-guard. func (s *PostgresStore) GetInFlightLadderCommitUSDHr(ctx context.Context, configID string, asOf time.Time) (*float64, error) { - var total float64 - err := s.db.QueryRow(ctx, ` - SELECT COALESCE(SUM(amount_usd_hr), 0) - FROM ladder_tranches - WHERE config_id = $1 - AND (status = $2 OR (status = $3 AND scheduled_date > $4)) - `, configID, string(ladder.TrancheStatusFired), string(ladder.TrancheStatusScheduled), ladderGraceCutoff(asOf)).Scan(&total) + total, err := inFlightLadderCommit(ctx, s.db, configID, asOf) if err != nil { - return nil, fmt.Errorf("GetInFlightLadderCommitUSDHr config_id=%s: %w", configID, err) + return nil, err } return &total, nil } diff --git a/internal/config/store_postgres_ladder_lifecycle.go b/internal/config/store_postgres_ladder_lifecycle.go index c2741089..75dcd47e 100644 --- a/internal/config/store_postgres_ladder_lifecycle.go +++ b/internal/config/store_postgres_ladder_lifecycle.go @@ -94,7 +94,11 @@ func (s *PostgresStore) TransitionLadderTrancheTx(ctx context.Context, tx pgx.Tx // returns an error the whole transaction rolls back and nothing is claimed, so // a claim and its execution row can never disagree after a crash. Rows locked // by a concurrent claim are skipped (FOR UPDATE SKIP LOCKED), so exactly one -// caller wins each tranche. +// caller wins each tranche. The whole claim runs under the config's ladder +// lock (taken first), including the create callback: with a caller asOf earlier +// than the planner's, a claim could otherwise fire a row the planner's netting +// read had already excluded as expired, and the row would be bought AND +// re-planned. A callback slower than ladderLockTimeout makes planners fail loudly. func (s *PostgresStore) ClaimDueLadderTranches( ctx context.Context, configID string, @@ -106,7 +110,7 @@ func (s *PostgresStore) ClaimDueLadderTranches( return nil, fmt.Errorf("ClaimDueLadderTranches limit must be positive, got %d", limit) } var claimed []LadderTrancheDB - err := s.WithTx(ctx, func(tx pgx.Tx) error { + err := s.withLadderConfigLock(ctx, configID, func(tx pgx.Tx) error { due, err := selectDueLadderTranchesTx(ctx, tx, configID, asOf, limit) if err != nil { return err diff --git a/internal/config/store_postgres_ladder_lock.go b/internal/config/store_postgres_ladder_lock.go new file mode 100644 index 00000000..77d85f8f --- /dev/null +++ b/internal/config/store_postgres_ladder_lock.go @@ -0,0 +1,153 @@ +package config + +import ( + "context" + "errors" + "fmt" + "log" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "github.com/LeanerCloud/cloud-commitments-go/pkg/ladder" +) + +// ladderConfigLockPrefix namespaces the per-config ladder advisory lock. It is +// a distinct text input to the same hashtextextended(text, 0) bigint key space +// as planStepProviderLockPrefix and the purchase-submit lock; a 64-bit hash +// collision could only cause a spurious wait, never a correctness break. +const ladderConfigLockPrefix = "cudly:ladder-config:" + +// ladderLockTimeout bounds only the advisory-lock wait so a stuck holder fails +// loudly instead of queueing every later run. Package var so a test can shrink it. +var ladderLockTimeout = 30 * time.Second + +// LadderPlanBuilder computes a run and its new tranches from the config's +// in-flight commitment. It runs while the per-config lock is held inside the +// persist transaction: it must be pure compute (no provider or network I/O). +type LadderPlanBuilder func(inFlightUSDHr float64) (*LadderRunDB, []LadderTrancheDB, error) + +func ladderConfigLockKey(configID string) string { return ladderConfigLockPrefix + configID } + +// withLadderConfigLock runs fn in a READ COMMITTED transaction that holds the +// config's ladder lock. It is the only way planning (PlanAndSaveLadderRun) and +// claiming (ClaimDueLadderTranches) serialize on one config. +// +// Lock order: this lock is taken FIRST. Whatever fn takes afterwards (provider +// plan-step locks, LockPurchasePlanTx, row locks) comes after it; code holding +// an execution or tranche row lock must never take this lock. +// +// Isolation is set explicitly: under REPEATABLE READ the snapshot is fixed at +// the first statement, before the wait, so the in-flight read would be stale. +func (s *PostgresStore) withLadderConfigLock(ctx context.Context, configID string, fn func(tx pgx.Tx) error) error { + if configID == "" { + return errors.New("ladder config lock: empty config id") + } + tx, err := s.db.Begin(ctx) + if err != nil { + return fmt.Errorf("failed to begin transaction: %w", err) + } + defer func() { + // ErrTxClosed after Commit is expected; the deferred call also covers a panic in fn. + if rbErr := tx.Rollback(ctx); rbErr != nil && !errors.Is(rbErr, pgx.ErrTxClosed) { + log.Printf("ladder config lock %s: rollback: %v", configID, rbErr) + } + }() + + if err := acquireLadderConfigLock(ctx, tx, configID); err != nil { + return err + } + if err := fn(tx); err != nil { + return err + } + if err := tx.Commit(ctx); err != nil { + return fmt.Errorf("failed to commit transaction: %w", err) + } + return nil +} + +// acquireLadderConfigLock must be the first thing run in tx: the isolation +// level can only be set before any query. +func acquireLadderConfigLock(ctx context.Context, tx pgx.Tx, configID string) error { + if _, err := tx.Exec(ctx, "SET TRANSACTION ISOLATION LEVEL READ COMMITTED"); err != nil { + return fmt.Errorf("ladder config lock %s: set isolation: %w", configID, err) + } + if _, err := tx.Exec(ctx, fmt.Sprintf("SET LOCAL lock_timeout = %d", ladderLockTimeout.Milliseconds())); err != nil { + return fmt.Errorf("ladder config lock %s: set lock_timeout: %w", configID, err) + } + if _, err := tx.Exec(ctx, "SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", ladderConfigLockKey(configID)); err != nil { + return fmt.Errorf("ladder config lock %s: %w", configID, err) + } + // The timeout was for the advisory wait only; do not leak it into fn's statements. + if _, err := tx.Exec(ctx, "SET LOCAL lock_timeout TO DEFAULT"); err != nil { + return fmt.Errorf("ladder config lock %s: reset lock_timeout: %w", configID, err) + } + return nil +} + +// PlanAndSaveLadderRun reads the config's in-flight commitment, lets build turn +// it into a run plus tranches, and persists both, all in ONE transaction under +// the config's ladder lock. Two overlapping runs for one config therefore +// cannot both plan the same gap: the second waits, then reads the first's +// committed tranches as in-flight. build must be pure compute. The run's +// ConfigID must equal configID and its StartedAt must equal asOf, which bounds +// both the netting read and the overdue sweep. +func (s *PostgresStore) PlanAndSaveLadderRun(ctx context.Context, configID string, asOf time.Time, build LadderPlanBuilder) (*LadderRunDB, error) { + var result LadderRunDB + err := s.withLadderConfigLock(ctx, configID, func(tx pgx.Tx) error { + inFlight, err := inFlightLadderCommit(ctx, tx, configID, asOf) + if err != nil { + return err + } + run, tranches, err := build(inFlight) + if err != nil { + return err + } + if run == nil { + return errors.New("ladder plan builder returned a nil run") + } + if run.ConfigID == nil || *run.ConfigID != configID { + return fmt.Errorf("ladder run config_id does not match locked config %s", configID) + } + // The netting read above and the overdue sweep in the insert must use + // the same grace cutoff, or a row could be counted and canceled (or neither). + if !run.StartedAt.Equal(asOf) { + return fmt.Errorf("ladder run started_at %s does not match asOf %s", run.StartedAt, asOf) + } + if run.ID == "" { + run.ID = uuid.New().String() + } + saved, err := insertLadderRunWithTranchesTx(ctx, tx, run, tranches) + if err != nil { + return err + } + result = *saved + return nil + }) + if err != nil { + return nil, err + } + return &result, nil +} + +// queryRower is satisfied by pgx.Tx and the pool connection. +type queryRower interface { + QueryRow(ctx context.Context, sql string, args ...any) pgx.Row +} + +// inFlightLadderCommit sums the config's in-flight commitment; see +// GetInFlightLadderCommitUSDHr for the row classes counted and excluded. +func inFlightLadderCommit(ctx context.Context, q queryRower, configID string, asOf time.Time) (float64, error) { + var total float64 + err := q.QueryRow(ctx, ` + SELECT COALESCE(SUM(amount_usd_hr), 0) + FROM ladder_tranches + WHERE config_id = $1 + AND (status = $2 OR (status = $3 AND scheduled_date > $4)) + `, configID, string(ladder.TrancheStatusFired), string(ladder.TrancheStatusScheduled), ladderGraceCutoff(asOf)).Scan(&total) + if err != nil { + return 0, fmt.Errorf("in-flight ladder commit config_id=%s: %w", configID, err) + } + return total, nil +} diff --git a/internal/config/store_postgres_ladder_lock_integration_test.go b/internal/config/store_postgres_ladder_lock_integration_test.go new file mode 100644 index 00000000..74bc7892 --- /dev/null +++ b/internal/config/store_postgres_ladder_lock_integration_test.go @@ -0,0 +1,233 @@ +//go:build integration + +package config + +// Real PostgreSQL tests for the per-config ladder lock. Waiting is observed in +// pg_locks (this config's advisory key, granted=false), never with sleeps. + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/LeanerCloud/cloud-commitments-go/pkg/ladder" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// ladderLockWaiters counts sessions blocked on this config's ladder lock. The +// bigint advisory key is split across pg_locks classid (high 32 bits) and +// objid (low 32 bits). +func ladderLockWaiters(ctx context.Context, t *testing.T, store *PostgresStore, cfgID string) int { + t.Helper() + var n int + require.NoError(t, store.db.QueryRow(ctx, ` + SELECT count(*) FROM pg_locks l, (SELECT hashtextextended($1, 0) AS k) h + WHERE l.locktype = 'advisory' AND l.objsubid = 1 AND NOT l.granted + AND l.classid = ((h.k >> 32) & 4294967295)::oid + AND l.objid = (h.k & 4294967295)::oid`, ladderConfigLockKey(cfgID)).Scan(&n)) + return n +} + +func waitForLadderWaiters(ctx context.Context, t *testing.T, store *PostgresStore, cfgID string, want int) { + t.Helper() + require.Eventually(t, func() bool { return ladderLockWaiters(ctx, t, store, cfgID) == want }, + 10*time.Second, 20*time.Millisecond, "expected %d waiter(s) on the ladder lock", want) +} + +// lockTestAsOf is the fixed run start shared by lockTestRun and the PlanAndSave +// calls (they must match). +var lockTestAsOf = time.Now().UTC().Truncate(time.Microsecond) + +func lockTestRun(cfgID string) *LadderRunDB { + return &LadderRunDB{ConfigID: &cfgID, StartedAt: lockTestAsOf, Status: ladder.RunStatusPlanned} +} + +// holdLadderLock runs PlanAndSaveLadderRun with a build that blocks until the +// returned release func is called, so the caller owns the lock meanwhile. +func holdLadderLock(ctx context.Context, t *testing.T, store *PostgresStore, cfgID string) (release func() error) { + t.Helper() + entered, gate, done := make(chan struct{}), make(chan struct{}), make(chan error, 1) + go func() { + _, err := store.PlanAndSaveLadderRun(ctx, cfgID, lockTestAsOf, func(float64) (*LadderRunDB, []LadderTrancheDB, error) { + close(entered) + <-gate + return lockTestRun(cfgID), nil, nil + }) + done <- err + }() + select { + case <-entered: + case <-time.After(10 * time.Second): + t.Fatal("holder never entered build") + } + return func() error { close(gate); return <-done } +} + +func countRuns(ctx context.Context, t *testing.T, store *PostgresStore, cfgID string) int { + t.Helper() + var n int + require.NoError(t, store.db.QueryRow(ctx, `SELECT count(*) FROM ladder_runs WHERE config_id = $1`, cfgID).Scan(&n)) + return n +} + +func TestLadderLock_ClaimWaitsForPlanner(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + store := setupLadderStore(ctx, t) + cfgID := seedLadderConfigWithExtID(ctx, t, store, "711111111111") + require.NoError(t, store.SaveLadderTranches(ctx, []LadderTrancheDB{lifecycleTranche(cfgID, 1, ladder.TrancheStatusScheduled, time.Now().UTC().Add(-time.Hour))})) + release := holdLadderLock(ctx, t, store, cfgID) + + var claimEntered atomic.Bool + inner := newExecCreator(ctx, t, store, new(atomic.Int64)) + claimDone := make(chan error, 1) + go func() { + _, err := store.ClaimDueLadderTranches(ctx, cfgID, time.Now().UTC(), 1, + func(c context.Context, tx pgx.Tx, tr LadderTrancheDB) (string, error) { + claimEntered.Store(true) + return inner(c, tx, tr) + }) + claimDone <- err + }() + waitForLadderWaiters(ctx, t, store, cfgID, 1) + assert.False(t, claimEntered.Load(), "claim must not reach its callback while the planner holds the lock") + require.NoError(t, release()) + require.NoError(t, <-claimDone) + assert.True(t, claimEntered.Load(), "claim proceeds once the planner releases") +} + +func TestLadderLock_IsPerConfig(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + store := setupLadderStore(ctx, t) + a := seedLadderConfigWithExtID(ctx, t, store, "721111111111") + b := seedLadderConfigWithExtID(ctx, t, store, "722222222222") + release := holdLadderLock(ctx, t, store, a) + _, err := store.PlanAndSaveLadderRun(ctx, b, lockTestAsOf, func(float64) (*LadderRunDB, []LadderTrancheDB, error) { + return lockTestRun(b), nil, nil + }) + require.NoError(t, err, "a held lock on config A must not block config B") + require.NoError(t, release()) +} + +func TestLadderLock_ReleasedOnBuildErrorPanicAndCancel(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + store := setupLadderStore(ctx, t) + cfgID := seedLadderConfigWithExtID(ctx, t, store, "731111111111") + ok := func(float64) (*LadderRunDB, []LadderTrancheDB, error) { return lockTestRun(cfgID), nil, nil } //nolint:unparam // matches LadderPlanBuilder + plan := func(c context.Context, b LadderPlanBuilder) error { + _, err := store.PlanAndSaveLadderRun(c, cfgID, lockTestAsOf, b) + return err + } + + t.Run("build error", func(t *testing.T) { + err := plan(ctx, func(float64) (*LadderRunDB, []LadderTrancheDB, error) { return nil, nil, errors.New("boom") }) + require.ErrorContains(t, err, "boom") + before := countRuns(ctx, t, store, cfgID) + require.NoError(t, plan(ctx, ok)) + assert.Equal(t, before+1, countRuns(ctx, t, store, cfgID)) + }) + t.Run("build panic", func(t *testing.T) { + assert.Panics(t, func() { + _ = plan(ctx, func(float64) (*LadderRunDB, []LadderTrancheDB, error) { panic("boom") }) + }) + before := countRuns(ctx, t, store, cfgID) + require.NoError(t, plan(ctx, ok), "the lock must be free after a panic") + assert.Equal(t, before+1, countRuns(ctx, t, store, cfgID)) + }) + t.Run("ctx canceled while waiting", func(t *testing.T) { + before := countRuns(ctx, t, store, cfgID) + release := holdLadderLock(ctx, t, store, cfgID) + waitCtx, cancelWait := context.WithCancel(ctx) + errc := make(chan error, 1) + go func() { errc <- plan(waitCtx, ok) }() + waitForLadderWaiters(ctx, t, store, cfgID, 1) + cancelWait() + require.Error(t, <-errc) + waitForLadderWaiters(ctx, t, store, cfgID, 0) + require.NoError(t, release()) + assert.Equal(t, before+1, countRuns(ctx, t, store, cfgID), "only the holder's run exists; the canceled waiter wrote nothing") + require.NoError(t, plan(ctx, ok)) + }) +} + +func TestLadderLock_EmptyConfigIDRejected(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + store := setupLadderStore(ctx, t) + built := false + _, err := store.PlanAndSaveLadderRun(ctx, "", time.Now().UTC(), func(float64) (*LadderRunDB, []LadderTrancheDB, error) { + built = true + return nil, nil, nil + }) + require.Error(t, err) + assert.False(t, built) + _, err = store.ClaimDueLadderTranches(ctx, "", time.Now().UTC(), 1, nil) + require.Error(t, err) +} + +func TestLadderLock_RejectsNilRunAndStartedAtMismatch(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + store := setupLadderStore(ctx, t) + cfgID := seedLadderConfigWithExtID(ctx, t, store, "751111111111") + _, err := store.PlanAndSaveLadderRun(ctx, cfgID, lockTestAsOf, func(float64) (*LadderRunDB, []LadderTrancheDB, error) { return nil, nil, nil }) + require.ErrorContains(t, err, "nil run") + _, err = store.PlanAndSaveLadderRun(ctx, cfgID, lockTestAsOf.Add(time.Second), func(float64) (*LadderRunDB, []LadderTrancheDB, error) { + return lockTestRun(cfgID), nil, nil + }) + require.ErrorContains(t, err, "does not match asOf") + assert.Equal(t, 0, countRuns(ctx, t, store, cfgID)) +} + +// The lock_timeout bounds only the advisory wait, and the transaction is READ +// COMMITTED even when the database default is REPEATABLE READ. +func TestLadderLock_TimeoutBoundsOnlyTheWaitAndIsolationIsReadCommitted(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + store := setupLadderStore(ctx, t) + cfgID := seedLadderConfigWithExtID(ctx, t, store, "741111111111") + + old := ladderLockTimeout + ladderLockTimeout = 300 * time.Millisecond + defer func() { ladderLockTimeout = old }() + + release := holdLadderLock(ctx, t, store, cfgID) + _, err := store.PlanAndSaveLadderRun(ctx, cfgID, lockTestAsOf, func(float64) (*LadderRunDB, []LadderTrancheDB, error) { + return lockTestRun(cfgID), nil, nil + }) + var pgErr *pgconn.PgError + require.ErrorAs(t, err, &pgErr, "a stuck holder must fail the waiter loudly") + assert.Equal(t, "55P03", pgErr.Code) + require.NoError(t, release()) + + var dbName string + require.NoError(t, store.db.QueryRow(ctx, `SELECT current_database()`).Scan(&dbName)) + _, err = store.db.Exec(ctx, `ALTER DATABASE `+pgx.Identifier{dbName}.Sanitize()+` SET default_transaction_isolation = 'repeatable read'`) + require.NoError(t, err) + t.Cleanup(func() { + _, _ = store.db.Exec(context.Background(), `ALTER DATABASE `+pgx.Identifier{dbName}.Sanitize()+` RESET default_transaction_isolation`) + }) + pool := store.db.(interface{ Pool() *pgxpool.Pool }).Pool() + pool.Reset() // new connections pick up the database default + + past := time.Now().UTC().Add(-time.Hour) + require.NoError(t, store.SaveLadderTranches(ctx, []LadderTrancheDB{lifecycleTranche(cfgID, 1, ladder.TrancheStatusScheduled, past)})) + inner := newExecCreator(ctx, t, store, new(atomic.Int64)) + var lockTimeout, isolation string + _, err = store.ClaimDueLadderTranches(ctx, cfgID, time.Now().UTC(), 1, func(c context.Context, tx pgx.Tx, tr LadderTrancheDB) (string, error) { + require.NoError(t, tx.QueryRow(c, `SHOW lock_timeout`).Scan(&lockTimeout)) + require.NoError(t, tx.QueryRow(c, `SHOW transaction_isolation`).Scan(&isolation)) + return inner(c, tx, tr) + }) + require.NoError(t, err) + assert.Equal(t, "0", lockTimeout, "the wait timeout must not leak into later statements") + assert.Equal(t, "read committed", isolation) +} diff --git a/internal/mocks/stores.go b/internal/mocks/stores.go index 791704a1..c9d2b4eb 100644 --- a/internal/mocks/stores.go +++ b/internal/mocks/stores.go @@ -1928,6 +1928,27 @@ func (m *MockConfigStore) SaveLadderRun(ctx context.Context, run *config.LadderR return v, args.Error(1) } +// PlanAndSaveLadderRun mocks the PlanAndSaveLadderRun operation. With no +// expectation registered it runs build with zero in-flight and returns the run +// it produced (transaction succeeds), so tests that only care about other +// calls stay green. +func (m *MockConfigStore) PlanAndSaveLadderRun(ctx context.Context, configID string, asOf time.Time, build config.LadderPlanBuilder) (*config.LadderRunDB, error) { + m.record("PlanAndSaveLadderRun", ctx, configID, asOf) + if !isExpected(&m.Mock, "PlanAndSaveLadderRun") { + run, _, err := build(0) + return run, err + } + args := m.Called(ctx, configID, asOf) + if args.Get(0) == nil { + return nil, args.Error(1) + } + v, ok := args.Get(0).(*config.LadderRunDB) + if !ok { + panic(fmt.Sprintf("mock: expected *config.LadderRunDB, got %T", args.Get(0))) + } + return v, args.Error(1) +} + // SaveLadderRunWithTranches mocks the SaveLadderRunWithTranches operation. // Returns the run unchanged (transaction succeeds) when no expectation is // registered, so tests that only care about other calls stay green. diff --git a/internal/server/test_helpers_test.go b/internal/server/test_helpers_test.go index 831d8537..cc06c0d6 100644 --- a/internal/server/test_helpers_test.go +++ b/internal/server/test_helpers_test.go @@ -416,6 +416,10 @@ func (m *mockConfigStoreForHealth) UpsertLadderConfig(_ context.Context, cfg *co func (m *mockConfigStoreForHealth) SaveLadderRun(_ context.Context, run *config.LadderRunDB) (*config.LadderRunDB, error) { return run, nil } +func (m *mockConfigStoreForHealth) PlanAndSaveLadderRun(_ context.Context, _ string, _ time.Time, build config.LadderPlanBuilder) (*config.LadderRunDB, error) { + run, _, err := build(0) + return run, err +} func (m *mockConfigStoreForHealth) SaveLadderRunWithTranches(_ context.Context, run *config.LadderRunDB, _ []config.LadderTrancheDB) (*config.LadderRunDB, error) { return run, nil }