From e5b62f0fead6f062c700cc6654b46bd6280ce620 Mon Sep 17 00:00:00 2001 From: Lucas Carlson Date: Fri, 9 Oct 2026 13:41:32 -0700 Subject: [PATCH 1/5] docs: add four tested problem guides Developers search for a problem, not for a virtual actor library. Four guides now start from one: race conditions in Rails, running jobs in order for each customer, expiring reservations, and saving state and queuing work together. Each one reproduces the failure, shows the plain Rails fix first (a unique index, a conditional update, row and optimistic locks, a sequence number, Sidekiq capsules, Solid Queue concurrency limits, a transactional outbox), and only then shows an actor where ordering, durable state, reminders, and recovery are needed together. Codex with GPT-6 Astra wrote the prose from briefs that held only verified facts and cited sources. Every code block in a guide is a file under examples/guides/, and the tests in test/guides/ prove each claim. Each test was watched to fail first, and each guard in the examples was removed once to confirm that a test fails without it. A document test fails when a guide does not embed the current example files, contains an em dash or an en dash, or links to a missing file. The concurrency helper limits the threads that act at once to the free connections in the pool. On PostgreSQL with the default pool of five, ten waiting synchronous callers otherwise held every connection and the actor turn could not get one ("could not obtain a connection from the pool within 5.000 seconds"). Active Job joins the development and test bundle for the job examples, pinned to the Rails line in compatibility runs. --- Gemfile | 1 + Gemfile.lock | 8 + README.md | 11 + docs/agents.md | 7 + docs/guides/expiring-reservations.md | 167 +++++++++++++ docs/guides/ordered-jobs.md | 197 ++++++++++++++++ docs/guides/race-conditions.md | 220 ++++++++++++++++++ docs/guides/transactional-outbox.md | 156 +++++++++++++ .../expiring_reservations/seat_inventory.rb | 75 ++++++ examples/guides/ordered_jobs/account.rb | 23 ++ .../guides/ordered_jobs/apply_entry_job.rb | 9 + .../ordered_jobs/apply_entry_unordered_job.rb | 7 + .../guides/ordered_jobs/ledger_account.rb | 25 ++ examples/guides/race_conditions/event.rb | 15 ++ .../guides/race_conditions/event_tickets.rb | 54 +++++ .../guides/transactional_outbox/checkout.rb | 22 ++ examples/guides/transactional_outbox/order.rb | 17 ++ .../transactional_outbox/outbox_message.rb | 14 ++ .../transactional_outbox/shipment_job.rb | 8 + .../transactional_outbox/solid_objects.rb | 12 + test/guides/expiring_reservations_test.rb | 113 +++++++++ test/guides/guide_documents_test.rb | 71 ++++++ test/guides/guide_test_helper.rb | 86 +++++++ test/guides/ordered_jobs_test.rb | 107 +++++++++ test/guides/race_conditions_test.rb | 116 +++++++++ test/guides/transactional_outbox_test.rb | 141 +++++++++++ 26 files changed, 1682 insertions(+) create mode 100644 docs/guides/expiring-reservations.md create mode 100644 docs/guides/ordered-jobs.md create mode 100644 docs/guides/race-conditions.md create mode 100644 docs/guides/transactional-outbox.md create mode 100644 examples/guides/expiring_reservations/seat_inventory.rb create mode 100644 examples/guides/ordered_jobs/account.rb create mode 100644 examples/guides/ordered_jobs/apply_entry_job.rb create mode 100644 examples/guides/ordered_jobs/apply_entry_unordered_job.rb create mode 100644 examples/guides/ordered_jobs/ledger_account.rb create mode 100644 examples/guides/race_conditions/event.rb create mode 100644 examples/guides/race_conditions/event_tickets.rb create mode 100644 examples/guides/transactional_outbox/checkout.rb create mode 100644 examples/guides/transactional_outbox/order.rb create mode 100644 examples/guides/transactional_outbox/outbox_message.rb create mode 100644 examples/guides/transactional_outbox/shipment_job.rb create mode 100644 examples/guides/transactional_outbox/solid_objects.rb create mode 100644 test/guides/expiring_reservations_test.rb create mode 100644 test/guides/guide_documents_test.rb create mode 100644 test/guides/guide_test_helper.rb create mode 100644 test/guides/ordered_jobs_test.rb create mode 100644 test/guides/race_conditions_test.rb create mode 100644 test/guides/transactional_outbox_test.rb diff --git a/Gemfile b/Gemfile index 4877c89..aff459a 100644 --- a/Gemfile +++ b/Gemfile @@ -14,6 +14,7 @@ if rails_version end group :development, :test do + gem "activejob", rails_version ? "~> #{rails_version}.0" : ">= 7.1", require: false gem "mysql2", ">= 0.5", require: false gem "pg", ">= 1.5", require: false gem "redis", ">= 5.0", require: false diff --git a/Gemfile.lock b/Gemfile.lock index 76d0e8d..c68d8e2 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -36,6 +36,9 @@ GEM erubi (~> 1.11) rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) + activejob (8.1.3.1) + activesupport (= 8.1.3.1) + globalid (>= 0.3.6) activemodel (8.1.3.1) activesupport (= 8.1.3.1) activerecord (8.1.3.1) @@ -78,6 +81,8 @@ GEM ffi (1.17.4-x86_64-linux-gnu) ffi (1.17.4-x86_64-linux-musl) fileutils (1.8.0) + globalid (1.4.0) + activesupport (>= 6.1) i18n (1.15.2) concurrent-ruby (~> 1.0) io-console (0.8.2) @@ -283,6 +288,7 @@ PLATFORMS x86_64-linux-musl DEPENDENCIES + activejob (>= 7.1) benchmark brakeman minitest @@ -301,6 +307,7 @@ CHECKSUMS actioncable (8.1.3.1) sha256=e318528295c878a3efdfe25f0f2267c80cb7a76eba41bb5f64d44aa380a3d91b actionpack (8.1.3.1) sha256=974cb7154548e81f470b1b0f247b99cb38e87825899dca58610596e2817723d0 actionview (8.1.3.1) sha256=2da68b8414c47b43bfbed1ce69c5afe1c04f78c267aacb5660a4cab5ca12cfb6 + activejob (8.1.3.1) sha256=1c8dd275df930df40deecffec63d913a550a33fd94bd298f69721dd96939954a activemodel (8.1.3.1) sha256=99cc02ce2faec371d14440949d85787ebd23a907c9baef0a9d4bcd4d21888f88 activerecord (8.1.3.1) sha256=0a2fb6c28f4938f6b013a3a549bec0a7e37d535f3dc8990e804bcc3258c0403b activesupport (8.1.3.1) sha256=85458765f25ea48b9019c46b6bb3fa5683197bf4280d9f06710a6e8d7a831376 @@ -326,6 +333,7 @@ CHECKSUMS ffi (1.17.4-x86_64-linux-gnu) sha256=9d3db14c2eae074b382fa9c083fe95aec6e0a1451da249eab096c34002bc752d ffi (1.17.4-x86_64-linux-musl) sha256=3fdf9888483de005f8ef8d1cf2d3b20d86626af206cbf780f6a6a12439a9c49e fileutils (1.8.0) sha256=8c6b1df54e2540bdb2f39258f08af78853aa70bad52b4d394bbc6424593c6e02 + globalid (1.4.0) sha256=037f12fbf1d9d7a014d501c2d5c77356fd4ddd96d7a7991d6700bba96706f427 i18n (1.15.2) sha256=00f9eb62412fe593b2a65a97daa75300d37abb8f7202ec748e94b6d46a9dd1b5 io-console (0.8.2) sha256=d6e3ae7a7cc7574f4b8893b4fca2162e57a825b223a177b7afa236c5ef9814cc irb (1.18.0) sha256=de9454a0703a54704b9811a5ef31a60c86949fbf4013fcf244fabc7c775248e3 diff --git a/README.md b/README.md index 234da25..712492d 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,7 @@ And so much more. - [Good uses](#good-uses) - [When a transaction is better](#when-a-transaction-is-better) - [Guarantees and boundaries](#guarantees-and-boundaries) +- [Guides](#guides) - [Read more](#read-more) - [Status and license](#status-and-license) @@ -164,6 +165,16 @@ SQL and should be allowed to enjoy that. Exactly once is not hiding in a more advanced configuration. Read the [correctness contract](docs/correctness.md) before using important data. +## Guides + +Each guide starts from a problem, shows the plain Rails fix first, and tests +every claim in [`test/guides/`](test/guides/): + +- [Prevent race conditions in Rails](docs/guides/race-conditions.md) +- [Run jobs in order for each customer](docs/guides/ordered-jobs.md) +- [Expiring reservations](docs/guides/expiring-reservations.md) +- [Save state and queue work together](docs/guides/transactional-outbox.md) + ## Read more - [Five-minute Rails guide](https://solidobjects.dev/5min/rails) diff --git a/docs/agents.md b/docs/agents.md index ac63991..8fce4a6 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -30,6 +30,13 @@ Select a simpler tool in these cases: The full checklist is in [Choosing Solid Objects](fit.md). The category guide is [Virtual actors in Ruby on Rails](virtual-actors.md). +Problem guides compare the plain Rails fix with an actor and test each claim: + +- [Prevent race conditions in Rails](guides/race-conditions.md) +- [Run jobs in order for each customer](guides/ordered-jobs.md) +- [Expiring reservations](guides/expiring-reservations.md) +- [Save state and queue work together](guides/transactional-outbox.md) + ## 2. Package identity | Item | Value | diff --git a/docs/guides/expiring-reservations.md b/docs/guides/expiring-reservations.md new file mode 100644 index 0000000..9c64b18 --- /dev/null +++ b/docs/guides/expiring-reservations.md @@ -0,0 +1,167 @@ +# Expiring reservations in Rails + +A reservation holds stock for a short time, until the buyer confirms it or the hold expires. The stock must never go below zero. A retry must not take stock twice. A late expiry must not cancel a newer state. Solid Objects (the gem solid_objects) puts the stock and its holds in one actor, with durable reminders for the deadlines. + +## Put the stock under the right identity + +An actor for each reservation cannot prevent an oversold show. Two reservations can take the same stock. Two reservation actors are two identities. They run at the same time, and neither actor sees the other actor's hold. + +Put the stock and all of its holds in the actor that owns the stock. This guide uses one actor for each show. + +## The plain SQL design + +- Store each hold in a table with its seats and an `expires_at` time. +- Count free seats as the capacity minus confirmed seats minus holds whose `expires_at` is still in the future. +- Insert a hold inside a transaction that locks the show row. Two holds cannot then both see the last seat. +- This design needs no timer. An old hold no longer counts when its time passes. + +This design is not enough when an action must occur at the deadline. You need a timer for these actions: + +- Release a payment authorization. +- Tell the buyer that the hold expired. +- Update state that other code reads without the clock. +- Start the next step of a workflow. + +The timer, a confirmation, and a client retry can all touch the same hold. Each path must take the same lock. Each path must handle a duplicate or late delivery. + +## The actor + +```ruby +class SeatInventory < SolidObjects::Actor + HOLD_DURATION = 15.minutes + EXTENSION = 5.minutes + MAX_EXTENSIONS = 2 + + attribute :capacity, default: 0 + attribute :holds, default: -> { {} } + attribute :confirmed, default: -> { {} } + + query :seats_left do + seats_available + end + + def open_show(capacity:) + self.capacity = capacity if self.capacity.zero? + seats_available + end + + def hold(hold_id:, buyer:, seats:) + return hold_result(hold_id) if holds.key?(hold_id) + return { status: "confirmed" } if confirmed.key?(hold_id) + reject(:not_enough_seats, "Only #{seats_available} seats are left") if seats > seats_available + + deadline = HOLD_DURATION.from_now + self.holds = holds.merge( + hold_id => { "buyer" => buyer, "seats" => seats, "expires_at" => deadline.to_i, "extensions" => 0 } + ) + schedule(at: deadline, key: hold_id).expire(hold_id:, expires_at: deadline.to_i) + hold_result(hold_id) + end + + def extend_hold(hold_id:) + hold = holds[hold_id] + reject(:no_hold, "The hold expired or does not exist") unless hold + reject(:extension_limit, "The hold cannot be extended again") if hold.fetch("extensions") >= MAX_EXTENSIONS + + deadline = Time.at(hold.fetch("expires_at")) + EXTENSION + self.holds = holds.merge( + hold_id => hold.merge("expires_at" => deadline.to_i, "extensions" => hold.fetch("extensions") + 1) + ) + schedule(at: deadline, key: hold_id).expire(hold_id:, expires_at: deadline.to_i) + hold_result(hold_id) + end + + def confirm(hold_id:) + return { status: "confirmed" } if confirmed.key?(hold_id) + + hold = holds[hold_id] + reject(:no_hold, "The hold expired or does not exist") unless hold + + self.holds = holds.except(hold_id) + self.confirmed = confirmed.merge(hold_id => hold.fetch("seats")) + unschedule(:expire, key: hold_id) + { status: "confirmed" } + end + + def expire(hold_id:, expires_at:) + return seats_available unless holds.dig(hold_id, "expires_at") == expires_at + + self.holds = holds.except(hold_id) + seats_available + end + + private + + def seats_available + capacity - holds.values.sum { |hold| hold.fetch("seats") } - confirmed.values.sum + end + + def hold_result(hold_id) + { status: "held", expires_at: holds.dig(hold_id, "expires_at") } + end +end +``` + +- `open_show` sets the capacity once. +- `hold` takes seats and records a deadline 15 minutes from now. It schedules one reminder with the hold ID as its key. A retry with the same hold ID returns the same hold and takes no more seats. When too few seats remain, `hold` rejects the request with the code `not_enough_seats`. +- `extend_hold` adds 5 minutes, at most two times. It schedules the reminder again with the same key, which moves the alarm. It rejects a third extension with the code `extension_limit`. +- `confirm` moves the hold to `confirmed` and cancels the reminder with `unschedule`. A confirmation retry returns the same result. After expiry, `confirm` rejects the request with the code `no_hold`. +- `expire` releases the seats only if the deadline in the message still matches the hold. After an extension, an expiry for the old deadline does nothing. +- `seats_left` is a query. A query runs as an ordered read in the actor mailbox. `reference.snapshot` reads the committed state without a message row. + +All state stays bounded by the show capacity. + +## Deadlines that survive a restart + +Use durable reminders for persistent timers in Rails. + +`schedule(at:, key:)` stores the reminder in the database in the same commit as the state change. A reminder is one named alarm for each actor and key. A new schedule with the same key moves the alarm. `unschedule(:expire, key: hold_id)` cancels it. + +Reminders run only while `bundle exec solid_objects start` runs. A reminder that falls due while the process is stopped runs after the process starts again. A reminder runs an ordinary actor message, so it runs in order with the other calls for that show. + +Delivery is at least once, so `expire` checks the deadline before it changes anything. See [reminders](../reminders.md). + +## What the tests prove + +- Eight concurrent holds request one seat each against a capacity of five. Five succeed, and three receive the rejection code `not_enough_seats`. +- A hold retry takes its seats once. +- An extension moves the reminder to the new deadline. At 16 minutes, nothing runs. At 21 minutes, the expiry runs and the seats return. +- An expiry for the old deadline changes nothing after an extension. +- The actor rejects a third extension with the code `extension_limit`. +- A confirmation retry returns the same result. The hold moves to `confirmed` once, and the confirmation cancels the reminder. +- A confirmation after expiry receives the rejection code `no_hold`. +- A hold that falls due while the runtime is stopped expires after a restart. + +See [the tests for this guide](../../test/guides/expiring_reservations_test.rb). + +## Payment and other side effects + +Do not call a payment provider inside the actor. An effect can run more than once. + +- Stage the payment with `emit`. +- Pass `context.id` to the provider as the idempotency key. +- Confirm the hold in the actor after the payment succeeds. + +See [the transactional outbox guide](transactional-outbox.md). + +## Run it in production + +- Authorization: the generated policies deny every call. Write a policy. Pass `authorization_context:` on each call. See [authorization policies](../authorization.md). +- Run `bundle exec solid_objects start` beside the web process. +- Every actor call creates a durable message row. Solid Objects keeps terminal message history for 30 days by default. See [retention and backups](../operations.md#retention-and-backups). + +## Limits + +- Solid Objects requires Ruby 3.3 or newer and Rails 7.1 or newer. +- Delivery is at least once. Each operation must be safe to run again. +- There are no transactions across actors. A reservation that spans two shows needs its own design. +- One busy show runs its calls one at a time. Many shows run at the same time. +- The gem is pre-1.0 and makes no production-ready claim. + +## More information + +- [The example file](../../examples/guides/expiring_reservations/seat_inventory.rb) +- [The tests for this guide](../../test/guides/expiring_reservations_test.rb) +- [Prevent race conditions in Rails](race-conditions.md) +- [Correctness and delivery semantics](../correctness.md) +- [Virtual actors in Ruby on Rails](../virtual-actors.md) diff --git a/docs/guides/ordered-jobs.md b/docs/guides/ordered-jobs.md new file mode 100644 index 0000000..3f9fe41 --- /dev/null +++ b/docs/guides/ordered-jobs.md @@ -0,0 +1,197 @@ +# Run jobs in order for each customer in Rails + +A Rails job queue does not keep one customer's job order when several workers run jobs or a job retries. You can serialize one queue, limit concurrency for each customer, or check a sequence number in the database. Solid Objects (the `solid_objects` gem) fits when each customer needs ordered commands, durable state, reminders, and recovery after a restart. Different customers still progress at the same time. + +## Reproduce the problem + +An account starts with a balance of 0 cents. The caller enqueues a deposit of 100 cents, then a withdrawal of 80 cents. If the withdrawal runs first, it fails because the balance is 0 cents. + +```ruby +class Account < ApplicationRecord + class InsufficientFunds < StandardError; end + class OutOfSequence < StandardError; end + + def apply_entry!(kind:, amount_cents:) + change = (kind == "deposit") ? amount_cents : -amount_cents + raise InsufficientFunds, "balance #{balance_cents}, change #{change}" if balance_cents + change < 0 + + update!(balance_cents: balance_cents + change) + end + + def apply_entry_in_sequence!(sequence:, kind:, amount_cents:) + with_lock do + raise OutOfSequence, "expected #{next_sequence}, received #{sequence}" if sequence > next_sequence + next if sequence < next_sequence + + apply_entry!(kind:, amount_cents:) + update!(next_sequence: next_sequence + 1) + end + end +end +``` + +`apply_entry!` adds a deposit or subtracts a withdrawal, then updates the balance. It raises `Account::InsufficientFunds` if the result is below 0. The later section uses `apply_entry_in_sequence!` to fix the order. + +The plain job calls `apply_entry!`: + +```ruby +class ApplyEntryUnorderedJob < ApplicationJob + def perform(account_id:, kind:, amount_cents:) + Account.find(account_id).apply_entry!(kind:, amount_cents:) + end +end +``` + +Order breaks for these reasons: + +- Several workers take jobs from the queue at the same time. +- A job that fails and retries runs after newer jobs. +- A job that waits for a lock can finish after a later job. + +The test performs the withdrawal job before the deposit job. The withdrawal raises `Account::InsufficientFunds`. See [the ordered jobs tests](../../test/guides/ordered_jobs_test.rb). + +## Native options + +| Option | What it gives | What it costs | +| --- | --- | --- | +| One serial queue (a Sidekiq capsule with concurrency 1) | One job at a time for the queues of that capsule, in each Sidekiq process | Every customer in that queue waits for every other customer. Sidekiq sets concurrency for each process. One order across the deployment needs one process for that queue. | +| Solid Queue `limits_concurrency` with the customer as the key | At most `to` jobs at a time for each key (`to` is 1 by default) | Solid Queue gives “no guarantee about the order of execution, only about jobs being performed at the same time”. | +| A sequence number and a row lock | Order for each customer | The code that enqueues must assign the sequence numbers. An early job raises an exception and retries. A job that fails on each attempt holds back the later jobs. | + +Sidekiq supports capsules from Sidekiq 7.0. A capsule can provide serial execution for a queue. The wiki says, “Do not declare a capsule for each queue.” Concurrency applies to each process. By default, one Sidekiq process creates five threads ([checked October 9, 2026](https://github.com/sidekiq/sidekiq/wiki/Advanced-Options#capsules)). + +```ruby +Sidekiq.configure_server do |config| + config.capsule("unsafe") do |cap| + cap.concurrency = 1 + cap.queues = %w[queue_a queue_b] # strict priority + # cap.queues = %w[queue_a,3 queue_b,1] # weighted + end +end +``` + +Solid Queue accepts `key`, `to`, `duration`, and `on_conflict` for concurrency controls. It requires `key` and sets `to` to 1 by default. A blocked job stays blocked until another job finishes or the duration expires. These controls do not guarantee execution order. Solid Queue does not use queue order to unblock jobs ([checked October 9, 2026](https://github.com/rails/solid_queue#concurrency-controls)). + +If one serial queue is fast enough for your volume, it is the simplest choice. If order does not matter, `limits_concurrency` is enough for one job at a time per customer. + +## Keep order with a sequence number + +`apply_entry_in_sequence!` locks the account row and checks the sequence number: + +- If the entry arrives early, the method raises `Account::OutOfSequence`. +- If the account already applied that sequence, the method skips the entry. +- If the entry has the expected sequence, the method applies it and moves `next_sequence` forward. + +The job calls this method: + +```ruby +class ApplyEntryJob < ApplicationJob + retry_on Account::OutOfSequence, wait: 5.seconds, attempts: 20 + + def perform(account_id:, sequence:, kind:, amount_cents:) + Account.find(account_id).apply_entry_in_sequence!(sequence:, kind:, amount_cents:) + end +end +``` + +`retry_on` re-enqueues the job when its entry arrives early. + +The tests show these results: + +- Sequence 2 first raises `Account::OutOfSequence`. +- Sequences 1 and 2 then apply in order. +- A repeated sequence 1 causes no change. +- The job enqueues a retry when its entry arrives early. + +This approach has two costs: + +- The caller allocates the sequence numbers. +- A broken entry blocks the account until someone fixes it. + +## One actor for each account + +There is one `LedgerAccount` actor for each account ID. + +```ruby +class LedgerAccount < SolidObjects::Actor + RECENT_ENTRY_LIMIT = 100 + + attribute :balance_cents, default: 0 + attribute :recent_entry_ids, default: -> { [] } + attribute :statement_balance_cents, default: nil + + def apply(entry_id:, kind:, amount_cents:) + return balance_cents if recent_entry_ids.include?(entry_id) + + change = (kind == "deposit") ? amount_cents : -amount_cents + reject(:insufficient_funds, "The balance is too low for this withdrawal") if balance_cents + change < 0 + + self.balance_cents += change + self.recent_entry_ids = (recent_entry_ids + [ entry_id ]).last(RECENT_ENTRY_LIMIT) + schedule(at: 1.day.from_now, key: "daily").close_statement + balance_cents + end + + def close_statement + self.statement_balance_cents = balance_cents + end +end +``` + +The caller sends each entry with `async` and an idempotency key: + +```ruby +LedgerAccount.ref(account.id.to_s) + .async(idempotency_key: entry.id.to_s, authorization_context: Current.user) + .apply(entry_id: entry.id.to_s, kind: entry.kind, amount_cents: entry.amount_cents) +``` + +The runtime stores the message in SQL before `async` returns. Messages for one actor run one at a time, in the order the runtime receives them. Different accounts run at the same time on different workers. + +The actor treats a business rejection and an exception differently: + +- `reject` ends an entry with a business result, such as `insufficient_funds`. The runtime does not retry the rejection. The next entry runs. +- An exception causes the runtime to retry that message. The message holds back later messages for that account until it succeeds or moves to dead letters. + +A repeated enqueue with the same idempotency key creates one message. A repeated delivery runs the method again. `apply` checks `recent_entry_ids` and returns without a change if the list contains the entry ID. + +`schedule(at: 1.day.from_now, key: "daily")` keeps one statement reminder for each account. Each new entry moves the reminder. The database stores the reminder, and the reminder runs after a restart. + +You do not assign sequence numbers. The actor mailbox gives the order. + +## What the tests prove + +- The tests enqueue five entries for two accounts in one mixed order. Two workers process them. Each account applies its own entries in enqueue order. The balances are 25 and 0 cents. +- The actor rejects a withdrawal that is too large. The deposit after it applies. +- Two enqueues share the same idempotency key. One more direct delivery repeats the entry. The account applies the entry once. +- The statement reminder runs after the test resets the caller process. + +## Choose + +- Independent jobs: use a normal queue. +- One job at a time for each customer, order not important: use `limits_concurrency`. +- Low volume and one order for everything: use one serial queue. +- Order for each customer plus durable state, reminders, and recovery: use an actor for each customer. + +## Run it in production + +- Run `bundle exec solid_objects start`. Async messages and reminders run only while this process runs. The database keeps unfinished work in SQL while the process does not run. +- The generated policies deny every call. Write a policy. Pass `authorization_context:` to `async`. See [authorization policies](../authorization.md). +- A message that fails on every attempt moves to dead letters after five attempts by default. Later messages then run. An operator can retry a dead letter. See [dead letters, retry, and redrive](../operations.md#dead-letters-retry-and-redrive). +- The actor remembers the idempotency keys of its last finished turns. It retains 64 keys by default. See [retention and backups](../operations.md#retention-and-backups). + +## Limits + +- Solid Objects requires Ruby 3.3 or newer and Rails 7.1 or newer. +- Delivery is at least once. Each operation must be safe to run again. +- The gem provides no transactions across actors. A transfer between two accounts needs its own design, for example one SQL transaction on normal tables. +- One busy account runs its entries one at a time. +- The gem is pre-1.0 and makes no production-ready claim. + +## More information + +- [The example files](../../examples/guides/ordered_jobs/ledger_account.rb) +- [The tests for this guide](../../test/guides/ordered_jobs_test.rb) +- [Correctness and delivery semantics](../correctness.md) +- [Reminders](../reminders.md) +- [Prevent race conditions in Rails](race-conditions.md) diff --git a/docs/guides/race-conditions.md b/docs/guides/race-conditions.md new file mode 100644 index 0000000..7f7a3f3 --- /dev/null +++ b/docs/guides/race-conditions.md @@ -0,0 +1,220 @@ +# Prevent race conditions in Rails + +Most Rails races need a database constraint, an atomic update, a row lock, or optimistic locking. A virtual actor helps when one resource has a lifecycle. Requests, jobs, and timers change it, and these changes must occur in order and survive a restart. Solid Objects (the `solid_objects` gem) provides that actor on the SQL database that the application already uses. + +## Choose the right tool + +| Problem | Start with | When Solid Objects becomes relevant | +| --- | --- | --- | +| Duplicate records | A unique index in the database | A larger entity lifecycle also needs ordered durable work | +| Concurrent increments or an inventory decrement | An atomic SQL update or a short transaction | The operation is part of holds, expiry, retries, and later commands | +| Two people edit from an old form | Optimistic locking or a revision check | The entity also needs coordination across jobs and requests | +| Several database changes in one request | A transaction and the correct row locks | Work must continue after that transaction and survive failures | +| Commands that arrive through requests, jobs, and reminders | An explicit coordination design | This is the main use for an actor. The rest of this guide shows it | + +## Use the Rails tools first + +### Unique index + +A Rails uniqueness validation does not create a uniqueness constraint in the database. Two database connections can create two records with the same value. The validation alone does not stop duplicates. + +Create a unique index on the column in the database. See the [Rails Guides, section 2.10: uniqueness](https://guides.rubyonrails.org/active_record_validations.html) (checked October 9, 2026). + +### Pessimistic locking + +Rails supports row-level locks through `SELECT … FOR UPDATE`. The `with_lock` method wraps the block in a transaction. It reloads the object with a lock before it runs the block. + +```ruby +event.with_lock do + event.update!(seats_available: event.seats_available - 1) if event.seats_available.positive? +end +``` + +See the [Rails pessimistic locking reference](https://api.rubyonrails.org/classes/ActiveRecord/Locking/Pessimistic.html) (checked October 9, 2026). + +### Optimistic locking + +Active Record uses an integer `lock_version` column for optimistic locking. Each update increments `lock_version`. A stale save raises `ActiveRecord::StaleObjectError`. + +The Rails documentation recommends a hidden `lock_version` field in the form. That field lets the check work across web requests. See the [Rails optimistic locking reference](https://api.rubyonrails.org/classes/ActiveRecord/Locking/Optimistic.html) (checked October 9, 2026). + +## A worked example: ticket holds + +An event has a fixed number of seats. Buyers hold seats before they pay. + +### Step 1: Reproduce the race + +Two requests load the same event. Each request sees one free seat. Each request writes the value it computed. Both holds succeed, although the event has only one seat. + +```ruby +class Event < ApplicationRecord + def hold_seat_unsafely + return false if seats_available.zero? + + update!(seats_available: seats_available - 1) + true + end + + def hold_seat + self.class.where(id:).where("seats_available > 0") + .update_all("seats_available = seats_available - 1") == 1 + end +end +``` + +The `hold_seat_unsafely` method shows the race. The `hold_seat` method provides the fix in step 2. + +The test reproduces the race without timed delays. It loads two copies of the event, then holds a seat with each copy. Both holds succeed. See [the race condition tests](../../test/guides/race_conditions_test.rb). + +### Step 2: Fix it with one SQL statement + +The `hold_seat` method lets the database decide. The `UPDATE` changes the row only when a seat is free. The method returns `true` only when the statement changes one row. + +In the test, ten requests load the event before they wait at a barrier. Then they try to hold seats at the same time. The event has three seats. Exactly three holds succeed. + +Stop here if a hold never expires and no later operation changes it. An atomic SQL update is the correct fix for a simple counter. + +### Step 3: The lifecycle needs more than a lock + +A real ticket hold has more requirements: + +- A hold expires after 10 minutes if the buyer does not pay. +- The buyer confirms the hold after payment. +- A client retries a hold or a confirmation after a timeout. +- The process restarts before the holds expire. +- An expiry that arrives late must not release a newer hold. + +A pure SQL design needs a holds table, an expiry job, retry keys, and recovery after a restart. Every path must take the same lock in the same order. + +### Step 4: One actor owns the event + +There is one `EventTickets` actor for each event ID. Calls for one actor run one at a time, in order. Different events can run at the same time. + +```ruby +class EventTickets < SolidObjects::Actor + HOLD_DURATION = 10.minutes + + attribute :opened, default: false + attribute :seats_available, default: 0 + attribute :holds, default: -> { {} } + attribute :sold, default: -> { [] } + attribute :title, default: "" + attribute :revision, default: 0 + + def open_sales(seats:) + return seats_available if opened + + self.opened = true + self.seats_available = seats + end + + def hold(buyer:, hold_id:) + return { held: true, hold_id: } if holds.dig(buyer, "hold_id") == hold_id + return { held: false, reason: "already_held" } if holds.key?(buyer) + return { held: false, reason: "sold_out" } if seats_available.zero? + + self.seats_available -= 1 + self.holds = holds.merge(buyer => { "hold_id" => hold_id }) + schedule(at: HOLD_DURATION.from_now, key: buyer).expire(buyer:, hold_id:) + { held: true, hold_id: } + end + + def confirm(buyer:, hold_id:) + return { confirmed: true } if sold.include?(hold_id) + reject(:no_hold, "The hold expired or does not exist") unless holds.dig(buyer, "hold_id") == hold_id + + self.holds = holds.except(buyer) + self.sold = sold + [ hold_id ] + unschedule(:expire, key: buyer) + { confirmed: true } + end + + def expire(buyer:, hold_id:) + return seats_available unless holds.dig(buyer, "hold_id") == hold_id + + self.holds = holds.except(buyer) + self.seats_available += 1 + end + + def update_details(title:, base_revision:) + reject(:stale_revision, "Reload the event and try again") unless base_revision == revision + + self.title = title + self.revision += 1 + end +end +``` + +The actor owns the lifecycle: + +- The `hold` method takes a seat and schedules a reminder with `schedule(at:, key: buyer)`. The database stores the reminder. +- The `confirm` method moves the hold to `sold` and cancels the reminder with `unschedule`. +- The `expire` method releases the seat only when the `hold_id` still matches. An expiry for an old hold does nothing. +- A retry of `hold` with the same IDs returns the same result while that hold exists. A retry of `confirm` with the same IDs returns the same result after the confirmation. These retries make no further changes. +- The `reject` method ends the call with a business result. The caller receives `SolidObjects::Rejected`. The runtime does not retry a rejection. + +Call the actor from a controller: + +```ruby +result = EventTickets.ref(event.id.to_s).hold( + buyer: Current.user.id.to_s, + hold_id: params.require(:hold_id), + authorization_context: Current.user +) +``` + +### Step 5: What the tests prove + +- **Concurrent holds:** Ten concurrent hold calls against three seats produce exactly three successful holds. +- **Restart before expiry:** The test resets the caller process. It runs due reminders at 9 minutes and at 11 minutes. Nothing runs at 9 minutes. The expiry runs at 11 minutes, and the seat returns. +- **Stale expiry:** The first hold expires. The buyer holds again with a new hold ID. The old expiry arrives again, and the new hold stays. +- **Retries:** Two identical holds and two identical confirmations sell one seat. +- **Confirmation after expiry:** The actor rejects the confirmation with the code `no_hold`. +- **Stale form:** The actor rejects the update with the code `stale_revision`. The next section explains this check. + +## Serial execution does not stop a stale form + +An actor runs one call at a time. A stale form can still replace newer data. If two people load revision 0 and both submit, the second submit still runs after the first. + +Use a domain operation or a revision check. The example's `update_details` method rejects a call when `base_revision` differs from the current revision. + +## Run it in production + +### Authorization + +The install generator creates policies that deny every call. Write a policy that checks the caller and the actor type. For example, the policy can check a signed-in user. + +Pass `authorization_context:` on each call. An actor ID is not a permission. See [authorization policies](../authorization.md). + +### The runtime process + +Reminders run only while `bundle exec solid_objects start` runs. SQL keeps the unfinished work while the runtime process is down. The runtime process runs an overdue reminder after it starts again. See [reminders](../reminders.md). + +### Retention cost + +Every actor call creates a durable message row. The default `message_retention` keeps terminal message history for 30 days. Actor state stays until you destroy the actor or configure `instance_retention_by_actor_type`. + +In this example, the event capacity bounds the state. See [retention and backups](../operations.md#retention-and-backups). + +### External effects + +Do not call a payment provider inside the actor. Use `emit` and an effect handler. Pass `context.id` to the provider as the idempotency key. + +Delivery is at least once, so an effect can run more than once. See [effect idempotency](../agents.md#8-make-external-effects-idempotent). + +## Limits + +- Solid Objects requires Ruby 3.3 or newer and Rails 7.1 or newer. +- Delivery is at least once, not exactly once. Write each operation so that it can run again. +- There are no transactions across actors. A change that must touch two events atomically needs one SQL transaction on normal tables. +- One busy event runs its calls one at a time. Many events can run at the same time. +- The gem is pre-1.0 and makes no production-ready claim. + +## More information + +- [The example files](../../examples/guides/race_conditions/event_tickets.rb) +- [The tests for this guide](../../test/guides/race_conditions_test.rb) +- [Correctness and delivery semantics](../correctness.md) +- [Virtual actors in Ruby on Rails](../virtual-actors.md) +- [Expiring reservations](expiring-reservations.md) +- [Ordered jobs for each customer](ordered-jobs.md) diff --git a/docs/guides/transactional-outbox.md b/docs/guides/transactional-outbox.md new file mode 100644 index 0000000..72c09a1 --- /dev/null +++ b/docs/guides/transactional-outbox.md @@ -0,0 +1,156 @@ +# Save state and queue work together in Rails + +A database commit and a job enqueue are two separate steps. If the process stops between them, the database keeps the data, but the queue receives no work. The transactional outbox pattern writes the work into the same database transaction as the data. A separate process delivers the work later. A Solid Objects actor, from the `solid_objects` gem, commits its state, its database writes, and its staged effects in one transaction. + +## Reproduce the lost job + +```ruby +class Order < ApplicationRecord + def self.place_and_enqueue!(reference:, total_cents:) + order = create!(reference:, total_cents:) + ShipmentJob.perform_later(order_id: order.id) + order + end + + def self.place_with_outbox!(reference:, total_cents:) + transaction do + order = create!(reference:, total_cents:) + OutboxMessage.create!(name: "request_shipment", arguments: { "order_id" => order.id }) + order + end + end +end +``` + +```ruby +class ShipmentJob < ApplicationJob + def perform(order_id:) + order = Order.find(order_id) + ShippingProvider.create_shipment(idempotency_key: "order-#{order.id}", order_reference: order.reference) + end +end +``` + +In `place_and_enqueue!`, `create!` commits the order. Then `perform_later` sends the job to the queue. The test uses a queue adapter that stops the process before the job reaches the queue. The order exists, but no job exists. + +`after_commit` and `enqueue_after_transaction_commit` have the same gap: they enqueue the job after the database commit. + +The Rails Guides state that `enqueue_after_transaction_commit` defers the enqueue until the Active Record transaction commits successfully. If the transaction rolls back, Rails does not enqueue the job. Rails 8 configures Solid Queue on a separate database by default. With this default, the job row and the order row use two databases. See [the Rails Guides, section 6.6.1](https://guides.rubyonrails.org/active_job_basics.html) (checked October 9, 2026). + +[The transactional outbox tests](../../test/guides/transactional_outbox_test.rb) demonstrate the crash gap. + +## The plain outbox pattern + +In `place_with_outbox!`, the model writes the order and an outbox row in one transaction. + +```ruby +class OutboxMessage < ApplicationRecord + JOBS = { "request_shipment" => ShipmentJob }.freeze + + scope :pending, -> { where(delivered_at: nil).order(:id) } + + def self.relay(limit: 100) + pending.limit(limit).each do |message| + JOBS.fetch(message.name).perform_later(**message.arguments.symbolize_keys) + message.update!(delivered_at: Time.current) + end + end +end +``` + +The `relay` method uses these steps: + +1. It sends each undelivered row to the queue. +2. It marks the row as delivered. + +If the process stops after the enqueue and before the mark, the next relay sends the row again. The job must be idempotent. `ShipmentJob` passes `order-` to the provider as the idempotency key. + +Rails Event Store uses this pattern. Its scheduler writes the job into the same database table within the same transaction. A separate `res_outbox` process sends those rows to the background jobs tool. See [Rails Event Store](https://railseventstore.org/docs/advanced-topics/outbox) (checked October 9, 2026). + +The plain outbox pattern is a good choice for an application that does not use actors. + +## The atomic boundary of an actor turn + +```ruby +class Checkout < SolidObjects::Actor + attribute :status, default: "open" + attribute :total_cents, default: 0 + attribute :shipment_id, default: nil + + def place(total_cents:) + return status unless status == "open" + + self.status = "placed" + self.total_cents = total_cents + commit_action(:record_order, reference: actor_id, total_cents:) + emit(:request_shipment, order_reference: actor_id, on_success: :shipment_requested) + status + end + + def shipment_requested(effect_id:, arguments:, result:) + self.status = "shipping" + self.shipment_id = result.fetch("shipment_id") + end +end +``` + +```ruby +SolidObjects.register_commit_action(:record_order) do |arguments, _context| + Order.create!(reference: arguments.fetch("reference"), total_cents: arguments.fetch("total_cents")) +end + +SolidObjects.register_effect(:request_shipment) do |arguments, context| + ShippingProvider.create_shipment( + idempotency_key: context.id, + order_reference: arguments.fetch("order_reference") + ) +end +``` + +One successful turn commits the actor state, the staged effects, the same-database commit actions, the reminders, and the outbound messages together. Actor Ruby code and external I/O run outside the actor-state transaction. + +- `commit_action(:record_order, ...)` writes the `orders` row inside that transaction. A commit action requires Solid Objects and `ActiveRecord::Base` to share one connection pool. It can run again after a database rollback. Keep each commit action deterministic, bounded, and database-only. +- `emit(:request_shipment, ...)` stages an effect row in the same transaction. After the commit, an effect worker calls the handler outside any database transaction. +- The handler passes `context.id` to the provider as the idempotency key. `context.id` is the effect ID. It stays the same on every attempt. +- `on_success: :shipment_requested` sends the provider result back to the actor as a normal message. +- If the turn raises, it commits no state, no order row, and no effect row. +- `place` returns early when the checkout is not open. A repeated call stages no additional order or effect. + +## What Solid Objects does not do + +- Solid Objects does not wrap ordinary Active Record writes. Code outside an actor keeps its own transactions. +- A direct Active Record write inside an actor operation raises `SolidObjects::ApplicationWriteForbidden`. A registered commit action provides the only path for application row writes inside the actor commit. +- A commit action is unavailable when Solid Objects uses a separate database. Use `emit` with an idempotent effect consumer. Two databases cannot share one transaction. +- Solid Objects does not provide exactly-once delivery. An effect can run more than once. + +## What the tests prove + +- A stop before the enqueue saves the order and loses the job. +- A failure between the order insert and the outbox insert saves neither row. +- The relay sends a message twice after a lost mark, and the provider creates one shipment. +- A failed turn keeps no state, no order row, and no effect row. +- A successful turn stores the state, the order row, and the effect row together. +- Repeated `place` calls stage one order and one effect in total. +- The provider response fails to reach the effect worker once. The effect runs again with the same `context.id`. The provider creates one shipment, and the actor records the shipment ID. + +## Run it in production + +- Run `bundle exec solid_objects start`. Effects and their callbacks run only while this process runs. The SQL database keeps undelivered effects while the process does not run. +- Register effects and commit actions in an initializer at boot. +- The generated policies deny every call. Write a policy. Pass `authorization_context:` on each call. See [authorization policies](../authorization.md). +- See [effect recovery](../effect-recovery.md) for effects whose worker stopped during a call. + +## Limits + +- Solid Objects requires Ruby 3.3 or newer and Rails 7.1 or newer. +- Delivery is at least once. Each effect handler must deduplicate with `context.id`. +- Solid Objects provides no transactions across actors. +- The gem is pre-1.0 and makes no production-ready claim. + +## More information + +- [The example files](../../examples/guides/transactional_outbox/checkout.rb) +- [The tests for this guide](../../test/guides/transactional_outbox_test.rb) +- [Correctness and delivery semantics](../correctness.md) +- [Expiring reservations](expiring-reservations.md) +- [Prevent race conditions in Rails](race-conditions.md) diff --git a/examples/guides/expiring_reservations/seat_inventory.rb b/examples/guides/expiring_reservations/seat_inventory.rb new file mode 100644 index 0000000..f7ca4eb --- /dev/null +++ b/examples/guides/expiring_reservations/seat_inventory.rb @@ -0,0 +1,75 @@ +# rbs_inline: enabled + +class SeatInventory < SolidObjects::Actor + HOLD_DURATION = 15.minutes + EXTENSION = 5.minutes + MAX_EXTENSIONS = 2 + + attribute :capacity, default: 0 + attribute :holds, default: -> { {} } + attribute :confirmed, default: -> { {} } + + query :seats_left do + seats_available + end + + def open_show(capacity:) + self.capacity = capacity if self.capacity.zero? + seats_available + end + + def hold(hold_id:, buyer:, seats:) + return hold_result(hold_id) if holds.key?(hold_id) + return { status: "confirmed" } if confirmed.key?(hold_id) + reject(:not_enough_seats, "Only #{seats_available} seats are left") if seats > seats_available + + deadline = HOLD_DURATION.from_now + self.holds = holds.merge( + hold_id => { "buyer" => buyer, "seats" => seats, "expires_at" => deadline.to_i, "extensions" => 0 } + ) + schedule(at: deadline, key: hold_id).expire(hold_id:, expires_at: deadline.to_i) + hold_result(hold_id) + end + + def extend_hold(hold_id:) + hold = holds[hold_id] + reject(:no_hold, "The hold expired or does not exist") unless hold + reject(:extension_limit, "The hold cannot be extended again") if hold.fetch("extensions") >= MAX_EXTENSIONS + + deadline = Time.at(hold.fetch("expires_at")) + EXTENSION + self.holds = holds.merge( + hold_id => hold.merge("expires_at" => deadline.to_i, "extensions" => hold.fetch("extensions") + 1) + ) + schedule(at: deadline, key: hold_id).expire(hold_id:, expires_at: deadline.to_i) + hold_result(hold_id) + end + + def confirm(hold_id:) + return { status: "confirmed" } if confirmed.key?(hold_id) + + hold = holds[hold_id] + reject(:no_hold, "The hold expired or does not exist") unless hold + + self.holds = holds.except(hold_id) + self.confirmed = confirmed.merge(hold_id => hold.fetch("seats")) + unschedule(:expire, key: hold_id) + { status: "confirmed" } + end + + def expire(hold_id:, expires_at:) + return seats_available unless holds.dig(hold_id, "expires_at") == expires_at + + self.holds = holds.except(hold_id) + seats_available + end + + private + + def seats_available + capacity - holds.values.sum { |hold| hold.fetch("seats") } - confirmed.values.sum + end + + def hold_result(hold_id) + { status: "held", expires_at: holds.dig(hold_id, "expires_at") } + end +end diff --git a/examples/guides/ordered_jobs/account.rb b/examples/guides/ordered_jobs/account.rb new file mode 100644 index 0000000..b51356b --- /dev/null +++ b/examples/guides/ordered_jobs/account.rb @@ -0,0 +1,23 @@ +# rbs_inline: enabled + +class Account < ApplicationRecord + class InsufficientFunds < StandardError; end + class OutOfSequence < StandardError; end + + def apply_entry!(kind:, amount_cents:) + change = (kind == "deposit") ? amount_cents : -amount_cents + raise InsufficientFunds, "balance #{balance_cents}, change #{change}" if balance_cents + change < 0 + + update!(balance_cents: balance_cents + change) + end + + def apply_entry_in_sequence!(sequence:, kind:, amount_cents:) + with_lock do + raise OutOfSequence, "expected #{next_sequence}, received #{sequence}" if sequence > next_sequence + next if sequence < next_sequence + + apply_entry!(kind:, amount_cents:) + update!(next_sequence: next_sequence + 1) + end + end +end diff --git a/examples/guides/ordered_jobs/apply_entry_job.rb b/examples/guides/ordered_jobs/apply_entry_job.rb new file mode 100644 index 0000000..30e462f --- /dev/null +++ b/examples/guides/ordered_jobs/apply_entry_job.rb @@ -0,0 +1,9 @@ +# rbs_inline: enabled + +class ApplyEntryJob < ApplicationJob + retry_on Account::OutOfSequence, wait: 5.seconds, attempts: 20 + + def perform(account_id:, sequence:, kind:, amount_cents:) + Account.find(account_id).apply_entry_in_sequence!(sequence:, kind:, amount_cents:) + end +end diff --git a/examples/guides/ordered_jobs/apply_entry_unordered_job.rb b/examples/guides/ordered_jobs/apply_entry_unordered_job.rb new file mode 100644 index 0000000..be18d60 --- /dev/null +++ b/examples/guides/ordered_jobs/apply_entry_unordered_job.rb @@ -0,0 +1,7 @@ +# rbs_inline: enabled + +class ApplyEntryUnorderedJob < ApplicationJob + def perform(account_id:, kind:, amount_cents:) + Account.find(account_id).apply_entry!(kind:, amount_cents:) + end +end diff --git a/examples/guides/ordered_jobs/ledger_account.rb b/examples/guides/ordered_jobs/ledger_account.rb new file mode 100644 index 0000000..9a44904 --- /dev/null +++ b/examples/guides/ordered_jobs/ledger_account.rb @@ -0,0 +1,25 @@ +# rbs_inline: enabled + +class LedgerAccount < SolidObjects::Actor + RECENT_ENTRY_LIMIT = 100 + + attribute :balance_cents, default: 0 + attribute :recent_entry_ids, default: -> { [] } + attribute :statement_balance_cents, default: nil + + def apply(entry_id:, kind:, amount_cents:) + return balance_cents if recent_entry_ids.include?(entry_id) + + change = (kind == "deposit") ? amount_cents : -amount_cents + reject(:insufficient_funds, "The balance is too low for this withdrawal") if balance_cents + change < 0 + + self.balance_cents += change + self.recent_entry_ids = (recent_entry_ids + [ entry_id ]).last(RECENT_ENTRY_LIMIT) + schedule(at: 1.day.from_now, key: "daily").close_statement + balance_cents + end + + def close_statement + self.statement_balance_cents = balance_cents + end +end diff --git a/examples/guides/race_conditions/event.rb b/examples/guides/race_conditions/event.rb new file mode 100644 index 0000000..27e4522 --- /dev/null +++ b/examples/guides/race_conditions/event.rb @@ -0,0 +1,15 @@ +# rbs_inline: enabled + +class Event < ApplicationRecord + def hold_seat_unsafely + return false if seats_available.zero? + + update!(seats_available: seats_available - 1) + true + end + + def hold_seat + self.class.where(id:).where("seats_available > 0") + .update_all("seats_available = seats_available - 1") == 1 + end +end diff --git a/examples/guides/race_conditions/event_tickets.rb b/examples/guides/race_conditions/event_tickets.rb new file mode 100644 index 0000000..f1c054f --- /dev/null +++ b/examples/guides/race_conditions/event_tickets.rb @@ -0,0 +1,54 @@ +# rbs_inline: enabled + +class EventTickets < SolidObjects::Actor + HOLD_DURATION = 10.minutes + + attribute :opened, default: false + attribute :seats_available, default: 0 + attribute :holds, default: -> { {} } + attribute :sold, default: -> { [] } + attribute :title, default: "" + attribute :revision, default: 0 + + def open_sales(seats:) + return seats_available if opened + + self.opened = true + self.seats_available = seats + end + + def hold(buyer:, hold_id:) + return { held: true, hold_id: } if holds.dig(buyer, "hold_id") == hold_id + return { held: false, reason: "already_held" } if holds.key?(buyer) + return { held: false, reason: "sold_out" } if seats_available.zero? + + self.seats_available -= 1 + self.holds = holds.merge(buyer => { "hold_id" => hold_id }) + schedule(at: HOLD_DURATION.from_now, key: buyer).expire(buyer:, hold_id:) + { held: true, hold_id: } + end + + def confirm(buyer:, hold_id:) + return { confirmed: true } if sold.include?(hold_id) + reject(:no_hold, "The hold expired or does not exist") unless holds.dig(buyer, "hold_id") == hold_id + + self.holds = holds.except(buyer) + self.sold = sold + [ hold_id ] + unschedule(:expire, key: buyer) + { confirmed: true } + end + + def expire(buyer:, hold_id:) + return seats_available unless holds.dig(buyer, "hold_id") == hold_id + + self.holds = holds.except(buyer) + self.seats_available += 1 + end + + def update_details(title:, base_revision:) + reject(:stale_revision, "Reload the event and try again") unless base_revision == revision + + self.title = title + self.revision += 1 + end +end diff --git a/examples/guides/transactional_outbox/checkout.rb b/examples/guides/transactional_outbox/checkout.rb new file mode 100644 index 0000000..958a53a --- /dev/null +++ b/examples/guides/transactional_outbox/checkout.rb @@ -0,0 +1,22 @@ +# rbs_inline: enabled + +class Checkout < SolidObjects::Actor + attribute :status, default: "open" + attribute :total_cents, default: 0 + attribute :shipment_id, default: nil + + def place(total_cents:) + return status unless status == "open" + + self.status = "placed" + self.total_cents = total_cents + commit_action(:record_order, reference: actor_id, total_cents:) + emit(:request_shipment, order_reference: actor_id, on_success: :shipment_requested) + status + end + + def shipment_requested(effect_id:, arguments:, result:) + self.status = "shipping" + self.shipment_id = result.fetch("shipment_id") + end +end diff --git a/examples/guides/transactional_outbox/order.rb b/examples/guides/transactional_outbox/order.rb new file mode 100644 index 0000000..58bc7d9 --- /dev/null +++ b/examples/guides/transactional_outbox/order.rb @@ -0,0 +1,17 @@ +# rbs_inline: enabled + +class Order < ApplicationRecord + def self.place_and_enqueue!(reference:, total_cents:) + order = create!(reference:, total_cents:) + ShipmentJob.perform_later(order_id: order.id) + order + end + + def self.place_with_outbox!(reference:, total_cents:) + transaction do + order = create!(reference:, total_cents:) + OutboxMessage.create!(name: "request_shipment", arguments: { "order_id" => order.id }) + order + end + end +end diff --git a/examples/guides/transactional_outbox/outbox_message.rb b/examples/guides/transactional_outbox/outbox_message.rb new file mode 100644 index 0000000..04a1a96 --- /dev/null +++ b/examples/guides/transactional_outbox/outbox_message.rb @@ -0,0 +1,14 @@ +# rbs_inline: enabled + +class OutboxMessage < ApplicationRecord + JOBS = { "request_shipment" => ShipmentJob }.freeze + + scope :pending, -> { where(delivered_at: nil).order(:id) } + + def self.relay(limit: 100) + pending.limit(limit).each do |message| + JOBS.fetch(message.name).perform_later(**message.arguments.symbolize_keys) + message.update!(delivered_at: Time.current) + end + end +end diff --git a/examples/guides/transactional_outbox/shipment_job.rb b/examples/guides/transactional_outbox/shipment_job.rb new file mode 100644 index 0000000..6beef29 --- /dev/null +++ b/examples/guides/transactional_outbox/shipment_job.rb @@ -0,0 +1,8 @@ +# rbs_inline: enabled + +class ShipmentJob < ApplicationJob + def perform(order_id:) + order = Order.find(order_id) + ShippingProvider.create_shipment(idempotency_key: "order-#{order.id}", order_reference: order.reference) + end +end diff --git a/examples/guides/transactional_outbox/solid_objects.rb b/examples/guides/transactional_outbox/solid_objects.rb new file mode 100644 index 0000000..de99ae0 --- /dev/null +++ b/examples/guides/transactional_outbox/solid_objects.rb @@ -0,0 +1,12 @@ +# rbs_inline: enabled + +SolidObjects.register_commit_action(:record_order) do |arguments, _context| + Order.create!(reference: arguments.fetch("reference"), total_cents: arguments.fetch("total_cents")) +end + +SolidObjects.register_effect(:request_shipment) do |arguments, context| + ShippingProvider.create_shipment( + idempotency_key: context.id, + order_reference: arguments.fetch("order_reference") + ) +end diff --git a/test/guides/expiring_reservations_test.rb b/test/guides/expiring_reservations_test.rb new file mode 100644 index 0000000..6e9808d --- /dev/null +++ b/test/guides/expiring_reservations_test.rb @@ -0,0 +1,113 @@ +# frozen_string_literal: true + +require_relative "guide_test_helper" +require_relative "../../examples/guides/expiring_reservations/seat_inventory" + +class ExpiringReservationsGuideTest < ActiveSupport::TestCase + include SolidObjects::TestHelper + include GuideTestSupport + + setup do + SeatInventory.ensure_registered! + end + + test "concurrent holds never take more seats than the show has" do + show = SeatInventory.ref("show-concurrent") + show.open_show(capacity: 5) + + results = concurrently(8) do |index| + show.hold(hold_id: "hold-#{index}", buyer: "buyer-#{index}", seats: 1).fetch("status") + rescue SolidObjects::Rejected => rejection + rejection.code + end + + assert_equal 5, results.count("held") + assert_equal 3, results.count("not_enough_seats") + assert_equal 0, show.seats_left + end + + test "a retried hold takes its seats once" do + show = SeatInventory.ref("show-retry") + show.open_show(capacity: 4) + + 2.times { show.hold(hold_id: "hold-1", buyer: "ada", seats: 3) } + + assert_equal 1, show.seats_left + end + + test "extending a hold moves its reminder to the new deadline" do + show = SeatInventory.ref("show-extend") + show.open_show(capacity: 2) + held = show.hold(hold_id: "hold-1", buyer: "ada", seats: 2) + + extended = show.extend_hold(hold_id: "hold-1") + + assert_equal held.fetch("expires_at") + SeatInventory::EXTENSION.to_i, extended.fetch("expires_at") + assert_equal extended.fetch("expires_at"), + reminder_due_at(actor_id: "show-extend", operation: :expire, key: "hold-1").to_i + assert_equal 0, run_due_reminders(now: 16.minutes.from_now) + assert_equal 0, show.seats_left + assert_equal 1, run_due_reminders(now: 21.minutes.from_now) + drain_solid_objects(roles: [ :actors ]) + assert_equal 2, show.seats_left + end + + test "an expiry for the old deadline changes nothing after an extension" do + show = SeatInventory.ref("show-stale") + show.open_show(capacity: 2) + held = show.hold(hold_id: "hold-1", buyer: "ada", seats: 2) + show.extend_hold(hold_id: "hold-1") + + show.expire(hold_id: "hold-1", expires_at: held.fetch("expires_at")) + + assert_equal 0, show.seats_left + end + + test "a hold can be extended only a limited number of times" do + show = SeatInventory.ref("show-limit") + show.open_show(capacity: 1) + show.hold(hold_id: "hold-1", buyer: "ada", seats: 1) + SeatInventory::MAX_EXTENSIONS.times { show.extend_hold(hold_id: "hold-1") } + + error = assert_raises(SolidObjects::Rejected) { show.extend_hold(hold_id: "hold-1") } + + assert_equal "extension_limit", error.code + end + + test "a retried confirmation applies once and cancels the expiry" do + show = SeatInventory.ref("show-confirm") + show.open_show(capacity: 3) + show.hold(hold_id: "hold-1", buyer: "ada", seats: 2) + + results = 2.times.map { show.confirm(hold_id: "hold-1").fetch("status") } + + assert_equal [ "confirmed", "confirmed" ], results + assert_nil reminder_due_at(actor_id: "show-confirm", operation: :expire, key: "hold-1") + assert_equal 1, show.seats_left + end + + test "a confirmation after the hold expired is rejected" do + show = SeatInventory.ref("show-late") + show.open_show(capacity: 1) + show.hold(hold_id: "hold-1", buyer: "ada", seats: 1) + run_due_reminders(now: 16.minutes.from_now) + drain_solid_objects(roles: [ :actors ]) + + error = assert_raises(SolidObjects::Rejected) { show.confirm(hold_id: "hold-1") } + + assert_equal "no_hold", error.code + assert_equal 1, show.seats_left + end + + test "a hold that falls due while the runtime is stopped expires after a restart" do + show = SeatInventory.ref("show-restart") + show.open_show(capacity: 1) + show.hold(hold_id: "hold-1", buyer: "ada", seats: 1) + SolidObjects.reset_caller_process! + + run_due_reminders(now: 2.hours.from_now) + drain_solid_objects(roles: [ :actors ]) + + assert_equal 1, show.seats_left + end +end diff --git a/test/guides/guide_documents_test.rb b/test/guides/guide_documents_test.rb new file mode 100644 index 0000000..61c3207 --- /dev/null +++ b/test/guides/guide_documents_test.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +require "test_helper" + +class GuideDocumentsTest < ActiveSupport::TestCase + ROOT = File.expand_path("../..", __dir__) + GUIDES = { + "docs/guides/race-conditions.md" => [ + "examples/guides/race_conditions/event.rb", + "examples/guides/race_conditions/event_tickets.rb" + ], + "docs/guides/ordered-jobs.md" => [ + "examples/guides/ordered_jobs/account.rb", + "examples/guides/ordered_jobs/apply_entry_unordered_job.rb", + "examples/guides/ordered_jobs/apply_entry_job.rb", + "examples/guides/ordered_jobs/ledger_account.rb" + ], + "docs/guides/expiring-reservations.md" => [ + "examples/guides/expiring_reservations/seat_inventory.rb" + ], + "docs/guides/transactional-outbox.md" => [ + "examples/guides/transactional_outbox/order.rb", + "examples/guides/transactional_outbox/shipment_job.rb", + "examples/guides/transactional_outbox/outbox_message.rb", + "examples/guides/transactional_outbox/checkout.rb", + "examples/guides/transactional_outbox/solid_objects.rb" + ] + }.freeze + DASHES = [ "–", "—" ].freeze + LINK = /(?(index) { index }) + ready = Queue.new + start = Queue.new + results = Queue.new + errors = Queue.new + connection_slots = Queue.new + [ count, ActiveRecord::Base.connection_pool.size - 1 ].min.times { connection_slots << true } + threads = count.times.map do |index| + Thread.new do + prepared = ActiveRecord::Base.connection_pool.with_connection { prepare.call(index) } + ready << true + start.pop + connection_slots.pop + begin + results << ActiveRecord::Base.connection_pool.with_connection { yield(prepared) } + ensure + connection_slots << true + end + rescue => error + ready << true + errors << error + end + end + count.times { ready.pop } + count.times { start << true } + threads.each(&:join) + raise errors.pop unless errors.empty? + + Array.new(results.size) { results.pop } + end +end diff --git a/test/guides/ordered_jobs_test.rb b/test/guides/ordered_jobs_test.rb new file mode 100644 index 0000000..8ad095d --- /dev/null +++ b/test/guides/ordered_jobs_test.rb @@ -0,0 +1,107 @@ +# frozen_string_literal: true + +require_relative "guide_test_helper" +require_relative "../../examples/guides/ordered_jobs/account" +require_relative "../../examples/guides/ordered_jobs/apply_entry_unordered_job" +require_relative "../../examples/guides/ordered_jobs/apply_entry_job" +require_relative "../../examples/guides/ordered_jobs/ledger_account" + +class OrderedJobsGuideTest < ActiveSupport::TestCase + include SolidObjects::TestHelper + include ActiveJob::TestHelper + include GuideTestSupport + + setup do + GuideSchema.reset + LedgerAccount.ensure_registered! + end + + test "jobs that run in a different order than they were enqueued reject a valid withdrawal" do + account = Account.create!(balance_cents: 0) + + assert_raises(Account::InsufficientFunds) do + ApplyEntryUnorderedJob.perform_now(account_id: account.id, kind: "withdrawal", amount_cents: 80) + end + ApplyEntryUnorderedJob.perform_now(account_id: account.id, kind: "deposit", amount_cents: 100) + + assert_equal 100, account.reload.balance_cents + end + + test "a sequence guard applies entries in order and ignores a repeated entry" do + account = Account.create!(balance_cents: 0) + + assert_raises(Account::OutOfSequence) do + account.apply_entry_in_sequence!(sequence: 2, kind: "withdrawal", amount_cents: 80) + end + account.apply_entry_in_sequence!(sequence: 1, kind: "deposit", amount_cents: 100) + account.apply_entry_in_sequence!(sequence: 2, kind: "withdrawal", amount_cents: 80) + account.apply_entry_in_sequence!(sequence: 1, kind: "deposit", amount_cents: 100) + + assert_equal 20, account.reload.balance_cents + assert_equal 3, account.next_sequence + end + + test "the sequenced job enqueues a retry when its entry arrives early" do + account = Account.create!(balance_cents: 0) + + assert_enqueued_jobs 1, only: ApplyEntryJob do + ApplyEntryJob.perform_now(account_id: account.id, sequence: 2, kind: "withdrawal", amount_cents: 80) + end + assert_equal 0, account.reload.balance_cents + end + + test "each account applies its entries in enqueue order while two workers run" do + entries = [ + [ "alice", "alice-1", "deposit", 100 ], + [ "bob", "bob-1", "deposit", 50 ], + [ "alice", "alice-2", "withdrawal", 80 ], + [ "bob", "bob-2", "withdrawal", 50 ], + [ "alice", "alice-3", "deposit", 5 ] + ] + entries.each do |account_id, entry_id, kind, amount_cents| + LedgerAccount.ref(account_id).async(idempotency_key: entry_id).apply(entry_id:, kind:, amount_cents:) + end + + concurrently(2) { SolidObjects::Worker.new.tap(&:run_until_idle).stop } + + assert_equal [ "alice-1", "alice-2", "alice-3" ], LedgerAccount.ref("alice").snapshot.recent_entry_ids + assert_equal 25, LedgerAccount.ref("alice").snapshot.balance_cents + assert_equal [ "bob-1", "bob-2" ], LedgerAccount.ref("bob").snapshot.recent_entry_ids + assert_equal 0, LedgerAccount.ref("bob").snapshot.balance_cents + end + + test "a rejected entry does not block the entries after it" do + account = LedgerAccount.ref("carol") + account.async(idempotency_key: "carol-1").apply(entry_id: "carol-1", kind: "withdrawal", amount_cents: 10) + account.async(idempotency_key: "carol-2").apply(entry_id: "carol-2", kind: "deposit", amount_cents: 30) + + drain_solid_objects(roles: [ :actors ]) + + assert_equal "rejected", account.find_by(idempotency_key: "carol-1").outcome.status + assert_equal 30, account.snapshot.balance_cents + end + + test "a repeated enqueue and a repeated delivery apply an entry once" do + account = LedgerAccount.ref("dave") + 2.times do + account.async(idempotency_key: "dave-1").apply(entry_id: "dave-1", kind: "deposit", amount_cents: 40) + end + drain_solid_objects(roles: [ :actors ]) + + account.apply(entry_id: "dave-1", kind: "deposit", amount_cents: 40) + + assert_equal 40, account.snapshot.balance_cents + assert_equal [ "dave-1" ], account.snapshot.recent_entry_ids + end + + test "the statement reminder runs after a restart" do + account = LedgerAccount.ref("erin") + account.apply(entry_id: "erin-1", kind: "deposit", amount_cents: 70) + SolidObjects.reset_caller_process! + + run_due_reminders(now: 25.hours.from_now) + drain_solid_objects(roles: [ :actors ]) + + assert_equal 70, account.snapshot.statement_balance_cents + end +end diff --git a/test/guides/race_conditions_test.rb b/test/guides/race_conditions_test.rb new file mode 100644 index 0000000..312773a --- /dev/null +++ b/test/guides/race_conditions_test.rb @@ -0,0 +1,116 @@ +# frozen_string_literal: true + +require_relative "guide_test_helper" +require_relative "../../examples/guides/race_conditions/event" +require_relative "../../examples/guides/race_conditions/event_tickets" + +class RaceConditionsGuideTest < ActiveSupport::TestCase + include SolidObjects::TestHelper + include GuideTestSupport + + setup do + GuideSchema.reset + EventTickets.ensure_registered! + end + + test "two requests that loaded the same event both hold the last seat" do + event = Event.create!(seats_available: 1) + first_request = Event.find(event.id) + second_request = Event.find(event.id) + + assert first_request.hold_seat_unsafely + assert second_request.hold_seat_unsafely + assert_equal 0, event.reload.seats_available + end + + test "a conditional update holds no more seats than the event has" do + event = Event.create!(seats_available: 3) + + results = concurrently(10, prepare: ->(_index) { Event.find(event.id) }) do |request_copy| + request_copy.hold_seat + end + + assert_equal 3, results.count(true) + assert_equal 0, event.reload.seats_available + end + + test "the actor holds no more seats than it has under concurrent requests" do + tickets = EventTickets.ref("event-concurrent") + tickets.open_sales(seats: 3) + + results = concurrently(10) do |index| + tickets.hold(buyer: "buyer-#{index}", hold_id: "hold-#{index}") + end + + assert_equal 3, results.count { |result| result.fetch("held") } + assert_equal 0, tickets.snapshot.seats_available + assert_equal 3, tickets.snapshot.holds.length + end + + test "a hold expires after a restart when its reminder is due" do + tickets = EventTickets.ref("event-restart") + tickets.open_sales(seats: 1) + tickets.hold(buyer: "ada", hold_id: "hold-1") + SolidObjects.reset_caller_process! + + assert_equal 0, run_due_reminders(now: 9.minutes.from_now) + assert_equal 1, run_due_reminders(now: 11.minutes.from_now) + drain_solid_objects(roles: [ :actors ]) + + assert_equal 1, tickets.snapshot.seats_available + assert_empty tickets.snapshot.holds + end + + test "a stale expiry for an old hold changes nothing" do + tickets = EventTickets.ref("event-stale") + tickets.open_sales(seats: 1) + tickets.hold(buyer: "ada", hold_id: "hold-1") + run_due_reminders(now: 11.minutes.from_now) + drain_solid_objects(roles: [ :actors ]) + tickets.hold(buyer: "ada", hold_id: "hold-2") + + tickets.expire(buyer: "ada", hold_id: "hold-1") + + assert_equal 0, tickets.snapshot.seats_available + assert_equal "hold-2", tickets.snapshot.holds.dig("ada", "hold_id") + end + + test "a retried hold and a retried confirmation apply once" do + tickets = EventTickets.ref("event-retry") + tickets.open_sales(seats: 2) + + holds = 2.times.map { tickets.hold(buyer: "ada", hold_id: "hold-1") } + confirmations = 2.times.map { tickets.confirm(buyer: "ada", hold_id: "hold-1") } + + assert_equal [ true, true ], holds.map { |result| result.fetch("held") } + assert_equal [ true, true ], confirmations.map { |result| result.fetch("confirmed") } + assert_equal 1, tickets.snapshot.seats_available + assert_equal [ "hold-1" ], tickets.snapshot.sold + assert_nil reminder_due_at(actor_id: "event-retry", operation: :expire, key: "ada") + end + + test "a confirmation after the hold expired is rejected" do + tickets = EventTickets.ref("event-late") + tickets.open_sales(seats: 1) + tickets.hold(buyer: "ada", hold_id: "hold-1") + run_due_reminders(now: 11.minutes.from_now) + drain_solid_objects(roles: [ :actors ]) + + error = assert_raises(SolidObjects::Rejected) { tickets.confirm(buyer: "ada", hold_id: "hold-1") } + + assert_equal "no_hold", error.code + assert_equal 1, tickets.snapshot.seats_available + end + + test "serial execution alone does not stop a stale form, so a revision check does" do + tickets = EventTickets.ref("event-details") + tickets.update_details(title: "Spring show", base_revision: 0) + + error = assert_raises(SolidObjects::Rejected) do + tickets.update_details(title: "Old title from a stale form", base_revision: 0) + end + + assert_equal "stale_revision", error.code + assert_equal "Spring show", tickets.snapshot.title + end +end diff --git a/test/guides/transactional_outbox_test.rb b/test/guides/transactional_outbox_test.rb new file mode 100644 index 0000000..33d8ecb --- /dev/null +++ b/test/guides/transactional_outbox_test.rb @@ -0,0 +1,141 @@ +# frozen_string_literal: true + +require_relative "guide_test_helper" +require_relative "../../examples/guides/transactional_outbox/shipment_job" +require_relative "../../examples/guides/transactional_outbox/order" +require_relative "../../examples/guides/transactional_outbox/outbox_message" +require_relative "../../examples/guides/transactional_outbox/checkout" + +module ShippingProvider + class LostResponse < StandardError; end + + class << self + attr_accessor :shipments, :calls, :lose_next_response + + def reset + self.shipments = {} + self.calls = [] + self.lose_next_response = false + end + + def create_shipment(idempotency_key:, order_reference:) + calls << idempotency_key + shipments[idempotency_key] ||= { "shipment_id" => "shipment-#{shipments.length + 1}", "order" => order_reference } + if lose_next_response + self.lose_next_response = false + raise LostResponse, "the worker stopped before it recorded the result" + end + + shipments.fetch(idempotency_key) + end + end +end + +class TransactionalOutboxGuideTest < ActiveSupport::TestCase + include SolidObjects::TestHelper + include ActiveJob::TestHelper + include GuideTestSupport + + class ProcessStopped < StandardError; end + + class StoppedProcessAdapter + def enqueue(*) + raise ProcessStopped, "the process stopped before the job reached the queue" + end + + def enqueue_at(*) + raise ProcessStopped, "the process stopped before the job reached the queue" + end + end + + setup do + GuideSchema.reset + ShippingProvider.reset + SolidObjects.configuration.retry_delay = ->(_attempt) { 0 } + load File.expand_path("../../examples/guides/transactional_outbox/solid_objects.rb", __dir__) + Checkout.ensure_registered! + end + + test "an order commits but its job is lost when the process stops before the enqueue" do + test_adapter = ShipmentJob.queue_adapter + ShipmentJob.queue_adapter = StoppedProcessAdapter.new + + assert_raises(ProcessStopped) { Order.place_and_enqueue!(reference: "order-1", total_cents: 1_500) } + + assert_equal 1, Order.count + assert_empty test_adapter.enqueued_jobs + ensure + ShipmentJob.queue_adapter = test_adapter + end + + test "the outbox message commits with the order or not at all" do + OutboxMessage.define_singleton_method(:create!) { |**| raise ProcessStopped, "the disk is full" } + + assert_raises(ProcessStopped) { Order.place_with_outbox!(reference: "order-rolled-back", total_cents: 1_500) } + OutboxMessage.singleton_class.remove_method(:create!) + Order.place_with_outbox!(reference: "order-2", total_cents: 1_500) + + assert_equal [ "order-2" ], Order.pluck(:reference) + assert_equal 1, OutboxMessage.pending.count + ensure + OutboxMessage.singleton_class.remove_method(:create!) if OutboxMessage.singleton_class.method_defined?(:create!, false) + end + + test "the relay can send a message twice, so the job must be idempotent" do + order = Order.place_with_outbox!(reference: "order-3", total_cents: 1_500) + OutboxMessage.relay + OutboxMessage.update_all(delivered_at: nil) + OutboxMessage.relay + + perform_enqueued_jobs + + assert_equal [ "order-#{order.id}", "order-#{order.id}" ], ShippingProvider.calls + assert_equal 1, ShippingProvider.shipments.length + assert_equal 0, OutboxMessage.pending.count + end + + test "a failed turn keeps no state, no order row, and no shipment request" do + checkout = Checkout.ref("cart-failed") + + assert_raises(SolidObjects::MessageFailed) { checkout.place(total_cents: nil) } + + assert_equal "open", checkout.snapshot.status + assert_equal 0, Order.count + assert_equal 0, SolidObjects::Effect.count + end + + test "a committed turn stores the state, the order row, and the shipment request together" do + checkout = Checkout.ref("cart-placed") + + checkout.place(total_cents: 2_400) + + assert_equal "placed", checkout.snapshot.status + assert_equal [ "cart-placed" ], Order.pluck(:reference) + assert_equal [ "request_shipment" ], SolidObjects::Effect.pluck(:name) + end + + test "a repeated place call stages one order and one shipment request" do + checkout = Checkout.ref("cart-repeated") + + 2.times { checkout.place(total_cents: 2_400) } + + assert_equal 1, Order.count + assert_equal 1, SolidObjects::Effect.count + end + + test "the effect runs again after a lost response and the provider creates one shipment" do + checkout = Checkout.ref("cart-shipping") + checkout.place(total_cents: 2_400) + ShippingProvider.lose_next_response = true + + drain_solid_objects(roles: [ :effects, :actors ]) + assert_equal [ "pending" ], SolidObjects::Effect.pluck(:status) + drain_solid_objects(roles: [ :effects, :actors ]) + + assert_equal 2, ShippingProvider.calls.length + assert_equal 1, ShippingProvider.calls.uniq.length + assert_equal 1, ShippingProvider.shipments.length + assert_equal "shipping", checkout.snapshot.status + assert_equal "shipment-1", checkout.snapshot.shipment_id + end +end From e94eb79b5df08aca623b77c7d020042470b62741 Mon Sep 17 00:00:00 2001 From: Lucas Carlson Date: Fri, 9 Oct 2026 13:41:33 -0700 Subject: [PATCH 2/5] chore: prepare version 0.17.3 --- CHANGELOG.md | 17 +++++++++++++++++ Gemfile.lock | 4 ++-- lib/solid_objects/version.rb | 2 +- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eecddc1..1f8fb28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,22 @@ # Changelog +## 0.17.3 - 2026-10-09 + +- Add four problem guides in `docs/guides/`: preventing race conditions in + Rails, running jobs in order for each customer, expiring reservations, and + saving state and queuing work together. Each guide reproduces the failure, + shows the plain Rails fix first, and then shows a Solid Objects actor where + it adds value. The README and the agent guide list them. +- Each guide embeds tested example files from `examples/guides/`. The tests in + `test/guides/` prove each claim: the race and its SQL fix, ordered entries + under two workers, a reminder that runs after a restart, a stale expiry that + changes nothing, an atomic actor turn, and an effect that runs twice with one + idempotency key. A document test fails when a guide does not embed the + current example files, contains an em dash or an en dash, or links to a + missing file. +- Add `activejob` to the development and test bundle for the guide examples. + Compatibility runs pin it to the Rails line under test. + ## 0.17.2 - 2026-10-08 - The README names the agent guide at the start of Installation, and the agent diff --git a/Gemfile.lock b/Gemfile.lock index c68d8e2..4ed22ff 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - solid_objects (0.17.2) + solid_objects (0.17.3) actioncable (>= 7.1) actionpack (>= 7.1) actionview (>= 7.1) @@ -392,7 +392,7 @@ CHECKSUMS rubocop-rails-omakase (1.1.0) sha256=2af73ac8ee5852de2919abbd2618af9c15c19b512c4cfc1f9a5d3b6ef009109d ruby-progressbar (1.13.0) sha256=80fc9c47a9b640d6834e0dc7b3c94c9df37f08cb072b7761e4a71e22cff29b33 securerandom (0.4.1) sha256=cc5193d414a4341b6e225f0cb4446aceca8e50d5e1888743fac16987638ea0b1 - solid_objects (0.17.2) + solid_objects (0.17.3) sqlite3 (2.9.5-aarch64-linux-gnu) sha256=78075b6337d3d182c6d2b4691049ed45cd220826160c9ea18946bf6a1de200dc sqlite3 (2.9.5-aarch64-linux-musl) sha256=18c801185deb4adc01ddb281e8f672a39e3d1729979ca91e39439cd3eac0402d sqlite3 (2.9.5-arm-linux-gnu) sha256=1bdfca0c7d63998c60b0f4a8e3c8df2d33800ccc4abd2d612eddbbbc92a4c48b diff --git a/lib/solid_objects/version.rb b/lib/solid_objects/version.rb index ac83448..da3f728 100644 --- a/lib/solid_objects/version.rb +++ b/lib/solid_objects/version.rb @@ -1,5 +1,5 @@ # rbs_inline: enabled module SolidObjects - VERSION = "0.17.2" + VERSION = "0.17.3" end From bc462f4d52d3689ad77f25f556c005f10ce7f19e Mon Sep 17 00:00:00 2001 From: Lucas Carlson Date: Fri, 9 Oct 2026 14:04:06 -0700 Subject: [PATCH 3/5] fix: enforce deadlines and record ledger entries Review found three gaps in the guide examples, and Rails 7.1 and 7.2 found two in the tests. - EventTickets and SeatInventory now store each deadline and check it in the operation, so a hold past its deadline cannot be confirmed or extended and frees its seats even when the expiry reminder has not run. The reminder only cleans up. New tests travel past the deadline without delivering a reminder; both failed before the change. - SeatInventory rejects a seat count that is not a positive integer. A hold for -5 seats had added five seats to the show. - LedgerAccount kept the last 100 entry IDs in state, so a repeated entry after 100 newer entries changed the balance again. Each entry now writes a ledger_entries row through a commit action in the same transaction, with a unique index, and apply checks that row first. A test applies 102 entries and repeats the first; it failed before. - On Rails 7.1 and 7.2, ActiveJob::TestHelper ignores a queue adapter assigned inside the test, so the lost-job test raised nothing. It now replaces perform_later for that test. - On Rails 7.1 SQLite, two workers that registered their processes at the same time raised "database is locked". The test now creates both workers before the threads start. - The Sidekiq snippet no longer carries the commented-out weighted line. Codex with GPT-6 Astra revised the three affected guides from change lists, and the guides embed the new example files. --- docs/guides/expiring-reservations.md | 36 ++++++++----- docs/guides/ordered-jobs.md | 53 +++++++++++++++---- docs/guides/race-conditions.md | 23 ++++++-- .../expiring_reservations/seat_inventory.rb | 20 ++++--- .../ordered_jobs/create_ledger_entries.rb | 14 +++++ .../guides/ordered_jobs/ledger_account.rb | 7 +-- examples/guides/ordered_jobs/ledger_entry.rb | 4 ++ examples/guides/ordered_jobs/solid_objects.rb | 10 ++++ .../guides/race_conditions/event_tickets.rb | 16 +++++- test/guides/expiring_reservations_test.rb | 23 ++++++++ test/guides/guide_documents_test.rb | 2 + test/guides/guide_test_helper.rb | 5 +- test/guides/ordered_jobs_test.rb | 25 +++++++-- test/guides/race_conditions_test.rb | 13 +++++ test/guides/transactional_outbox_test.rb | 18 ++----- 15 files changed, 209 insertions(+), 60 deletions(-) create mode 100644 examples/guides/ordered_jobs/create_ledger_entries.rb create mode 100644 examples/guides/ordered_jobs/ledger_entry.rb create mode 100644 examples/guides/ordered_jobs/solid_objects.rb diff --git a/docs/guides/expiring-reservations.md b/docs/guides/expiring-reservations.md index 9c64b18..8e7c6d6 100644 --- a/docs/guides/expiring-reservations.md +++ b/docs/guides/expiring-reservations.md @@ -46,7 +46,8 @@ class SeatInventory < SolidObjects::Actor end def hold(hold_id:, buyer:, seats:) - return hold_result(hold_id) if holds.key?(hold_id) + reject(:invalid_seats, "Hold at least one seat") unless seats.is_a?(Integer) && seats.positive? + return hold_result(hold_id) if active_hold?(hold_id) return { status: "confirmed" } if confirmed.key?(hold_id) reject(:not_enough_seats, "Only #{seats_available} seats are left") if seats > seats_available @@ -59,8 +60,9 @@ class SeatInventory < SolidObjects::Actor end def extend_hold(hold_id:) - hold = holds[hold_id] - reject(:no_hold, "The hold expired or does not exist") unless hold + reject(:no_hold, "The hold expired or does not exist") unless active_hold?(hold_id) + + hold = holds.fetch(hold_id) reject(:extension_limit, "The hold cannot be extended again") if hold.fetch("extensions") >= MAX_EXTENSIONS deadline = Time.at(hold.fetch("expires_at")) + EXTENSION @@ -73,10 +75,9 @@ class SeatInventory < SolidObjects::Actor def confirm(hold_id:) return { status: "confirmed" } if confirmed.key?(hold_id) + reject(:no_hold, "The hold expired or does not exist") unless active_hold?(hold_id) - hold = holds[hold_id] - reject(:no_hold, "The hold expired or does not exist") unless hold - + hold = holds.fetch(hold_id) self.holds = holds.except(hold_id) self.confirmed = confirmed.merge(hold_id => hold.fetch("seats")) unschedule(:expire, key: hold_id) @@ -92,8 +93,13 @@ class SeatInventory < SolidObjects::Actor private + def active_hold?(hold_id) + holds.key?(hold_id) && holds.dig(hold_id, "expires_at") > Time.current.to_i + end + def seats_available - capacity - holds.values.sum { |hold| hold.fetch("seats") } - confirmed.values.sum + held_seats = holds.each_key.select { |hold_id| active_hold?(hold_id) }.sum { |hold_id| holds.dig(hold_id, "seats") } + capacity - held_seats - confirmed.values.sum end def hold_result(hold_id) @@ -103,19 +109,19 @@ end ``` - `open_show` sets the capacity once. -- `hold` takes seats and records a deadline 15 minutes from now. It schedules one reminder with the hold ID as its key. A retry with the same hold ID returns the same hold and takes no more seats. When too few seats remain, `hold` rejects the request with the code `not_enough_seats`. -- `extend_hold` adds 5 minutes, at most two times. It schedules the reminder again with the same key, which moves the alarm. It rejects a third extension with the code `extension_limit`. -- `confirm` moves the hold to `confirmed` and cancels the reminder with `unschedule`. A confirmation retry returns the same result. After expiry, `confirm` rejects the request with the code `no_hold`. -- `expire` releases the seats only if the deadline in the message still matches the hold. After an extension, an expiry for the old deadline does nothing. -- `seats_left` is a query. A query runs as an ordered read in the actor mailbox. `reference.snapshot` reads the committed state without a message row. +- `hold` takes seats and records a deadline 15 minutes from now. It schedules one reminder with the hold ID as its key. A retry with the same hold ID returns the same hold and takes no more seats. When too few seats remain, `hold` rejects the request with the code `not_enough_seats`. `hold` rejects a seat count that is not a positive integer with the code `invalid_seats`. Without this check, a hold for -5 seats adds seats to the show. +- `extend_hold` adds 5 minutes, at most two times. It schedules the reminder again with the same key, which moves the alarm. It rejects a third extension with the code `extension_limit`. It accepts only an active hold: a hold whose stored deadline is still in the future. After the deadline, it rejects the request with the code `no_hold`, even before the expiry reminder runs. A stopped or slow runtime process does not extend a hold. +- `confirm` moves the hold to `confirmed` and cancels the reminder with `unschedule`. A confirmation retry returns the same result. `confirm` accepts only an active hold whose stored deadline is still in the future. After the deadline, it rejects the request with the code `no_hold`, even before the expiry reminder runs. +- The stored deadline is the rule. `expire` removes the old hold from the state only if the deadline in the message still matches the hold. After an extension, an expiry for the old deadline does nothing. +- `seats_left` is a query. A query runs as an ordered read in the actor mailbox. `reference.snapshot` reads the committed state without a message row. A hold past its deadline no longer counts against the seats. `seats_left` and new holds see the seats again at the deadline. -All state stays bounded by the show capacity. +Active holds and confirmed seats are bounded by the show capacity. An expired hold stays in the state until its reminder runs. ## Deadlines that survive a restart Use durable reminders for persistent timers in Rails. -`schedule(at:, key:)` stores the reminder in the database in the same commit as the state change. A reminder is one named alarm for each actor and key. A new schedule with the same key moves the alarm. `unschedule(:expire, key: hold_id)` cancels it. +`schedule(at:, key:)` stores the reminder in the database in the same commit as the state change. A reminder is one named alarm for each actor and key. A new schedule with the same key moves the alarm. `unschedule(:expire, key: hold_id)` cancels it. The deadline check in `confirm` and `extend_hold` does not wait for the reminder. Reminders run only while `bundle exec solid_objects start` runs. A reminder that falls due while the process is stopped runs after the process starts again. A reminder runs an ordinary actor message, so it runs in order with the other calls for that show. @@ -130,6 +136,8 @@ Delivery is at least once, so `expire` checks the deadline before it changes any - The actor rejects a third extension with the code `extension_limit`. - A confirmation retry returns the same result. The hold moves to `confirmed` once, and the confirmation cancels the reminder. - A confirmation after expiry receives the rejection code `no_hold`. +- Past the deadline, before the reminder runs: the test moves the clock 16 minutes forward and delivers no reminder. The actor rejects the confirmation and the extension with `no_hold`, and both seats are free. +- A hold for zero seats or for -5 seats is rejected with `invalid_seats`, and the free seats do not change. - A hold that falls due while the runtime is stopped expires after a restart. See [the tests for this guide](../../test/guides/expiring_reservations_test.rb). diff --git a/docs/guides/ordered-jobs.md b/docs/guides/ordered-jobs.md index 3f9fe41..324eaf9 100644 --- a/docs/guides/ordered-jobs.md +++ b/docs/guides/ordered-jobs.md @@ -65,11 +65,12 @@ Sidekiq.configure_server do |config| config.capsule("unsafe") do |cap| cap.concurrency = 1 cap.queues = %w[queue_a queue_b] # strict priority - # cap.queues = %w[queue_a,3 queue_b,1] # weighted end end ``` +This snippet comes from the Sidekiq wiki. + Solid Queue accepts `key`, `to`, `duration`, and `on_conflict` for concurrency controls. It requires `key` and sets `to` to 1 by default. A blocked job stays blocked until another job finishes or the duration expires. These controls do not guarantee execution order. Solid Queue does not use queue order to unblock jobs ([checked October 9, 2026](https://github.com/rails/solid_queue#concurrency-controls)). If one serial queue is fast enough for your volume, it is the simplest choice. If order does not matter, `limits_concurrency` is enough for one job at a time per customer. @@ -113,21 +114,50 @@ This approach has two costs: There is one `LedgerAccount` actor for each account ID. ```ruby -class LedgerAccount < SolidObjects::Actor - RECENT_ENTRY_LIMIT = 100 +class CreateLedgerEntries < ActiveRecord::Migration[7.1] + def change + create_table :ledger_entries do |table| + table.string :account_id, null: false + table.string :entry_id, null: false + table.string :kind, null: false + table.integer :amount_cents, null: false + table.timestamps + end + add_index :ledger_entries, [ :account_id, :entry_id ], unique: true + end +end +``` + +The migration creates a `ledger_entries` table with a unique index on the account ID and the entry ID. `LedgerEntry` is a plain Active Record model for this table. +```ruby +SolidObjects.register_commit_action(:record_ledger_entry) do |arguments, _context| + LedgerEntry.create!( + account_id: arguments.fetch("account_id"), + entry_id: arguments.fetch("entry_id"), + kind: arguments.fetch("kind"), + amount_cents: arguments.fetch("amount_cents") + ) +end +``` + +The initializer registers the commit action `record_ledger_entry`. A commit action writes application rows inside the actor transaction. The entry row and the new balance commit together or not at all. A commit action needs Solid Objects and `ActiveRecord::Base` to share one connection pool. + +Commit actions can run again after a database rollback. Keep them deterministic, bounded, and database-only. Actor handlers can read application records. They cannot write them directly. + +```ruby +class LedgerAccount < SolidObjects::Actor attribute :balance_cents, default: 0 - attribute :recent_entry_ids, default: -> { [] } attribute :statement_balance_cents, default: nil def apply(entry_id:, kind:, amount_cents:) - return balance_cents if recent_entry_ids.include?(entry_id) + return balance_cents if LedgerEntry.exists?(account_id: actor_id, entry_id:) change = (kind == "deposit") ? amount_cents : -amount_cents reject(:insufficient_funds, "The balance is too low for this withdrawal") if balance_cents + change < 0 self.balance_cents += change - self.recent_entry_ids = (recent_entry_ids + [ entry_id ]).last(RECENT_ENTRY_LIMIT) + commit_action(:record_ledger_entry, account_id: actor_id, entry_id:, kind:, amount_cents:) schedule(at: 1.day.from_now, key: "daily").close_statement balance_cents end @@ -138,6 +168,10 @@ class LedgerAccount < SolidObjects::Actor end ``` +`apply` first checks `LedgerEntry.exists?` for the account and the entry. If the row exists, `apply` returns the balance and changes nothing. Calls for one account run one at a time, so this check and the insert cannot race. + +The unique index provides another check. A duplicate insert fails the turn. The actor does not apply the entry twice. + The caller sends each entry with `async` and an idempotency key: ```ruby @@ -153,7 +187,7 @@ The actor treats a business rejection and an exception differently: - `reject` ends an entry with a business result, such as `insufficient_funds`. The runtime does not retry the rejection. The next entry runs. - An exception causes the runtime to retry that message. The message holds back later messages for that account until it succeeds or moves to dead letters. -A repeated enqueue with the same idempotency key creates one message. A repeated delivery runs the method again. `apply` checks `recent_entry_ids` and returns without a change if the list contains the entry ID. +A repeated enqueue with the same idempotency key creates one message while the first message row exists. A repeated delivery runs the method again. In both cases, the `ledger_entries` row stops a second change. This record does not expire. `schedule(at: 1.day.from_now, key: "daily")` keeps one statement reminder for each account. Each new entry moves the reminder. The database stores the reminder, and the reminder runs after a restart. @@ -161,9 +195,10 @@ You do not assign sequence numbers. The actor mailbox gives the order. ## What the tests prove -- The tests enqueue five entries for two accounts in one mixed order. Two workers process them. Each account applies its own entries in enqueue order. The balances are 25 and 0 cents. +- The tests enqueue five entries for two accounts in one mixed order. Two workers process them. Each account applies its own entries in enqueue order. The tests read the order from the `ledger_entries` rows of each account. The balances are 25 and 0 cents. - The actor rejects a withdrawal that is too large. The deposit after it applies. - Two enqueues share the same idempotency key. One more direct delivery repeats the entry. The account applies the entry once. +- The account applies 102 entries. Then the first entry arrives again. The balance does not change. - The statement reminder runs after the test resets the caller process. ## Choose @@ -178,7 +213,7 @@ You do not assign sequence numbers. The actor mailbox gives the order. - Run `bundle exec solid_objects start`. Async messages and reminders run only while this process runs. The database keeps unfinished work in SQL while the process does not run. - The generated policies deny every call. Write a policy. Pass `authorization_context:` to `async`. See [authorization policies](../authorization.md). - A message that fails on every attempt moves to dead letters after five attempts by default. Later messages then run. An operator can retry a dead letter. See [dead letters, retry, and redrive](../operations.md#dead-letters-retry-and-redrive). -- The actor remembers the idempotency keys of its last finished turns. It retains 64 keys by default. See [retention and backups](../operations.md#retention-and-backups). +- The runtime remembers the idempotency keys of the last 64 finished turns for each actor. That window is not enough for money. The `ledger_entries` table is the durable record. See [retention and backups](../operations.md#retention-and-backups). ## Limits diff --git a/docs/guides/race-conditions.md b/docs/guides/race-conditions.md index 7f7a3f3..bc879c3 100644 --- a/docs/guides/race-conditions.md +++ b/docs/guides/race-conditions.md @@ -109,18 +109,22 @@ class EventTickets < SolidObjects::Actor end def hold(buyer:, hold_id:) + release_expired_holds return { held: true, hold_id: } if holds.dig(buyer, "hold_id") == hold_id return { held: false, reason: "already_held" } if holds.key?(buyer) return { held: false, reason: "sold_out" } if seats_available.zero? + deadline = HOLD_DURATION.from_now self.seats_available -= 1 - self.holds = holds.merge(buyer => { "hold_id" => hold_id }) - schedule(at: HOLD_DURATION.from_now, key: buyer).expire(buyer:, hold_id:) + self.holds = holds.merge(buyer => { "hold_id" => hold_id, "expires_at" => deadline.to_i }) + schedule(at: deadline, key: buyer).expire(buyer:, hold_id:) { held: true, hold_id: } end def confirm(buyer:, hold_id:) return { confirmed: true } if sold.include?(hold_id) + + release_expired_holds reject(:no_hold, "The hold expired or does not exist") unless holds.dig(buyer, "hold_id") == hold_id self.holds = holds.except(buyer) @@ -142,14 +146,24 @@ class EventTickets < SolidObjects::Actor self.title = title self.revision += 1 end + + private + + def release_expired_holds + expired_buyers = holds.select { |_buyer, hold| hold.fetch("expires_at") <= Time.current.to_i }.keys + self.holds = holds.except(*expired_buyers) + self.seats_available += expired_buyers.length + end end ``` The actor owns the lifecycle: -- The `hold` method takes a seat and schedules a reminder with `schedule(at:, key: buyer)`. The database stores the reminder. +- The `hold` method takes a seat and stores the deadline, `expires_at`, beside `hold_id` in the hold. It schedules a reminder with `schedule(at:, key: buyer)`. The database stores the reminder. +- The `hold` and `confirm` methods first call the private `release_expired_holds` method. This method removes each hold at or past its deadline and returns its seat. +- The stored deadline is the rule. After the deadline, the actor rejects confirmation, even before the reminder runs. A stopped or slow runtime process does not extend a hold. - The `confirm` method moves the hold to `sold` and cancels the reminder with `unschedule`. -- The `expire` method releases the seat only when the `hold_id` still matches. An expiry for an old hold does nothing. +- The `expire` method releases the seat only when the `hold_id` still matches. An expiry for an old hold does nothing. The reminder cleans up the hold if no other call releases it first. - A retry of `hold` with the same IDs returns the same result while that hold exists. A retry of `confirm` with the same IDs returns the same result after the confirmation. These retries make no further changes. - The `reject` method ends the call with a business result. The caller receives `SolidObjects::Rejected`. The runtime does not retry a rejection. @@ -170,6 +184,7 @@ result = EventTickets.ref(event.id.to_s).hold( - **Stale expiry:** The first hold expires. The buyer holds again with a new hold ID. The old expiry arrives again, and the new hold stays. - **Retries:** Two identical holds and two identical confirmations sell one seat. - **Confirmation after expiry:** The actor rejects the confirmation with the code `no_hold`. +- **Past the deadline, before the reminder runs:** The test moves the clock 11 minutes forward and delivers no reminder. The actor rejects the confirmation with `no_hold`, and another buyer holds the seat. - **Stale form:** The actor rejects the update with the code `stale_revision`. The next section explains this check. ## Serial execution does not stop a stale form diff --git a/examples/guides/expiring_reservations/seat_inventory.rb b/examples/guides/expiring_reservations/seat_inventory.rb index f7ca4eb..eb5c316 100644 --- a/examples/guides/expiring_reservations/seat_inventory.rb +++ b/examples/guides/expiring_reservations/seat_inventory.rb @@ -19,7 +19,8 @@ def open_show(capacity:) end def hold(hold_id:, buyer:, seats:) - return hold_result(hold_id) if holds.key?(hold_id) + reject(:invalid_seats, "Hold at least one seat") unless seats.is_a?(Integer) && seats.positive? + return hold_result(hold_id) if active_hold?(hold_id) return { status: "confirmed" } if confirmed.key?(hold_id) reject(:not_enough_seats, "Only #{seats_available} seats are left") if seats > seats_available @@ -32,8 +33,9 @@ def hold(hold_id:, buyer:, seats:) end def extend_hold(hold_id:) - hold = holds[hold_id] - reject(:no_hold, "The hold expired or does not exist") unless hold + reject(:no_hold, "The hold expired or does not exist") unless active_hold?(hold_id) + + hold = holds.fetch(hold_id) reject(:extension_limit, "The hold cannot be extended again") if hold.fetch("extensions") >= MAX_EXTENSIONS deadline = Time.at(hold.fetch("expires_at")) + EXTENSION @@ -46,10 +48,9 @@ def extend_hold(hold_id:) def confirm(hold_id:) return { status: "confirmed" } if confirmed.key?(hold_id) + reject(:no_hold, "The hold expired or does not exist") unless active_hold?(hold_id) - hold = holds[hold_id] - reject(:no_hold, "The hold expired or does not exist") unless hold - + hold = holds.fetch(hold_id) self.holds = holds.except(hold_id) self.confirmed = confirmed.merge(hold_id => hold.fetch("seats")) unschedule(:expire, key: hold_id) @@ -65,8 +66,13 @@ def expire(hold_id:, expires_at:) private + def active_hold?(hold_id) + holds.key?(hold_id) && holds.dig(hold_id, "expires_at") > Time.current.to_i + end + def seats_available - capacity - holds.values.sum { |hold| hold.fetch("seats") } - confirmed.values.sum + held_seats = holds.each_key.select { |hold_id| active_hold?(hold_id) }.sum { |hold_id| holds.dig(hold_id, "seats") } + capacity - held_seats - confirmed.values.sum end def hold_result(hold_id) diff --git a/examples/guides/ordered_jobs/create_ledger_entries.rb b/examples/guides/ordered_jobs/create_ledger_entries.rb new file mode 100644 index 0000000..d96eafa --- /dev/null +++ b/examples/guides/ordered_jobs/create_ledger_entries.rb @@ -0,0 +1,14 @@ +# rbs_inline: enabled + +class CreateLedgerEntries < ActiveRecord::Migration[7.1] + def change + create_table :ledger_entries do |table| + table.string :account_id, null: false + table.string :entry_id, null: false + table.string :kind, null: false + table.integer :amount_cents, null: false + table.timestamps + end + add_index :ledger_entries, [ :account_id, :entry_id ], unique: true + end +end diff --git a/examples/guides/ordered_jobs/ledger_account.rb b/examples/guides/ordered_jobs/ledger_account.rb index 9a44904..011b78b 100644 --- a/examples/guides/ordered_jobs/ledger_account.rb +++ b/examples/guides/ordered_jobs/ledger_account.rb @@ -1,20 +1,17 @@ # rbs_inline: enabled class LedgerAccount < SolidObjects::Actor - RECENT_ENTRY_LIMIT = 100 - attribute :balance_cents, default: 0 - attribute :recent_entry_ids, default: -> { [] } attribute :statement_balance_cents, default: nil def apply(entry_id:, kind:, amount_cents:) - return balance_cents if recent_entry_ids.include?(entry_id) + return balance_cents if LedgerEntry.exists?(account_id: actor_id, entry_id:) change = (kind == "deposit") ? amount_cents : -amount_cents reject(:insufficient_funds, "The balance is too low for this withdrawal") if balance_cents + change < 0 self.balance_cents += change - self.recent_entry_ids = (recent_entry_ids + [ entry_id ]).last(RECENT_ENTRY_LIMIT) + commit_action(:record_ledger_entry, account_id: actor_id, entry_id:, kind:, amount_cents:) schedule(at: 1.day.from_now, key: "daily").close_statement balance_cents end diff --git a/examples/guides/ordered_jobs/ledger_entry.rb b/examples/guides/ordered_jobs/ledger_entry.rb new file mode 100644 index 0000000..953e1a6 --- /dev/null +++ b/examples/guides/ordered_jobs/ledger_entry.rb @@ -0,0 +1,4 @@ +# rbs_inline: enabled + +class LedgerEntry < ApplicationRecord +end diff --git a/examples/guides/ordered_jobs/solid_objects.rb b/examples/guides/ordered_jobs/solid_objects.rb new file mode 100644 index 0000000..30c122e --- /dev/null +++ b/examples/guides/ordered_jobs/solid_objects.rb @@ -0,0 +1,10 @@ +# rbs_inline: enabled + +SolidObjects.register_commit_action(:record_ledger_entry) do |arguments, _context| + LedgerEntry.create!( + account_id: arguments.fetch("account_id"), + entry_id: arguments.fetch("entry_id"), + kind: arguments.fetch("kind"), + amount_cents: arguments.fetch("amount_cents") + ) +end diff --git a/examples/guides/race_conditions/event_tickets.rb b/examples/guides/race_conditions/event_tickets.rb index f1c054f..f183fb0 100644 --- a/examples/guides/race_conditions/event_tickets.rb +++ b/examples/guides/race_conditions/event_tickets.rb @@ -18,18 +18,22 @@ def open_sales(seats:) end def hold(buyer:, hold_id:) + release_expired_holds return { held: true, hold_id: } if holds.dig(buyer, "hold_id") == hold_id return { held: false, reason: "already_held" } if holds.key?(buyer) return { held: false, reason: "sold_out" } if seats_available.zero? + deadline = HOLD_DURATION.from_now self.seats_available -= 1 - self.holds = holds.merge(buyer => { "hold_id" => hold_id }) - schedule(at: HOLD_DURATION.from_now, key: buyer).expire(buyer:, hold_id:) + self.holds = holds.merge(buyer => { "hold_id" => hold_id, "expires_at" => deadline.to_i }) + schedule(at: deadline, key: buyer).expire(buyer:, hold_id:) { held: true, hold_id: } end def confirm(buyer:, hold_id:) return { confirmed: true } if sold.include?(hold_id) + + release_expired_holds reject(:no_hold, "The hold expired or does not exist") unless holds.dig(buyer, "hold_id") == hold_id self.holds = holds.except(buyer) @@ -51,4 +55,12 @@ def update_details(title:, base_revision:) self.title = title self.revision += 1 end + + private + + def release_expired_holds + expired_buyers = holds.select { |_buyer, hold| hold.fetch("expires_at") <= Time.current.to_i }.keys + self.holds = holds.except(*expired_buyers) + self.seats_available += expired_buyers.length + end end diff --git a/test/guides/expiring_reservations_test.rb b/test/guides/expiring_reservations_test.rb index 6e9808d..8229b94 100644 --- a/test/guides/expiring_reservations_test.rb +++ b/test/guides/expiring_reservations_test.rb @@ -99,6 +99,29 @@ class ExpiringReservationsGuideTest < ActiveSupport::TestCase assert_equal 1, show.seats_left end + test "a hold past its deadline cannot be confirmed or extended before the reminder runs" do + show = SeatInventory.ref("show-deadline") + show.open_show(capacity: 2) + show.hold(hold_id: "hold-1", buyer: "ada", seats: 2) + + travel 16.minutes do + assert_equal "no_hold", assert_raises(SolidObjects::Rejected) { show.confirm(hold_id: "hold-1") }.code + assert_equal "no_hold", assert_raises(SolidObjects::Rejected) { show.extend_hold(hold_id: "hold-1") }.code + assert_equal 2, show.seats_left + end + end + + test "a hold for zero seats or a negative number of seats is rejected" do + show = SeatInventory.ref("show-invalid") + show.open_show(capacity: 5) + + [ 0, -5 ].each do |seats| + error = assert_raises(SolidObjects::Rejected) { show.hold(hold_id: "hold-#{seats}", buyer: "ada", seats:) } + assert_equal "invalid_seats", error.code + end + assert_equal 5, show.seats_left + end + test "a hold that falls due while the runtime is stopped expires after a restart" do show = SeatInventory.ref("show-restart") show.open_show(capacity: 1) diff --git a/test/guides/guide_documents_test.rb b/test/guides/guide_documents_test.rb index 61c3207..aef7b16 100644 --- a/test/guides/guide_documents_test.rb +++ b/test/guides/guide_documents_test.rb @@ -13,6 +13,8 @@ class GuideDocumentsTest < ActiveSupport::TestCase "examples/guides/ordered_jobs/account.rb", "examples/guides/ordered_jobs/apply_entry_unordered_job.rb", "examples/guides/ordered_jobs/apply_entry_job.rb", + "examples/guides/ordered_jobs/create_ledger_entries.rb", + "examples/guides/ordered_jobs/solid_objects.rb", "examples/guides/ordered_jobs/ledger_account.rb" ], "docs/guides/expiring-reservations.md" => [ diff --git a/test/guides/guide_test_helper.rb b/test/guides/guide_test_helper.rb index 66b5025..8faf602 100644 --- a/test/guides/guide_test_helper.rb +++ b/test/guides/guide_test_helper.rb @@ -3,6 +3,7 @@ require "database_test_helper" require "active_job" require "solid_objects/test_helper" +require_relative "../../examples/guides/ordered_jobs/create_ledger_entries" ActiveJob::Base.queue_adapter = :test ActiveJob::Base.logger = Logger.new(nil) @@ -27,6 +28,8 @@ def self.create table.integer :next_sequence, null: false, default: 1 end + CreateLedgerEntries.new.migrate(:up) unless connection.table_exists?(:ledger_entries) + connection.create_table(:orders, if_not_exists: true) do |table| table.string :reference, null: false table.integer :total_cents, null: false @@ -40,7 +43,7 @@ def self.create end def self.reset - [ "events", "accounts", "orders", "outbox_messages" ].each do |table| + [ "events", "accounts", "ledger_entries", "orders", "outbox_messages" ].each do |table| ActiveRecord::Base.connection.execute("DELETE FROM #{table}") end end diff --git a/test/guides/ordered_jobs_test.rb b/test/guides/ordered_jobs_test.rb index 8ad095d..4ca9a7f 100644 --- a/test/guides/ordered_jobs_test.rb +++ b/test/guides/ordered_jobs_test.rb @@ -4,6 +4,7 @@ require_relative "../../examples/guides/ordered_jobs/account" require_relative "../../examples/guides/ordered_jobs/apply_entry_unordered_job" require_relative "../../examples/guides/ordered_jobs/apply_entry_job" +require_relative "../../examples/guides/ordered_jobs/ledger_entry" require_relative "../../examples/guides/ordered_jobs/ledger_account" class OrderedJobsGuideTest < ActiveSupport::TestCase @@ -13,9 +14,14 @@ class OrderedJobsGuideTest < ActiveSupport::TestCase setup do GuideSchema.reset + load File.expand_path("../../examples/guides/ordered_jobs/solid_objects.rb", __dir__) LedgerAccount.ensure_registered! end + def applied_entry_ids(account_id) + LedgerEntry.where(account_id:).order(:id).pluck(:entry_id) + end + test "jobs that run in a different order than they were enqueued reject a valid withdrawal" do account = Account.create!(balance_cents: 0) @@ -62,11 +68,13 @@ class OrderedJobsGuideTest < ActiveSupport::TestCase LedgerAccount.ref(account_id).async(idempotency_key: entry_id).apply(entry_id:, kind:, amount_cents:) end - concurrently(2) { SolidObjects::Worker.new.tap(&:run_until_idle).stop } + workers = Array.new(2) { SolidObjects::Worker.new } + concurrently(2, prepare: ->(index) { workers.fetch(index) }) { |worker| worker.run_until_idle } + workers.each(&:stop) - assert_equal [ "alice-1", "alice-2", "alice-3" ], LedgerAccount.ref("alice").snapshot.recent_entry_ids + assert_equal [ "alice-1", "alice-2", "alice-3" ], applied_entry_ids("alice") assert_equal 25, LedgerAccount.ref("alice").snapshot.balance_cents - assert_equal [ "bob-1", "bob-2" ], LedgerAccount.ref("bob").snapshot.recent_entry_ids + assert_equal [ "bob-1", "bob-2" ], applied_entry_ids("bob") assert_equal 0, LedgerAccount.ref("bob").snapshot.balance_cents end @@ -91,7 +99,16 @@ class OrderedJobsGuideTest < ActiveSupport::TestCase account.apply(entry_id: "dave-1", kind: "deposit", amount_cents: 40) assert_equal 40, account.snapshot.balance_cents - assert_equal [ "dave-1" ], account.snapshot.recent_entry_ids + assert_equal [ "dave-1" ], applied_entry_ids("dave") + end + + test "a repeated entry after more than one hundred newer entries still applies once" do + account = LedgerAccount.ref("frank") + 102.times { |index| account.apply(entry_id: "frank-#{index}", kind: "deposit", amount_cents: 1) } + + account.apply(entry_id: "frank-0", kind: "deposit", amount_cents: 1) + + assert_equal 102, account.snapshot.balance_cents end test "the statement reminder runs after a restart" do diff --git a/test/guides/race_conditions_test.rb b/test/guides/race_conditions_test.rb index 312773a..8feb5ff 100644 --- a/test/guides/race_conditions_test.rb +++ b/test/guides/race_conditions_test.rb @@ -102,6 +102,19 @@ class RaceConditionsGuideTest < ActiveSupport::TestCase assert_equal 1, tickets.snapshot.seats_available end + test "a hold past its deadline cannot be confirmed and frees its seat before the reminder runs" do + tickets = EventTickets.ref("event-deadline") + tickets.open_sales(seats: 1) + tickets.hold(buyer: "ada", hold_id: "hold-1") + + travel 11.minutes do + error = assert_raises(SolidObjects::Rejected) { tickets.confirm(buyer: "ada", hold_id: "hold-1") } + + assert_equal "no_hold", error.code + assert tickets.hold(buyer: "grace", hold_id: "hold-2").fetch("held") + end + end + test "serial execution alone does not stop a stale form, so a revision check does" do tickets = EventTickets.ref("event-details") tickets.update_details(title: "Spring show", base_revision: 0) diff --git a/test/guides/transactional_outbox_test.rb b/test/guides/transactional_outbox_test.rb index 33d8ecb..1f147a8 100644 --- a/test/guides/transactional_outbox_test.rb +++ b/test/guides/transactional_outbox_test.rb @@ -38,16 +38,6 @@ class TransactionalOutboxGuideTest < ActiveSupport::TestCase class ProcessStopped < StandardError; end - class StoppedProcessAdapter - def enqueue(*) - raise ProcessStopped, "the process stopped before the job reached the queue" - end - - def enqueue_at(*) - raise ProcessStopped, "the process stopped before the job reached the queue" - end - end - setup do GuideSchema.reset ShippingProvider.reset @@ -57,15 +47,15 @@ def enqueue_at(*) end test "an order commits but its job is lost when the process stops before the enqueue" do - test_adapter = ShipmentJob.queue_adapter - ShipmentJob.queue_adapter = StoppedProcessAdapter.new + ShipmentJob.define_singleton_method(:perform_later) { |**| raise ProcessStopped, "the process stopped before the enqueue" } assert_raises(ProcessStopped) { Order.place_and_enqueue!(reference: "order-1", total_cents: 1_500) } + ShipmentJob.singleton_class.remove_method(:perform_later) assert_equal 1, Order.count - assert_empty test_adapter.enqueued_jobs + assert_no_enqueued_jobs ensure - ShipmentJob.queue_adapter = test_adapter + ShipmentJob.singleton_class.remove_method(:perform_later) if ShipmentJob.singleton_class.method_defined?(:perform_later, false) end test "the outbox message commits with the order or not at all" do From 891983113f6e1b84750834cf60efba657810024b Mon Sep 17 00:00:00 2001 From: Lucas Carlson Date: Fri, 9 Oct 2026 14:14:18 -0700 Subject: [PATCH 4/5] fix: check the deadline in an old ticket expiry A retry with the same buyer and hold ID after the deadline creates a fresh hold. When the scheduler had already queued the old expiry, that expiry removed the fresh hold, because expire checked only the hold ID. The reminder now passes the stored deadline, and expire releases a seat only when the hold ID and the deadline both match, as SeatInventory does. A new test retries after the deadline and then delivers the old expiry; with the deadline check removed, it fails. --- docs/guides/race-conditions.md | 9 +++++---- .../guides/race_conditions/event_tickets.rb | 6 +++--- test/guides/race_conditions_test.rb | 18 +++++++++++++++++- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/docs/guides/race-conditions.md b/docs/guides/race-conditions.md index bc879c3..cb6acd7 100644 --- a/docs/guides/race-conditions.md +++ b/docs/guides/race-conditions.md @@ -117,7 +117,7 @@ class EventTickets < SolidObjects::Actor deadline = HOLD_DURATION.from_now self.seats_available -= 1 self.holds = holds.merge(buyer => { "hold_id" => hold_id, "expires_at" => deadline.to_i }) - schedule(at: deadline, key: buyer).expire(buyer:, hold_id:) + schedule(at: deadline, key: buyer).expire(buyer:, hold_id:, expires_at: deadline.to_i) { held: true, hold_id: } end @@ -133,8 +133,8 @@ class EventTickets < SolidObjects::Actor { confirmed: true } end - def expire(buyer:, hold_id:) - return seats_available unless holds.dig(buyer, "hold_id") == hold_id + def expire(buyer:, hold_id:, expires_at:) + return seats_available unless holds[buyer] == { "hold_id" => hold_id, "expires_at" => expires_at } self.holds = holds.except(buyer) self.seats_available += 1 @@ -163,7 +163,7 @@ The actor owns the lifecycle: - The `hold` and `confirm` methods first call the private `release_expired_holds` method. This method removes each hold at or past its deadline and returns its seat. - The stored deadline is the rule. After the deadline, the actor rejects confirmation, even before the reminder runs. A stopped or slow runtime process does not extend a hold. - The `confirm` method moves the hold to `sold` and cancels the reminder with `unschedule`. -- The `expire` method releases the seat only when the `hold_id` still matches. An expiry for an old hold does nothing. The reminder cleans up the hold if no other call releases it first. +- The reminder passes the stored deadline: `expire(buyer:, hold_id:, expires_at:)`. The `expire` method releases the seat only when both the hold ID and the deadline still match the stored hold. An expiry for an old hold does nothing. An expiry that the scheduler queued before a retry created a fresh hold with the same hold ID also does nothing. The reminder cleans up the hold if no other call releases it first. - A retry of `hold` with the same IDs returns the same result while that hold exists. A retry of `confirm` with the same IDs returns the same result after the confirmation. These retries make no further changes. - The `reject` method ends the call with a business result. The caller receives `SolidObjects::Rejected`. The runtime does not retry a rejection. @@ -185,6 +185,7 @@ result = EventTickets.ref(event.id.to_s).hold( - **Retries:** Two identical holds and two identical confirmations sell one seat. - **Confirmation after expiry:** The actor rejects the confirmation with the code `no_hold`. - **Past the deadline, before the reminder runs:** The test moves the clock 11 minutes forward and delivers no reminder. The actor rejects the confirmation with `no_hold`, and another buyer holds the seat. +- **A retry with the same hold ID after the deadline:** the test moves the clock 11 minutes forward. The retry creates a fresh hold, and then the old expiry arrives. The fresh hold stays. - **Stale form:** The actor rejects the update with the code `stale_revision`. The next section explains this check. ## Serial execution does not stop a stale form diff --git a/examples/guides/race_conditions/event_tickets.rb b/examples/guides/race_conditions/event_tickets.rb index f183fb0..e611f29 100644 --- a/examples/guides/race_conditions/event_tickets.rb +++ b/examples/guides/race_conditions/event_tickets.rb @@ -26,7 +26,7 @@ def hold(buyer:, hold_id:) deadline = HOLD_DURATION.from_now self.seats_available -= 1 self.holds = holds.merge(buyer => { "hold_id" => hold_id, "expires_at" => deadline.to_i }) - schedule(at: deadline, key: buyer).expire(buyer:, hold_id:) + schedule(at: deadline, key: buyer).expire(buyer:, hold_id:, expires_at: deadline.to_i) { held: true, hold_id: } end @@ -42,8 +42,8 @@ def confirm(buyer:, hold_id:) { confirmed: true } end - def expire(buyer:, hold_id:) - return seats_available unless holds.dig(buyer, "hold_id") == hold_id + def expire(buyer:, hold_id:, expires_at:) + return seats_available unless holds[buyer] == { "hold_id" => hold_id, "expires_at" => expires_at } self.holds = holds.except(buyer) self.seats_available += 1 diff --git a/test/guides/race_conditions_test.rb b/test/guides/race_conditions_test.rb index 8feb5ff..b5a3602 100644 --- a/test/guides/race_conditions_test.rb +++ b/test/guides/race_conditions_test.rb @@ -65,11 +65,12 @@ class RaceConditionsGuideTest < ActiveSupport::TestCase tickets = EventTickets.ref("event-stale") tickets.open_sales(seats: 1) tickets.hold(buyer: "ada", hold_id: "hold-1") + first_deadline = tickets.snapshot.holds.dig("ada", "expires_at") run_due_reminders(now: 11.minutes.from_now) drain_solid_objects(roles: [ :actors ]) tickets.hold(buyer: "ada", hold_id: "hold-2") - tickets.expire(buyer: "ada", hold_id: "hold-1") + tickets.expire(buyer: "ada", hold_id: "hold-1", expires_at: first_deadline) assert_equal 0, tickets.snapshot.seats_available assert_equal "hold-2", tickets.snapshot.holds.dig("ada", "hold_id") @@ -115,6 +116,21 @@ class RaceConditionsGuideTest < ActiveSupport::TestCase end end + test "an old expiry queued after a retry with the same hold ID keeps the new hold" do + tickets = EventTickets.ref("event-retry-after-deadline") + tickets.open_sales(seats: 1) + tickets.hold(buyer: "ada", hold_id: "hold-1") + first_deadline = tickets.snapshot.holds.dig("ada", "expires_at") + + travel 11.minutes do + assert tickets.hold(buyer: "ada", hold_id: "hold-1").fetch("held") + tickets.expire(buyer: "ada", hold_id: "hold-1", expires_at: first_deadline) + + assert_equal 0, tickets.snapshot.seats_available + assert_equal "hold-1", tickets.snapshot.holds.dig("ada", "hold_id") + end + end + test "serial execution alone does not stop a stale form, so a revision check does" do tickets = EventTickets.ref("event-details") tickets.update_details(title: "Spring show", base_revision: 0) From 8f077a1affa6195b008a9d3e181aec723d045925 Mon Sep 17 00:00:00 2001 From: Lucas Carlson Date: Fri, 9 Oct 2026 14:14:35 -0700 Subject: [PATCH 5/5] ci: pull database images from the ECR Public mirror GitHub runners hit the Docker Hub pull rate limit for anonymous pulls (toomanyrequests) on the Redis, PostgreSQL, and MySQL service containers. The ECR Public mirror serves the same official images without that limit. --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c7fd656..6e76fd9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,7 +66,7 @@ jobs: timeout-minutes: 15 services: redis: - image: redis:7 + image: public.ecr.aws/docker/library/redis:7 ports: - 6379:6379 options: >- @@ -87,7 +87,7 @@ jobs: timeout-minutes: 15 services: postgres: - image: postgres:18 + image: public.ecr.aws/docker/library/postgres:18 env: POSTGRES_USER: solid_objects POSTGRES_PASSWORD: solid_objects @@ -121,7 +121,7 @@ jobs: client: [ mysql2, trilogy ] services: mysql: - image: mysql:8.4 + image: public.ecr.aws/docker/library/mysql:8.4 env: MYSQL_USER: solid_objects MYSQL_PASSWORD: solid_objects