From 638d6a660b2c095f39c87527cefa21904268053e Mon Sep 17 00:00:00 2001 From: Dhruv Kumar Singh Date: Sat, 1 Aug 2026 22:36:28 +0530 Subject: [PATCH 1/2] Run Hypothesis workloads under the seed explorer Hypothesis picks the workload, explore() runs it under a range of seeds, and the property is that no seed broke it. The racy variant proves the division of labor: Hypothesis shrinks the workload to two writers and one payload while the reported seed replays the schedule on its own, with the same trace hash. Seeds stay out of the strategies on purpose - a seed has no size to shrink toward, and two shrinkers aimed at one failure fight. Hypothesis joins the dev group only; the package gains no dependency. --- pyproject.toml | 2 +- tests/test_hypothesis_recipe.py | 187 ++++++++++++++++++++++++++++++++ uv.lock | 64 +++++++++++ 3 files changed, 252 insertions(+), 1 deletion(-) create mode 100644 tests/test_hypothesis_recipe.py diff --git a/pyproject.toml b/pyproject.toml index eccb899..3014eda 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ Repository = "https://github.com/dhruvl/simloop" Issues = "https://github.com/dhruvl/simloop/issues" [dependency-groups] -dev = ["pytest>=8", "mypy>=1.11"] +dev = ["pytest>=8", "mypy>=1.11", "hypothesis>=6"] # Third-party libraries the compatibility probes drive. Dev-only and pinned # exactly, so docs/compatibility.md's version column describes what actually # ran; never installed by a plain `uv run pytest`. diff --git a/tests/test_hypothesis_recipe.py b/tests/test_hypothesis_recipe.py new file mode 100644 index 0000000..6fe4d8b --- /dev/null +++ b/tests/test_hypothesis_recipe.py @@ -0,0 +1,187 @@ +"""Hypothesis over the data, simloop over the schedule. + +The two searches are meant to compose: Hypothesis picks the workload, simloop +picks the interleaving that workload runs under. This file is that claim as a +test rather than a sentence — a correct log that survives every generated +workload under every seed, and a racy one where the combination finds the bug, +shrinks the *workload* to its minimum, and hands back a seed that replays the +*schedule* on its own. + +Seeds are deliberately not a Hypothesis strategy: see docs/cookbook.md for why +the two shrinkers must not be pointed at the same failure. +""" + +from __future__ import annotations + +import asyncio +from collections.abc import Sequence + +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +from simloop import explore, sim_test + +# Seeds each generated workload is explored under. Small on purpose: this runs +# once per example, so the file's cost is examples x seeds, and a race two +# tasks deep does not need a long search to turn up. +SEEDS = 8 + +# Payload alphabet and sizes kept tiny — they exist to be shrunk, not to +# exercise string handling. +_PAYLOADS = st.lists( + st.text(alphabet="abc", min_size=1, max_size=3), min_size=1, max_size=3 +) +_WRITERS = st.integers(min_value=1, max_value=4) +# How long a write is in flight, in virtual seconds. 0.0 means "yield once", +# where the interleaving is the seed's to choose; the longer draws park every +# writer on a timer, which is a different shape of workload and costs nothing +# in wall time. +_DELAYS = st.sampled_from((0.0, 0.001, 0.010)) + +# Every Hypothesis test here shares one configuration. `deadline=None` because +# a simulated run's wall-clock duration is meaningless — it compresses whatever +# virtual time the workload asks for, so a per-example time limit measures the +# machine and not the code. `derandomize` and `database=None` are what make CI +# reproducible: the examples come from a hash of the test rather than from the +# clock, and no example from a previous run is replayed out of `.hypothesis/`. +_SETTINGS = settings( + deadline=None, derandomize=True, database=None, max_examples=25 +) + + +async def _reserve_then_write(log: list[str], payload: str, delay: float) -> None: + """Append to ``log`` by reserving an index and then writing to it. + + The gap between the two is the whole bug: a writer that reserves an index + while another writer holds the same one overwrites its append instead of + adding to it. + """ + index = len(log) + await asyncio.sleep(delay) + log[index : index + 1] = [payload] + + +async def _write_batch( + log: list[str], + lock: asyncio.Lock, + payloads: Sequence[str], + delay: float, + guarded: bool, +) -> None: + for payload in payloads: + if guarded: + async with lock: + await _reserve_then_write(log, payload, delay) + else: + await _reserve_then_write(log, payload, delay) + + +async def _replicate( + writers: int, payloads: Sequence[str], delay: float, *, guarded: bool +) -> None: + """Concurrent writers appending their payloads to one shared log. + + The invariant is the only thing a caller cares about: every append that was + started is in the log. ``guarded`` is the fix — holding the lock across the + reserve and the write — and the two versions run the same appending code. + """ + loop = asyncio.get_running_loop() + lock = asyncio.Lock() + log: list[str] = [] + batches = [ + loop.create_task(_write_batch(log, lock, payloads, delay, guarded)) + for _ in range(writers) + ] + await asyncio.gather(*batches) + expected = writers * len(payloads) + assert len(log) == expected, f"lost {expected - len(log)} of {expected} appends" + + +@_SETTINGS +@given(writers=_WRITERS, payloads=_PAYLOADS, delay=_DELAYS) +def test_the_locked_log_holds_across_examples_and_seeds( + writers: int, payloads: list[str], delay: float +) -> None: + # The recipe itself: Hypothesis names a workload, explore() runs it under + # every seed, and the report — not an exception — is what says whether any + # schedule broke it. + report = explore( + lambda: _replicate(writers, payloads, delay, guarded=True), range(SEEDS) + ) + assert report is None, report.render() + + +def test_hypothesis_shrinks_the_workload_while_the_seed_pins_the_schedule() -> None: + # Hypothesis reports its minimal falsifying example by running it one last + # time, so the last workload recorded here is the one it settled on. + failed: list[tuple[int, list[str], float, int, str]] = [] + + @_SETTINGS + @given(writers=_WRITERS, payloads=_PAYLOADS, delay=_DELAYS) + def every_append_survives( + writers: int, payloads: list[str], delay: float + ) -> None: + report = explore( + lambda: _replicate(writers, payloads, delay, guarded=False), + range(SEEDS), + ) + if report is not None: + failed.append( + (writers, payloads, delay, report.seed, report.trace_hash) + ) + raise AssertionError(report.render()) + + with pytest.raises(AssertionError): + every_append_survives() + + writers, payloads, delay, seed, trace_hash = failed[-1] + # Two writers, one payload each, and a write that merely yields: the + # smallest workload that can lose an append. Hypothesis got there by + # shrinking the data — nothing about the schedule was shrunk, and the + # seeds it explored were the same range(SEEDS) every time. + assert (writers, payloads, delay) == (2, ["a"], 0.0) + # Bigger workloads failed on the way here — more writers, longer payload + # lists, writes that sit on a timer — and the reported example is the + # smallest of everything the search saw fail. + assert any(entry[0] > 2 or len(entry[1]) > 1 or entry[2] > 0.0 for entry in failed) + assert min(entry[:3] for entry in failed) == (writers, payloads, delay) + + # And the schedule half of the report stands on its own: the seed alone + # reproduces the failure, with the same trace, without the seeds around it. + replay = explore( + lambda: _replicate(writers, payloads, delay, guarded=False), [seed] + ) + assert replay is not None + assert replay.seed == seed + assert replay.seeds_passed == 0 + assert replay.trace_hash == trace_hash + assert isinstance(replay.exception, AssertionError) + assert "lost 1 of 2 appends" in str(replay.exception) + + +@_SETTINGS +@given(writers=_WRITERS, payloads=_PAYLOADS, delay=_DELAYS) +@sim_test(seeds=SEEDS) +async def test_the_decorators_stack_into_the_short_form( + writers: int, payloads: list[str], delay: float +) -> None: + # The same recipe written the short way: @sim_test turns the coroutine + # into a synchronous test that explores seeds, and @given calls that test + # once per example. It works because the wrapper passes its arguments + # through to the coroutine — one example, one exploration. + await _replicate(writers, payloads, delay, guarded=True) + + +def test_the_same_workload_and_seed_pin_one_schedule() -> None: + # What lets Hypothesis shrink at all: for a fixed workload, the seed range + # is a pure function, so a workload that failed and then shrank cannot have + # changed its verdict for a reason the search cannot see. + def run() -> str: + report = explore( + lambda: _replicate(2, ["a"], 0.0, guarded=False), range(SEEDS) + ) + assert report is not None + return f"{report.seed}:{report.trace_hash}" + + assert run() == run() diff --git a/uv.lock b/uv.lock index 1f27ae2..134e024 100644 --- a/uv.lock +++ b/uv.lock @@ -335,6 +335,59 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] +[[package]] +name = "hypothesis" +version = "6.164.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sortedcontainers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7a/ac/7b76103bd74d8457e4de0c6a6c3a26ac6327016438bde125e0a3de83a5b8/hypothesis-6.164.0.tar.gz", hash = "sha256:5d63d263d8c71b571638c18d9591f6e34b836c60a12469e9d9105c1c785f00f1", size = 492022, upload-time = "2026-07-30T12:39:49.085Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7c/fe/d5b75a55892b33e72945f82efc71f645d29c0bfdb9f00727f7535a52edcc/hypothesis-6.164.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:14b861ac3353f8643b82a3ba76b8a0a54d2a06160c32b9a1f64a8ab41b179089", size = 771561, upload-time = "2026-07-30T12:39:00.404Z" }, + { url = "https://files.pythonhosted.org/packages/1c/b0/2f01e9efc7267446bad0e2a68f7472daa174a72553d213b16aefe44b2bda/hypothesis-6.164.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:3d8c8bb00a4b86ae90b9ad41f3e1c99d016ec3e64c0ff9d676a4bb7be4f56948", size = 767079, upload-time = "2026-07-30T12:39:23.123Z" }, + { url = "https://files.pythonhosted.org/packages/c3/26/d7bcd26b58e1df2bd39116b924b2a72676215d9650e68cbff9a629c3ce30/hypothesis-6.164.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e80e3ba8eaf37664eaa0f2625cef120b330b128a7df570210cf8be4f5ae65aaa", size = 1096364, upload-time = "2026-07-30T12:38:49.972Z" }, + { url = "https://files.pythonhosted.org/packages/9d/17/99fe7ea866935da83444c3ef7885a14fc7349d96ff61c6faebd37ef4edf2/hypothesis-6.164.0-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8cdf70f821e2d2f3a0bccaab29830aea8aefb63a77806e7e91246fb65a10c8d3", size = 1124963, upload-time = "2026-07-30T12:39:13.1Z" }, + { url = "https://files.pythonhosted.org/packages/38/e8/df08be6296cbc1271d44e81f8ff9dcd6267a07552fb768e0fdc166e93d40/hypothesis-6.164.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bcc3743e22b3cffa7267b4bc74d03628606e4a115495728e986a7be220987315", size = 1145886, upload-time = "2026-07-30T12:39:45.612Z" }, + { url = "https://files.pythonhosted.org/packages/4e/72/d5cf6fbfac40891d4281f630e16a6eb217ff56f97e350a06e0fd9322aa6a/hypothesis-6.164.0-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:730f09d4afcd8a918b3d589bfb6421e3b41c057aa57652a773ef4f512cc60836", size = 1101181, upload-time = "2026-07-30T12:39:05.194Z" }, + { url = "https://files.pythonhosted.org/packages/fb/ff/7ceb002329febffb678b65835ca6e9479a916325d088aadb0210d07f8252/hypothesis-6.164.0-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9651cb48cb5a995295b442138d15d381547b935dcb0066fca7148a7955347400", size = 1137970, upload-time = "2026-07-30T12:39:16.076Z" }, + { url = "https://files.pythonhosted.org/packages/7c/8f/c12c697b73ca9ca24d8a913879e3e0a9db86479754c7221554247c701565/hypothesis-6.164.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:51d161d2655dd86143b370c577267b5b7b4c2e8fcb8a3f22c1a787572aad707c", size = 1270184, upload-time = "2026-07-30T12:38:54.436Z" }, + { url = "https://files.pythonhosted.org/packages/0e/2f/93f1c850c794fc9c80f5e61b3b20652126b865e6f57b348ae530446aadc7/hypothesis-6.164.0-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:e8a250552390128b57e3afe55035ce2c2cb1f6f0919817657854244f071bc5be", size = 1397987, upload-time = "2026-07-30T12:38:21.113Z" }, + { url = "https://files.pythonhosted.org/packages/c5/b8/bab2546325e15e87c8518dfbca263c81dbc35d566c516d66c9da98a38b77/hypothesis-6.164.0-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:570cd51944e1cc3443847d8afa3d17fcf8aac475a1f744c9e7318a5ad7ef5c9f", size = 1270755, upload-time = "2026-07-30T12:38:51.571Z" }, + { url = "https://files.pythonhosted.org/packages/6a/4e/ea97dd39678a42dc5a24e3e2a64d3b950fad9fb1dcce8d7be5afb52a0335/hypothesis-6.164.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:3a423e543055b3de5af7a7624c4285422541658367211fa293a3a57dd0ad01ba", size = 1312888, upload-time = "2026-07-30T12:38:30.847Z" }, + { url = "https://files.pythonhosted.org/packages/44/84/a6f2d5b12b23d65f16eb398750e430065f9d1f40f4418569e3b87ef58d23/hypothesis-6.164.0-cp310-abi3-win32.whl", hash = "sha256:f5e51490b2ce64c66138f24477d83c71b6224ab0ef65700da10187c464b54e94", size = 657401, upload-time = "2026-07-30T12:39:11.581Z" }, + { url = "https://files.pythonhosted.org/packages/f5/d3/c5ee410daa594cac2d3fe1fbe5473f2390e35f4369e168a817e43341ce2f/hypothesis-6.164.0-cp310-abi3-win_amd64.whl", hash = "sha256:c9059dfbb039342b6590bbce207f90e0f9a80fdf45a404c68c2d3e598be78ab3", size = 663566, upload-time = "2026-07-30T12:39:30.27Z" }, + { url = "https://files.pythonhosted.org/packages/90/91/4942fe3f2f08b920368ed5a2937346259e843e382205513b4a0e70d2de9d/hypothesis-6.164.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:6bc3373fe550cf4d7cadb94ceaeb91e431e1418a96b7baa330487366eaa67d3c", size = 773152, upload-time = "2026-07-30T12:38:33.328Z" }, + { url = "https://files.pythonhosted.org/packages/eb/df/e66d052386a2b6c3e2f3eab32a02d7de3c9c59cd21d5dd58c08ecfa715f0/hypothesis-6.164.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2780297ca68929b153eff7effb2ebe67e9487d2fd9f49fa961007f8f2d236c9e", size = 764713, upload-time = "2026-07-30T12:38:48.59Z" }, + { url = "https://files.pythonhosted.org/packages/f5/d5/5a50d14b8f04809e973c4dea884b367fef3663ff253c1205fa9e96229ef9/hypothesis-6.164.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b400bb4eb5a4a1e19cd5af3cc63817909e6b54b4603e04022bdba46860913d7", size = 1095160, upload-time = "2026-07-30T12:38:58.925Z" }, + { url = "https://files.pythonhosted.org/packages/58/01/781b19ce4382ec239c4dc6ec3bd9f195e69e5570f2814bbf04b5781ecb18/hypothesis-6.164.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7fca6632933fc506dd96926d9383483e4c0066c7ff62c748d059a3276da761e7", size = 1145199, upload-time = "2026-07-30T12:39:09.904Z" }, + { url = "https://files.pythonhosted.org/packages/e9/64/30e016863515ca01c1c738b05dd50491353d3ccae6432362e56e0c15d0da/hypothesis-6.164.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b9e1f6e89e5ec34735b727f3ce41d12e7f3b8efc162c91c8a225e10b54b504b4", size = 1267980, upload-time = "2026-07-30T12:38:18.733Z" }, + { url = "https://files.pythonhosted.org/packages/84/23/17eb8d67d59ecd3a820c905fbdf514e371dd7d01631e62a304cdd5793abe/hypothesis-6.164.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:51b0f967f608707b24ed37a298174ae6eec7899bfe3f271d1c3062c39ad66c06", size = 1312181, upload-time = "2026-07-30T12:38:36.056Z" }, + { url = "https://files.pythonhosted.org/packages/42/69/cff9f3cd9524252adda7c8e0e129dfc176e72f64fdf0bf1552d1ea43d78d/hypothesis-6.164.0-cp312-cp312-win_amd64.whl", hash = "sha256:5770df7d518bf867a9379e9081abd9e44db1d15473430e26a0946438c08c5926", size = 660690, upload-time = "2026-07-30T12:38:28.107Z" }, + { url = "https://files.pythonhosted.org/packages/ba/b4/729697380a22dc2ce8feae3c64b08bf3bd3c27e99c3706cb9bdac40c6fc8/hypothesis-6.164.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:29e7cb48974cb9fd87602e20625c890385793c6b56c18a957085a9c291f56ef8", size = 773046, upload-time = "2026-07-30T12:39:40.473Z" }, + { url = "https://files.pythonhosted.org/packages/38/35/72374f02d90dfda198afd8aac6b1e7d1184506f97e62ebcf3d2c1e5bf761/hypothesis-6.164.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1ff8c3819345be8dd15ee6588ee9383869a54c9a3d2232cce5e26b456424135d", size = 764659, upload-time = "2026-07-30T12:38:55.896Z" }, + { url = "https://files.pythonhosted.org/packages/6e/75/fb26388915d71e5949b98ccd0c9d95edcbe6b45d0370f177d43633d81ae2/hypothesis-6.164.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:33e88be13fac3ff7cb789a0b4cc43d99fb297db085f529fbb363188141c7d5bf", size = 1095078, upload-time = "2026-07-30T12:38:34.677Z" }, + { url = "https://files.pythonhosted.org/packages/be/63/f6da6e39667d39a1e44c5df82fbe6cff070c29aaffa9beb62a5322e7d8ae/hypothesis-6.164.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2e296d03a77355ce2e1c32e85a636b555edf0ddaaef277f98f1b84fe38a4595", size = 1145015, upload-time = "2026-07-30T12:39:26.487Z" }, + { url = "https://files.pythonhosted.org/packages/88/c7/55ba09727da3d9a60628c50e31e6083a36f403cb230f5e1a7bd1749a5c39/hypothesis-6.164.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:53698a1b246714539dd0ecc2d556cde613d74e9f7385ec4109e0651ab2d382d6", size = 1268027, upload-time = "2026-07-30T12:38:25.676Z" }, + { url = "https://files.pythonhosted.org/packages/ff/35/4789cade332f799b0e8f2f7ea0fe2aae6157a85e60f74497e316dd17a7e3/hypothesis-6.164.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:004c92c4b869f8e258f0641101b7743cae8420436f4465383f681c086ef95c9d", size = 1311895, upload-time = "2026-07-30T12:39:14.621Z" }, + { url = "https://files.pythonhosted.org/packages/12/8a/18d85e624f8631aec42daa8a2f07c6edcedb7385b2c0f375ba8a30cbd065/hypothesis-6.164.0-cp313-cp313-win_amd64.whl", hash = "sha256:4878f81fa92a580d3e16b53e64e01a9d9fe1dca5973783558493a003138dbd36", size = 660656, upload-time = "2026-07-30T12:38:37.696Z" }, + { url = "https://files.pythonhosted.org/packages/c7/06/3c144d427799c7c72befb0bb3b199d419a89b96e1002fd8f0cc94c84ffb7/hypothesis-6.164.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:9110010bdf6deb3ba9134f8ce8b683e8bb0fba108a351045c96d60c410eb6963", size = 773254, upload-time = "2026-07-30T12:38:38.919Z" }, + { url = "https://files.pythonhosted.org/packages/74/2d/b61a10d9e70df04aa7e8f34efef8e4afe364e8995c59f894e1c35b428214/hypothesis-6.164.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4df103e5d32b47d574c6e857d45361e2cba5a198d6dae4e4ee1bd248b3a2cbfa", size = 764786, upload-time = "2026-07-30T12:38:24.464Z" }, + { url = "https://files.pythonhosted.org/packages/99/68/7f80ac7bdffe78686135311c919534be411d4565c2a5ba38fd389880c553/hypothesis-6.164.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4abec95020960c0ed08e5be318d2bcdde79f2c6fc7785e368a9389d31d3e802a", size = 1095578, upload-time = "2026-07-30T12:38:57.422Z" }, + { url = "https://files.pythonhosted.org/packages/45/f9/97dcbac776bcf33cb4241b52111527821f707b60a84d03d0ea670b09a134/hypothesis-6.164.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9b106756cc9abd50ab1632541ea7b7223792d877a084726aa0304237d758181e", size = 1145207, upload-time = "2026-07-30T12:38:23.387Z" }, + { url = "https://files.pythonhosted.org/packages/a4/df/68184b6f71540435c895cf35ad1d67a3634a887c597ab38d3372c0d20186/hypothesis-6.164.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:11c4aab2ae6757fc4bc3bbf009487e24fd3490365817bbf40b9ec85a7e02fabb", size = 1268357, upload-time = "2026-07-30T12:38:52.946Z" }, + { url = "https://files.pythonhosted.org/packages/a0/76/6a6851dc8af89a5c0418937d38456417b2a1fc9db15c992b9cb43d53a7a3/hypothesis-6.164.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:4713edecbc0969557ca135769a36d1e524c8e3b7a2b271de48d98fa29f681bf6", size = 1312183, upload-time = "2026-07-30T12:39:28.604Z" }, + { url = "https://files.pythonhosted.org/packages/2f/19/83adeb1f8f045bd8a1ab9822d0c3db28b337d37fff01d809fcd6e3ea70f8/hypothesis-6.164.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:e6882d316c390d33c55ec8f1675f35ab238d7c0473ccf8d235c69eaef6c621b9", size = 604771, upload-time = "2026-07-30T12:39:33.579Z" }, + { url = "https://files.pythonhosted.org/packages/0b/62/fcb48ebfbccdc5b695de175b9d1d344b3688782150f0603124bb70c0891b/hypothesis-6.164.0-cp314-cp314-win_amd64.whl", hash = "sha256:7c3357633b38bca8c927fd90d02b39a0a3f35f24cdbcfb2fb1dcf69a3f63bd85", size = 660570, upload-time = "2026-07-30T12:39:43.898Z" }, + { url = "https://files.pythonhosted.org/packages/42/61/5857da7db0435fa69df658a9eafba62eb8a1319454005ce2a0d97f6f9e4d/hypothesis-6.164.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:53152cb549f52d661c47768d0d12a192ef26a7a9758a7f13b8ec41e8e63d6325", size = 771839, upload-time = "2026-07-30T12:38:26.842Z" }, + { url = "https://files.pythonhosted.org/packages/6c/9c/22292a9dab1c544362d1759244132c7d71a9d9d5eda5d454ec735fba6bd3/hypothesis-6.164.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:cee7898ad84b63da6506ae48483bb36f319a25ea4c2b1d2df47d021cc4080c24", size = 763363, upload-time = "2026-07-30T12:38:20.042Z" }, + { url = "https://files.pythonhosted.org/packages/e8/29/cc0c6e9a065a32f93fe52dde746232f007d2cabf619d4e7b1b37bd34c424/hypothesis-6.164.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0def33f0d236e54144a5218997e4492925144d4615f25fdbb4ac8e47b7b709e6", size = 1094171, upload-time = "2026-07-30T12:39:35.158Z" }, + { url = "https://files.pythonhosted.org/packages/a7/59/37040d0776a29d4bc6d0ca9a50ca2755200007e4a8ddc27b010115b69c85/hypothesis-6.164.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:471fd80d70f2df606b1320276168bc2c6007a586124a1d81628264ccb9266f68", size = 1144089, upload-time = "2026-07-30T12:38:43.024Z" }, + { url = "https://files.pythonhosted.org/packages/fa/10/5235ed3c090a2f12fa15cc1d08e5a36cfa31bc0607c45199b0806e930ab4/hypothesis-6.164.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2eb285756aee62890fd08d6e97cf77651dfe7c093ceac094df52120a7a8dbe68", size = 1266595, upload-time = "2026-07-30T12:39:36.979Z" }, + { url = "https://files.pythonhosted.org/packages/7f/97/ffc4cee4dfdffe658e839d5f4df72ae3fa7bfea9401550b475d9700e0ee2/hypothesis-6.164.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7c5215b5568968c35c6e124e5a4a8068f80419d6171414ddf735b49e1df1ab59", size = 1310998, upload-time = "2026-07-30T12:38:45.788Z" }, + { url = "https://files.pythonhosted.org/packages/dd/08/681d4a272cd2812151581c3328e41a80a34e420d676e419a25b4b9dc2291/hypothesis-6.164.0-cp314-cp314t-win_amd64.whl", hash = "sha256:a845e59fae87bb47a6fb84e0d5adb5679b3b55042fc3f8791da91486103cfbf0", size = 660724, upload-time = "2026-07-30T12:38:40.341Z" }, +] + [[package]] name = "idna" version = "3.18" @@ -721,6 +774,7 @@ source = { editable = "." } [package.dev-dependencies] dev = [ + { name = "hypothesis" }, { name = "mypy" }, { name = "pytest" }, ] @@ -735,6 +789,7 @@ probes = [ [package.metadata.requires-dev] dev = [ + { name = "hypothesis", specifier = ">=6" }, { name = "mypy", specifier = ">=1.11" }, { name = "pytest", specifier = ">=8" }, ] @@ -745,6 +800,15 @@ probes = [ { name = "websockets", specifier = "==17.0.1" }, ] +[[package]] +name = "sortedcontainers" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, +] + [[package]] name = "typing-extensions" version = "4.16.0" From 2fcc40c807b015d5aef6e37ef2abba2509dd0c49 Mon Sep 17 00:00:00 2001 From: Dhruv Kumar Singh Date: Sat, 1 Aug 2026 22:36:28 +0530 Subject: [PATCH 2/2] Write the Hypothesis recipe into a cookbook The first cookbook page: the worked example behind the recipe test, the settings CI needs (deadline off, derandomized, no example database) and why, what a failure hands back from each half, what the composition costs, and its honest limits. README gains the short form and the link. --- CHANGELOG.md | 10 +++ README.md | 23 +++++ docs/cookbook.md | 223 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 256 insertions(+) create mode 100644 docs/cookbook.md diff --git a/CHANGELOG.md b/CHANGELOG.md index a530c67..32e6741 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -78,6 +78,16 @@ wire protocol under a SimLoop, and `docs/compatibility.md` publishes what each one did, verbatim. Dev-only — the probes are never packaged, their pinned dependencies live in their own group, and CI does not run them. +- Composing with Hypothesis is a documented recipe with a test behind it: + `docs/cookbook.md` walks through `@given` generating the workload while + `explore()` runs it under a range of seeds, and + `tests/test_hypothesis_recipe.py` runs that composition in CI — a green case + across examples and seeds, and a planted bug where Hypothesis shrinks the + workload to its minimum while the reported seed replays the schedule on its + own. It is a recipe rather than an integration: nothing was added to the + package, seeds are deliberately not a strategy (a seed has no size to shrink + toward, and two shrinkers aimed at one failure fight), and Hypothesis is a + dev dependency of this repository rather than something simloop imports. - `server.sockets` on a simulated server answers with an empty tuple instead of not existing, which is all aiohttp's `web.TCPSite` and websockets' `serve()` need to start; both now run their documented diff --git a/README.md b/README.md index 467edde..f3fcb6b 100644 --- a/README.md +++ b/README.md @@ -189,6 +189,29 @@ events are drawn, and the page says so when there were more. `simloop.timeline_html(events)` renders the same page from any trace you are holding. +## Compose with Hypothesis + +Hypothesis searches the data; simloop searches the schedule. `@given` builds +the workload, `explore()` runs that workload under a range of seeds, and the +property is "no seed broke it": + +```python +@settings(deadline=None, derandomize=True, database=None) +@given(writers=st.integers(1, 4), payloads=st.lists(st.text(), min_size=1)) +def test_the_log_keeps_every_append(writers, payloads): + report = explore(lambda: replicate(writers, payloads), range(8)) + assert report is None, report.render() +``` + +A failure then arrives in two halves: Hypothesis reports the smallest workload +that still breaks, simloop reports the seed that breaks it and the command +that replays it. Seeds stay out of the strategies on purpose — a seed has no +size to shrink toward, and two shrinkers aimed at one failure fight. The +worked example, the settings CI needs and the honest limits are in +[docs/cookbook.md](https://github.com/dhruvl/simloop/blob/main/docs/cookbook.md), +and the composition is a test in this repository rather than a claim. It stays +a recipe: simloop has no Hypothesis dependency and no integration package. + ## What the simulation gives you - **Seeded scheduling** — the ready queue's execution order comes from a diff --git a/docs/cookbook.md b/docs/cookbook.md new file mode 100644 index 0000000..cd77b9e --- /dev/null +++ b/docs/cookbook.md @@ -0,0 +1,223 @@ +# Cookbook + +Recipes that are known to work, because each one is also a test in this +repository. One so far. + +## Hypothesis for the data, simloop for the schedule + +A concurrency bug usually needs two things to go wrong at once: a workload +that can race — enough clients, the right payloads, a timeout short enough to +matter — and an interleaving that makes it race. Property-based testing and +simulation testing each search one of those and neither searches the other. +[Hypothesis](https://hypothesis.readthedocs.io/) generates and minimizes +*data*; simloop enumerates *schedules* and replays the one that failed. + +They compose without an integration layer. Hypothesis picks the workload, +`explore()` runs that workload under a range of seeds, and the property is +"no seed broke it". There is no simloop-Hypothesis package to install and +nothing in simloop knows Hypothesis exists — the whole recipe is the shape of +one test function, which is why this is a cookbook page rather than a module. + +The worked example below is +[tests/test_hypothesis_recipe.py](https://github.com/dhruvl/simloop/blob/main/tests/test_hypothesis_recipe.py), +which runs in simloop's CI on every commit. + +### The workload + +Writers appending to one shared log, where appending is "read the length, +then write at that index" — with an `await` in the gap, the way a real write +has network or disk in the middle: + +```python +async def reserve_then_write(log, payload, delay): + index = len(log) + await asyncio.sleep(delay) + log[index : index + 1] = [payload] + + +async def replicate(writers, payloads, delay, *, guarded): + loop = asyncio.get_running_loop() + lock = asyncio.Lock() + log = [] + + async def write_batch(): + for payload in payloads: + if guarded: + async with lock: + await reserve_then_write(log, payload, delay) + else: + await reserve_then_write(log, payload, delay) + + await asyncio.gather(*[loop.create_task(write_batch()) for _ in range(writers)]) + expected = writers * len(payloads) + assert len(log) == expected, f"lost {expected - len(log)} of {expected} appends" +``` + +Three parameters — how many writers, what they write, how long a write takes +in virtual seconds — and one invariant: every append that started is in the +log. `guarded=True` holds the lock across the reserve and the write, which is +the fix. + +### The recipe + +```python +from hypothesis import given, settings +from hypothesis import strategies as st + +from simloop import explore + +SEEDS = 8 + + +@settings(deadline=None, derandomize=True, database=None, max_examples=25) +@given( + writers=st.integers(min_value=1, max_value=4), + payloads=st.lists(st.text(alphabet="abc", min_size=1, max_size=3), + min_size=1, max_size=3), + delay=st.sampled_from((0.0, 0.001, 0.010)), +) +def test_the_log_keeps_every_append(writers, payloads, delay): + report = explore( + lambda: replicate(writers, payloads, delay, guarded=True), range(SEEDS) + ) + assert report is None, report.render() +``` + +Read it as one sentence: for every workload Hypothesis can build, none of the +first `SEEDS` schedules loses an append. `explore()` returns a +[`SeedReport`](supported-api.md#exploring-schedules) for the first seed that +failed and `None` when they all passed, so the property is a plain `is None` +and the report — failing seed, replay command, trace tail, schedule diff — is +the assertion message. + +Drop the lock (`guarded=False`) and the combination finds the bug, at which +point the two searches divide the reproduction between them. + +### The short form + +`@sim_test` and `@given` stack, and the arguments flow through: + +```python +@settings(deadline=None, derandomize=True, database=None, max_examples=25) +@given(writers=st.integers(min_value=1, max_value=4)) +@sim_test(seeds=8) +async def test_the_log_keeps_every_append(writers): + await replicate(writers, ["a"], 0.0, guarded=True) +``` + +`@sim_test` turns the coroutine into a synchronous test that explores seeds +and re-raises the first failure with the report attached; `@given` calls that +test once per example. Use this form when you want the pytest options +(`--simloop-seeds`, `--simloop-replay`, `--simloop-shrink`, +`--simloop-timeline`) to reach the exploration; use the explicit `explore()` +form when you want the report as a value — to assert on the failing seed, or +to keep exploring after one. + +One option does not survive the stack: `--simloop-jobs` refuses any test that +takes arguments, because worker processes cannot rebuild them. Its message +talks about fixtures; a Hypothesis argument is the same problem. + +### What a failure gives you + +Both halves land in the same output. Hypothesis prints the minimal workload +it could still fail with, and simloop's report names the seed and the command +that replays it: + +``` +E AssertionError: lost 1 of 2 appends +E assert 1 == 2 +E + where 1 = len(['a']) +E simloop: failed at seed 0 (0 seeds passed first) +E replay: pytest 'tests/test_log.py::test_the_log_keeps_every_append' --simloop-replay=0 +E +E last 20 trace events: +E [t=0.0000] run seq=2 driver TaskStepMethWrapper +E ... +E Failing test case: test_the_log_keeps_every_append( +E writers=2, +E ) +``` + +Two writers is the smallest workload that can lose an append, and seed 0 is a +schedule where it does. Nothing about that pair is approximate: rerun the same +workload at that one seed and you get the same failure, the same trace and the +same trace hash. + +The one thing to know when you turn this into a regression test: +`--simloop-replay=SEED` pins the schedule, not the example. To pin both, write +the minimized workload down as an explicit case — `@example(writers=2)`, or a +plain non-Hypothesis test calling `explore()` with those arguments — and keep +the property test for the search. + +### Why the seed is not a strategy + +The obvious next move is `seed=st.integers()`, and it is a mistake. It puts +both searches inside one shrinker, and they minimize incompatible things. + +- **A seed has no size.** Hypothesis shrinks toward smaller values because + smaller usually means simpler, and for a seed it means nothing at all: a + seed is an opaque index into the space of schedules, so seed 0 is not a + simpler failure than seed 8,172 — it is a different one, and usually one + that does not reproduce. The shrinker spends its budget wandering between + unrelated schedules instead of cutting down the workload. +- **The property stops being a function.** With a fixed seed range, "this + workload fails" is deterministic — the same arguments always produce the + same verdict. Draw the seeds too and the same workload passes or fails + depending on what was drawn, which is exactly the flakiness Hypothesis + cannot shrink through: it will abandon a shrink it cannot reproduce and say + so. +- **The schedule already has its own shrinker, and it works on the right + object.** Minimizing an interleaving is not minimizing a number; it is + editing the recorded scheduling decisions back toward FIFO order and + keeping the ones that matter. That is `--simloop-shrink`, and it runs on + the failing seed after the fact. + +So: Hypothesis owns the data, `range(SEEDS)` owns the schedules, and the two +shrinkers never meet. If a workload needs more schedule coverage, raise +`SEEDS` (or `--simloop-seeds` in CI) — that is a knob, not a search space. + +### Settings that matter + +`deadline=None` is not optional. Hypothesis's per-example deadline (200 ms by +default) is a wall-clock measurement, and under simulation wall-clock time is +not the thing being tested: a virtual `asyncio.sleep(300)` costs nothing, one +example is many simulated runs, and how long they take is a statement about +the machine. On a busy CI runner the deadline fires as a flaky failure with +nothing behind it. + +`derandomize=True` and `database=None` are what make the suite reproducible. +Derandomizing derives the examples from a hash of the test, so every run tests +the same workloads until the code or the version changes; `database=None` +stops Hypothesis from replaying examples out of a local `.hypothesis/` +directory that CI does not have. Together with the fixed seed range, the test +is then a pure function of the repository: a green run means something, and a +red one reproduces on the first try. + +That is the CI story, not the only story. Locally, the database is worth +having — it remembers the workload that failed and tries it first next time — +and dropping `derandomize` widens the search across runs. Both are reasonable +in a nightly sweep. Neither belongs in a test that is supposed to give the +same answer on every machine. + +### What it costs + +One example is one full exploration, so the workload runs `max_examples × +SEEDS` times. The numbers above are 25 × 8 = 200 runs, which is a couple of +seconds for a workload this size because virtual time is free. It multiplies, +though: 500 examples over 1,000 seeds is half a million runs, so raise the two +knobs deliberately and separately. More examples buys workload variety; more +seeds buys schedule coverage for the workloads you already have. + +### Honest limits + +- The minimal example belongs to Hypothesis, and which minimum it reports can + change when Hypothesis does. simloop's own test asserts on the exact + shrunk workload on purpose — that assertion is what proves the shrinking is + real — and the version is pinned in `uv.lock`. Asserting on a minimum is a + choice to make knowingly, not a default. +- This page covers `@given`. Hypothesis's stateful testing drives its own + run loop, and nothing here says what a `RuleBasedStateMachine` does on top + of a simulated one; it is untried rather than known to work. +- Hypothesis is not a dependency of simloop and never will be under this + recipe. It is a dev dependency of this repository, so that the recipe can + be tested.