Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 24 additions & 8 deletions clients/backoff.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,23 @@ def _jitter_source(
return None



def compute_backoff_delay(
attempt: int,
*,
base_delay: float,
max_delay: float,
jitter: bool = True,
rng: random.Random | None = None,
) -> float:
"""Return exponential backoff delay for attempt, optionally with jitter."""
delay = min(max_delay, base_delay * (2 ** attempt))
if jitter:
jitter_roll = rng.random() if rng is not None else random.random()
delay = delay * (0.5 + jitter_roll)
return delay


def retry_with_backoff(
fn: Callable[[], object],
*,
Expand All @@ -48,14 +65,13 @@ def retry_with_backoff(
except errors as exc:
if attempt >= retries:
raise
delay = min(max_delay, base_delay * (2 ** attempt))
if jitter:
jitter_roll = (
jitter_rng.random()
if jitter_rng is not None
else random.random()
)
delay = delay * (0.5 + jitter_roll)
delay = compute_backoff_delay(
attempt,
base_delay=base_delay,
max_delay=max_delay,
jitter=jitter,
rng=jitter_rng,
)
# Prefer Retry-After when present (ThrottleError).
retry_after = getattr(exc, "retry_after", None)
if retry_after is not None:
Expand Down
8 changes: 8 additions & 0 deletions clients/circuit_breaker.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,14 @@ def state(self) -> str:
return "half_open"
return "open"

def seconds_until_half_open(self) -> float:
"""Return seconds until half-open; 0.0 when closed or half-open."""
if self.opened_at is None:
return 0.0
elapsed = self._monotonic() - self.opened_at
remaining = self.recovery_timeout - elapsed
return remaining if remaining > 0 else 0.0

def reset(self) -> None:
"""Clear failure count and close the circuit."""
self.failures = 0
Expand Down
32 changes: 31 additions & 1 deletion tests/test_backoff.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
import random
import unittest

from clients.backoff import retry_with_backoff
from clients.backoff import compute_backoff_delay, retry_with_backoff
from clients.clock import FakeClock
from scenarios.retry.handler import TransientError
from scenarios.throttle.handler import ThrottleError, reset as throttle_reset
Expand Down Expand Up @@ -172,6 +172,36 @@ def flaky():
)
self.assertAlmostEqual(clock.monotonic(), expected)

def test_compute_backoff_delay_without_jitter(self):
self.assertAlmostEqual(
compute_backoff_delay(
0, base_delay=0.01, max_delay=0.2, jitter=False
),
0.01,
)
self.assertAlmostEqual(
compute_backoff_delay(
1, base_delay=0.01, max_delay=0.2, jitter=False
),
0.02,
)
self.assertAlmostEqual(
compute_backoff_delay(
10, base_delay=0.01, max_delay=0.2, jitter=False
),
0.2,
)

def test_compute_backoff_delay_with_rng_jitter(self):
rng = random.Random(42)
roll = random.Random(42).random()
expected = 0.01 * (0.5 + roll)
actual = compute_backoff_delay(
0, base_delay=0.01, max_delay=0.2, jitter=True, rng=rng
)
self.assertAlmostEqual(actual, expected)



if __name__ == "__main__":
unittest.main()
52 changes: 52 additions & 0 deletions tests/test_circuit_breaker.py
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,58 @@ def boom():
self.assertFalse(breaker.is_open)


def test_seconds_until_half_open_closed_is_zero(self):
clock = FakeClock()
breaker = CircuitBreaker(
failure_threshold=1,
recovery_timeout=0.5,
watch=(TransientError,),
clock=clock,
)
self.assertEqual(breaker.seconds_until_half_open(), 0.0)
self.assertEqual(breaker.state, "closed")

def test_seconds_until_half_open_while_open(self):
clock = FakeClock()
breaker = CircuitBreaker(
failure_threshold=1,
recovery_timeout=0.5,
watch=(TransientError,),
clock=clock,
)

def boom():
raise TransientError("x")

with self.assertRaises(TransientError):
breaker.call(boom)
self.assertEqual(breaker.state, "open")
self.assertAlmostEqual(breaker.seconds_until_half_open(), 0.5)
clock.sleep(0.2)
self.assertAlmostEqual(breaker.seconds_until_half_open(), 0.3)
# read-only: state and opened_at unchanged by accessor
self.assertEqual(breaker.state, "open")
self.assertIsNotNone(breaker.opened_at)
self.assertEqual(breaker.failures, 1)

def test_seconds_until_half_open_zero_when_half_open(self):
clock = FakeClock()
breaker = CircuitBreaker(
failure_threshold=1,
recovery_timeout=0.5,
watch=(TransientError,),
clock=clock,
)

def boom():
raise TransientError("x")

with self.assertRaises(TransientError):
breaker.call(boom)
clock.sleep(0.5)
self.assertEqual(breaker.state, "half_open")
self.assertEqual(breaker.seconds_until_half_open(), 0.0)



if __name__ == "__main__":
Expand Down
Loading