Skip to content
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 11 additions & 4 deletions tests/unit/crawlers/_basic/test_basic_crawler.py
Original file line number Diff line number Diff line change
Expand Up @@ -1869,8 +1869,11 @@ async def test_crawler_multiple_stops_in_parallel() -> None:
]
processed_urls = []

# Set concurrency to 2 to ensure two urls are being visited in parallel.
crawler = BasicCrawler(concurrency_settings=ConcurrencySettings(desired_concurrency=2, max_concurrency=2))
# Set concurrency to 2 to ensure two urls are being visited in parallel. An overloaded machine still runs
# `min_concurrency` tasks.
crawler = BasicCrawler(
concurrency_settings=ConcurrencySettings(min_concurrency=2, desired_concurrency=2, max_concurrency=2)
)

both_handlers_started = asyncio.Barrier(2)
only_one_handler_at_a_time = asyncio.Semaphore(1)
Expand Down Expand Up @@ -2155,7 +2158,8 @@ async def handler(context: BasicCrawlingContext) -> None:
async def test_pause_after_abort_on_error_ends_run() -> None:
"""`pause()` after a request fails with `abort_on_error` waits for the request in progress, and the run ends."""
crawler = BasicCrawler(
concurrency_settings=ConcurrencySettings(desired_concurrency=2, max_concurrency=2),
# An overloaded machine still runs `min_concurrency` tasks, so the second request starts on a busy machine too.
concurrency_settings=ConcurrencySettings(min_concurrency=2, desired_concurrency=2, max_concurrency=2),
abort_on_error=True,
max_request_retries=0,
)
Expand Down Expand Up @@ -2478,7 +2482,10 @@ async def test_context_use_state_race_condition_in_handlers(key_value_store: Key

Result should be incremented by 2.
Method `use_state` must be implemented in a way that prevents race conditions in such scenario."""
crawler = BasicCrawler()
# An overloaded machine still runs `min_concurrency` tasks, so both handlers run in parallel on a busy machine too.
crawler = BasicCrawler(
concurrency_settings=ConcurrencySettings(min_concurrency=2, desired_concurrency=2, max_concurrency=2)
)
store = await crawler.get_key_value_store()
await store.set_value(f'{BasicCrawler._CRAWLEE_STATE_KEY}_0', {'counter': 0})
handler_barrier = asyncio.Barrier(2)
Expand Down
Loading