From 8764dccb7ec19ee456e8ea2ac58a6941cd1264f9 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Thu, 12 Nov 2026 09:17:12 +0100 Subject: [PATCH 1/7] fix: list child runs started without a `run_name` in `Actor.child_runs` --- .rules.md | 2 +- src/apify/_actor.py | 178 +++++++++------------- src/apify/_child_runs.py | 108 ++++++++++--- tests/unit/actor/test_actor_child_runs.py | 145 ++++++++++++++++-- tests/unit/actor/test_actor_helpers.py | 90 +++++++---- 5 files changed, 356 insertions(+), 167 deletions(-) diff --git a/.rules.md b/.rules.md index a91d9d86..29ccbe1c 100644 --- a/.rules.md +++ b/.rules.md @@ -22,7 +22,7 @@ Don't introduce a shared Pydantic base model to centralize `model_config`. Each - **`Actor`** (`_actor.py`) is a `lazy-object-proxy` wrapping `_ActorType`. It works both as a class (`Actor.is_at_home()`) and as a context manager (`async with Actor:`); on `__aenter__` the proxy's `__wrapped__` is replaced with the active `_ActorType` instance. - **`Configuration`** (`_configuration.py`) extends Crawlee's `Configuration` and is populated from `APIFY_*` environment variables. -- **Named child runs** (`_child_runs.py`) — runs started with `run_name` are persisted in the default key-value store so they can be reattached after a migration or resurrection; exposed through `Actor.child_runs`. +- **Child runs** (`_child_runs.py`) — runs started with `run_name` are persisted in the default key-value store under their name, so they can be reattached after a migration or resurrection; runs started without a name are persisted under their run ID. All of them are exposed through `Actor.child_runs`. - **Storage clients** (`storage_clients/`) implement Crawlee's storage client interface: `ApifyStorageClient` talks to the Apify API (separate sub-clients for single vs. shared request queues), `FileSystemStorageClient` adds Apify-specific key-value store behavior, and `SmartApifyStorageClient` delegates to the cloud client on the platform and to the local one elsewhere. - **Events** — `ApifyEventManager` receives platform events over a WebSocket. - **Errors** — `apify.errors` only re-exports the exceptions from `apify_client.errors`; the SDK defines no error classes of its own, and API calls raise the client exceptions unchanged. diff --git a/src/apify/_actor.py b/src/apify/_actor.py index bb17a6e7..dac57fda 100644 --- a/src/apify/_actor.py +++ b/src/apify/_actor.py @@ -433,11 +433,11 @@ def apify_client(self) -> ApifyClientAsync: @property @_ensure_context def child_runs(self) -> dict[str, RunClientAsync]: - """Clients for the named child runs of this Actor run, keyed by the run name. + """Clients for the child runs of this Actor run, keyed by the run name, or by the run ID for an unnamed run. - Every run started by `Actor.start`, `Actor.call`, `Actor.start_task` or `Actor.call_task` with a `run_name` is - included, even one started before a migration or resurrection of this Actor run. Runs started without a - `run_name` are not tracked. Each client points to the current run under its name: + Every run started by `Actor.start`, `Actor.call`, `Actor.start_task` or `Actor.call_task` is included, even one + started before a migration or resurrection of this Actor run. The client of a named run points to the current + run under its name: ```python run = await Actor.child_runs['my-child'].wait_for_finish() @@ -1103,7 +1103,8 @@ async def start( to the recorded run. A `SUCCEEDED` run is returned as is, an `ABORTED` or `TIMED-OUT` one is resurrected, and a new run is started only when nothing is recorded under the name, or the recorded run `FAILED` or no longer exists. Using `run_name` to attach to a run of a different Actor, task or - with a different input raises a `ValueError`. + with a different input raises a `ValueError`. A run started without `run_name` is recorded under its + run ID, so it is in `Actor.child_runs`, but no later call reattaches to it. abort_with_parent: If true, the child run is gracefully aborted when this Actor run is gracefully aborted. It requires `run_name`, and the value is recorded under it, replacing the one from an earlier call. Once this Actor run is being gracefully aborted, a call with it raises a `RuntimeError` and @@ -1136,9 +1137,6 @@ async def start( webhooks=to_client_representations(webhooks), ) - if run_name is None: - return await start_run() - run, _ = await self._find_or_start_child_run( run_name, actor_id=actor_id, @@ -1304,7 +1302,8 @@ async def call( to the recorded run. A `SUCCEEDED` run is returned as is, an `ABORTED` or `TIMED-OUT` one is resurrected, and a new run is started only when nothing is recorded under the name, or the recorded run `FAILED` or no longer exists. Using `run_name` to attach to a run of a different Actor, task or - with a different input raises a `ValueError`. + with a different input raises a `ValueError`. A run started without `run_name` is recorded under its + run ID, so it is in `Actor.child_runs`, but no later call reattaches to it. abort_with_parent: If true, the child run is gracefully aborted when this Actor run is gracefully aborted. It requires `run_name`, and the value is recorded under it, replacing the one from an earlier call. Once this Actor run is being gracefully aborted, a call with it raises a `RuntimeError` and @@ -1323,53 +1322,38 @@ async def call( client = self.new_client(token=token) if token else self.apify_client actor_client = client.actor(actor_id) + start_run = partial( + actor_client.start, + run_input=run_input, + content_type=content_type, + build=build, + max_items=max_items, + max_total_charge_usd=max_total_charge_usd, + restart_on_error=restart_on_error, + memory_mbytes=memory_mbytes, + run_timeout=self._resolve_run_timeout(timeout), + force_permission_level=force_permission_level, + webhooks=to_client_representations(webhooks), + ) - if run_name is None: - run = await actor_client.call( - run_input=run_input, - content_type=content_type, - build=build, - max_items=max_items, - max_total_charge_usd=max_total_charge_usd, - restart_on_error=restart_on_error, - memory_mbytes=memory_mbytes, - run_timeout=self._resolve_run_timeout(timeout), - force_permission_level=force_permission_level, - webhooks=to_client_representations(webhooks), - wait_duration=wait, - logger=logger, - ) - else: - started_run, is_new = await self._find_or_start_child_run( - run_name, - actor_id=actor_id, - run_input=run_input, - client=client, - start_run=partial( - actor_client.start, - run_input=run_input, - content_type=content_type, - build=build, - max_items=max_items, - max_total_charge_usd=max_total_charge_usd, - restart_on_error=restart_on_error, - memory_mbytes=memory_mbytes, - run_timeout=self._resolve_run_timeout(timeout), - force_permission_level=force_permission_level, - webhooks=to_client_representations(webhooks), - ), - build=build, - max_items=max_items, - max_total_charge_usd=max_total_charge_usd, - restart_on_error=restart_on_error, - memory_mbytes=memory_mbytes, - timeout=timeout, - abort_with_parent=abort_with_parent, - ) - # The earlier attempt of this call already streamed the log of a reattached or resurrected run. - run = await self._wait_for_child_run( - run_name, client.run(started_run.id), started_run, wait=wait, logger=logger, from_start=is_new - ) + started_run, is_new = await self._find_or_start_child_run( + run_name, + actor_id=actor_id, + run_input=run_input, + client=client, + start_run=start_run, + build=build, + max_items=max_items, + max_total_charge_usd=max_total_charge_usd, + restart_on_error=restart_on_error, + memory_mbytes=memory_mbytes, + timeout=timeout, + abort_with_parent=abort_with_parent, + ) + # The earlier attempt of this call already streamed the log of a reattached or resurrected run. + run = await self._wait_for_child_run( + run_name, client.run(started_run.id), started_run, wait=wait, logger=logger, from_start=is_new + ) if run is None: raise RuntimeError(f'Failed to call Actor with ID "{actor_id}".') @@ -1378,7 +1362,7 @@ async def call( async def _find_or_start_child_run( self, - name: str, + name: str | None, *, actor_id: str | None = None, task_id: str | None = None, @@ -1393,6 +1377,9 @@ async def _find_or_start_child_run( timeout: timedelta | Literal['inherit'] | None, abort_with_parent: bool, ) -> tuple[Run, bool]: + if name is None: + run = await self._child_run_registry.start_unnamed(client=client, start_run=start_run) + return run, True return await self._child_run_registry.find_or_start( name, actor_id=actor_id, @@ -1416,7 +1403,7 @@ async def _abort_child_runs(self) -> None: async def _wait_for_child_run( self, - name: str, + name: str | None, run_client: RunClientAsync, run: Run, *, @@ -1491,7 +1478,8 @@ async def start_task( to the recorded run. A `SUCCEEDED` run is returned as is, an `ABORTED` or `TIMED-OUT` one is resurrected, and a new run is started only when nothing is recorded under the name, or the recorded run `FAILED` or no longer exists. Using `run_name` to attach to a run of a different Actor, task or - with a different input raises a `ValueError`. + with a different input raises a `ValueError`. A run started without `run_name` is recorded under its + run ID, so it is in `Actor.child_runs`, but no later call reattaches to it. abort_with_parent: If true, the child run is gracefully aborted when this Actor run is gracefully aborted. It requires `run_name`, and the value is recorded under it, replacing the one from an earlier call. Once this Actor run is being gracefully aborted, a call with it raises a `RuntimeError` and @@ -1521,9 +1509,6 @@ async def start_task( webhooks=to_client_representations(webhooks), ) - if run_name is None: - return await start_run() - run, _ = await self._find_or_start_child_run( run_name, task_id=task_id, @@ -1591,7 +1576,8 @@ async def call_task( to the recorded run. A `SUCCEEDED` run is returned as is, an `ABORTED` or `TIMED-OUT` one is resurrected, and a new run is started only when nothing is recorded under the name, or the recorded run `FAILED` or no longer exists. Using `run_name` to attach to a run of a different Actor, task or - with a different input raises a `ValueError`. + with a different input raises a `ValueError`. A run started without `run_name` is recorded under its + run ID, so it is in `Actor.child_runs`, but no later call reattaches to it. abort_with_parent: If true, the child run is gracefully aborted when this Actor run is gracefully aborted. It requires `run_name`, and the value is recorded under it, replacing the one from an earlier call. Once this Actor run is being gracefully aborted, a call with it raises a `RuntimeError` and @@ -1610,47 +1596,35 @@ async def call_task( client = self.new_client(token=token) if token else self.apify_client task_client = client.task(task_id) + start_run = partial( + task_client.start, + task_input=task_input, + build=build, + max_items=max_items, + max_total_charge_usd=max_total_charge_usd, + restart_on_error=restart_on_error, + memory_mbytes=memory_mbytes, + run_timeout=self._resolve_run_timeout(timeout), + webhooks=to_client_representations(webhooks), + ) - if run_name is None: - run = await task_client.call( - task_input=task_input, - build=build, - max_items=max_items, - max_total_charge_usd=max_total_charge_usd, - restart_on_error=restart_on_error, - memory_mbytes=memory_mbytes, - run_timeout=self._resolve_run_timeout(timeout), - webhooks=to_client_representations(webhooks), - wait_duration=wait, - ) - else: - started_run, _ = await self._find_or_start_child_run( - run_name, - task_id=task_id, - run_input=task_input, - client=client, - start_run=partial( - task_client.start, - task_input=task_input, - build=build, - max_items=max_items, - max_total_charge_usd=max_total_charge_usd, - restart_on_error=restart_on_error, - memory_mbytes=memory_mbytes, - run_timeout=self._resolve_run_timeout(timeout), - webhooks=to_client_representations(webhooks), - ), - build=build, - max_items=max_items, - max_total_charge_usd=max_total_charge_usd, - restart_on_error=restart_on_error, - memory_mbytes=memory_mbytes, - timeout=timeout, - abort_with_parent=abort_with_parent, - ) - run = await self._wait_for_child_run( - run_name, client.run(started_run.id), started_run, wait=wait, logger=None, from_start=False - ) + started_run, _ = await self._find_or_start_child_run( + run_name, + task_id=task_id, + run_input=task_input, + client=client, + start_run=start_run, + build=build, + max_items=max_items, + max_total_charge_usd=max_total_charge_usd, + restart_on_error=restart_on_error, + memory_mbytes=memory_mbytes, + timeout=timeout, + abort_with_parent=abort_with_parent, + ) + run = await self._wait_for_child_run( + run_name, client.run(started_run.id), started_run, wait=wait, logger=None, from_start=False + ) if run is None: raise RuntimeError(f'Failed to call Task with ID "{task_id}".') diff --git a/src/apify/_child_runs.py b/src/apify/_child_runs.py index 4b3bc52d..e10f77d0 100644 --- a/src/apify/_child_runs.py +++ b/src/apify/_child_runs.py @@ -25,7 +25,10 @@ logger = getLogger(__name__) CHILD_RUNS_KEY = '__ACTOR_CHILD_RUNS' -"""Key in the default key-value store under which the child run registry is persisted.""" +"""Key in the default key-value store under which the named child runs are persisted.""" + +UNNAMED_CHILD_RUNS_KEY = 'SDK_UNNAMED_CHILD_RUNS' +"""Key in the default key-value store under which the child runs started without a name are persisted.""" _SETTLING_STATUSES = frozenset({'ABORTING', 'TIMING-OUT'}) """Statuses that end as `ABORTED` / `TIMED-OUT` shortly, and are resurrectable once they do.""" @@ -96,23 +99,38 @@ async def _get_recorded_run(run_client: RunClientAsync) -> Run | None: _records_adapter = TypeAdapter(dict[str, ChildRunRecord]) +_unnamed_records_adapter = TypeAdapter(dict[str, ChildRunSnapshot]) + + +def _malformed_registry_error(key: str) -> ValueError: + return ValueError(f'The child run registry under the "{key}" key in the default key-value store is malformed.') + class ChildRunRegistry: - """Persisted name -> run map that lets named child runs survive a migration or resurrection of the parent. + """Persisted record of the child runs of this Actor run, which survives a migration or resurrection of the parent. - Every change is written to the key-value store right away. A hard kill of the parent between the platform - starting the child and that write can still orphan the child, since nothing but the platform knows about it. + A named child run is recorded under its name, so a later start under the same name can reattach to it. A child run + started without a name is recorded under its run ID. Every change is written to the key-value store right away. A + hard kill of the parent between the platform starting the child and that write can still orphan the child, since + nothing but the platform knows about it. """ def __init__(self, open_key_value_store: Callable[[], Awaitable[KeyValueStore]]) -> None: self._open_key_value_store = open_key_value_store + self._key_value_store: KeyValueStore | None = None + """The default key-value store the records were loaded from, which every change is written to.""" self._records: dict[str, ChildRunRecord] | None = None + """Records of the child runs by name, or `None` until they are loaded.""" + self._unnamed_records: dict[str, ChildRunSnapshot] = {} + """Records of the child runs started without a name, by run ID. Loaded together with `_records`.""" self._lock = asyncio.Lock() """Guards loading the records and writing them back to the key-value store.""" self._name_locks: WeakValueDictionary[str, asyncio.Lock] = WeakValueDictionary() """Serializes `find_or_start` per name. A lock is dropped once no call under its name holds it.""" self._clients: dict[str, ApifyClientAsync] = {} """Client the run under each name was last started or looked up with. Lost on a migration.""" + self._unnamed_clients: dict[str, ApifyClientAsync] = {} + """Client each run started without a name was started with, by run ID. Lost on a migration.""" self._parent_aborting = False """Whether this Actor run is being gracefully aborted, after which no run marked `abort_with_parent` starts.""" @@ -215,28 +233,58 @@ async def find_or_start( self._clients[name] = client return run, False + async def start_unnamed(self, *, client: ApifyClientAsync, start_run: Callable[[], Awaitable[Run]]) -> Run: + """Start a child run without a name and record it under its run ID. + + Args: + client: Client the run is started with. + start_run: Starts a new run of the Actor or task. + + Returns: + The started run. + """ + await self._load() + run = await start_run() + self._unnamed_clients[run.id] = client + await self._save_unnamed(ChildRunSnapshot(run_id=run.id, status=run.status, started_at=run.started_at)) + return run + def run_clients(self, default_client: ApifyClientAsync) -> dict[str, RunClientAsync]: - """Return a client for the current run under each recorded name. + """Return a client for the current run under each recorded name, and for each run recorded without a name. - Each client comes from the client its name was last started or looked up with in this process, so a run started - with a custom token uses that token. + A named run is keyed by its name, a run started without a name by its run ID. Each client comes from the client + its run was last started or looked up with in this process, so a run started with a custom token uses that + token. Args: - default_client: Client used for a name not started or looked up in this process, e.g. one recorded before a + default_client: Client used for a run not started or looked up in this process, e.g. one recorded before a migration. """ - return { + unnamed = { + run_id: self._unnamed_clients.get(run_id, default_client).run(run_id) for run_id in self._unnamed_records + } + named = { name: self._clients.get(name, default_client).run(record.run_id) for name, record in (self._records or {}).items() } + return {**unnamed, **named} - async def update(self, name: str, run: Run) -> None: - """Record the latest observed status of the run recorded under `name`. + async def update(self, name: str | None, run: Run) -> None: + """Record the latest observed status of the run recorded under `name`, or of a run started without a name. Args: - name: Name of the child run. - run: The run as just returned by the API. Nothing is recorded unless it is the current run under `name`. + name: Name of the child run, or `None` for a run started without a name. + run: The run as just returned by the API. Nothing is recorded unless it is the current run under `name`, or + a recorded run started without a name. """ + if name is None: + await self._load() + unnamed_record = self._unnamed_records.get(run.id) + if unnamed_record is None or unnamed_record.status == run.status: + return + await self._save_unnamed(unnamed_record.model_copy(update={'status': run.status})) + return + record = (await self._load()).get(name) if record is None or record.run_id != run.id or record.status == run.status: return @@ -298,24 +346,42 @@ async def load(self) -> dict[str, ChildRunRecord]: """Read the records from the default key-value store, replacing any read before.""" async with self._lock: key_value_store = await self._open_key_value_store() + self._key_value_store = key_value_store stored = await key_value_store.get_value(CHILD_RUNS_KEY) + stored_unnamed = await key_value_store.get_value(UNNAMED_CHILD_RUNS_KEY) try: - self._records = _records_adapter.validate_python(stored or {}) + records = _records_adapter.validate_python(stored or {}) except ValidationError as exc: - raise ValueError( - f'The child run registry under the "{CHILD_RUNS_KEY}" key in the default key-value store ' - 'is malformed.' - ) from exc - return self._records + raise _malformed_registry_error(CHILD_RUNS_KEY) from exc + try: + unnamed_records = _unnamed_records_adapter.validate_python(stored_unnamed or {}) + except ValidationError as exc: + raise _malformed_registry_error(UNNAMED_CHILD_RUNS_KEY) from exc + self._records = records + self._unnamed_records = unnamed_records + return records async def _load(self) -> dict[str, ChildRunRecord]: return self._records if self._records is not None else await self.load() + def _loaded_key_value_store(self) -> KeyValueStore: + if self._key_value_store is None: + raise RuntimeError('The child run registry was not loaded.') + return self._key_value_store + async def _save(self, name: str, record: ChildRunRecord) -> None: records = await self._load() - key_value_store = await self._open_key_value_store() async with self._lock: records[name] = record - await key_value_store.set_value( + await self._loaded_key_value_store().set_value( CHILD_RUNS_KEY, _records_adapter.dump_python(records, by_alias=True, mode='json') ) + + async def _save_unnamed(self, record: ChildRunSnapshot) -> None: + await self._load() + async with self._lock: + self._unnamed_records[record.run_id] = record + await self._loaded_key_value_store().set_value( + UNNAMED_CHILD_RUNS_KEY, + _unnamed_records_adapter.dump_python(self._unnamed_records, by_alias=True, mode='json'), + ) diff --git a/tests/unit/actor/test_actor_child_runs.py b/tests/unit/actor/test_actor_child_runs.py index e4bd5d2c..d6933f47 100644 --- a/tests/unit/actor/test_actor_child_runs.py +++ b/tests/unit/actor/test_actor_child_runs.py @@ -14,7 +14,7 @@ from apify import Actor, Configuration, _child_runs from apify._actor import _ActorType -from apify._child_runs import CHILD_RUNS_KEY, ChildRunRegistry, checksum_request +from apify._child_runs import CHILD_RUNS_KEY, UNNAMED_CHILD_RUNS_KEY, ChildRunRegistry, checksum_request from apify.events import ApifyEventManager if TYPE_CHECKING: @@ -68,6 +68,11 @@ def stored_record( } +def stored_unnamed_record(run_id: str, status: str) -> dict[str, Any]: + """Build a record of a child run started without a name, as it is stored in the default KVS.""" + return {'runId': run_id, 'status': status, 'startedAt': STARTED_AT} + + @pytest.fixture def apify_event_manager() -> ApifyEventManager: """Make the Actor use `ApifyEventManager`, which delivers `ABORTING` on the platform, without a websocket.""" @@ -104,16 +109,84 @@ async def test_named_start_records_run_in_kvs(apify_client_async_patcher: ApifyC assert stored == {'scrape-eu': stored_record('new-run', 'READY')} -async def test_unnamed_start_is_not_recorded(apify_client_async_patcher: ApifyClientAsyncPatcher) -> None: - """A start without a name leaves the registry untouched.""" - apify_client_async_patcher.patch('actor', 'start', return_value=make_run('new-run', 'READY')) +@pytest.mark.parametrize( + ('method', 'resource'), + [ + pytest.param('start', 'actor', id='start'), + pytest.param('start_task', 'task', id='start task'), + ], +) +async def test_unnamed_start_records_run_under_its_id( + apify_client_async_patcher: ApifyClientAsyncPatcher, method: str, resource: str +) -> None: + """A start without a name persists the run under its run ID, apart from the named runs.""" + apify_client_async_patcher.patch(resource, 'start', return_value=make_run('new-run', 'READY')) + + async with Actor: + await getattr(Actor, method)('some-id') + kvs = await Actor.open_key_value_store() + stored_named = await kvs.get_value(CHILD_RUNS_KEY) + stored_unnamed = await kvs.get_value(UNNAMED_CHILD_RUNS_KEY) + + assert stored_named is None + assert stored_unnamed == {'new-run': stored_unnamed_record('new-run', 'READY')} + + +async def test_unnamed_starts_never_reattach(apify_client_async_patcher: ApifyClientAsyncPatcher) -> None: + """Each start without a name starts a new run, even with a recorded active run of the same Actor.""" + run_ids = iter(['first-run', 'second-run']) + apify_client_async_patcher.patch( + 'actor', 'start', replacement_method=AsyncMock(side_effect=lambda *_, **__: make_run(next(run_ids), 'READY')) + ) async with Actor: + await Actor.start('some-actor') await Actor.start('some-actor') kvs = await Actor.open_key_value_store() - stored = await kvs.get_value(CHILD_RUNS_KEY) + stored = await kvs.get_value(UNNAMED_CHILD_RUNS_KEY) - assert stored is None + assert stored.keys() == {'first-run', 'second-run'} + + +@pytest.mark.parametrize( + ('method', 'resource'), + [ + pytest.param('call', 'actor', id='call'), + pytest.param('call_task', 'task', id='call task'), + ], +) +async def test_unnamed_call_records_finished_status( + apify_client_async_patcher: ApifyClientAsyncPatcher, method: str, resource: str +) -> None: + """A call without a name records the run when it starts and stores its status once it finishes.""" + apify_client_async_patcher.patch(resource, 'start', return_value=make_run('new-run', 'READY')) + apify_client_async_patcher.patch('run', 'wait_for_finish', return_value=make_run('new-run', 'SUCCEEDED')) + kwargs: dict[str, Any] = {'logger': None} if method == 'call' else {} + + async with Actor: + run = await getattr(Actor, method)('some-id', **kwargs) + kvs = await Actor.open_key_value_store() + stored = await kvs.get_value(UNNAMED_CHILD_RUNS_KEY) + + assert run.status == 'SUCCEEDED' + assert stored == {'new-run': stored_unnamed_record('new-run', 'SUCCEEDED')} + assert apify_client_async_patcher.calls[resource]['call'] == [] + + +async def test_unnamed_call_streams_log_from_start(apify_client_async_patcher: ApifyClientAsyncPatcher) -> None: + """A call without a name streams the log of its new run from the start, with the given wait duration.""" + get_streamed_log = Mock(return_value=MagicMock()) + apify_client_async_patcher.patch('actor', 'start', return_value=make_run('new-run', 'READY')) + apify_client_async_patcher.patch('run', 'wait_for_finish', return_value=make_run('new-run', 'SUCCEEDED')) + apify_client_async_patcher.patch('run', 'get_status_message_watcher', return_value=MagicMock()) + apify_client_async_patcher.patch('run', 'get_streamed_log', replacement_method=get_streamed_log) + + async with Actor: + await Actor.call('some-actor', wait=timedelta(minutes=1)) + + assert get_streamed_log.call_args.kwargs['from_start'] is True + [(_, kwargs)] = apify_client_async_patcher.calls['run']['wait_for_finish'] + assert kwargs['wait_duration'] == timedelta(minutes=1) @pytest.mark.parametrize( @@ -390,13 +463,20 @@ async def test_named_call_without_logger_only_waits(apify_client_async_patcher: get_status_message_watcher.assert_not_called() -async def test_init_rejects_malformed_registry() -> None: +@pytest.mark.parametrize( + 'key', + [ + pytest.param(CHILD_RUNS_KEY, id='named runs'), + pytest.param(UNNAMED_CHILD_RUNS_KEY, id='unnamed runs'), + ], +) +async def test_init_rejects_malformed_registry(key: str) -> None: """Init raises a `ValueError` naming the key when the registry in the default KVS is malformed, and tears down.""" async with Actor: kvs = await Actor.open_key_value_store() - await kvs.set_value(CHILD_RUNS_KEY, {'scrape-eu': {'runId': 'old-run'}}) + await kvs.set_value(key, {'some-run': {'runId': 'old-run'}}) - with pytest.raises(ValueError, match=CHILD_RUNS_KEY): + with pytest.raises(ValueError, match=key): await Actor.init() assert not Actor._active @@ -676,13 +756,28 @@ async def test_named_call_records_finished_status(apify_client_async_patcher: Ap assert stored == {'scrape-eu': stored_record('new-run', 'FAILED')} -async def test_child_runs_is_empty_without_named_runs(apify_client_async_patcher: ApifyClientAsyncPatcher) -> None: - """`Actor.child_runs` is empty when no run was started with a `run_name`.""" - apify_client_async_patcher.patch('actor', 'start', return_value=make_run('new-run', 'READY')) +async def test_child_runs_is_empty_without_child_runs() -> None: + """`Actor.child_runs` is empty when no child run was started.""" + async with Actor: + assert Actor.child_runs == {} + + +async def test_child_runs_keys_unnamed_run_by_run_id(apify_client_async_patcher: ApifyClientAsyncPatcher) -> None: + """A run started without a name shows up in `Actor.child_runs` under its run ID, next to the named runs.""" + run_ids = iter(['unnamed-run', 'named-run']) + apify_client_async_patcher.patch( + 'actor', 'start', replacement_method=AsyncMock(side_effect=lambda *_, **__: make_run(next(run_ids), 'READY')) + ) async with Actor: await Actor.start('some-actor') - assert Actor.child_runs == {} + await Actor.start('some-actor', run_name='scrape-eu') + child_runs = Actor.child_runs + + assert {key: run_client.resource_id for key, run_client in child_runs.items()} == { + 'unnamed-run': 'unnamed-run', + 'scrape-eu': 'named-run', + } def test_child_runs_requires_initialized_actor() -> None: @@ -702,6 +797,7 @@ async def test_child_runs_includes_runs_recorded_before_init() -> None: 'scrape-us': stored_record('us-run', 'FAILED', actor_id=None, task_id='some-task'), }, ) + await kvs.set_value(UNNAMED_CHILD_RUNS_KEY, {'unnamed-run': stored_unnamed_record('unnamed-run', 'RUNNING')}) async with Actor: child_runs = Actor.child_runs @@ -709,6 +805,7 @@ async def test_child_runs_includes_runs_recorded_before_init() -> None: assert {name: run_client.resource_id for name, run_client in child_runs.items()} == { 'scrape-eu': 'eu-run', 'scrape-us': 'us-run', + 'unnamed-run': 'unnamed-run', } @@ -753,6 +850,28 @@ def recording_new_client(self: _ActorType, **kwargs: Any) -> ApifyClientAsync: assert child_runs['recorded']._http_client is default_http_client +async def test_child_runs_uses_client_of_unnamed_start( + apify_client_async_patcher: ApifyClientAsyncPatcher, monkeypatch: pytest.MonkeyPatch +) -> None: + """A run started without a name with a custom token gets a client with that token in `Actor.child_runs`.""" + apify_client_async_patcher.patch('actor', 'start', return_value=make_run('custom-run', 'READY')) + new_client = _ActorType.new_client + clients_by_token: dict[str | None, ApifyClientAsync] = {} + + def recording_new_client(self: _ActorType, **kwargs: Any) -> ApifyClientAsync: + client = new_client(self, **kwargs) + clients_by_token[kwargs.get('token')] = client + return client + + monkeypatch.setattr(_ActorType, 'new_client', recording_new_client) + + async with Actor: + await Actor.start('some-actor', token='custom-token') + child_runs = Actor.child_runs + + assert child_runs['custom-run']._http_client is clients_by_token['custom-token']._http_client + + async def test_child_runs_keeps_client_of_name_after_rejected_reuse( apify_client_async_patcher: ApifyClientAsyncPatcher, ) -> None: diff --git a/tests/unit/actor/test_actor_helpers.py b/tests/unit/actor/test_actor_helpers.py index 2d740e4b..30f142e9 100644 --- a/tests/unit/actor/test_actor_helpers.py +++ b/tests/unit/actor/test_actor_helpers.py @@ -6,7 +6,7 @@ from datetime import UTC, datetime, timedelta from decimal import Decimal from typing import TYPE_CHECKING -from unittest.mock import AsyncMock, Mock, patch +from unittest.mock import AsyncMock, MagicMock, Mock, patch import pytest @@ -106,26 +106,32 @@ async def test_new_client_uses_configured_api_public_base_url(monkeypatch: pytes async def test_call_actor(apify_client_async_patcher: ApifyClientAsyncPatcher, fake_actor_run: Run) -> None: - apify_client_async_patcher.patch('actor', 'call', return_value=fake_actor_run) + """`Actor.call` starts the Actor through the client's `actor.start` and waits for the run.""" + apify_client_async_patcher.patch('actor', 'start', return_value=fake_actor_run) + apify_client_async_patcher.patch('run', 'wait_for_finish', return_value=fake_actor_run) actor_id = 'some-actor-id' async with Actor: - await Actor.call(actor_id) + await Actor.call(actor_id, logger=None) - assert len(apify_client_async_patcher.calls['actor']['call']) == 1 + assert len(apify_client_async_patcher.calls['actor']['start']) == 1 # The first argument is ActorClientAsync, which was called, let's check its id. - assert apify_client_async_patcher.calls['actor']['call'][0][0][0].resource_id == actor_id + assert apify_client_async_patcher.calls['actor']['start'][0][0][0].resource_id == actor_id + assert len(apify_client_async_patcher.calls['run']['wait_for_finish']) == 1 async def test_call_actor_task(apify_client_async_patcher: ApifyClientAsyncPatcher, fake_actor_run: Run) -> None: - apify_client_async_patcher.patch('task', 'call', return_value=fake_actor_run) + """`Actor.call_task` starts the task through the client's `task.start` and waits for the run.""" + apify_client_async_patcher.patch('task', 'start', return_value=fake_actor_run) + apify_client_async_patcher.patch('run', 'wait_for_finish', return_value=fake_actor_run) task_id = 'some-task-id' async with Actor: await Actor.call_task(task_id) - assert len(apify_client_async_patcher.calls['task']['call']) == 1 - assert apify_client_async_patcher.calls['task']['call'][0][0][0].resource_id == task_id + assert len(apify_client_async_patcher.calls['task']['start']) == 1 + assert apify_client_async_patcher.calls['task']['start'][0][0][0].resource_id == task_id + assert len(apify_client_async_patcher.calls['run']['wait_for_finish']) == 1 async def test_start_actor_task(apify_client_async_patcher: ApifyClientAsyncPatcher, fake_actor_run: Run) -> None: @@ -154,13 +160,20 @@ async def test_start_actor(apify_client_async_patcher: ApifyClientAsyncPatcher, assert apify_client_async_patcher.calls['actor']['start'][0][0][0].resource_id == actor_id +def patch_run_wait(apify_client_async_patcher: ApifyClientAsyncPatcher, run: Run) -> None: + """Make `Actor.call` and `Actor.call_task` get `run` back from the wait, without streaming its log.""" + apify_client_async_patcher.patch('run', 'wait_for_finish', return_value=run) + apify_client_async_patcher.patch('run', 'get_status_message_watcher', return_value=MagicMock()) + apify_client_async_patcher.patch('run', 'get_streamed_log', return_value=MagicMock()) + + @pytest.mark.parametrize( ('client_resource', 'client_method', 'sdk_method'), [ pytest.param('actor', 'start', 'start', id='start'), - pytest.param('actor', 'call', 'call', id='call'), + pytest.param('actor', 'start', 'call', id='call'), pytest.param('task', 'start', 'start_task', id='start_task'), - pytest.param('task', 'call', 'call_task', id='call_task'), + pytest.param('task', 'start', 'call_task', id='call_task'), ], ) async def test_max_items_forwarded_to_client( @@ -172,6 +185,7 @@ async def test_max_items_forwarded_to_client( ) -> None: """`max_items` passed to any of the run-starting helpers reaches the API client.""" apify_client_async_patcher.patch(client_resource, client_method, return_value=fake_actor_run) + patch_run_wait(apify_client_async_patcher, fake_actor_run) async with Actor: with pytest.warns(FutureWarning): @@ -184,9 +198,9 @@ async def test_max_items_forwarded_to_client( ('client_resource', 'client_method', 'sdk_method'), [ pytest.param('actor', 'start', 'start', id='start'), - pytest.param('actor', 'call', 'call', id='call'), + pytest.param('actor', 'start', 'call', id='call'), pytest.param('task', 'start', 'start_task', id='start_task'), - pytest.param('task', 'call', 'call_task', id='call_task'), + pytest.param('task', 'start', 'call_task', id='call_task'), pytest.param('run', 'resurrect', 'resurrect', id='resurrect'), ], ) @@ -199,6 +213,7 @@ async def test_max_items_warns_deprecated( ) -> None: """Passing `max_items` emits a `FutureWarning` that recommends `max_total_charge_usd` and points at the caller.""" apify_client_async_patcher.patch(client_resource, client_method, return_value=fake_actor_run) + patch_run_wait(apify_client_async_patcher, fake_actor_run) async with Actor: with pytest.warns(FutureWarning, match='`max_total_charge_usd`') as record: @@ -212,9 +227,9 @@ async def test_max_items_warns_deprecated( ('client_resource', 'client_method', 'sdk_method'), [ pytest.param('actor', 'start', 'start', id='start'), - pytest.param('actor', 'call', 'call', id='call'), + pytest.param('actor', 'start', 'call', id='call'), pytest.param('task', 'start', 'start_task', id='start_task'), - pytest.param('task', 'call', 'call_task', id='call_task'), + pytest.param('task', 'start', 'call_task', id='call_task'), pytest.param('run', 'resurrect', 'resurrect', id='resurrect'), ], ) @@ -227,6 +242,7 @@ async def test_no_max_items_warning_without_max_items( ) -> None: """The `max_items` deprecation warning is not emitted when `max_items` is not passed.""" apify_client_async_patcher.patch(client_resource, client_method, return_value=fake_actor_run) + patch_run_wait(apify_client_async_patcher, fake_actor_run) async with Actor: with warnings.catch_warnings(): @@ -238,9 +254,9 @@ async def test_no_max_items_warning_without_max_items( ('client_resource', 'client_method', 'sdk_method'), [ pytest.param('actor', 'start', 'start', id='start'), - pytest.param('actor', 'call', 'call', id='call'), + pytest.param('actor', 'start', 'call', id='call'), pytest.param('task', 'start', 'start_task', id='start_task'), - pytest.param('task', 'call', 'call_task', id='call_task'), + pytest.param('task', 'start', 'call_task', id='call_task'), ], ) async def test_max_total_charge_usd_forwarded_to_client( @@ -252,6 +268,7 @@ async def test_max_total_charge_usd_forwarded_to_client( ) -> None: """`max_total_charge_usd` passed to any of the run-starting helpers reaches the API client.""" apify_client_async_patcher.patch(client_resource, client_method, return_value=fake_actor_run) + patch_run_wait(apify_client_async_patcher, fake_actor_run) async with Actor: await getattr(Actor, sdk_method)('some-id', max_total_charge_usd=Decimal('2.5')) @@ -497,9 +514,9 @@ async def listener(_data: object) -> None: _ACTOR_REMOTE_METHODS = [ pytest.param('actor', 'start', 'start', 'some-actor-id', id='start'), - pytest.param('actor', 'call', 'call', 'some-actor-id', id='call'), + pytest.param('actor', 'start', 'call', 'some-actor-id', id='call'), pytest.param('task', 'start', 'start_task', 'some-task-id', id='start_task'), - pytest.param('task', 'call', 'call_task', 'some-task-id', id='call_task'), + pytest.param('task', 'start', 'call_task', 'some-task-id', id='call_task'), ] @@ -515,6 +532,7 @@ async def test_remote_method_with_webhooks( ) -> None: """Test that start/call/start_task/call_task correctly serialize webhooks.""" apify_client_async_patcher.patch(client_resource, client_method, return_value=fake_actor_run) + patch_run_wait(apify_client_async_patcher, fake_actor_run) async with Actor: actor_method = getattr(Actor, actor_method_name) @@ -542,6 +560,7 @@ async def test_remote_method_with_timedelta_timeout( ) -> None: """Test that start/call/start_task/call_task accept a timedelta timeout.""" apify_client_async_patcher.patch(client_resource, client_method, return_value=fake_actor_run) + patch_run_wait(apify_client_async_patcher, fake_actor_run) async with Actor: actor_method = getattr(Actor, actor_method_name) @@ -565,6 +584,7 @@ async def test_remote_method_with_invalid_timeout( ) -> None: """Test that start/call/start_task/call_task raise ValueError for invalid timeout.""" apify_client_async_patcher.patch(client_resource, client_method, return_value=fake_actor_run) + patch_run_wait(apify_client_async_patcher, fake_actor_run) async with Actor: actor_method = getattr(Actor, actor_method_name) @@ -619,45 +639,55 @@ async def test_get_remaining_time_returns_positive_when_timeout_in_future() -> N assert result <= timedelta(minutes=5) -@pytest.mark.parametrize('method_name', ['start', 'call']) +@pytest.mark.parametrize( + 'method_name', + [ + pytest.param('start', id='start'), + pytest.param('call', id='call'), + ], +) async def test_actor_start_and_call_skipped_when_no_inherited_time_remains( apify_client_async_patcher: ApifyClientAsyncPatcher, + fake_actor_run: Run, method_name: str, ) -> None: """Test that Actor.start/Actor.call with `timeout='inherit'` is skipped when the run is past its timeout.""" - apify_client_async_patcher.patch('actor', method_name, return_value=Mock()) + apify_client_async_patcher.patch('actor', 'start', return_value=fake_actor_run) + apify_client_async_patcher.patch('run', 'wait_for_finish', return_value=fake_actor_run) + kwargs = {'logger': None} if method_name == 'call' else {} async with Actor: Actor.configuration.is_at_home = True Actor.configuration.timeout_at = datetime.now(tz=UTC) - timedelta(minutes=5) - await getattr(Actor, method_name)('some-actor-id', timeout='inherit') + await getattr(Actor, method_name)('some-actor-id', timeout='inherit', **kwargs) - assert len(apify_client_async_patcher.calls['actor'][method_name]) == 1 - assert apify_client_async_patcher.calls['actor'][method_name][0][1]['run_timeout'] == timedelta(seconds=1) + assert len(apify_client_async_patcher.calls['actor']['start']) == 1 + assert apify_client_async_patcher.calls['actor']['start'][0][1]['run_timeout'] == timedelta(seconds=1) @pytest.mark.parametrize( - ('client_method', 'actor_method_name'), + 'actor_method_name', [ - pytest.param('start', 'start_task', id='start_task'), - pytest.param('call', 'call_task', id='call_task'), + pytest.param('start_task', id='start task'), + pytest.param('call_task', id='call task'), ], ) async def test_actor_task_methods_skipped_when_no_inherited_time_remains( apify_client_async_patcher: ApifyClientAsyncPatcher, - client_method: str, + fake_actor_run: Run, actor_method_name: str, ) -> None: """Test that start_task/call_task with `timeout='inherit'` is skipped when the run is past its timeout.""" - apify_client_async_patcher.patch('task', client_method, return_value=Mock()) + apify_client_async_patcher.patch('task', 'start', return_value=fake_actor_run) + apify_client_async_patcher.patch('run', 'wait_for_finish', return_value=fake_actor_run) async with Actor: Actor.configuration.is_at_home = True Actor.configuration.timeout_at = datetime.now(tz=UTC) - timedelta(minutes=5) await getattr(Actor, actor_method_name)('some-task-id', timeout='inherit') - assert len(apify_client_async_patcher.calls['task'][client_method]) == 1 - assert apify_client_async_patcher.calls['task'][client_method][0][1]['run_timeout'] == timedelta(seconds=1) + assert len(apify_client_async_patcher.calls['task']['start']) == 1 + assert apify_client_async_patcher.calls['task']['start'][0][1]['run_timeout'] == timedelta(seconds=1) async def test_reboot_runs_all_listeners_even_when_one_fails( From 8561c26b4ff1ae7f8e9859b077278a29c386e1bd Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Thu, 12 Nov 2026 12:10:13 +0100 Subject: [PATCH 2/7] refactor: store child runs without a `run_name` under the `__ACTOR_CHILD_RUNS` key --- src/apify/_child_runs.py | 77 ++++++----------------- tests/unit/actor/test_actor_child_runs.py | 38 ++++++----- 2 files changed, 38 insertions(+), 77 deletions(-) diff --git a/src/apify/_child_runs.py b/src/apify/_child_runs.py index e10f77d0..74e0c7d5 100644 --- a/src/apify/_child_runs.py +++ b/src/apify/_child_runs.py @@ -25,10 +25,7 @@ logger = getLogger(__name__) CHILD_RUNS_KEY = '__ACTOR_CHILD_RUNS' -"""Key in the default key-value store under which the named child runs are persisted.""" - -UNNAMED_CHILD_RUNS_KEY = 'SDK_UNNAMED_CHILD_RUNS' -"""Key in the default key-value store under which the child runs started without a name are persisted.""" +"""Key in the default key-value store under which the child runs are persisted.""" _SETTLING_STATUSES = frozenset({'ABORTING', 'TIMING-OUT'}) """Statuses that end as `ABORTED` / `TIMED-OUT` shortly, and are resurrectable once they do.""" @@ -59,10 +56,10 @@ class ChildRunSnapshot(BaseModel): class ChildRunRecord(ChildRunSnapshot): - """The current run tracked under a name in the child run registry, with the runs it replaced.""" + """The current run tracked under a name or run ID in the child run registry, with the runs it replaced.""" - checksum: str - """Hash of the Actor or task and the input the current run was started with.""" + checksum: str | None = None + """Hash of the Actor or task and the input the current run was started with, or `None` for an unnamed run.""" history: list[ChildRunSnapshot] = Field(default_factory=list) """Earlier runs under this name that failed or went missing and were replaced by a new run, oldest first.""" @@ -99,12 +96,6 @@ async def _get_recorded_run(run_client: RunClientAsync) -> Run | None: _records_adapter = TypeAdapter(dict[str, ChildRunRecord]) -_unnamed_records_adapter = TypeAdapter(dict[str, ChildRunSnapshot]) - - -def _malformed_registry_error(key: str) -> ValueError: - return ValueError(f'The child run registry under the "{key}" key in the default key-value store is malformed.') - class ChildRunRegistry: """Persisted record of the child runs of this Actor run, which survives a migration or resurrection of the parent. @@ -120,17 +111,13 @@ def __init__(self, open_key_value_store: Callable[[], Awaitable[KeyValueStore]]) self._key_value_store: KeyValueStore | None = None """The default key-value store the records were loaded from, which every change is written to.""" self._records: dict[str, ChildRunRecord] | None = None - """Records of the child runs by name, or `None` until they are loaded.""" - self._unnamed_records: dict[str, ChildRunSnapshot] = {} - """Records of the child runs started without a name, by run ID. Loaded together with `_records`.""" + """Records of the child runs by name, or by run ID for a run started without a name. `None` until loaded.""" self._lock = asyncio.Lock() """Guards loading the records and writing them back to the key-value store.""" self._name_locks: WeakValueDictionary[str, asyncio.Lock] = WeakValueDictionary() """Serializes `find_or_start` per name. A lock is dropped once no call under its name holds it.""" self._clients: dict[str, ApifyClientAsync] = {} - """Client the run under each name was last started or looked up with. Lost on a migration.""" - self._unnamed_clients: dict[str, ApifyClientAsync] = {} - """Client each run started without a name was started with, by run ID. Lost on a migration.""" + """Client the run under each name or run ID was last started or looked up with. Lost on a migration.""" self._parent_aborting = False """Whether this Actor run is being gracefully aborted, after which no run marked `abort_with_parent` starts.""" @@ -245,8 +232,8 @@ async def start_unnamed(self, *, client: ApifyClientAsync, start_run: Callable[[ """ await self._load() run = await start_run() - self._unnamed_clients[run.id] = client - await self._save_unnamed(ChildRunSnapshot(run_id=run.id, status=run.status, started_at=run.started_at)) + self._clients[run.id] = client + await self._save(run.id, ChildRunRecord(run_id=run.id, status=run.status, started_at=run.started_at)) return run def run_clients(self, default_client: ApifyClientAsync) -> dict[str, RunClientAsync]: @@ -260,14 +247,10 @@ def run_clients(self, default_client: ApifyClientAsync) -> dict[str, RunClientAs default_client: Client used for a run not started or looked up in this process, e.g. one recorded before a migration. """ - unnamed = { - run_id: self._unnamed_clients.get(run_id, default_client).run(run_id) for run_id in self._unnamed_records + return { + key: self._clients.get(key, default_client).run(record.run_id) + for key, record in (self._records or {}).items() } - named = { - name: self._clients.get(name, default_client).run(record.run_id) - for name, record in (self._records or {}).items() - } - return {**unnamed, **named} async def update(self, name: str | None, run: Run) -> None: """Record the latest observed status of the run recorded under `name`, or of a run started without a name. @@ -277,18 +260,11 @@ async def update(self, name: str | None, run: Run) -> None: run: The run as just returned by the API. Nothing is recorded unless it is the current run under `name`, or a recorded run started without a name. """ - if name is None: - await self._load() - unnamed_record = self._unnamed_records.get(run.id) - if unnamed_record is None or unnamed_record.status == run.status: - return - await self._save_unnamed(unnamed_record.model_copy(update={'status': run.status})) - return - - record = (await self._load()).get(name) + key = run.id if name is None else name + record = (await self._load()).get(key) if record is None or record.run_id != run.id or record.status == run.status: return - await self._save(name, record.model_copy(update={'status': run.status})) + await self._save(key, record.model_copy(update={'status': run.status})) async def abort_runs_with_parent(self, client: ApifyClientAsync) -> None: """Gracefully abort every recorded run marked `abort_with_parent`, and refuse to start any more of them. @@ -348,18 +324,14 @@ async def load(self) -> dict[str, ChildRunRecord]: key_value_store = await self._open_key_value_store() self._key_value_store = key_value_store stored = await key_value_store.get_value(CHILD_RUNS_KEY) - stored_unnamed = await key_value_store.get_value(UNNAMED_CHILD_RUNS_KEY) try: - records = _records_adapter.validate_python(stored or {}) + self._records = _records_adapter.validate_python(stored or {}) except ValidationError as exc: - raise _malformed_registry_error(CHILD_RUNS_KEY) from exc - try: - unnamed_records = _unnamed_records_adapter.validate_python(stored_unnamed or {}) - except ValidationError as exc: - raise _malformed_registry_error(UNNAMED_CHILD_RUNS_KEY) from exc - self._records = records - self._unnamed_records = unnamed_records - return records + raise ValueError( + f'The child run registry under the "{CHILD_RUNS_KEY}" key in the default key-value store ' + 'is malformed.' + ) from exc + return self._records async def _load(self) -> dict[str, ChildRunRecord]: return self._records if self._records is not None else await self.load() @@ -376,12 +348,3 @@ async def _save(self, name: str, record: ChildRunRecord) -> None: await self._loaded_key_value_store().set_value( CHILD_RUNS_KEY, _records_adapter.dump_python(records, by_alias=True, mode='json') ) - - async def _save_unnamed(self, record: ChildRunSnapshot) -> None: - await self._load() - async with self._lock: - self._unnamed_records[record.run_id] = record - await self._loaded_key_value_store().set_value( - UNNAMED_CHILD_RUNS_KEY, - _unnamed_records_adapter.dump_python(self._unnamed_records, by_alias=True, mode='json'), - ) diff --git a/tests/unit/actor/test_actor_child_runs.py b/tests/unit/actor/test_actor_child_runs.py index d6933f47..f0e62d15 100644 --- a/tests/unit/actor/test_actor_child_runs.py +++ b/tests/unit/actor/test_actor_child_runs.py @@ -14,7 +14,7 @@ from apify import Actor, Configuration, _child_runs from apify._actor import _ActorType -from apify._child_runs import CHILD_RUNS_KEY, UNNAMED_CHILD_RUNS_KEY, ChildRunRegistry, checksum_request +from apify._child_runs import CHILD_RUNS_KEY, ChildRunRegistry, checksum_request from apify.events import ApifyEventManager if TYPE_CHECKING: @@ -70,7 +70,14 @@ def stored_record( def stored_unnamed_record(run_id: str, status: str) -> dict[str, Any]: """Build a record of a child run started without a name, as it is stored in the default KVS.""" - return {'runId': run_id, 'status': status, 'startedAt': STARTED_AT} + return { + 'runId': run_id, + 'status': status, + 'startedAt': STARTED_AT, + 'checksum': None, + 'history': [], + 'abortWithParent': False, + } @pytest.fixture @@ -119,17 +126,15 @@ async def test_named_start_records_run_in_kvs(apify_client_async_patcher: ApifyC async def test_unnamed_start_records_run_under_its_id( apify_client_async_patcher: ApifyClientAsyncPatcher, method: str, resource: str ) -> None: - """A start without a name persists the run under its run ID, apart from the named runs.""" + """A start without a name persists the run under its run ID.""" apify_client_async_patcher.patch(resource, 'start', return_value=make_run('new-run', 'READY')) async with Actor: await getattr(Actor, method)('some-id') kvs = await Actor.open_key_value_store() - stored_named = await kvs.get_value(CHILD_RUNS_KEY) - stored_unnamed = await kvs.get_value(UNNAMED_CHILD_RUNS_KEY) + stored = await kvs.get_value(CHILD_RUNS_KEY) - assert stored_named is None - assert stored_unnamed == {'new-run': stored_unnamed_record('new-run', 'READY')} + assert stored == {'new-run': stored_unnamed_record('new-run', 'READY')} async def test_unnamed_starts_never_reattach(apify_client_async_patcher: ApifyClientAsyncPatcher) -> None: @@ -143,7 +148,7 @@ async def test_unnamed_starts_never_reattach(apify_client_async_patcher: ApifyCl await Actor.start('some-actor') await Actor.start('some-actor') kvs = await Actor.open_key_value_store() - stored = await kvs.get_value(UNNAMED_CHILD_RUNS_KEY) + stored = await kvs.get_value(CHILD_RUNS_KEY) assert stored.keys() == {'first-run', 'second-run'} @@ -166,7 +171,7 @@ async def test_unnamed_call_records_finished_status( async with Actor: run = await getattr(Actor, method)('some-id', **kwargs) kvs = await Actor.open_key_value_store() - stored = await kvs.get_value(UNNAMED_CHILD_RUNS_KEY) + stored = await kvs.get_value(CHILD_RUNS_KEY) assert run.status == 'SUCCEEDED' assert stored == {'new-run': stored_unnamed_record('new-run', 'SUCCEEDED')} @@ -463,20 +468,13 @@ async def test_named_call_without_logger_only_waits(apify_client_async_patcher: get_status_message_watcher.assert_not_called() -@pytest.mark.parametrize( - 'key', - [ - pytest.param(CHILD_RUNS_KEY, id='named runs'), - pytest.param(UNNAMED_CHILD_RUNS_KEY, id='unnamed runs'), - ], -) -async def test_init_rejects_malformed_registry(key: str) -> None: +async def test_init_rejects_malformed_registry() -> None: """Init raises a `ValueError` naming the key when the registry in the default KVS is malformed, and tears down.""" async with Actor: kvs = await Actor.open_key_value_store() - await kvs.set_value(key, {'some-run': {'runId': 'old-run'}}) + await kvs.set_value(CHILD_RUNS_KEY, {'scrape-eu': {'runId': 'old-run'}}) - with pytest.raises(ValueError, match=key): + with pytest.raises(ValueError, match=CHILD_RUNS_KEY): await Actor.init() assert not Actor._active @@ -795,9 +793,9 @@ async def test_child_runs_includes_runs_recorded_before_init() -> None: { 'scrape-eu': stored_record('eu-run', 'RUNNING'), 'scrape-us': stored_record('us-run', 'FAILED', actor_id=None, task_id='some-task'), + 'unnamed-run': stored_unnamed_record('unnamed-run', 'RUNNING'), }, ) - await kvs.set_value(UNNAMED_CHILD_RUNS_KEY, {'unnamed-run': stored_unnamed_record('unnamed-run', 'RUNNING')}) async with Actor: child_runs = Actor.child_runs From 3b3fd8d31a5ab9544910acb03b1f00e3914d2a21 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Thu, 12 Nov 2026 12:21:28 +0100 Subject: [PATCH 3/7] refactor: store the run name in each child run record --- src/apify/_child_runs.py | 8 +++++ tests/unit/actor/test_actor_child_runs.py | 39 ++++++++++++++++++++--- 2 files changed, 42 insertions(+), 5 deletions(-) diff --git a/src/apify/_child_runs.py b/src/apify/_child_runs.py index 74e0c7d5..b901e156 100644 --- a/src/apify/_child_runs.py +++ b/src/apify/_child_runs.py @@ -58,6 +58,9 @@ class ChildRunSnapshot(BaseModel): class ChildRunRecord(ChildRunSnapshot): """The current run tracked under a name or run ID in the child run registry, with the runs it replaced.""" + run_name: str | None = None + """Name the run was started under, or `None` for an unnamed run.""" + checksum: str | None = None """Hash of the Actor or task and the input the current run was started with, or `None` for an unnamed run.""" @@ -311,6 +314,7 @@ async def _start( run_id=run.id, status=run.status, started_at=run.started_at, + run_name=name, checksum=checksum, history=history, abort_with_parent=abort_with_parent, @@ -331,6 +335,10 @@ async def load(self) -> dict[str, ChildRunRecord]: f'The child run registry under the "{CHILD_RUNS_KEY}" key in the default key-value store ' 'is malformed.' ) from exc + # A record with a checksum and no `run_name` is a named record written by an older SDK version. + for key, record in self._records.items(): + if record.run_name is None and record.checksum is not None: + record.run_name = key return self._records async def _load(self) -> dict[str, ChildRunRecord]: diff --git a/tests/unit/actor/test_actor_child_runs.py b/tests/unit/actor/test_actor_child_runs.py index f0e62d15..f10e7bd6 100644 --- a/tests/unit/actor/test_actor_child_runs.py +++ b/tests/unit/actor/test_actor_child_runs.py @@ -51,6 +51,7 @@ def stored_record( run_id: str, status: str, *, + run_name: str = 'scrape-eu', actor_id: str | None = 'some-actor', task_id: str | None = None, run_input: Any = None, @@ -62,6 +63,7 @@ def stored_record( 'runId': run_id, 'status': status, 'startedAt': STARTED_AT, + 'runName': run_name, 'checksum': checksum_request(actor_id=actor_id, task_id=task_id, run_input=run_input), 'history': history or [], 'abortWithParent': abort_with_parent, @@ -74,6 +76,7 @@ def stored_unnamed_record(run_id: str, status: str) -> dict[str, Any]: 'runId': run_id, 'status': status, 'startedAt': STARTED_AT, + 'runName': None, 'checksum': None, 'history': [], 'abortWithParent': False, @@ -98,7 +101,7 @@ async def record_child_run( ) -> None: """Seed the registry the way an earlier attempt of this Actor run would have left it, and reload it like init.""" kvs = await Actor.open_key_value_store() - record = stored_record(run_id, 'RUNNING', actor_id=actor_id, task_id=task_id, run_input=run_input) + record = stored_record(run_id, 'RUNNING', run_name=name, actor_id=actor_id, task_id=task_id, run_input=run_input) await kvs.set_value(CHILD_RUNS_KEY, {name: record}) await Actor._child_run_registry.load() @@ -741,6 +744,25 @@ async def test_reattached_run_status_is_recorded(apify_client_async_patcher: Api assert stored == {'scrape-eu': stored_record('old-run', 'SUCCEEDED')} +async def test_record_without_run_name_gets_its_key_as_run_name( + apify_client_async_patcher: ApifyClientAsyncPatcher, +) -> None: + """A named record stored without `runName` is reattached and stored again with its key as the run name.""" + apify_client_async_patcher.patch('run', 'get', return_value=make_run('old-run', 'SUCCEEDED')) + record = stored_record('old-run', 'RUNNING') + del record['runName'] + + async with Actor: + kvs = await Actor.open_key_value_store() + await kvs.set_value(CHILD_RUNS_KEY, {'scrape-eu': record}) + await Actor._child_run_registry.load() + run = await Actor.start('some-actor', run_name='scrape-eu') + stored = await kvs.get_value(CHILD_RUNS_KEY) + + assert run.id == 'old-run' + assert stored == {'scrape-eu': stored_record('old-run', 'SUCCEEDED')} + + async def test_named_call_records_finished_status(apify_client_async_patcher: ApifyClientAsyncPatcher) -> None: """A named call stores the status of the run once it finishes.""" apify_client_async_patcher.patch('actor', 'start', return_value=make_run('new-run', 'READY')) @@ -792,7 +814,9 @@ async def test_child_runs_includes_runs_recorded_before_init() -> None: CHILD_RUNS_KEY, { 'scrape-eu': stored_record('eu-run', 'RUNNING'), - 'scrape-us': stored_record('us-run', 'FAILED', actor_id=None, task_id='some-task'), + 'scrape-us': stored_record( + 'us-run', 'FAILED', run_name='scrape-us', actor_id=None, task_id='some-task' + ), 'unnamed-run': stored_unnamed_record('unnamed-run', 'RUNNING'), }, ) @@ -997,7 +1021,7 @@ async def abort_run(run_client: Any, *_args: Any, **_kwargs: Any) -> Run: await kvs.set_value( CHILD_RUNS_KEY, { - name: stored_record(run_id, 'RUNNING', abort_with_parent=marked) + name: stored_record(run_id, 'RUNNING', run_name=name, abort_with_parent=marked) for name, run_id, marked in [ ('running', 'running-run', True), ('ready', 'ready-run', True), @@ -1030,7 +1054,9 @@ async def test_aborting_event_skips_a_child_run_that_no_longer_exists( async with Actor: kvs = await Actor.open_key_value_store() - await kvs.set_value(CHILD_RUNS_KEY, {'gone': stored_record('gone-run', 'RUNNING', abort_with_parent=True)}) + await kvs.set_value( + CHILD_RUNS_KEY, {'gone': stored_record('gone-run', 'RUNNING', run_name='gone', abort_with_parent=True)} + ) await Actor._child_run_registry.load() apify_event_manager.emit(event=Event.ABORTING, event_data=EventAbortingData()) await apify_event_manager.wait_for_all_listeners_to_complete() @@ -1057,7 +1083,10 @@ async def abort_run(run_client: Any, *_args: Any, **_kwargs: Any) -> Run: kvs = await Actor.open_key_value_store() await kvs.set_value( CHILD_RUNS_KEY, - {name: stored_record(f'{name}-run', 'RUNNING', abort_with_parent=True) for name in ['broken', 'healthy']}, + { + name: stored_record(f'{name}-run', 'RUNNING', run_name=name, abort_with_parent=True) + for name in ['broken', 'healthy'] + }, ) await Actor._child_run_registry.load() apify_event_manager.emit(event=Event.ABORTING, event_data=EventAbortingData()) From 9b87e98e5627515f4f04ec44ead9d17e03e8d6a8 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Thu, 12 Nov 2026 11:40:38 +0100 Subject: [PATCH 4/7] feat: reattach child runs started without a `run_name` after a migration --- .rules.md | 2 +- src/apify/_actor.py | 59 +++++-- src/apify/_child_runs.py | 180 ++++++++++++---------- tests/unit/actor/test_actor_child_runs.py | 158 ++++++++++++++----- 4 files changed, 262 insertions(+), 137 deletions(-) diff --git a/.rules.md b/.rules.md index 29ccbe1c..388a018f 100644 --- a/.rules.md +++ b/.rules.md @@ -22,7 +22,7 @@ Don't introduce a shared Pydantic base model to centralize `model_config`. Each - **`Actor`** (`_actor.py`) is a `lazy-object-proxy` wrapping `_ActorType`. It works both as a class (`Actor.is_at_home()`) and as a context manager (`async with Actor:`); on `__aenter__` the proxy's `__wrapped__` is replaced with the active `_ActorType` instance. - **`Configuration`** (`_configuration.py`) extends Crawlee's `Configuration` and is populated from `APIFY_*` environment variables. -- **Child runs** (`_child_runs.py`) — runs started with `run_name` are persisted in the default key-value store under their name, so they can be reattached after a migration or resurrection; runs started without a name are persisted under their run ID. All of them are exposed through `Actor.child_runs`. +- **Child runs** (`_child_runs.py`) — runs are persisted in the default key-value store under their `run_name`, or without one under an implicit key (a hash of the Actor or task, input and run options, plus the number of identical starts before it), so they can be reattached after a migration or resurrection. All of them are exposed through `Actor.child_runs`. - **Storage clients** (`storage_clients/`) implement Crawlee's storage client interface: `ApifyStorageClient` talks to the Apify API (separate sub-clients for single vs. shared request queues), `FileSystemStorageClient` adds Apify-specific key-value store behavior, and `SmartApifyStorageClient` delegates to the cloud client on the platform and to the local one elsewhere. - **Events** — `ApifyEventManager` receives platform events over a WebSocket. - **Errors** — `apify.errors` only re-exports the exceptions from `apify_client.errors`; the SDK defines no error classes of its own, and API calls raise the client exceptions unchanged. diff --git a/src/apify/_actor.py b/src/apify/_actor.py index dac57fda..2510a46a 100644 --- a/src/apify/_actor.py +++ b/src/apify/_actor.py @@ -433,7 +433,7 @@ def apify_client(self) -> ApifyClientAsync: @property @_ensure_context def child_runs(self) -> dict[str, RunClientAsync]: - """Clients for the child runs of this Actor run, keyed by the run name, or by the run ID for an unnamed run. + """Clients for the child runs of this Actor run, keyed by the run name, or by an implicit key without one. Every run started by `Actor.start`, `Actor.call`, `Actor.start_task` or `Actor.call_task` is included, even one started before a migration or resurrection of this Actor run. The client of a named run points to the current @@ -1103,8 +1103,10 @@ async def start( to the recorded run. A `SUCCEEDED` run is returned as is, an `ABORTED` or `TIMED-OUT` one is resurrected, and a new run is started only when nothing is recorded under the name, or the recorded run `FAILED` or no longer exists. Using `run_name` to attach to a run of a different Actor, task or - with a different input raises a `ValueError`. A run started without `run_name` is recorded under its - run ID, so it is in `Actor.child_runs`, but no later call reattaches to it. + with a different input raises a `ValueError`. A run started without `run_name` is recorded under an + implicit key computed from the Actor or task, the input, the run options and the number of identical + calls without `run_name` before it, so after a migration the n-th identical call reattaches to the + n-th recorded run. abort_with_parent: If true, the child run is gracefully aborted when this Actor run is gracefully aborted. It requires `run_name`, and the value is recorded under it, replacing the one from an earlier call. Once this Actor run is being gracefully aborted, a call with it raises a `RuntimeError` and @@ -1143,12 +1145,15 @@ async def start( run_input=run_input, client=client, start_run=start_run, + content_type=content_type, + force_permission_level=force_permission_level, build=build, max_items=max_items, max_total_charge_usd=max_total_charge_usd, restart_on_error=restart_on_error, memory_mbytes=memory_mbytes, timeout=timeout, + webhooks=webhooks, abort_with_parent=abort_with_parent, ) return run @@ -1302,8 +1307,10 @@ async def call( to the recorded run. A `SUCCEEDED` run is returned as is, an `ABORTED` or `TIMED-OUT` one is resurrected, and a new run is started only when nothing is recorded under the name, or the recorded run `FAILED` or no longer exists. Using `run_name` to attach to a run of a different Actor, task or - with a different input raises a `ValueError`. A run started without `run_name` is recorded under its - run ID, so it is in `Actor.child_runs`, but no later call reattaches to it. + with a different input raises a `ValueError`. A run started without `run_name` is recorded under an + implicit key computed from the Actor or task, the input, the run options and the number of identical + calls without `run_name` before it, so after a migration the n-th identical call reattaches to the + n-th recorded run. abort_with_parent: If true, the child run is gracefully aborted when this Actor run is gracefully aborted. It requires `run_name`, and the value is recorded under it, replacing the one from an earlier call. Once this Actor run is being gracefully aborted, a call with it raises a `RuntimeError` and @@ -1342,17 +1349,20 @@ async def call( run_input=run_input, client=client, start_run=start_run, + content_type=content_type, + force_permission_level=force_permission_level, build=build, max_items=max_items, max_total_charge_usd=max_total_charge_usd, restart_on_error=restart_on_error, memory_mbytes=memory_mbytes, timeout=timeout, + webhooks=webhooks, abort_with_parent=abort_with_parent, ) # The earlier attempt of this call already streamed the log of a reattached or resurrected run. run = await self._wait_for_child_run( - run_name, client.run(started_run.id), started_run, wait=wait, logger=logger, from_start=is_new + client.run(started_run.id), started_run, wait=wait, logger=logger, from_start=is_new ) if run is None: @@ -1369,22 +1379,34 @@ async def _find_or_start_child_run( run_input: Any, client: ApifyClientAsync, start_run: Callable[[], Awaitable[Run]], + content_type: str | None = None, + force_permission_level: ActorPermissionLevel | None = None, build: str | None, max_items: int | None, max_total_charge_usd: Decimal | None, restart_on_error: bool | None, memory_mbytes: int | None, timeout: timedelta | Literal['inherit'] | None, + webhooks: list[Webhook] | None, abort_with_parent: bool, ) -> tuple[Run, bool]: - if name is None: - run = await self._child_run_registry.start_unnamed(client=client, start_run=start_run) - return run, True + run_options = { + 'content_type': content_type, + 'build': build, + 'max_items': max_items, + 'max_total_charge_usd': max_total_charge_usd, + 'restart_on_error': restart_on_error, + 'memory_mbytes': memory_mbytes, + 'timeout': timeout, + 'force_permission_level': force_permission_level, + 'webhooks': [asdict(webhook) for webhook in webhooks] if webhooks else None, + } return await self._child_run_registry.find_or_start( name, actor_id=actor_id, task_id=task_id, run_input=run_input, + run_options=run_options, client=client, start_run=start_run, resurrect_run=lambda run_id: client.run(run_id).resurrect( @@ -1403,7 +1425,6 @@ async def _abort_child_runs(self) -> None: async def _wait_for_child_run( self, - name: str | None, run_client: RunClientAsync, run: Run, *, @@ -1425,7 +1446,7 @@ async def _wait_for_child_run( finished_run = await run_client.wait_for_finish(wait_duration=wait) if finished_run is not None: - await self._child_run_registry.update(name, finished_run) + await self._child_run_registry.update(finished_run) return finished_run @_ensure_context @@ -1478,8 +1499,10 @@ async def start_task( to the recorded run. A `SUCCEEDED` run is returned as is, an `ABORTED` or `TIMED-OUT` one is resurrected, and a new run is started only when nothing is recorded under the name, or the recorded run `FAILED` or no longer exists. Using `run_name` to attach to a run of a different Actor, task or - with a different input raises a `ValueError`. A run started without `run_name` is recorded under its - run ID, so it is in `Actor.child_runs`, but no later call reattaches to it. + with a different input raises a `ValueError`. A run started without `run_name` is recorded under an + implicit key computed from the Actor or task, the input, the run options and the number of identical + calls without `run_name` before it, so after a migration the n-th identical call reattaches to the + n-th recorded run. abort_with_parent: If true, the child run is gracefully aborted when this Actor run is gracefully aborted. It requires `run_name`, and the value is recorded under it, replacing the one from an earlier call. Once this Actor run is being gracefully aborted, a call with it raises a `RuntimeError` and @@ -1521,6 +1544,7 @@ async def start_task( restart_on_error=restart_on_error, memory_mbytes=memory_mbytes, timeout=timeout, + webhooks=webhooks, abort_with_parent=abort_with_parent, ) return run @@ -1576,8 +1600,10 @@ async def call_task( to the recorded run. A `SUCCEEDED` run is returned as is, an `ABORTED` or `TIMED-OUT` one is resurrected, and a new run is started only when nothing is recorded under the name, or the recorded run `FAILED` or no longer exists. Using `run_name` to attach to a run of a different Actor, task or - with a different input raises a `ValueError`. A run started without `run_name` is recorded under its - run ID, so it is in `Actor.child_runs`, but no later call reattaches to it. + with a different input raises a `ValueError`. A run started without `run_name` is recorded under an + implicit key computed from the Actor or task, the input, the run options and the number of identical + calls without `run_name` before it, so after a migration the n-th identical call reattaches to the + n-th recorded run. abort_with_parent: If true, the child run is gracefully aborted when this Actor run is gracefully aborted. It requires `run_name`, and the value is recorded under it, replacing the one from an earlier call. Once this Actor run is being gracefully aborted, a call with it raises a `RuntimeError` and @@ -1620,10 +1646,11 @@ async def call_task( restart_on_error=restart_on_error, memory_mbytes=memory_mbytes, timeout=timeout, + webhooks=webhooks, abort_with_parent=abort_with_parent, ) run = await self._wait_for_child_run( - run_name, client.run(started_run.id), started_run, wait=wait, logger=None, from_start=False + client.run(started_run.id), started_run, wait=wait, logger=None, from_start=False ) if run is None: diff --git a/src/apify/_child_runs.py b/src/apify/_child_runs.py index b901e156..20a70cfc 100644 --- a/src/apify/_child_runs.py +++ b/src/apify/_child_runs.py @@ -3,6 +3,7 @@ import asyncio import hashlib import json +from collections import Counter from datetime import datetime from logging import getLogger from typing import TYPE_CHECKING, Any @@ -25,7 +26,7 @@ logger = getLogger(__name__) CHILD_RUNS_KEY = '__ACTOR_CHILD_RUNS' -"""Key in the default key-value store under which the child runs are persisted.""" +"""Key in the default key-value store under which the child run registry is persisted.""" _SETTLING_STATUSES = frozenset({'ABORTING', 'TIMING-OUT'}) """Statuses that end as `ABORTED` / `TIMED-OUT` shortly, and are resurrectable once they do.""" @@ -39,6 +40,9 @@ _NOT_FOUND_RETRY_INTERVAL_SECS = 0.25 +_IMPLICIT_KEY_HASH_LENGTH = 16 +"""Number of hex digits of the request hash kept in the key of a child run started without a name.""" + class ChildRunSnapshot(BaseModel): """A child run as last observed by this Actor run.""" @@ -56,13 +60,13 @@ class ChildRunSnapshot(BaseModel): class ChildRunRecord(ChildRunSnapshot): - """The current run tracked under a name or run ID in the child run registry, with the runs it replaced.""" + """The current run tracked under a name or implicit key in the child run registry, with the runs it replaced.""" run_name: str | None = None - """Name the run was started under, or `None` for an unnamed run.""" + """Name the run was started under, or `None` for a run started without a name.""" - checksum: str | None = None - """Hash of the Actor or task and the input the current run was started with, or `None` for an unnamed run.""" + checksum: str + """Hash of the Actor or task and the input the current run was started with.""" history: list[ChildRunSnapshot] = Field(default_factory=list) """Earlier runs under this name that failed or went missing and were replaced by a new run, oldest first.""" @@ -71,13 +75,22 @@ class ChildRunRecord(ChildRunSnapshot): """Whether the current run is aborted when this Actor run is gracefully aborted.""" -def checksum_request(*, actor_id: str | None, task_id: str | None, run_input: Any) -> str: - """Hash the Actor or task and the input of a named start, in the same JSON shape as the JS SDK.""" +def checksum_request( + *, actor_id: str | None, task_id: str | None, run_input: Any, run_options: dict[str, Any] | None = None +) -> str: + """Hash the Actor or task, the input and the run options of a start. + + Without run options, the hashed JSON has the same shape as in the JS SDK. Only the options set to anything but + `None` are hashed, so a run option added to the SDK later does not change the hash of a start that does not use it. + """ request: dict[str, Any] = ( {'type': 'actor', 'id': actor_id} if actor_id is not None else {'type': 'task', 'id': task_id} ) if run_input is not None: request['input'] = run_input + options = {name: value for name, value in (run_options or {}).items() if value is not None} + if options: + request['options'] = options serialized = json.dumps(request, sort_keys=True, separators=(',', ':'), ensure_ascii=False, default=str) return hashlib.sha256(serialized.encode()).hexdigest() @@ -104,9 +117,11 @@ class ChildRunRegistry: """Persisted record of the child runs of this Actor run, which survives a migration or resurrection of the parent. A named child run is recorded under its name, so a later start under the same name can reattach to it. A child run - started without a name is recorded under its run ID. Every change is written to the key-value store right away. A - hard kill of the parent between the platform starting the child and that write can still orphan the child, since - nothing but the platform knows about it. + started without a name is recorded under an implicit key: a hash of its Actor or task, input and run options, and + the number of identical starts without a name before it in this process, as in `3f2a9c0e1b7d4a65-0`. After a + migration, the starts run again in the same order, so the n-th identical start reattaches to the n-th recorded run. + Every change is written to the key-value store right away. A hard kill of the parent between the platform starting + the child and that write can still orphan the child, since nothing but the platform knows about it. """ def __init__(self, open_key_value_store: Callable[[], Awaitable[KeyValueStore]]) -> None: @@ -114,29 +129,35 @@ def __init__(self, open_key_value_store: Callable[[], Awaitable[KeyValueStore]]) self._key_value_store: KeyValueStore | None = None """The default key-value store the records were loaded from, which every change is written to.""" self._records: dict[str, ChildRunRecord] | None = None - """Records of the child runs by name, or by run ID for a run started without a name. `None` until loaded.""" + """Records of the child runs by name, or by implicit key for a run started without a name. + + `None` until loaded. + """ self._lock = asyncio.Lock() """Guards loading the records and writing them back to the key-value store.""" self._name_locks: WeakValueDictionary[str, asyncio.Lock] = WeakValueDictionary() - """Serializes `find_or_start` per name. A lock is dropped once no call under its name holds it.""" + """Serializes `find_or_start` per key. A lock is dropped once no call under its key holds it.""" self._clients: dict[str, ApifyClientAsync] = {} - """Client the run under each name or run ID was last started or looked up with. Lost on a migration.""" + """Client the run under each key was last started or looked up with. Lost on a migration.""" + self._implicit_starts: Counter[str] = Counter() + """Number of starts without a name in this process, by the hash part of their implicit key.""" self._parent_aborting = False """Whether this Actor run is being gracefully aborted, after which no run marked `abort_with_parent` starts.""" async def find_or_start( self, - name: str, + name: str | None, *, actor_id: str | None = None, task_id: str | None = None, run_input: Any, + run_options: dict[str, Any] | None = None, client: ApifyClientAsync, start_run: Callable[[], Awaitable[Run]], resurrect_run: Callable[[str], Awaitable[Run]], abort_with_parent: bool = False, ) -> tuple[Run, bool]: - """Return the run recorded under `name`, or start one when there is none to reuse. + """Return the run recorded under `name` or the implicit key, or start one when there is none to reuse. A recorded run that is `READY` or `RUNNING` is reattached and one that `SUCCEEDED` is returned as is. An `ABORTED` or `TIMED-OUT` run is resurrected, since Actors are expected to resume from their state. @@ -144,16 +165,17 @@ async def find_or_start( task or input. Args: - name: Name of the child run, unique within the parent run. + name: Name of the child run, unique within the parent run, or `None` to use the implicit key. actor_id: The Actor to start. task_id: The task to start, in place of `actor_id`. run_input: Input of the run. The recorded run is reused only if it was started with the same Actor or task and input. + run_options: Options the run is started with. Without `name`, they are part of the implicit key. client: Client used to look up the recorded run. start_run: Starts a new run of the Actor or task. resurrect_run: Resurrects the recorded run, given its ID. - abort_with_parent: Whether to abort the run when this Actor run is gracefully aborted. It replaces the value - recorded under `name`. + abort_with_parent: Whether to abort the run when this Actor run is gracefully aborted. It replaces the + recorded value. Returns: The run, and whether it was newly started. @@ -164,22 +186,34 @@ async def find_or_start( started, resurrected or reattached. """ checksum = checksum_request(actor_id=actor_id, task_id=task_id, run_input=run_input) - - async with self._name_locks.setdefault(name, asyncio.Lock()): + key = name + if key is None: + request_hash = checksum_request( + actor_id=actor_id, task_id=task_id, run_input=run_input, run_options=run_options + )[:_IMPLICIT_KEY_HASH_LENGTH] + key = f'{request_hash}-{self._implicit_starts[request_hash]}' + self._implicit_starts[request_hash] += 1 + + async with self._name_locks.setdefault(key, asyncio.Lock()): if abort_with_parent and self._parent_aborting: raise RuntimeError( - f'Child run "{name}" with `abort_with_parent` was not started, since this Actor run is being ' + f'Child run "{key}" with `abort_with_parent` was not started, since this Actor run is being ' 'aborted.' ) records = await self._load() - record = records.get(name) + record = records.get(key) if record is None: run = await self._start( - name, checksum=checksum, start_run=start_run, history=[], abort_with_parent=abort_with_parent + key, + run_name=name, + checksum=checksum, + start_run=start_run, + history=[], + abort_with_parent=abort_with_parent, ) - self._clients[name] = client + self._clients[key] = client return run, True run_client = client.run(record.run_id) @@ -195,56 +229,41 @@ async def find_or_start( started_at=record.started_at, ) run = await self._start( - name, + key, + run_name=name, checksum=checksum, start_run=start_run, history=[*record.history, replaced], abort_with_parent=abort_with_parent, ) - self._clients[name] = client + self._clients[key] = client return run, True if record.checksum != checksum: raise ValueError( - f'The run name "{name}" was already used for a different Actor, task or input. ' + f'The run name "{key}" was already used for a different Actor, task or input. ' 'Use a unique `run_name` for each child run.' ) if record.abort_with_parent != abort_with_parent: - await self._save(name, record.model_copy(update={'abort_with_parent': abort_with_parent})) + await self._save(key, record.model_copy(update={'abort_with_parent': abort_with_parent})) if run.status in _RESURRECTABLE_STATUSES: - logger.info('Resurrecting child run "%s"', name, extra={'run_id': run.id, 'status': run.status}) + logger.info('Resurrecting child run "%s"', key, extra={'run_id': run.id, 'status': run.status}) run = await resurrect_run(run.id) else: - logger.info('Reattaching to child run "%s"', name, extra={'run_id': run.id, 'status': run.status}) + logger.info('Reattaching to child run "%s"', key, extra={'run_id': run.id, 'status': run.status}) - await self.update(name, run) - self._clients[name] = client + await self._update(key, run) + self._clients[key] = client return run, False - async def start_unnamed(self, *, client: ApifyClientAsync, start_run: Callable[[], Awaitable[Run]]) -> Run: - """Start a child run without a name and record it under its run ID. - - Args: - client: Client the run is started with. - start_run: Starts a new run of the Actor or task. - - Returns: - The started run. - """ - await self._load() - run = await start_run() - self._clients[run.id] = client - await self._save(run.id, ChildRunRecord(run_id=run.id, status=run.status, started_at=run.started_at)) - return run - def run_clients(self, default_client: ApifyClientAsync) -> dict[str, RunClientAsync]: - """Return a client for the current run under each recorded name, and for each run recorded without a name. + """Return a client for the current run under each recorded key. - A named run is keyed by its name, a run started without a name by its run ID. Each client comes from the client - its run was last started or looked up with in this process, so a run started with a custom token uses that - token. + A named run is keyed by its name, a run started without a name by its implicit key. Each client comes from the + client its run was last started or looked up with in this process, so a run started with a custom token uses + that token. Args: default_client: Client used for a run not started or looked up in this process, e.g. one recorded before a @@ -255,15 +274,18 @@ def run_clients(self, default_client: ApifyClientAsync) -> dict[str, RunClientAs for key, record in (self._records or {}).items() } - async def update(self, name: str | None, run: Run) -> None: - """Record the latest observed status of the run recorded under `name`, or of a run started without a name. + async def update(self, run: Run) -> None: + """Record the latest observed status of a run. Args: - name: Name of the child run, or `None` for a run started without a name. - run: The run as just returned by the API. Nothing is recorded unless it is the current run under `name`, or - a recorded run started without a name. + run: The run as just returned by the API. Nothing is recorded unless it is the current run under some key. """ - key = run.id if name is None else name + records = await self._load() + key = next((key for key, record in records.items() if record.run_id == run.id), None) + if key is not None: + await self._update(key, run) + + async def _update(self, key: str, run: Run) -> None: record = (await self._load()).get(key) if record is None or record.run_id != run.id or record.status == run.status: return @@ -277,33 +299,34 @@ async def abort_runs_with_parent(self, client: ApifyClientAsync) -> None: the start records it. Args: - client: Client used for a name not started or looked up in this process, e.g. after a migration. + client: Client used for a run not started or looked up in this process, e.g. after a migration. """ self._parent_aborting = True records = await self._load() - # Names with a start in flight are not recorded yet, so their locks are awaited too. - await asyncio.gather(*(self._abort(name, client) for name in {*records, *self._name_locks})) + # Keys with a start in flight are not recorded yet, so their locks are awaited too. + await asyncio.gather(*(self._abort(key, client) for key in {*records, *self._name_locks})) - async def _abort(self, name: str, default_client: ApifyClientAsync) -> None: - async with self._name_locks.setdefault(name, asyncio.Lock()): - record = (await self._load()).get(name) + async def _abort(self, key: str, default_client: ApifyClientAsync) -> None: + async with self._name_locks.setdefault(key, asyncio.Lock()): + record = (await self._load()).get(key) if record is None or not record.abort_with_parent: return - run_client = self._clients.get(name, default_client).run(record.run_id) + run_client = self._clients.get(key, default_client).run(record.run_id) try: run = await run_client.abort(gracefully=True) except NotFoundError: - logger.debug('Child run "%s" no longer exists', name, extra={'run_id': record.run_id}) + logger.debug('Child run "%s" no longer exists', key, extra={'run_id': record.run_id}) except Exception: - logger.exception('Failed to abort child run "%s"', name, extra={'run_id': record.run_id}) + logger.exception('Failed to abort child run "%s"', key, extra={'run_id': record.run_id}) else: if run.status in _ABORTED_STATUSES: - logger.info('Aborted child run "%s" with the parent', name, extra={'run_id': record.run_id}) + logger.info('Aborted child run "%s" with the parent', key, extra={'run_id': record.run_id}) async def _start( self, - name: str, + key: str, *, + run_name: str | None, checksum: str, start_run: Callable[[], Awaitable[Run]], history: list[ChildRunSnapshot], @@ -314,17 +337,22 @@ async def _start( run_id=run.id, status=run.status, started_at=run.started_at, - run_name=name, + run_name=run_name, checksum=checksum, history=history, abort_with_parent=abort_with_parent, ) - await self._save(name, record) + await self._save(key, record) return run async def load(self) -> dict[str, ChildRunRecord]: - """Read the records from the default key-value store, replacing any read before.""" + """Read the records from the default key-value store, replacing any read before. + + The starts without a name are counted from zero again, since the code of this Actor run starts over after a + migration or resurrection, and so do the implicit keys. + """ async with self._lock: + self._implicit_starts.clear() key_value_store = await self._open_key_value_store() self._key_value_store = key_value_store stored = await key_value_store.get_value(CHILD_RUNS_KEY) @@ -335,10 +363,6 @@ async def load(self) -> dict[str, ChildRunRecord]: f'The child run registry under the "{CHILD_RUNS_KEY}" key in the default key-value store ' 'is malformed.' ) from exc - # A record with a checksum and no `run_name` is a named record written by an older SDK version. - for key, record in self._records.items(): - if record.run_name is None and record.checksum is not None: - record.run_name = key return self._records async def _load(self) -> dict[str, ChildRunRecord]: @@ -349,10 +373,10 @@ def _loaded_key_value_store(self) -> KeyValueStore: raise RuntimeError('The child run registry was not loaded.') return self._key_value_store - async def _save(self, name: str, record: ChildRunRecord) -> None: + async def _save(self, key: str, record: ChildRunRecord) -> None: records = await self._load() async with self._lock: - records[name] = record + records[key] = record await self._loaded_key_value_store().set_value( CHILD_RUNS_KEY, _records_adapter.dump_python(records, by_alias=True, mode='json') ) diff --git a/tests/unit/actor/test_actor_child_runs.py b/tests/unit/actor/test_actor_child_runs.py index f10e7bd6..9f4f8c7e 100644 --- a/tests/unit/actor/test_actor_child_runs.py +++ b/tests/unit/actor/test_actor_child_runs.py @@ -70,14 +70,29 @@ def stored_record( } -def stored_unnamed_record(run_id: str, status: str) -> dict[str, Any]: +def implicit_key( + *, + actor_id: str | None = 'some-actor', + task_id: str | None = None, + run_input: Any = None, + run_options: dict[str, Any] | None = None, + index: int = 0, +) -> str: + """Build the key a start without a name is recorded under, given how many identical starts came before it.""" + request_hash = checksum_request(actor_id=actor_id, task_id=task_id, run_input=run_input, run_options=run_options) + return f'{request_hash[:16]}-{index}' + + +def stored_unnamed_record( + run_id: str, status: str, *, actor_id: str | None = 'some-actor', task_id: str | None = None +) -> dict[str, Any]: """Build a record of a child run started without a name, as it is stored in the default KVS.""" return { 'runId': run_id, 'status': status, 'startedAt': STARTED_AT, 'runName': None, - 'checksum': None, + 'checksum': checksum_request(actor_id=actor_id, task_id=task_id, run_input=None), 'history': [], 'abortWithParent': False, } @@ -120,16 +135,16 @@ async def test_named_start_records_run_in_kvs(apify_client_async_patcher: ApifyC @pytest.mark.parametrize( - ('method', 'resource'), + ('method', 'resource', 'ids'), [ - pytest.param('start', 'actor', id='start'), - pytest.param('start_task', 'task', id='start task'), + pytest.param('start', 'actor', {'actor_id': 'some-id'}, id='start'), + pytest.param('start_task', 'task', {'actor_id': None, 'task_id': 'some-id'}, id='start task'), ], ) -async def test_unnamed_start_records_run_under_its_id( - apify_client_async_patcher: ApifyClientAsyncPatcher, method: str, resource: str +async def test_unnamed_start_records_run_under_implicit_key( + apify_client_async_patcher: ApifyClientAsyncPatcher, method: str, resource: str, ids: dict[str, Any] ) -> None: - """A start without a name persists the run under its run ID.""" + """A start without a name persists the run under the implicit key of its Actor or task.""" apify_client_async_patcher.patch(resource, 'start', return_value=make_run('new-run', 'READY')) async with Actor: @@ -137,34 +152,112 @@ async def test_unnamed_start_records_run_under_its_id( kvs = await Actor.open_key_value_store() stored = await kvs.get_value(CHILD_RUNS_KEY) - assert stored == {'new-run': stored_unnamed_record('new-run', 'READY')} + assert stored == {implicit_key(**ids): stored_unnamed_record('new-run', 'READY', **ids)} -async def test_unnamed_starts_never_reattach(apify_client_async_patcher: ApifyClientAsyncPatcher) -> None: - """Each start without a name starts a new run, even with a recorded active run of the same Actor.""" +async def test_identical_unnamed_starts_start_separate_runs( + apify_client_async_patcher: ApifyClientAsyncPatcher, +) -> None: + """Identical starts without a name each start a run, recorded under consecutive implicit keys.""" run_ids = iter(['first-run', 'second-run']) apify_client_async_patcher.patch( 'actor', 'start', replacement_method=AsyncMock(side_effect=lambda *_, **__: make_run(next(run_ids), 'READY')) ) + async with Actor: + first = await Actor.start('some-actor') + second = await Actor.start('some-actor') + kvs = await Actor.open_key_value_store() + stored = await kvs.get_value(CHILD_RUNS_KEY) + + assert (first.id, second.id) == ('first-run', 'second-run') + assert stored == { + implicit_key(index=0): stored_unnamed_record('first-run', 'READY'), + implicit_key(index=1): stored_unnamed_record('second-run', 'READY'), + } + + +async def test_unnamed_starts_reattach_in_order_after_migration( + apify_client_async_patcher: ApifyClientAsyncPatcher, +) -> None: + """After a migration, the n-th identical start without a name reattaches to the n-th recorded run.""" + apify_client_async_patcher.patch('actor', 'start', return_value=make_run('new-run', 'READY')) + apify_client_async_patcher.patch( + 'run', 'get', replacement_method=lambda run_client, *_, **__: make_run(run_client.resource_id, 'RUNNING') + ) + + async with Actor: + kvs = await Actor.open_key_value_store() + await kvs.set_value( + CHILD_RUNS_KEY, + { + implicit_key(index=0): stored_unnamed_record('first-run', 'RUNNING'), + implicit_key(index=1): stored_unnamed_record('second-run', 'RUNNING'), + }, + ) + await Actor._child_run_registry.load() + runs = [await Actor.start('some-actor') for _ in range(3)] + + assert [run.id for run in runs] == ['first-run', 'second-run', 'new-run'] + assert len(apify_client_async_patcher.calls['actor']['start']) == 1 + + +async def test_init_counts_unnamed_starts_from_zero(apify_client_async_patcher: ApifyClientAsyncPatcher) -> None: + """A new attempt of the Actor run counts the starts without a name from zero, so its first start reattaches.""" + apify_client_async_patcher.patch('actor', 'start', return_value=make_run('first-run', 'READY')) + apify_client_async_patcher.patch('run', 'get', return_value=make_run('first-run', 'RUNNING')) + async with Actor: await Actor.start('some-actor') + + async with Actor: + run = await Actor.start('some-actor') + + assert run.id == 'first-run' + assert len(apify_client_async_patcher.calls['actor']['start']) == 1 + + +async def test_unnamed_starts_with_different_run_options_get_different_keys( + apify_client_async_patcher: ApifyClientAsyncPatcher, +) -> None: + """The run options are part of the implicit key, and the token is not.""" + run_ids = iter(['default-run', 'big-run', 'token-run']) + apify_client_async_patcher.patch( + 'actor', 'start', replacement_method=AsyncMock(side_effect=lambda *_, **__: make_run(next(run_ids), 'READY')) + ) + + async with Actor: await Actor.start('some-actor') + await Actor.start('some-actor', memory_mbytes=4096) + await Actor.start('some-actor', token='other-token') kvs = await Actor.open_key_value_store() stored = await kvs.get_value(CHILD_RUNS_KEY) - assert stored.keys() == {'first-run', 'second-run'} + assert {key: record['runId'] for key, record in stored.items()} == { + implicit_key(index=0): 'default-run', + implicit_key(run_options={'memory_mbytes': 4096}): 'big-run', + implicit_key(index=1): 'token-run', + } + + +def test_checksum_ignores_run_options_set_to_none() -> None: + """Run options set to `None` do not change the request hash.""" + without_options = checksum_request(actor_id='some-actor', task_id=None, run_input={'a': 1}) + with_none_options = checksum_request( + actor_id='some-actor', task_id=None, run_input={'a': 1}, run_options={'build': None, 'memory_mbytes': None} + ) + assert with_none_options == without_options @pytest.mark.parametrize( - ('method', 'resource'), + ('method', 'resource', 'ids'), [ - pytest.param('call', 'actor', id='call'), - pytest.param('call_task', 'task', id='call task'), + pytest.param('call', 'actor', {'actor_id': 'some-id'}, id='call'), + pytest.param('call_task', 'task', {'actor_id': None, 'task_id': 'some-id'}, id='call task'), ], ) async def test_unnamed_call_records_finished_status( - apify_client_async_patcher: ApifyClientAsyncPatcher, method: str, resource: str + apify_client_async_patcher: ApifyClientAsyncPatcher, method: str, resource: str, ids: dict[str, Any] ) -> None: """A call without a name records the run when it starts and stores its status once it finishes.""" apify_client_async_patcher.patch(resource, 'start', return_value=make_run('new-run', 'READY')) @@ -177,7 +270,7 @@ async def test_unnamed_call_records_finished_status( stored = await kvs.get_value(CHILD_RUNS_KEY) assert run.status == 'SUCCEEDED' - assert stored == {'new-run': stored_unnamed_record('new-run', 'SUCCEEDED')} + assert stored == {implicit_key(**ids): stored_unnamed_record('new-run', 'SUCCEEDED', **ids)} assert apify_client_async_patcher.calls[resource]['call'] == [] @@ -744,25 +837,6 @@ async def test_reattached_run_status_is_recorded(apify_client_async_patcher: Api assert stored == {'scrape-eu': stored_record('old-run', 'SUCCEEDED')} -async def test_record_without_run_name_gets_its_key_as_run_name( - apify_client_async_patcher: ApifyClientAsyncPatcher, -) -> None: - """A named record stored without `runName` is reattached and stored again with its key as the run name.""" - apify_client_async_patcher.patch('run', 'get', return_value=make_run('old-run', 'SUCCEEDED')) - record = stored_record('old-run', 'RUNNING') - del record['runName'] - - async with Actor: - kvs = await Actor.open_key_value_store() - await kvs.set_value(CHILD_RUNS_KEY, {'scrape-eu': record}) - await Actor._child_run_registry.load() - run = await Actor.start('some-actor', run_name='scrape-eu') - stored = await kvs.get_value(CHILD_RUNS_KEY) - - assert run.id == 'old-run' - assert stored == {'scrape-eu': stored_record('old-run', 'SUCCEEDED')} - - async def test_named_call_records_finished_status(apify_client_async_patcher: ApifyClientAsyncPatcher) -> None: """A named call stores the status of the run once it finishes.""" apify_client_async_patcher.patch('actor', 'start', return_value=make_run('new-run', 'READY')) @@ -782,8 +856,8 @@ async def test_child_runs_is_empty_without_child_runs() -> None: assert Actor.child_runs == {} -async def test_child_runs_keys_unnamed_run_by_run_id(apify_client_async_patcher: ApifyClientAsyncPatcher) -> None: - """A run started without a name shows up in `Actor.child_runs` under its run ID, next to the named runs.""" +async def test_child_runs_keys_unnamed_run_by_implicit_key(apify_client_async_patcher: ApifyClientAsyncPatcher) -> None: + """A run started without a name shows up in `Actor.child_runs` under its implicit key, next to the named runs.""" run_ids = iter(['unnamed-run', 'named-run']) apify_client_async_patcher.patch( 'actor', 'start', replacement_method=AsyncMock(side_effect=lambda *_, **__: make_run(next(run_ids), 'READY')) @@ -795,7 +869,7 @@ async def test_child_runs_keys_unnamed_run_by_run_id(apify_client_async_patcher: child_runs = Actor.child_runs assert {key: run_client.resource_id for key, run_client in child_runs.items()} == { - 'unnamed-run': 'unnamed-run', + implicit_key(): 'unnamed-run', 'scrape-eu': 'named-run', } @@ -817,7 +891,7 @@ async def test_child_runs_includes_runs_recorded_before_init() -> None: 'scrape-us': stored_record( 'us-run', 'FAILED', run_name='scrape-us', actor_id=None, task_id='some-task' ), - 'unnamed-run': stored_unnamed_record('unnamed-run', 'RUNNING'), + implicit_key(): stored_unnamed_record('unnamed-run', 'RUNNING'), }, ) @@ -827,7 +901,7 @@ async def test_child_runs_includes_runs_recorded_before_init() -> None: assert {name: run_client.resource_id for name, run_client in child_runs.items()} == { 'scrape-eu': 'eu-run', 'scrape-us': 'us-run', - 'unnamed-run': 'unnamed-run', + implicit_key(): 'unnamed-run', } @@ -891,7 +965,7 @@ def recording_new_client(self: _ActorType, **kwargs: Any) -> ApifyClientAsync: await Actor.start('some-actor', token='custom-token') child_runs = Actor.child_runs - assert child_runs['custom-run']._http_client is clients_by_token['custom-token']._http_client + assert child_runs[implicit_key()]._http_client is clients_by_token['custom-token']._http_client async def test_child_runs_keeps_client_of_name_after_rejected_reuse( From 29081e590e3bba67f48f6be1e573d87d41cf2605 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Thu, 12 Nov 2026 12:05:00 +0100 Subject: [PATCH 5/7] feat: reuse the child run of an identical call without a `run_name` --- .rules.md | 2 +- src/apify/_actor.py | 24 ++++---- src/apify/_child_runs.py | 20 ++----- tests/unit/actor/test_actor_child_runs.py | 73 +++++++---------------- 4 files changed, 40 insertions(+), 79 deletions(-) diff --git a/.rules.md b/.rules.md index 388a018f..90c05491 100644 --- a/.rules.md +++ b/.rules.md @@ -22,7 +22,7 @@ Don't introduce a shared Pydantic base model to centralize `model_config`. Each - **`Actor`** (`_actor.py`) is a `lazy-object-proxy` wrapping `_ActorType`. It works both as a class (`Actor.is_at_home()`) and as a context manager (`async with Actor:`); on `__aenter__` the proxy's `__wrapped__` is replaced with the active `_ActorType` instance. - **`Configuration`** (`_configuration.py`) extends Crawlee's `Configuration` and is populated from `APIFY_*` environment variables. -- **Child runs** (`_child_runs.py`) — runs are persisted in the default key-value store under their `run_name`, or without one under an implicit key (a hash of the Actor or task, input and run options, plus the number of identical starts before it), so they can be reattached after a migration or resurrection. All of them are exposed through `Actor.child_runs`. +- **Child runs** (`_child_runs.py`) — runs are persisted in the default key-value store under their `run_name`, or without one under an implicit key (a hash of the Actor or task, input and run options, so identical calls reuse one run), so they can be reattached after a migration or resurrection. All of them are exposed through `Actor.child_runs`. - **Storage clients** (`storage_clients/`) implement Crawlee's storage client interface: `ApifyStorageClient` talks to the Apify API (separate sub-clients for single vs. shared request queues), `FileSystemStorageClient` adds Apify-specific key-value store behavior, and `SmartApifyStorageClient` delegates to the cloud client on the platform and to the local one elsewhere. - **Events** — `ApifyEventManager` receives platform events over a WebSocket. - **Errors** — `apify.errors` only re-exports the exceptions from `apify_client.errors`; the SDK defines no error classes of its own, and API calls raise the client exceptions unchanged. diff --git a/src/apify/_actor.py b/src/apify/_actor.py index 2510a46a..874c6184 100644 --- a/src/apify/_actor.py +++ b/src/apify/_actor.py @@ -1104,9 +1104,9 @@ async def start( resurrected, and a new run is started only when nothing is recorded under the name, or the recorded run `FAILED` or no longer exists. Using `run_name` to attach to a run of a different Actor, task or with a different input raises a `ValueError`. A run started without `run_name` is recorded under an - implicit key computed from the Actor or task, the input, the run options and the number of identical - calls without `run_name` before it, so after a migration the n-th identical call reattaches to the - n-th recorded run. + implicit key, a hash of the Actor or task, the input and the run options, and the same rules apply + to it. So an identical call without `run_name` reuses that run, also before a migration. To start a + new run for an identical call, e.g. when polling, pass a `run_name` that differs between the calls. abort_with_parent: If true, the child run is gracefully aborted when this Actor run is gracefully aborted. It requires `run_name`, and the value is recorded under it, replacing the one from an earlier call. Once this Actor run is being gracefully aborted, a call with it raises a `RuntimeError` and @@ -1308,9 +1308,9 @@ async def call( resurrected, and a new run is started only when nothing is recorded under the name, or the recorded run `FAILED` or no longer exists. Using `run_name` to attach to a run of a different Actor, task or with a different input raises a `ValueError`. A run started without `run_name` is recorded under an - implicit key computed from the Actor or task, the input, the run options and the number of identical - calls without `run_name` before it, so after a migration the n-th identical call reattaches to the - n-th recorded run. + implicit key, a hash of the Actor or task, the input and the run options, and the same rules apply + to it. So an identical call without `run_name` reuses that run, also before a migration. To start a + new run for an identical call, e.g. when polling, pass a `run_name` that differs between the calls. abort_with_parent: If true, the child run is gracefully aborted when this Actor run is gracefully aborted. It requires `run_name`, and the value is recorded under it, replacing the one from an earlier call. Once this Actor run is being gracefully aborted, a call with it raises a `RuntimeError` and @@ -1500,9 +1500,9 @@ async def start_task( resurrected, and a new run is started only when nothing is recorded under the name, or the recorded run `FAILED` or no longer exists. Using `run_name` to attach to a run of a different Actor, task or with a different input raises a `ValueError`. A run started without `run_name` is recorded under an - implicit key computed from the Actor or task, the input, the run options and the number of identical - calls without `run_name` before it, so after a migration the n-th identical call reattaches to the - n-th recorded run. + implicit key, a hash of the Actor or task, the input and the run options, and the same rules apply + to it. So an identical call without `run_name` reuses that run, also before a migration. To start a + new run for an identical call, e.g. when polling, pass a `run_name` that differs between the calls. abort_with_parent: If true, the child run is gracefully aborted when this Actor run is gracefully aborted. It requires `run_name`, and the value is recorded under it, replacing the one from an earlier call. Once this Actor run is being gracefully aborted, a call with it raises a `RuntimeError` and @@ -1601,9 +1601,9 @@ async def call_task( resurrected, and a new run is started only when nothing is recorded under the name, or the recorded run `FAILED` or no longer exists. Using `run_name` to attach to a run of a different Actor, task or with a different input raises a `ValueError`. A run started without `run_name` is recorded under an - implicit key computed from the Actor or task, the input, the run options and the number of identical - calls without `run_name` before it, so after a migration the n-th identical call reattaches to the - n-th recorded run. + implicit key, a hash of the Actor or task, the input and the run options, and the same rules apply + to it. So an identical call without `run_name` reuses that run, also before a migration. To start a + new run for an identical call, e.g. when polling, pass a `run_name` that differs between the calls. abort_with_parent: If true, the child run is gracefully aborted when this Actor run is gracefully aborted. It requires `run_name`, and the value is recorded under it, replacing the one from an earlier call. Once this Actor run is being gracefully aborted, a call with it raises a `RuntimeError` and diff --git a/src/apify/_child_runs.py b/src/apify/_child_runs.py index 20a70cfc..c29322dd 100644 --- a/src/apify/_child_runs.py +++ b/src/apify/_child_runs.py @@ -3,7 +3,6 @@ import asyncio import hashlib import json -from collections import Counter from datetime import datetime from logging import getLogger from typing import TYPE_CHECKING, Any @@ -117,9 +116,8 @@ class ChildRunRegistry: """Persisted record of the child runs of this Actor run, which survives a migration or resurrection of the parent. A named child run is recorded under its name, so a later start under the same name can reattach to it. A child run - started without a name is recorded under an implicit key: a hash of its Actor or task, input and run options, and - the number of identical starts without a name before it in this process, as in `3f2a9c0e1b7d4a65-0`. After a - migration, the starts run again in the same order, so the n-th identical start reattaches to the n-th recorded run. + started without a name is recorded under an implicit key, a hash of its Actor or task, input and run options, so + every identical start without a name reuses the same run, also after a migration. Every change is written to the key-value store right away. A hard kill of the parent between the platform starting the child and that write can still orphan the child, since nothing but the platform knows about it. """ @@ -139,8 +137,6 @@ def __init__(self, open_key_value_store: Callable[[], Awaitable[KeyValueStore]]) """Serializes `find_or_start` per key. A lock is dropped once no call under its key holds it.""" self._clients: dict[str, ApifyClientAsync] = {} """Client the run under each key was last started or looked up with. Lost on a migration.""" - self._implicit_starts: Counter[str] = Counter() - """Number of starts without a name in this process, by the hash part of their implicit key.""" self._parent_aborting = False """Whether this Actor run is being gracefully aborted, after which no run marked `abort_with_parent` starts.""" @@ -190,9 +186,8 @@ async def find_or_start( if key is None: request_hash = checksum_request( actor_id=actor_id, task_id=task_id, run_input=run_input, run_options=run_options - )[:_IMPLICIT_KEY_HASH_LENGTH] - key = f'{request_hash}-{self._implicit_starts[request_hash]}' - self._implicit_starts[request_hash] += 1 + ) + key = request_hash[:_IMPLICIT_KEY_HASH_LENGTH] async with self._name_locks.setdefault(key, asyncio.Lock()): if abort_with_parent and self._parent_aborting: @@ -346,13 +341,8 @@ async def _start( return run async def load(self) -> dict[str, ChildRunRecord]: - """Read the records from the default key-value store, replacing any read before. - - The starts without a name are counted from zero again, since the code of this Actor run starts over after a - migration or resurrection, and so do the implicit keys. - """ + """Read the records from the default key-value store, replacing any read before.""" async with self._lock: - self._implicit_starts.clear() key_value_store = await self._open_key_value_store() self._key_value_store = key_value_store stored = await key_value_store.get_value(CHILD_RUNS_KEY) diff --git a/tests/unit/actor/test_actor_child_runs.py b/tests/unit/actor/test_actor_child_runs.py index 9f4f8c7e..57efc3d0 100644 --- a/tests/unit/actor/test_actor_child_runs.py +++ b/tests/unit/actor/test_actor_child_runs.py @@ -76,11 +76,10 @@ def implicit_key( task_id: str | None = None, run_input: Any = None, run_options: dict[str, Any] | None = None, - index: int = 0, ) -> str: - """Build the key a start without a name is recorded under, given how many identical starts came before it.""" + """Build the key a start without a name is recorded under.""" request_hash = checksum_request(actor_id=actor_id, task_id=task_id, run_input=run_input, run_options=run_options) - return f'{request_hash[:16]}-{index}' + return request_hash[:16] def stored_unnamed_record( @@ -155,14 +154,10 @@ async def test_unnamed_start_records_run_under_implicit_key( assert stored == {implicit_key(**ids): stored_unnamed_record('new-run', 'READY', **ids)} -async def test_identical_unnamed_starts_start_separate_runs( - apify_client_async_patcher: ApifyClientAsyncPatcher, -) -> None: - """Identical starts without a name each start a run, recorded under consecutive implicit keys.""" - run_ids = iter(['first-run', 'second-run']) - apify_client_async_patcher.patch( - 'actor', 'start', replacement_method=AsyncMock(side_effect=lambda *_, **__: make_run(next(run_ids), 'READY')) - ) +async def test_identical_unnamed_starts_reuse_one_run(apify_client_async_patcher: ApifyClientAsyncPatcher) -> None: + """An identical start without a name reattaches to the run of the first one instead of starting another.""" + apify_client_async_patcher.patch('actor', 'start', return_value=make_run('first-run', 'READY')) + apify_client_async_patcher.patch('run', 'get', return_value=make_run('first-run', 'RUNNING')) async with Actor: first = await Actor.start('some-actor') @@ -170,73 +165,49 @@ async def test_identical_unnamed_starts_start_separate_runs( kvs = await Actor.open_key_value_store() stored = await kvs.get_value(CHILD_RUNS_KEY) - assert (first.id, second.id) == ('first-run', 'second-run') - assert stored == { - implicit_key(index=0): stored_unnamed_record('first-run', 'READY'), - implicit_key(index=1): stored_unnamed_record('second-run', 'READY'), - } + assert (first.id, second.id) == ('first-run', 'first-run') + assert len(apify_client_async_patcher.calls['actor']['start']) == 1 + assert stored == {implicit_key(): stored_unnamed_record('first-run', 'RUNNING')} -async def test_unnamed_starts_reattach_in_order_after_migration( - apify_client_async_patcher: ApifyClientAsyncPatcher, -) -> None: - """After a migration, the n-th identical start without a name reattaches to the n-th recorded run.""" +async def test_unnamed_start_reattaches_after_migration(apify_client_async_patcher: ApifyClientAsyncPatcher) -> None: + """After a migration, a start without a name reattaches to the run recorded under its implicit key.""" apify_client_async_patcher.patch('actor', 'start', return_value=make_run('new-run', 'READY')) - apify_client_async_patcher.patch( - 'run', 'get', replacement_method=lambda run_client, *_, **__: make_run(run_client.resource_id, 'RUNNING') - ) + apify_client_async_patcher.patch('run', 'get', return_value=make_run('old-run', 'RUNNING')) async with Actor: kvs = await Actor.open_key_value_store() - await kvs.set_value( - CHILD_RUNS_KEY, - { - implicit_key(index=0): stored_unnamed_record('first-run', 'RUNNING'), - implicit_key(index=1): stored_unnamed_record('second-run', 'RUNNING'), - }, - ) + await kvs.set_value(CHILD_RUNS_KEY, {implicit_key(): stored_unnamed_record('old-run', 'RUNNING')}) await Actor._child_run_registry.load() - runs = [await Actor.start('some-actor') for _ in range(3)] - - assert [run.id for run in runs] == ['first-run', 'second-run', 'new-run'] - assert len(apify_client_async_patcher.calls['actor']['start']) == 1 - - -async def test_init_counts_unnamed_starts_from_zero(apify_client_async_patcher: ApifyClientAsyncPatcher) -> None: - """A new attempt of the Actor run counts the starts without a name from zero, so its first start reattaches.""" - apify_client_async_patcher.patch('actor', 'start', return_value=make_run('first-run', 'READY')) - apify_client_async_patcher.patch('run', 'get', return_value=make_run('first-run', 'RUNNING')) - - async with Actor: - await Actor.start('some-actor') - - async with Actor: run = await Actor.start('some-actor') - assert run.id == 'first-run' - assert len(apify_client_async_patcher.calls['actor']['start']) == 1 + assert run.id == 'old-run' + assert apify_client_async_patcher.calls['actor']['start'] == [] async def test_unnamed_starts_with_different_run_options_get_different_keys( apify_client_async_patcher: ApifyClientAsyncPatcher, ) -> None: """The run options are part of the implicit key, and the token is not.""" - run_ids = iter(['default-run', 'big-run', 'token-run']) + run_ids = iter(['default-run', 'big-run']) apify_client_async_patcher.patch( 'actor', 'start', replacement_method=AsyncMock(side_effect=lambda *_, **__: make_run(next(run_ids), 'READY')) ) + apify_client_async_patcher.patch( + 'run', 'get', replacement_method=lambda run_client, *_, **__: make_run(run_client.resource_id, 'RUNNING') + ) async with Actor: await Actor.start('some-actor') await Actor.start('some-actor', memory_mbytes=4096) - await Actor.start('some-actor', token='other-token') + token_run = await Actor.start('some-actor', token='other-token') kvs = await Actor.open_key_value_store() stored = await kvs.get_value(CHILD_RUNS_KEY) + assert token_run.id == 'default-run' assert {key: record['runId'] for key, record in stored.items()} == { - implicit_key(index=0): 'default-run', + implicit_key(): 'default-run', implicit_key(run_options={'memory_mbytes': 4096}): 'big-run', - implicit_key(index=1): 'token-run', } From dbb6ea7489d72f03b2315fa669ada3ed6795ce5b Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Thu, 12 Nov 2026 12:14:26 +0100 Subject: [PATCH 6/7] feat: drop `startedAt` from child run records and hash the run options into `checksum` --- src/apify/_actor.py | 36 ++++++++------ src/apify/_child_runs.py | 32 +++++------- tests/unit/actor/test_actor_child_runs.py | 60 ++++++++++++++++------- 3 files changed, 73 insertions(+), 55 deletions(-) diff --git a/src/apify/_actor.py b/src/apify/_actor.py index 874c6184..8e4cb431 100644 --- a/src/apify/_actor.py +++ b/src/apify/_actor.py @@ -1103,10 +1103,11 @@ async def start( to the recorded run. A `SUCCEEDED` run is returned as is, an `ABORTED` or `TIMED-OUT` one is resurrected, and a new run is started only when nothing is recorded under the name, or the recorded run `FAILED` or no longer exists. Using `run_name` to attach to a run of a different Actor, task or - with a different input raises a `ValueError`. A run started without `run_name` is recorded under an - implicit key, a hash of the Actor or task, the input and the run options, and the same rules apply - to it. So an identical call without `run_name` reuses that run, also before a migration. To start a - new run for an identical call, e.g. when polling, pass a `run_name` that differs between the calls. + with a different input or run options raises a `ValueError`. A run started without `run_name` is + recorded under an implicit key, a hash of the Actor or task, the input and the run options, and the + same rules apply to it. So an identical call without `run_name` reuses that run, also before a + migration. To start a new run for an identical call, e.g. when polling, pass a `run_name` that differs + between the calls. abort_with_parent: If true, the child run is gracefully aborted when this Actor run is gracefully aborted. It requires `run_name`, and the value is recorded under it, replacing the one from an earlier call. Once this Actor run is being gracefully aborted, a call with it raises a `RuntimeError` and @@ -1307,10 +1308,11 @@ async def call( to the recorded run. A `SUCCEEDED` run is returned as is, an `ABORTED` or `TIMED-OUT` one is resurrected, and a new run is started only when nothing is recorded under the name, or the recorded run `FAILED` or no longer exists. Using `run_name` to attach to a run of a different Actor, task or - with a different input raises a `ValueError`. A run started without `run_name` is recorded under an - implicit key, a hash of the Actor or task, the input and the run options, and the same rules apply - to it. So an identical call without `run_name` reuses that run, also before a migration. To start a - new run for an identical call, e.g. when polling, pass a `run_name` that differs between the calls. + with a different input or run options raises a `ValueError`. A run started without `run_name` is + recorded under an implicit key, a hash of the Actor or task, the input and the run options, and the + same rules apply to it. So an identical call without `run_name` reuses that run, also before a + migration. To start a new run for an identical call, e.g. when polling, pass a `run_name` that differs + between the calls. abort_with_parent: If true, the child run is gracefully aborted when this Actor run is gracefully aborted. It requires `run_name`, and the value is recorded under it, replacing the one from an earlier call. Once this Actor run is being gracefully aborted, a call with it raises a `RuntimeError` and @@ -1499,10 +1501,11 @@ async def start_task( to the recorded run. A `SUCCEEDED` run is returned as is, an `ABORTED` or `TIMED-OUT` one is resurrected, and a new run is started only when nothing is recorded under the name, or the recorded run `FAILED` or no longer exists. Using `run_name` to attach to a run of a different Actor, task or - with a different input raises a `ValueError`. A run started without `run_name` is recorded under an - implicit key, a hash of the Actor or task, the input and the run options, and the same rules apply - to it. So an identical call without `run_name` reuses that run, also before a migration. To start a - new run for an identical call, e.g. when polling, pass a `run_name` that differs between the calls. + with a different input or run options raises a `ValueError`. A run started without `run_name` is + recorded under an implicit key, a hash of the Actor or task, the input and the run options, and the + same rules apply to it. So an identical call without `run_name` reuses that run, also before a + migration. To start a new run for an identical call, e.g. when polling, pass a `run_name` that differs + between the calls. abort_with_parent: If true, the child run is gracefully aborted when this Actor run is gracefully aborted. It requires `run_name`, and the value is recorded under it, replacing the one from an earlier call. Once this Actor run is being gracefully aborted, a call with it raises a `RuntimeError` and @@ -1600,10 +1603,11 @@ async def call_task( to the recorded run. A `SUCCEEDED` run is returned as is, an `ABORTED` or `TIMED-OUT` one is resurrected, and a new run is started only when nothing is recorded under the name, or the recorded run `FAILED` or no longer exists. Using `run_name` to attach to a run of a different Actor, task or - with a different input raises a `ValueError`. A run started without `run_name` is recorded under an - implicit key, a hash of the Actor or task, the input and the run options, and the same rules apply - to it. So an identical call without `run_name` reuses that run, also before a migration. To start a - new run for an identical call, e.g. when polling, pass a `run_name` that differs between the calls. + with a different input or run options raises a `ValueError`. A run started without `run_name` is + recorded under an implicit key, a hash of the Actor or task, the input and the run options, and the + same rules apply to it. So an identical call without `run_name` reuses that run, also before a + migration. To start a new run for an identical call, e.g. when polling, pass a `run_name` that differs + between the calls. abort_with_parent: If true, the child run is gracefully aborted when this Actor run is gracefully aborted. It requires `run_name`, and the value is recorded under it, replacing the one from an earlier call. Once this Actor run is being gracefully aborted, a call with it raises a `RuntimeError` and diff --git a/src/apify/_child_runs.py b/src/apify/_child_runs.py index c29322dd..f2ca6808 100644 --- a/src/apify/_child_runs.py +++ b/src/apify/_child_runs.py @@ -3,7 +3,6 @@ import asyncio import hashlib import json -from datetime import datetime from logging import getLogger from typing import TYPE_CHECKING, Any from weakref import WeakValueDictionary @@ -54,9 +53,6 @@ class ChildRunSnapshot(BaseModel): status: str """Last status of the run observed by this Actor run, or `LOST` once the platform no longer returned it.""" - started_at: datetime - """When the run started.""" - class ChildRunRecord(ChildRunSnapshot): """The current run tracked under a name or implicit key in the child run registry, with the runs it replaced.""" @@ -65,7 +61,7 @@ class ChildRunRecord(ChildRunSnapshot): """Name the run was started under, or `None` for a run started without a name.""" checksum: str - """Hash of the Actor or task and the input the current run was started with.""" + """Hash of the Actor or task, the input and the run options the current run was started with.""" history: list[ChildRunSnapshot] = Field(default_factory=list) """Earlier runs under this name that failed or went missing and were replaced by a new run, oldest first.""" @@ -157,16 +153,16 @@ async def find_or_start( A recorded run that is `READY` or `RUNNING` is reattached and one that `SUCCEEDED` is returned as is. An `ABORTED` or `TIMED-OUT` run is resurrected, since Actors are expected to resume from their state. - A `FAILED` run, or one the API no longer knows, is replaced by a new run under the same name, for any Actor, - task or input. + A `FAILED` run, or one the API no longer knows, is replaced by a new run under the same key, for any Actor, + task, input or run options. Args: name: Name of the child run, unique within the parent run, or `None` to use the implicit key. actor_id: The Actor to start. task_id: The task to start, in place of `actor_id`. - run_input: Input of the run. The recorded run is reused only if it was started with the same Actor or task - and input. - run_options: Options the run is started with. Without `name`, they are part of the implicit key. + run_input: Input of the run. The recorded run is reused only if it was started with the same Actor or task, + input and run options. + run_options: Options the run is started with. client: Client used to look up the recorded run. start_run: Starts a new run of the Actor or task. resurrect_run: Resurrects the recorded run, given its ID. @@ -177,17 +173,13 @@ async def find_or_start( The run, and whether it was newly started. Raises: - ValueError: If the recorded run would be reused, but was started for a different Actor, task or input. + ValueError: If the recorded run would be reused, but was started for a different Actor, task, input or + run options. RuntimeError: If `abort_with_parent` is set and this Actor run is being gracefully aborted. No run is started, resurrected or reattached. """ - checksum = checksum_request(actor_id=actor_id, task_id=task_id, run_input=run_input) - key = name - if key is None: - request_hash = checksum_request( - actor_id=actor_id, task_id=task_id, run_input=run_input, run_options=run_options - ) - key = request_hash[:_IMPLICIT_KEY_HASH_LENGTH] + checksum = checksum_request(actor_id=actor_id, task_id=task_id, run_input=run_input, run_options=run_options) + key = name if name is not None else checksum[:_IMPLICIT_KEY_HASH_LENGTH] async with self._name_locks.setdefault(key, asyncio.Lock()): if abort_with_parent and self._parent_aborting: @@ -221,7 +213,6 @@ async def find_or_start( replaced = ChildRunSnapshot( run_id=record.run_id, status=run.status if run is not None else 'LOST', - started_at=record.started_at, ) run = await self._start( key, @@ -236,7 +227,7 @@ async def find_or_start( if record.checksum != checksum: raise ValueError( - f'The run name "{key}" was already used for a different Actor, task or input. ' + f'The run name "{key}" was already used for a different Actor, task, input or run options. ' 'Use a unique `run_name` for each child run.' ) @@ -331,7 +322,6 @@ async def _start( record = ChildRunRecord( run_id=run.id, status=run.status, - started_at=run.started_at, run_name=run_name, checksum=checksum, history=history, diff --git a/tests/unit/actor/test_actor_child_runs.py b/tests/unit/actor/test_actor_child_runs.py index 57efc3d0..b4ad217f 100644 --- a/tests/unit/actor/test_actor_child_runs.py +++ b/tests/unit/actor/test_actor_child_runs.py @@ -55,16 +55,17 @@ def stored_record( actor_id: str | None = 'some-actor', task_id: str | None = None, run_input: Any = None, + run_options: dict[str, Any] | None = None, history: list[dict[str, str]] | None = None, abort_with_parent: bool = False, ) -> dict[str, Any]: """Build a registry record as it is stored in the default KVS.""" + checksum = checksum_request(actor_id=actor_id, task_id=task_id, run_input=run_input, run_options=run_options) return { 'runId': run_id, 'status': status, - 'startedAt': STARTED_AT, 'runName': run_name, - 'checksum': checksum_request(actor_id=actor_id, task_id=task_id, run_input=run_input), + 'checksum': checksum, 'history': history or [], 'abortWithParent': abort_with_parent, } @@ -89,7 +90,6 @@ def stored_unnamed_record( return { 'runId': run_id, 'status': status, - 'startedAt': STARTED_AT, 'runName': None, 'checksum': checksum_request(actor_id=actor_id, task_id=task_id, run_input=None), 'history': [], @@ -112,10 +112,19 @@ async def record_child_run( actor_id: str | None = 'some-actor', task_id: str | None = None, run_input: Any = None, + run_options: dict[str, Any] | None = None, ) -> None: """Seed the registry the way an earlier attempt of this Actor run would have left it, and reload it like init.""" kvs = await Actor.open_key_value_store() - record = stored_record(run_id, 'RUNNING', run_name=name, actor_id=actor_id, task_id=task_id, run_input=run_input) + record = stored_record( + run_id, + 'RUNNING', + run_name=name, + actor_id=actor_id, + task_id=task_id, + run_input=run_input, + run_options=run_options, + ) await kvs.set_value(CHILD_RUNS_KEY, {name: record}) await Actor._child_run_registry.load() @@ -301,7 +310,7 @@ async def test_named_start_resurrects_recorded_run( apify_client_async_patcher.patch('run', 'resurrect', return_value=make_run('old-run', 'RUNNING')) async with Actor: - await record_child_run('scrape-eu', 'old-run') + await record_child_run('scrape-eu', 'old-run', run_options={'memory_mbytes': 2048}) run = await Actor.start('some-actor', run_name='scrape-eu', memory_mbytes=2048) assert run.id == 'old-run' @@ -363,9 +372,7 @@ async def test_named_start_replaces_failed_or_missing_run( assert run.id == 'new-run' assert stored == { - 'scrape-eu': stored_record( - 'new-run', 'READY', history=[{'runId': 'old-run', 'status': replaced_status, 'startedAt': STARTED_AT}] - ) + 'scrape-eu': stored_record('new-run', 'READY', history=[{'runId': 'old-run', 'status': replaced_status}]) } @@ -378,12 +385,27 @@ async def test_named_start_rejects_name_recorded_for_another_actor( async with Actor: await record_child_run('scrape-eu', 'old-run', actor_id='other-actor') - with pytest.raises(ValueError, match='already used for a different Actor, task or input'): + with pytest.raises(ValueError, match='already used for a different Actor, task, input or run options'): await Actor.start('some-actor', run_name='scrape-eu') assert apify_client_async_patcher.calls['actor']['start'] == [] +async def test_named_start_rejects_name_recorded_with_other_run_options( + apify_client_async_patcher: ApifyClientAsyncPatcher, +) -> None: + """Reusing a name with different run options raises instead of attaching to the recorded run.""" + apify_client_async_patcher.patch('actor', 'start', return_value=make_run('new-run', 'READY')) + apify_client_async_patcher.patch('run', 'get', return_value=make_run('old-run', 'RUNNING')) + + async with Actor: + await record_child_run('scrape-eu', 'old-run', run_options={'memory_mbytes': 1024}) + with pytest.raises(ValueError, match='already used for a different Actor, task, input or run options'): + await Actor.start('some-actor', run_name='scrape-eu', memory_mbytes=4096) + + assert apify_client_async_patcher.calls['actor']['start'] == [] + + async def test_named_start_rejects_name_recorded_for_task( apify_client_async_patcher: ApifyClientAsyncPatcher, ) -> None: @@ -393,7 +415,7 @@ async def test_named_start_rejects_name_recorded_for_task( async with Actor: await record_child_run('scrape-eu', 'old-run', actor_id=None, task_id='some-task') - with pytest.raises(ValueError, match='already used for a different Actor, task or input'): + with pytest.raises(ValueError, match='already used for a different Actor, task, input or run options'): await Actor.start('some-actor', run_name='scrape-eu') assert apify_client_async_patcher.calls['actor']['start'] == [] @@ -506,7 +528,7 @@ async def test_named_call_waits_for_resurrected_run(apify_client_async_patcher: apify_client_async_patcher.patch('run', 'get_streamed_log', replacement_method=get_streamed_log) async with Actor: - await record_child_run('scrape-eu', 'old-run') + await record_child_run('scrape-eu', 'old-run', run_options={'timeout': timedelta(minutes=5)}) run = await Actor.call('some-actor', run_name='scrape-eu', timeout=timedelta(minutes=5)) assert run.id == 'old-run' @@ -586,7 +608,9 @@ async def test_named_call_task_resurrects_recorded_run(apify_client_async_patche apify_client_async_patcher.patch('run', 'wait_for_finish', return_value=make_run('old-run', 'SUCCEEDED')) async with Actor: - await record_child_run('scrape-eu', 'old-run', actor_id=None, task_id='some-task') + await record_child_run( + 'scrape-eu', 'old-run', actor_id=None, task_id='some-task', run_options={'memory_mbytes': 2048} + ) run = await Actor.call_task('some-task', run_name='scrape-eu', memory_mbytes=2048) assert run.id == 'old-run' @@ -604,7 +628,7 @@ async def test_named_call_task_rejects_name_recorded_for_actor( async with Actor: await record_child_run('scrape-eu', 'old-run') - with pytest.raises(ValueError, match='already used for a different Actor, task or input'): + with pytest.raises(ValueError, match='already used for a different Actor, task, input or run options'): await Actor.call_task('some-task', run_name='scrape-eu') assert apify_client_async_patcher.calls['task']['start'] == [] @@ -634,7 +658,7 @@ async def test_named_start_forwards_max_items_to_resurrect( apify_client_async_patcher.patch('run', 'resurrect', return_value=make_run('old-run', 'RUNNING')) async with Actor: - await record_child_run('scrape-eu', 'old-run') + await record_child_run('scrape-eu', 'old-run', run_options={'max_items': 10}) with pytest.warns(FutureWarning, match='max_items') as warnings: await Actor.start('some-actor', run_name='scrape-eu', max_items=10) @@ -715,7 +739,7 @@ async def test_named_start_rejects_name_recorded_with_other_input( async with Actor: await record_child_run('scrape-eu', 'old-run', run_input={'since': '2025-01-01'}) - with pytest.raises(ValueError, match='already used for a different Actor, task or input'): + with pytest.raises(ValueError, match='already used for a different Actor, task, input or run options'): await Actor.start('some-actor', {'since': '2026-01-01'}, run_name='scrape-eu') assert apify_client_async_patcher.calls['actor']['start'] == [] @@ -750,7 +774,7 @@ async def test_named_start_replaces_run_recorded_for_other_request( 'new-run', 'READY', run_input={'since': '2026-01-01'}, - history=[{'runId': 'old-run', 'status': 'FAILED', 'startedAt': STARTED_AT}], + history=[{'runId': 'old-run', 'status': 'FAILED'}], ) } @@ -948,7 +972,7 @@ async def test_child_runs_keeps_client_of_name_after_rejected_reuse( async with Actor: await Actor.start('some-actor', {'since': '2025-01-01'}, run_name='scrape-eu') - with pytest.raises(ValueError, match='already used for a different Actor, task or input'): + with pytest.raises(ValueError, match='already used for a different Actor, task, input or run options'): await Actor.start('some-actor', {'since': '2026-01-01'}, run_name='scrape-eu', token='other-token') child_runs = Actor.child_runs default_http_client = Actor.apify_client._http_client @@ -1178,7 +1202,7 @@ async def test_rejected_named_start_keeps_the_client_used_to_abort() -> None: kvs = await Actor.open_key_value_store() await kvs.set_value(CHILD_RUNS_KEY, {'scrape-eu': stored_record('old-run', 'RUNNING', abort_with_parent=True)}) registry = ChildRunRegistry(Actor.open_key_value_store) - with pytest.raises(ValueError, match='already used for a different Actor, task or input'): + with pytest.raises(ValueError, match='already used for a different Actor, task, input or run options'): await registry.find_or_start( 'scrape-eu', actor_id='other-actor', From 5ca3e05f080c4b56767f8cbc3f052c666cb8313e Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Thu, 12 Nov 2026 12:15:27 +0100 Subject: [PATCH 7/7] feat: drop `history` from child run records --- src/apify/_child_runs.py | 23 +++----------- tests/unit/actor/test_actor_child_runs.py | 38 ++++++++++++++++------- 2 files changed, 30 insertions(+), 31 deletions(-) diff --git a/src/apify/_child_runs.py b/src/apify/_child_runs.py index f2ca6808..5b6209e0 100644 --- a/src/apify/_child_runs.py +++ b/src/apify/_child_runs.py @@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Any from weakref import WeakValueDictionary -from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from pydantic.alias_generators import to_camel from apify_client.errors import NotFoundError @@ -42,8 +42,8 @@ """Number of hex digits of the request hash kept in the key of a child run started without a name.""" -class ChildRunSnapshot(BaseModel): - """A child run as last observed by this Actor run.""" +class ChildRunRecord(BaseModel): + """The current run tracked under a name or implicit key in the child run registry.""" model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel) @@ -51,11 +51,7 @@ class ChildRunSnapshot(BaseModel): """ID of the run.""" status: str - """Last status of the run observed by this Actor run, or `LOST` once the platform no longer returned it.""" - - -class ChildRunRecord(ChildRunSnapshot): - """The current run tracked under a name or implicit key in the child run registry, with the runs it replaced.""" + """Last status of the run observed by this Actor run.""" run_name: str | None = None """Name the run was started under, or `None` for a run started without a name.""" @@ -63,9 +59,6 @@ class ChildRunRecord(ChildRunSnapshot): checksum: str """Hash of the Actor or task, the input and the run options the current run was started with.""" - history: list[ChildRunSnapshot] = Field(default_factory=list) - """Earlier runs under this name that failed or went missing and were replaced by a new run, oldest first.""" - abort_with_parent: bool = False """Whether the current run is aborted when this Actor run is gracefully aborted.""" @@ -197,7 +190,6 @@ async def find_or_start( run_name=name, checksum=checksum, start_run=start_run, - history=[], abort_with_parent=abort_with_parent, ) self._clients[key] = client @@ -210,16 +202,11 @@ async def find_or_start( run = await run_client.wait_for_finish() if run is None or run.status == 'FAILED': - replaced = ChildRunSnapshot( - run_id=record.run_id, - status=run.status if run is not None else 'LOST', - ) run = await self._start( key, run_name=name, checksum=checksum, start_run=start_run, - history=[*record.history, replaced], abort_with_parent=abort_with_parent, ) self._clients[key] = client @@ -315,7 +302,6 @@ async def _start( run_name: str | None, checksum: str, start_run: Callable[[], Awaitable[Run]], - history: list[ChildRunSnapshot], abort_with_parent: bool, ) -> Run: run = await start_run() @@ -324,7 +310,6 @@ async def _start( status=run.status, run_name=run_name, checksum=checksum, - history=history, abort_with_parent=abort_with_parent, ) await self._save(key, record) diff --git a/tests/unit/actor/test_actor_child_runs.py b/tests/unit/actor/test_actor_child_runs.py index b4ad217f..583d4de8 100644 --- a/tests/unit/actor/test_actor_child_runs.py +++ b/tests/unit/actor/test_actor_child_runs.py @@ -56,7 +56,6 @@ def stored_record( task_id: str | None = None, run_input: Any = None, run_options: dict[str, Any] | None = None, - history: list[dict[str, str]] | None = None, abort_with_parent: bool = False, ) -> dict[str, Any]: """Build a registry record as it is stored in the default KVS.""" @@ -66,7 +65,6 @@ def stored_record( 'status': status, 'runName': run_name, 'checksum': checksum, - 'history': history or [], 'abortWithParent': abort_with_parent, } @@ -92,7 +90,6 @@ def stored_unnamed_record( 'status': status, 'runName': None, 'checksum': checksum_request(actor_id=actor_id, task_id=task_id, run_input=None), - 'history': [], 'abortWithParent': False, } @@ -347,19 +344,18 @@ async def test_named_start_resurrects_settling_run_after_it_finishes( @pytest.mark.parametrize( - ('recorded_run', 'replaced_status'), + 'recorded_run', [ - pytest.param(make_run('old-run', 'FAILED'), 'FAILED', id='failed'), - pytest.param(None, 'LOST', id='not found'), + pytest.param(make_run('old-run', 'FAILED'), id='failed'), + pytest.param(None, id='not found'), ], ) async def test_named_start_replaces_failed_or_missing_run( apify_client_async_patcher: ApifyClientAsyncPatcher, monkeypatch: pytest.MonkeyPatch, recorded_run: Run | None, - replaced_status: str, ) -> None: - """A recorded run that failed or no longer exists is replaced by a new run and kept in the history.""" + """A recorded run that failed or no longer exists is replaced by a new run under the same name.""" monkeypatch.setattr(_child_runs, '_NOT_FOUND_GRACE_SECS', 0) apify_client_async_patcher.patch('actor', 'start', return_value=make_run('new-run', 'READY')) apify_client_async_patcher.patch('run', 'get', return_value=recorded_run) @@ -371,9 +367,7 @@ async def test_named_start_replaces_failed_or_missing_run( stored = await kvs.get_value(CHILD_RUNS_KEY) assert run.id == 'new-run' - assert stored == { - 'scrape-eu': stored_record('new-run', 'READY', history=[{'runId': 'old-run', 'status': replaced_status}]) - } + assert stored == {'scrape-eu': stored_record('new-run', 'READY')} async def test_named_start_rejects_name_recorded_for_another_actor( @@ -391,6 +385,27 @@ async def test_named_start_rejects_name_recorded_for_another_actor( assert apify_client_async_patcher.calls['actor']['start'] == [] +async def test_named_start_reattaches_to_record_from_older_sdk( + apify_client_async_patcher: ApifyClientAsyncPatcher, +) -> None: + """A record with `startedAt` and `history` and without `runName`, as SDK 4.2 stores it, is still reattached.""" + apify_client_async_patcher.patch('actor', 'start', return_value=make_run('new-run', 'READY')) + apify_client_async_patcher.patch('run', 'get', return_value=make_run('old-run', 'RUNNING')) + record = stored_record('old-run', 'RUNNING') + del record['runName'] + record['startedAt'] = STARTED_AT + record['history'] = [{'runId': 'older-run', 'status': 'FAILED', 'startedAt': STARTED_AT}] + + async with Actor: + kvs = await Actor.open_key_value_store() + await kvs.set_value(CHILD_RUNS_KEY, {'scrape-eu': record}) + await Actor._child_run_registry.load() + run = await Actor.start('some-actor', run_name='scrape-eu') + + assert run.id == 'old-run' + assert apify_client_async_patcher.calls['actor']['start'] == [] + + async def test_named_start_rejects_name_recorded_with_other_run_options( apify_client_async_patcher: ApifyClientAsyncPatcher, ) -> None: @@ -774,7 +789,6 @@ async def test_named_start_replaces_run_recorded_for_other_request( 'new-run', 'READY', run_input={'since': '2026-01-01'}, - history=[{'runId': 'old-run', 'status': 'FAILED'}], ) }