diff --git a/.rules.md b/.rules.md index a91d9d86..90c05491 100644 --- a/.rules.md +++ b/.rules.md @@ -22,7 +22,7 @@ Don't introduce a shared Pydantic base model to centralize `model_config`. Each - **`Actor`** (`_actor.py`) is a `lazy-object-proxy` wrapping `_ActorType`. It works both as a class (`Actor.is_at_home()`) and as a context manager (`async with Actor:`); on `__aenter__` the proxy's `__wrapped__` is replaced with the active `_ActorType` instance. - **`Configuration`** (`_configuration.py`) extends Crawlee's `Configuration` and is populated from `APIFY_*` environment variables. -- **Named child runs** (`_child_runs.py`) — runs started with `run_name` are persisted in the default key-value store so they can be reattached after a migration or resurrection; exposed through `Actor.child_runs`. +- **Child runs** (`_child_runs.py`) — runs are persisted in the default key-value store under their `run_name`, or without one under an implicit key (a hash of the Actor or task, input and run options, so identical calls reuse one run), so they can be reattached after a migration or resurrection. All of them are exposed through `Actor.child_runs`. - **Storage clients** (`storage_clients/`) implement Crawlee's storage client interface: `ApifyStorageClient` talks to the Apify API (separate sub-clients for single vs. shared request queues), `FileSystemStorageClient` adds Apify-specific key-value store behavior, and `SmartApifyStorageClient` delegates to the cloud client on the platform and to the local one elsewhere. - **Events** — `ApifyEventManager` receives platform events over a WebSocket. - **Errors** — `apify.errors` only re-exports the exceptions from `apify_client.errors`; the SDK defines no error classes of its own, and API calls raise the client exceptions unchanged. diff --git a/src/apify/_actor.py b/src/apify/_actor.py index bb17a6e7..8e4cb431 100644 --- a/src/apify/_actor.py +++ b/src/apify/_actor.py @@ -433,11 +433,11 @@ def apify_client(self) -> ApifyClientAsync: @property @_ensure_context def child_runs(self) -> dict[str, RunClientAsync]: - """Clients for the named child runs of this Actor run, keyed by the run name. + """Clients for the child runs of this Actor run, keyed by the run name, or by an implicit key without one. - Every run started by `Actor.start`, `Actor.call`, `Actor.start_task` or `Actor.call_task` with a `run_name` is - included, even one started before a migration or resurrection of this Actor run. Runs started without a - `run_name` are not tracked. Each client points to the current run under its name: + Every run started by `Actor.start`, `Actor.call`, `Actor.start_task` or `Actor.call_task` is included, even one + started before a migration or resurrection of this Actor run. The client of a named run points to the current + run under its name: ```python run = await Actor.child_runs['my-child'].wait_for_finish() @@ -1103,7 +1103,11 @@ async def start( to the recorded run. A `SUCCEEDED` run is returned as is, an `ABORTED` or `TIMED-OUT` one is resurrected, and a new run is started only when nothing is recorded under the name, or the recorded run `FAILED` or no longer exists. Using `run_name` to attach to a run of a different Actor, task or - with a different input raises a `ValueError`. + with a different input or run options raises a `ValueError`. A run started without `run_name` is + recorded under an implicit key, a hash of the Actor or task, the input and the run options, and the + same rules apply to it. So an identical call without `run_name` reuses that run, also before a + migration. To start a new run for an identical call, e.g. when polling, pass a `run_name` that differs + between the calls. abort_with_parent: If true, the child run is gracefully aborted when this Actor run is gracefully aborted. It requires `run_name`, and the value is recorded under it, replacing the one from an earlier call. Once this Actor run is being gracefully aborted, a call with it raises a `RuntimeError` and @@ -1136,21 +1140,21 @@ async def start( webhooks=to_client_representations(webhooks), ) - if run_name is None: - return await start_run() - run, _ = await self._find_or_start_child_run( run_name, actor_id=actor_id, run_input=run_input, client=client, start_run=start_run, + content_type=content_type, + force_permission_level=force_permission_level, build=build, max_items=max_items, max_total_charge_usd=max_total_charge_usd, restart_on_error=restart_on_error, memory_mbytes=memory_mbytes, timeout=timeout, + webhooks=webhooks, abort_with_parent=abort_with_parent, ) return run @@ -1304,7 +1308,11 @@ async def call( to the recorded run. A `SUCCEEDED` run is returned as is, an `ABORTED` or `TIMED-OUT` one is resurrected, and a new run is started only when nothing is recorded under the name, or the recorded run `FAILED` or no longer exists. Using `run_name` to attach to a run of a different Actor, task or - with a different input raises a `ValueError`. + with a different input or run options raises a `ValueError`. A run started without `run_name` is + recorded under an implicit key, a hash of the Actor or task, the input and the run options, and the + same rules apply to it. So an identical call without `run_name` reuses that run, also before a + migration. To start a new run for an identical call, e.g. when polling, pass a `run_name` that differs + between the calls. abort_with_parent: If true, the child run is gracefully aborted when this Actor run is gracefully aborted. It requires `run_name`, and the value is recorded under it, replacing the one from an earlier call. Once this Actor run is being gracefully aborted, a call with it raises a `RuntimeError` and @@ -1323,53 +1331,41 @@ async def call( client = self.new_client(token=token) if token else self.apify_client actor_client = client.actor(actor_id) + start_run = partial( + actor_client.start, + run_input=run_input, + content_type=content_type, + build=build, + max_items=max_items, + max_total_charge_usd=max_total_charge_usd, + restart_on_error=restart_on_error, + memory_mbytes=memory_mbytes, + run_timeout=self._resolve_run_timeout(timeout), + force_permission_level=force_permission_level, + webhooks=to_client_representations(webhooks), + ) - if run_name is None: - run = await actor_client.call( - run_input=run_input, - content_type=content_type, - build=build, - max_items=max_items, - max_total_charge_usd=max_total_charge_usd, - restart_on_error=restart_on_error, - memory_mbytes=memory_mbytes, - run_timeout=self._resolve_run_timeout(timeout), - force_permission_level=force_permission_level, - webhooks=to_client_representations(webhooks), - wait_duration=wait, - logger=logger, - ) - else: - started_run, is_new = await self._find_or_start_child_run( - run_name, - actor_id=actor_id, - run_input=run_input, - client=client, - start_run=partial( - actor_client.start, - run_input=run_input, - content_type=content_type, - build=build, - max_items=max_items, - max_total_charge_usd=max_total_charge_usd, - restart_on_error=restart_on_error, - memory_mbytes=memory_mbytes, - run_timeout=self._resolve_run_timeout(timeout), - force_permission_level=force_permission_level, - webhooks=to_client_representations(webhooks), - ), - build=build, - max_items=max_items, - max_total_charge_usd=max_total_charge_usd, - restart_on_error=restart_on_error, - memory_mbytes=memory_mbytes, - timeout=timeout, - abort_with_parent=abort_with_parent, - ) - # The earlier attempt of this call already streamed the log of a reattached or resurrected run. - run = await self._wait_for_child_run( - run_name, client.run(started_run.id), started_run, wait=wait, logger=logger, from_start=is_new - ) + started_run, is_new = await self._find_or_start_child_run( + run_name, + actor_id=actor_id, + run_input=run_input, + client=client, + start_run=start_run, + content_type=content_type, + force_permission_level=force_permission_level, + build=build, + max_items=max_items, + max_total_charge_usd=max_total_charge_usd, + restart_on_error=restart_on_error, + memory_mbytes=memory_mbytes, + timeout=timeout, + webhooks=webhooks, + abort_with_parent=abort_with_parent, + ) + # The earlier attempt of this call already streamed the log of a reattached or resurrected run. + run = await self._wait_for_child_run( + client.run(started_run.id), started_run, wait=wait, logger=logger, from_start=is_new + ) if run is None: raise RuntimeError(f'Failed to call Actor with ID "{actor_id}".') @@ -1378,26 +1374,41 @@ async def call( async def _find_or_start_child_run( self, - name: str, + name: str | None, *, actor_id: str | None = None, task_id: str | None = None, run_input: Any, client: ApifyClientAsync, start_run: Callable[[], Awaitable[Run]], + content_type: str | None = None, + force_permission_level: ActorPermissionLevel | None = None, build: str | None, max_items: int | None, max_total_charge_usd: Decimal | None, restart_on_error: bool | None, memory_mbytes: int | None, timeout: timedelta | Literal['inherit'] | None, + webhooks: list[Webhook] | None, abort_with_parent: bool, ) -> tuple[Run, bool]: + run_options = { + 'content_type': content_type, + 'build': build, + 'max_items': max_items, + 'max_total_charge_usd': max_total_charge_usd, + 'restart_on_error': restart_on_error, + 'memory_mbytes': memory_mbytes, + 'timeout': timeout, + 'force_permission_level': force_permission_level, + 'webhooks': [asdict(webhook) for webhook in webhooks] if webhooks else None, + } return await self._child_run_registry.find_or_start( name, actor_id=actor_id, task_id=task_id, run_input=run_input, + run_options=run_options, client=client, start_run=start_run, resurrect_run=lambda run_id: client.run(run_id).resurrect( @@ -1416,7 +1427,6 @@ async def _abort_child_runs(self) -> None: async def _wait_for_child_run( self, - name: str, run_client: RunClientAsync, run: Run, *, @@ -1438,7 +1448,7 @@ async def _wait_for_child_run( finished_run = await run_client.wait_for_finish(wait_duration=wait) if finished_run is not None: - await self._child_run_registry.update(name, finished_run) + await self._child_run_registry.update(finished_run) return finished_run @_ensure_context @@ -1491,7 +1501,11 @@ async def start_task( to the recorded run. A `SUCCEEDED` run is returned as is, an `ABORTED` or `TIMED-OUT` one is resurrected, and a new run is started only when nothing is recorded under the name, or the recorded run `FAILED` or no longer exists. Using `run_name` to attach to a run of a different Actor, task or - with a different input raises a `ValueError`. + with a different input or run options raises a `ValueError`. A run started without `run_name` is + recorded under an implicit key, a hash of the Actor or task, the input and the run options, and the + same rules apply to it. So an identical call without `run_name` reuses that run, also before a + migration. To start a new run for an identical call, e.g. when polling, pass a `run_name` that differs + between the calls. abort_with_parent: If true, the child run is gracefully aborted when this Actor run is gracefully aborted. It requires `run_name`, and the value is recorded under it, replacing the one from an earlier call. Once this Actor run is being gracefully aborted, a call with it raises a `RuntimeError` and @@ -1521,9 +1535,6 @@ async def start_task( webhooks=to_client_representations(webhooks), ) - if run_name is None: - return await start_run() - run, _ = await self._find_or_start_child_run( run_name, task_id=task_id, @@ -1536,6 +1547,7 @@ async def start_task( restart_on_error=restart_on_error, memory_mbytes=memory_mbytes, timeout=timeout, + webhooks=webhooks, abort_with_parent=abort_with_parent, ) return run @@ -1591,7 +1603,11 @@ async def call_task( to the recorded run. A `SUCCEEDED` run is returned as is, an `ABORTED` or `TIMED-OUT` one is resurrected, and a new run is started only when nothing is recorded under the name, or the recorded run `FAILED` or no longer exists. Using `run_name` to attach to a run of a different Actor, task or - with a different input raises a `ValueError`. + with a different input or run options raises a `ValueError`. A run started without `run_name` is + recorded under an implicit key, a hash of the Actor or task, the input and the run options, and the + same rules apply to it. So an identical call without `run_name` reuses that run, also before a + migration. To start a new run for an identical call, e.g. when polling, pass a `run_name` that differs + between the calls. abort_with_parent: If true, the child run is gracefully aborted when this Actor run is gracefully aborted. It requires `run_name`, and the value is recorded under it, replacing the one from an earlier call. Once this Actor run is being gracefully aborted, a call with it raises a `RuntimeError` and @@ -1610,47 +1626,36 @@ async def call_task( client = self.new_client(token=token) if token else self.apify_client task_client = client.task(task_id) + start_run = partial( + task_client.start, + task_input=task_input, + build=build, + max_items=max_items, + max_total_charge_usd=max_total_charge_usd, + restart_on_error=restart_on_error, + memory_mbytes=memory_mbytes, + run_timeout=self._resolve_run_timeout(timeout), + webhooks=to_client_representations(webhooks), + ) - if run_name is None: - run = await task_client.call( - task_input=task_input, - build=build, - max_items=max_items, - max_total_charge_usd=max_total_charge_usd, - restart_on_error=restart_on_error, - memory_mbytes=memory_mbytes, - run_timeout=self._resolve_run_timeout(timeout), - webhooks=to_client_representations(webhooks), - wait_duration=wait, - ) - else: - started_run, _ = await self._find_or_start_child_run( - run_name, - task_id=task_id, - run_input=task_input, - client=client, - start_run=partial( - task_client.start, - task_input=task_input, - build=build, - max_items=max_items, - max_total_charge_usd=max_total_charge_usd, - restart_on_error=restart_on_error, - memory_mbytes=memory_mbytes, - run_timeout=self._resolve_run_timeout(timeout), - webhooks=to_client_representations(webhooks), - ), - build=build, - max_items=max_items, - max_total_charge_usd=max_total_charge_usd, - restart_on_error=restart_on_error, - memory_mbytes=memory_mbytes, - timeout=timeout, - abort_with_parent=abort_with_parent, - ) - run = await self._wait_for_child_run( - run_name, client.run(started_run.id), started_run, wait=wait, logger=None, from_start=False - ) + started_run, _ = await self._find_or_start_child_run( + run_name, + task_id=task_id, + run_input=task_input, + client=client, + start_run=start_run, + build=build, + max_items=max_items, + max_total_charge_usd=max_total_charge_usd, + restart_on_error=restart_on_error, + memory_mbytes=memory_mbytes, + timeout=timeout, + webhooks=webhooks, + abort_with_parent=abort_with_parent, + ) + run = await self._wait_for_child_run( + client.run(started_run.id), started_run, wait=wait, logger=None, from_start=False + ) if run is None: raise RuntimeError(f'Failed to call Task with ID "{task_id}".') diff --git a/src/apify/_child_runs.py b/src/apify/_child_runs.py index 4b3bc52d..5b6209e0 100644 --- a/src/apify/_child_runs.py +++ b/src/apify/_child_runs.py @@ -3,12 +3,11 @@ import asyncio import hashlib import json -from datetime import datetime from logging import getLogger from typing import TYPE_CHECKING, Any from weakref import WeakValueDictionary -from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from pydantic.alias_generators import to_camel from apify_client.errors import NotFoundError @@ -39,9 +38,12 @@ _NOT_FOUND_RETRY_INTERVAL_SECS = 0.25 +_IMPLICIT_KEY_HASH_LENGTH = 16 +"""Number of hex digits of the request hash kept in the key of a child run started without a name.""" -class ChildRunSnapshot(BaseModel): - """A child run as last observed by this Actor run.""" + +class ChildRunRecord(BaseModel): + """The current run tracked under a name or implicit key in the child run registry.""" model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel) @@ -49,32 +51,34 @@ class ChildRunSnapshot(BaseModel): """ID of the run.""" status: str - """Last status of the run observed by this Actor run, or `LOST` once the platform no longer returned it.""" - - started_at: datetime - """When the run started.""" + """Last status of the run observed by this Actor run.""" - -class ChildRunRecord(ChildRunSnapshot): - """The current run tracked under a name in the child run registry, with the runs it replaced.""" + run_name: str | None = None + """Name the run was started under, or `None` for a run started without a name.""" checksum: str - """Hash of the Actor or task and the input the current run was started with.""" - - history: list[ChildRunSnapshot] = Field(default_factory=list) - """Earlier runs under this name that failed or went missing and were replaced by a new run, oldest first.""" + """Hash of the Actor or task, the input and the run options the current run was started with.""" abort_with_parent: bool = False """Whether the current run is aborted when this Actor run is gracefully aborted.""" -def checksum_request(*, actor_id: str | None, task_id: str | None, run_input: Any) -> str: - """Hash the Actor or task and the input of a named start, in the same JSON shape as the JS SDK.""" +def checksum_request( + *, actor_id: str | None, task_id: str | None, run_input: Any, run_options: dict[str, Any] | None = None +) -> str: + """Hash the Actor or task, the input and the run options of a start. + + Without run options, the hashed JSON has the same shape as in the JS SDK. Only the options set to anything but + `None` are hashed, so a run option added to the SDK later does not change the hash of a start that does not use it. + """ request: dict[str, Any] = ( {'type': 'actor', 'id': actor_id} if actor_id is not None else {'type': 'task', 'id': task_id} ) if run_input is not None: request['input'] = run_input + options = {name: value for name, value in (run_options or {}).items() if value is not None} + if options: + request['options'] = options serialized = json.dumps(request, sort_keys=True, separators=(',', ':'), ensure_ascii=False, default=str) return hashlib.sha256(serialized.encode()).hexdigest() @@ -98,80 +102,97 @@ async def _get_recorded_run(run_client: RunClientAsync) -> Run | None: class ChildRunRegistry: - """Persisted name -> run map that lets named child runs survive a migration or resurrection of the parent. + """Persisted record of the child runs of this Actor run, which survives a migration or resurrection of the parent. - Every change is written to the key-value store right away. A hard kill of the parent between the platform - starting the child and that write can still orphan the child, since nothing but the platform knows about it. + A named child run is recorded under its name, so a later start under the same name can reattach to it. A child run + started without a name is recorded under an implicit key, a hash of its Actor or task, input and run options, so + every identical start without a name reuses the same run, also after a migration. + Every change is written to the key-value store right away. A hard kill of the parent between the platform starting + the child and that write can still orphan the child, since nothing but the platform knows about it. """ def __init__(self, open_key_value_store: Callable[[], Awaitable[KeyValueStore]]) -> None: self._open_key_value_store = open_key_value_store + self._key_value_store: KeyValueStore | None = None + """The default key-value store the records were loaded from, which every change is written to.""" self._records: dict[str, ChildRunRecord] | None = None + """Records of the child runs by name, or by implicit key for a run started without a name. + + `None` until loaded. + """ self._lock = asyncio.Lock() """Guards loading the records and writing them back to the key-value store.""" self._name_locks: WeakValueDictionary[str, asyncio.Lock] = WeakValueDictionary() - """Serializes `find_or_start` per name. A lock is dropped once no call under its name holds it.""" + """Serializes `find_or_start` per key. A lock is dropped once no call under its key holds it.""" self._clients: dict[str, ApifyClientAsync] = {} - """Client the run under each name was last started or looked up with. Lost on a migration.""" + """Client the run under each key was last started or looked up with. Lost on a migration.""" self._parent_aborting = False """Whether this Actor run is being gracefully aborted, after which no run marked `abort_with_parent` starts.""" async def find_or_start( self, - name: str, + name: str | None, *, actor_id: str | None = None, task_id: str | None = None, run_input: Any, + run_options: dict[str, Any] | None = None, client: ApifyClientAsync, start_run: Callable[[], Awaitable[Run]], resurrect_run: Callable[[str], Awaitable[Run]], abort_with_parent: bool = False, ) -> tuple[Run, bool]: - """Return the run recorded under `name`, or start one when there is none to reuse. + """Return the run recorded under `name` or the implicit key, or start one when there is none to reuse. A recorded run that is `READY` or `RUNNING` is reattached and one that `SUCCEEDED` is returned as is. An `ABORTED` or `TIMED-OUT` run is resurrected, since Actors are expected to resume from their state. - A `FAILED` run, or one the API no longer knows, is replaced by a new run under the same name, for any Actor, - task or input. + A `FAILED` run, or one the API no longer knows, is replaced by a new run under the same key, for any Actor, + task, input or run options. Args: - name: Name of the child run, unique within the parent run. + name: Name of the child run, unique within the parent run, or `None` to use the implicit key. actor_id: The Actor to start. task_id: The task to start, in place of `actor_id`. - run_input: Input of the run. The recorded run is reused only if it was started with the same Actor or task - and input. + run_input: Input of the run. The recorded run is reused only if it was started with the same Actor or task, + input and run options. + run_options: Options the run is started with. client: Client used to look up the recorded run. start_run: Starts a new run of the Actor or task. resurrect_run: Resurrects the recorded run, given its ID. - abort_with_parent: Whether to abort the run when this Actor run is gracefully aborted. It replaces the value - recorded under `name`. + abort_with_parent: Whether to abort the run when this Actor run is gracefully aborted. It replaces the + recorded value. Returns: The run, and whether it was newly started. Raises: - ValueError: If the recorded run would be reused, but was started for a different Actor, task or input. + ValueError: If the recorded run would be reused, but was started for a different Actor, task, input or + run options. RuntimeError: If `abort_with_parent` is set and this Actor run is being gracefully aborted. No run is started, resurrected or reattached. """ - checksum = checksum_request(actor_id=actor_id, task_id=task_id, run_input=run_input) + checksum = checksum_request(actor_id=actor_id, task_id=task_id, run_input=run_input, run_options=run_options) + key = name if name is not None else checksum[:_IMPLICIT_KEY_HASH_LENGTH] - async with self._name_locks.setdefault(name, asyncio.Lock()): + async with self._name_locks.setdefault(key, asyncio.Lock()): if abort_with_parent and self._parent_aborting: raise RuntimeError( - f'Child run "{name}" with `abort_with_parent` was not started, since this Actor run is being ' + f'Child run "{key}" with `abort_with_parent` was not started, since this Actor run is being ' 'aborted.' ) records = await self._load() - record = records.get(name) + record = records.get(key) if record is None: run = await self._start( - name, checksum=checksum, start_run=start_run, history=[], abort_with_parent=abort_with_parent + key, + run_name=name, + checksum=checksum, + start_run=start_run, + abort_with_parent=abort_with_parent, ) - self._clients[name] = client + self._clients[key] = client return run, True run_client = client.run(record.run_id) @@ -181,66 +202,67 @@ async def find_or_start( run = await run_client.wait_for_finish() if run is None or run.status == 'FAILED': - replaced = ChildRunSnapshot( - run_id=record.run_id, - status=run.status if run is not None else 'LOST', - started_at=record.started_at, - ) run = await self._start( - name, + key, + run_name=name, checksum=checksum, start_run=start_run, - history=[*record.history, replaced], abort_with_parent=abort_with_parent, ) - self._clients[name] = client + self._clients[key] = client return run, True if record.checksum != checksum: raise ValueError( - f'The run name "{name}" was already used for a different Actor, task or input. ' + f'The run name "{key}" was already used for a different Actor, task, input or run options. ' 'Use a unique `run_name` for each child run.' ) if record.abort_with_parent != abort_with_parent: - await self._save(name, record.model_copy(update={'abort_with_parent': abort_with_parent})) + await self._save(key, record.model_copy(update={'abort_with_parent': abort_with_parent})) if run.status in _RESURRECTABLE_STATUSES: - logger.info('Resurrecting child run "%s"', name, extra={'run_id': run.id, 'status': run.status}) + logger.info('Resurrecting child run "%s"', key, extra={'run_id': run.id, 'status': run.status}) run = await resurrect_run(run.id) else: - logger.info('Reattaching to child run "%s"', name, extra={'run_id': run.id, 'status': run.status}) + logger.info('Reattaching to child run "%s"', key, extra={'run_id': run.id, 'status': run.status}) - await self.update(name, run) - self._clients[name] = client + await self._update(key, run) + self._clients[key] = client return run, False def run_clients(self, default_client: ApifyClientAsync) -> dict[str, RunClientAsync]: - """Return a client for the current run under each recorded name. + """Return a client for the current run under each recorded key. - Each client comes from the client its name was last started or looked up with in this process, so a run started - with a custom token uses that token. + A named run is keyed by its name, a run started without a name by its implicit key. Each client comes from the + client its run was last started or looked up with in this process, so a run started with a custom token uses + that token. Args: - default_client: Client used for a name not started or looked up in this process, e.g. one recorded before a + default_client: Client used for a run not started or looked up in this process, e.g. one recorded before a migration. """ return { - name: self._clients.get(name, default_client).run(record.run_id) - for name, record in (self._records or {}).items() + key: self._clients.get(key, default_client).run(record.run_id) + for key, record in (self._records or {}).items() } - async def update(self, name: str, run: Run) -> None: - """Record the latest observed status of the run recorded under `name`. + async def update(self, run: Run) -> None: + """Record the latest observed status of a run. Args: - name: Name of the child run. - run: The run as just returned by the API. Nothing is recorded unless it is the current run under `name`. + run: The run as just returned by the API. Nothing is recorded unless it is the current run under some key. """ - record = (await self._load()).get(name) + records = await self._load() + key = next((key for key, record in records.items() if record.run_id == run.id), None) + if key is not None: + await self._update(key, run) + + async def _update(self, key: str, run: Run) -> None: + record = (await self._load()).get(key) if record is None or record.run_id != run.id or record.status == run.status: return - await self._save(name, record.model_copy(update={'status': run.status})) + await self._save(key, record.model_copy(update={'status': run.status})) async def abort_runs_with_parent(self, client: ApifyClientAsync) -> None: """Gracefully abort every recorded run marked `abort_with_parent`, and refuse to start any more of them. @@ -250,54 +272,54 @@ async def abort_runs_with_parent(self, client: ApifyClientAsync) -> None: the start records it. Args: - client: Client used for a name not started or looked up in this process, e.g. after a migration. + client: Client used for a run not started or looked up in this process, e.g. after a migration. """ self._parent_aborting = True records = await self._load() - # Names with a start in flight are not recorded yet, so their locks are awaited too. - await asyncio.gather(*(self._abort(name, client) for name in {*records, *self._name_locks})) + # Keys with a start in flight are not recorded yet, so their locks are awaited too. + await asyncio.gather(*(self._abort(key, client) for key in {*records, *self._name_locks})) - async def _abort(self, name: str, default_client: ApifyClientAsync) -> None: - async with self._name_locks.setdefault(name, asyncio.Lock()): - record = (await self._load()).get(name) + async def _abort(self, key: str, default_client: ApifyClientAsync) -> None: + async with self._name_locks.setdefault(key, asyncio.Lock()): + record = (await self._load()).get(key) if record is None or not record.abort_with_parent: return - run_client = self._clients.get(name, default_client).run(record.run_id) + run_client = self._clients.get(key, default_client).run(record.run_id) try: run = await run_client.abort(gracefully=True) except NotFoundError: - logger.debug('Child run "%s" no longer exists', name, extra={'run_id': record.run_id}) + logger.debug('Child run "%s" no longer exists', key, extra={'run_id': record.run_id}) except Exception: - logger.exception('Failed to abort child run "%s"', name, extra={'run_id': record.run_id}) + logger.exception('Failed to abort child run "%s"', key, extra={'run_id': record.run_id}) else: if run.status in _ABORTED_STATUSES: - logger.info('Aborted child run "%s" with the parent', name, extra={'run_id': record.run_id}) + logger.info('Aborted child run "%s" with the parent', key, extra={'run_id': record.run_id}) async def _start( self, - name: str, + key: str, *, + run_name: str | None, checksum: str, start_run: Callable[[], Awaitable[Run]], - history: list[ChildRunSnapshot], abort_with_parent: bool, ) -> Run: run = await start_run() record = ChildRunRecord( run_id=run.id, status=run.status, - started_at=run.started_at, + run_name=run_name, checksum=checksum, - history=history, abort_with_parent=abort_with_parent, ) - await self._save(name, record) + await self._save(key, record) return run async def load(self) -> dict[str, ChildRunRecord]: """Read the records from the default key-value store, replacing any read before.""" async with self._lock: key_value_store = await self._open_key_value_store() + self._key_value_store = key_value_store stored = await key_value_store.get_value(CHILD_RUNS_KEY) try: self._records = _records_adapter.validate_python(stored or {}) @@ -311,11 +333,15 @@ async def load(self) -> dict[str, ChildRunRecord]: async def _load(self) -> dict[str, ChildRunRecord]: return self._records if self._records is not None else await self.load() - async def _save(self, name: str, record: ChildRunRecord) -> None: + def _loaded_key_value_store(self) -> KeyValueStore: + if self._key_value_store is None: + raise RuntimeError('The child run registry was not loaded.') + return self._key_value_store + + async def _save(self, key: str, record: ChildRunRecord) -> None: records = await self._load() - key_value_store = await self._open_key_value_store() async with self._lock: - records[name] = record - await key_value_store.set_value( + records[key] = record + await self._loaded_key_value_store().set_value( CHILD_RUNS_KEY, _records_adapter.dump_python(records, by_alias=True, mode='json') ) diff --git a/tests/unit/actor/test_actor_child_runs.py b/tests/unit/actor/test_actor_child_runs.py index e4bd5d2c..583d4de8 100644 --- a/tests/unit/actor/test_actor_child_runs.py +++ b/tests/unit/actor/test_actor_child_runs.py @@ -51,23 +51,49 @@ def stored_record( run_id: str, status: str, *, + run_name: str = 'scrape-eu', actor_id: str | None = 'some-actor', task_id: str | None = None, run_input: Any = None, - history: list[dict[str, str]] | None = None, + run_options: dict[str, Any] | None = None, abort_with_parent: bool = False, ) -> dict[str, Any]: """Build a registry record as it is stored in the default KVS.""" + checksum = checksum_request(actor_id=actor_id, task_id=task_id, run_input=run_input, run_options=run_options) return { 'runId': run_id, 'status': status, - 'startedAt': STARTED_AT, - 'checksum': checksum_request(actor_id=actor_id, task_id=task_id, run_input=run_input), - 'history': history or [], + 'runName': run_name, + 'checksum': checksum, 'abortWithParent': abort_with_parent, } +def implicit_key( + *, + actor_id: str | None = 'some-actor', + task_id: str | None = None, + run_input: Any = None, + run_options: dict[str, Any] | None = None, +) -> str: + """Build the key a start without a name is recorded under.""" + request_hash = checksum_request(actor_id=actor_id, task_id=task_id, run_input=run_input, run_options=run_options) + return request_hash[:16] + + +def stored_unnamed_record( + run_id: str, status: str, *, actor_id: str | None = 'some-actor', task_id: str | None = None +) -> dict[str, Any]: + """Build a record of a child run started without a name, as it is stored in the default KVS.""" + return { + 'runId': run_id, + 'status': status, + 'runName': None, + 'checksum': checksum_request(actor_id=actor_id, task_id=task_id, run_input=None), + 'abortWithParent': False, + } + + @pytest.fixture def apify_event_manager() -> ApifyEventManager: """Make the Actor use `ApifyEventManager`, which delivers `ABORTING` on the platform, without a websocket.""" @@ -83,10 +109,19 @@ async def record_child_run( actor_id: str | None = 'some-actor', task_id: str | None = None, run_input: Any = None, + run_options: dict[str, Any] | None = None, ) -> None: """Seed the registry the way an earlier attempt of this Actor run would have left it, and reload it like init.""" kvs = await Actor.open_key_value_store() - record = stored_record(run_id, 'RUNNING', actor_id=actor_id, task_id=task_id, run_input=run_input) + record = stored_record( + run_id, + 'RUNNING', + run_name=name, + actor_id=actor_id, + task_id=task_id, + run_input=run_input, + run_options=run_options, + ) await kvs.set_value(CHILD_RUNS_KEY, {name: record}) await Actor._child_run_registry.load() @@ -104,16 +139,132 @@ async def test_named_start_records_run_in_kvs(apify_client_async_patcher: ApifyC assert stored == {'scrape-eu': stored_record('new-run', 'READY')} -async def test_unnamed_start_is_not_recorded(apify_client_async_patcher: ApifyClientAsyncPatcher) -> None: - """A start without a name leaves the registry untouched.""" +@pytest.mark.parametrize( + ('method', 'resource', 'ids'), + [ + pytest.param('start', 'actor', {'actor_id': 'some-id'}, id='start'), + pytest.param('start_task', 'task', {'actor_id': None, 'task_id': 'some-id'}, id='start task'), + ], +) +async def test_unnamed_start_records_run_under_implicit_key( + apify_client_async_patcher: ApifyClientAsyncPatcher, method: str, resource: str, ids: dict[str, Any] +) -> None: + """A start without a name persists the run under the implicit key of its Actor or task.""" + apify_client_async_patcher.patch(resource, 'start', return_value=make_run('new-run', 'READY')) + + async with Actor: + await getattr(Actor, method)('some-id') + kvs = await Actor.open_key_value_store() + stored = await kvs.get_value(CHILD_RUNS_KEY) + + assert stored == {implicit_key(**ids): stored_unnamed_record('new-run', 'READY', **ids)} + + +async def test_identical_unnamed_starts_reuse_one_run(apify_client_async_patcher: ApifyClientAsyncPatcher) -> None: + """An identical start without a name reattaches to the run of the first one instead of starting another.""" + apify_client_async_patcher.patch('actor', 'start', return_value=make_run('first-run', 'READY')) + apify_client_async_patcher.patch('run', 'get', return_value=make_run('first-run', 'RUNNING')) + + async with Actor: + first = await Actor.start('some-actor') + second = await Actor.start('some-actor') + kvs = await Actor.open_key_value_store() + stored = await kvs.get_value(CHILD_RUNS_KEY) + + assert (first.id, second.id) == ('first-run', 'first-run') + assert len(apify_client_async_patcher.calls['actor']['start']) == 1 + assert stored == {implicit_key(): stored_unnamed_record('first-run', 'RUNNING')} + + +async def test_unnamed_start_reattaches_after_migration(apify_client_async_patcher: ApifyClientAsyncPatcher) -> None: + """After a migration, a start without a name reattaches to the run recorded under its implicit key.""" apify_client_async_patcher.patch('actor', 'start', return_value=make_run('new-run', 'READY')) + apify_client_async_patcher.patch('run', 'get', return_value=make_run('old-run', 'RUNNING')) + + async with Actor: + kvs = await Actor.open_key_value_store() + await kvs.set_value(CHILD_RUNS_KEY, {implicit_key(): stored_unnamed_record('old-run', 'RUNNING')}) + await Actor._child_run_registry.load() + run = await Actor.start('some-actor') + + assert run.id == 'old-run' + assert apify_client_async_patcher.calls['actor']['start'] == [] + + +async def test_unnamed_starts_with_different_run_options_get_different_keys( + apify_client_async_patcher: ApifyClientAsyncPatcher, +) -> None: + """The run options are part of the implicit key, and the token is not.""" + run_ids = iter(['default-run', 'big-run']) + apify_client_async_patcher.patch( + 'actor', 'start', replacement_method=AsyncMock(side_effect=lambda *_, **__: make_run(next(run_ids), 'READY')) + ) + apify_client_async_patcher.patch( + 'run', 'get', replacement_method=lambda run_client, *_, **__: make_run(run_client.resource_id, 'RUNNING') + ) async with Actor: await Actor.start('some-actor') + await Actor.start('some-actor', memory_mbytes=4096) + token_run = await Actor.start('some-actor', token='other-token') kvs = await Actor.open_key_value_store() stored = await kvs.get_value(CHILD_RUNS_KEY) - assert stored is None + assert token_run.id == 'default-run' + assert {key: record['runId'] for key, record in stored.items()} == { + implicit_key(): 'default-run', + implicit_key(run_options={'memory_mbytes': 4096}): 'big-run', + } + + +def test_checksum_ignores_run_options_set_to_none() -> None: + """Run options set to `None` do not change the request hash.""" + without_options = checksum_request(actor_id='some-actor', task_id=None, run_input={'a': 1}) + with_none_options = checksum_request( + actor_id='some-actor', task_id=None, run_input={'a': 1}, run_options={'build': None, 'memory_mbytes': None} + ) + assert with_none_options == without_options + + +@pytest.mark.parametrize( + ('method', 'resource', 'ids'), + [ + pytest.param('call', 'actor', {'actor_id': 'some-id'}, id='call'), + pytest.param('call_task', 'task', {'actor_id': None, 'task_id': 'some-id'}, id='call task'), + ], +) +async def test_unnamed_call_records_finished_status( + apify_client_async_patcher: ApifyClientAsyncPatcher, method: str, resource: str, ids: dict[str, Any] +) -> None: + """A call without a name records the run when it starts and stores its status once it finishes.""" + apify_client_async_patcher.patch(resource, 'start', return_value=make_run('new-run', 'READY')) + apify_client_async_patcher.patch('run', 'wait_for_finish', return_value=make_run('new-run', 'SUCCEEDED')) + kwargs: dict[str, Any] = {'logger': None} if method == 'call' else {} + + async with Actor: + run = await getattr(Actor, method)('some-id', **kwargs) + kvs = await Actor.open_key_value_store() + stored = await kvs.get_value(CHILD_RUNS_KEY) + + assert run.status == 'SUCCEEDED' + assert stored == {implicit_key(**ids): stored_unnamed_record('new-run', 'SUCCEEDED', **ids)} + assert apify_client_async_patcher.calls[resource]['call'] == [] + + +async def test_unnamed_call_streams_log_from_start(apify_client_async_patcher: ApifyClientAsyncPatcher) -> None: + """A call without a name streams the log of its new run from the start, with the given wait duration.""" + get_streamed_log = Mock(return_value=MagicMock()) + apify_client_async_patcher.patch('actor', 'start', return_value=make_run('new-run', 'READY')) + apify_client_async_patcher.patch('run', 'wait_for_finish', return_value=make_run('new-run', 'SUCCEEDED')) + apify_client_async_patcher.patch('run', 'get_status_message_watcher', return_value=MagicMock()) + apify_client_async_patcher.patch('run', 'get_streamed_log', replacement_method=get_streamed_log) + + async with Actor: + await Actor.call('some-actor', wait=timedelta(minutes=1)) + + assert get_streamed_log.call_args.kwargs['from_start'] is True + [(_, kwargs)] = apify_client_async_patcher.calls['run']['wait_for_finish'] + assert kwargs['wait_duration'] == timedelta(minutes=1) @pytest.mark.parametrize( @@ -156,7 +307,7 @@ async def test_named_start_resurrects_recorded_run( apify_client_async_patcher.patch('run', 'resurrect', return_value=make_run('old-run', 'RUNNING')) async with Actor: - await record_child_run('scrape-eu', 'old-run') + await record_child_run('scrape-eu', 'old-run', run_options={'memory_mbytes': 2048}) run = await Actor.start('some-actor', run_name='scrape-eu', memory_mbytes=2048) assert run.id == 'old-run' @@ -193,19 +344,18 @@ async def test_named_start_resurrects_settling_run_after_it_finishes( @pytest.mark.parametrize( - ('recorded_run', 'replaced_status'), + 'recorded_run', [ - pytest.param(make_run('old-run', 'FAILED'), 'FAILED', id='failed'), - pytest.param(None, 'LOST', id='not found'), + pytest.param(make_run('old-run', 'FAILED'), id='failed'), + pytest.param(None, id='not found'), ], ) async def test_named_start_replaces_failed_or_missing_run( apify_client_async_patcher: ApifyClientAsyncPatcher, monkeypatch: pytest.MonkeyPatch, recorded_run: Run | None, - replaced_status: str, ) -> None: - """A recorded run that failed or no longer exists is replaced by a new run and kept in the history.""" + """A recorded run that failed or no longer exists is replaced by a new run under the same name.""" monkeypatch.setattr(_child_runs, '_NOT_FOUND_GRACE_SECS', 0) apify_client_async_patcher.patch('actor', 'start', return_value=make_run('new-run', 'READY')) apify_client_async_patcher.patch('run', 'get', return_value=recorded_run) @@ -217,11 +367,7 @@ async def test_named_start_replaces_failed_or_missing_run( stored = await kvs.get_value(CHILD_RUNS_KEY) assert run.id == 'new-run' - assert stored == { - 'scrape-eu': stored_record( - 'new-run', 'READY', history=[{'runId': 'old-run', 'status': replaced_status, 'startedAt': STARTED_AT}] - ) - } + assert stored == {'scrape-eu': stored_record('new-run', 'READY')} async def test_named_start_rejects_name_recorded_for_another_actor( @@ -233,12 +379,48 @@ async def test_named_start_rejects_name_recorded_for_another_actor( async with Actor: await record_child_run('scrape-eu', 'old-run', actor_id='other-actor') - with pytest.raises(ValueError, match='already used for a different Actor, task or input'): + with pytest.raises(ValueError, match='already used for a different Actor, task, input or run options'): await Actor.start('some-actor', run_name='scrape-eu') assert apify_client_async_patcher.calls['actor']['start'] == [] +async def test_named_start_reattaches_to_record_from_older_sdk( + apify_client_async_patcher: ApifyClientAsyncPatcher, +) -> None: + """A record with `startedAt` and `history` and without `runName`, as SDK 4.2 stores it, is still reattached.""" + apify_client_async_patcher.patch('actor', 'start', return_value=make_run('new-run', 'READY')) + apify_client_async_patcher.patch('run', 'get', return_value=make_run('old-run', 'RUNNING')) + record = stored_record('old-run', 'RUNNING') + del record['runName'] + record['startedAt'] = STARTED_AT + record['history'] = [{'runId': 'older-run', 'status': 'FAILED', 'startedAt': STARTED_AT}] + + async with Actor: + kvs = await Actor.open_key_value_store() + await kvs.set_value(CHILD_RUNS_KEY, {'scrape-eu': record}) + await Actor._child_run_registry.load() + run = await Actor.start('some-actor', run_name='scrape-eu') + + assert run.id == 'old-run' + assert apify_client_async_patcher.calls['actor']['start'] == [] + + +async def test_named_start_rejects_name_recorded_with_other_run_options( + apify_client_async_patcher: ApifyClientAsyncPatcher, +) -> None: + """Reusing a name with different run options raises instead of attaching to the recorded run.""" + apify_client_async_patcher.patch('actor', 'start', return_value=make_run('new-run', 'READY')) + apify_client_async_patcher.patch('run', 'get', return_value=make_run('old-run', 'RUNNING')) + + async with Actor: + await record_child_run('scrape-eu', 'old-run', run_options={'memory_mbytes': 1024}) + with pytest.raises(ValueError, match='already used for a different Actor, task, input or run options'): + await Actor.start('some-actor', run_name='scrape-eu', memory_mbytes=4096) + + assert apify_client_async_patcher.calls['actor']['start'] == [] + + async def test_named_start_rejects_name_recorded_for_task( apify_client_async_patcher: ApifyClientAsyncPatcher, ) -> None: @@ -248,7 +430,7 @@ async def test_named_start_rejects_name_recorded_for_task( async with Actor: await record_child_run('scrape-eu', 'old-run', actor_id=None, task_id='some-task') - with pytest.raises(ValueError, match='already used for a different Actor, task or input'): + with pytest.raises(ValueError, match='already used for a different Actor, task, input or run options'): await Actor.start('some-actor', run_name='scrape-eu') assert apify_client_async_patcher.calls['actor']['start'] == [] @@ -361,7 +543,7 @@ async def test_named_call_waits_for_resurrected_run(apify_client_async_patcher: apify_client_async_patcher.patch('run', 'get_streamed_log', replacement_method=get_streamed_log) async with Actor: - await record_child_run('scrape-eu', 'old-run') + await record_child_run('scrape-eu', 'old-run', run_options={'timeout': timedelta(minutes=5)}) run = await Actor.call('some-actor', run_name='scrape-eu', timeout=timedelta(minutes=5)) assert run.id == 'old-run' @@ -441,7 +623,9 @@ async def test_named_call_task_resurrects_recorded_run(apify_client_async_patche apify_client_async_patcher.patch('run', 'wait_for_finish', return_value=make_run('old-run', 'SUCCEEDED')) async with Actor: - await record_child_run('scrape-eu', 'old-run', actor_id=None, task_id='some-task') + await record_child_run( + 'scrape-eu', 'old-run', actor_id=None, task_id='some-task', run_options={'memory_mbytes': 2048} + ) run = await Actor.call_task('some-task', run_name='scrape-eu', memory_mbytes=2048) assert run.id == 'old-run' @@ -459,7 +643,7 @@ async def test_named_call_task_rejects_name_recorded_for_actor( async with Actor: await record_child_run('scrape-eu', 'old-run') - with pytest.raises(ValueError, match='already used for a different Actor, task or input'): + with pytest.raises(ValueError, match='already used for a different Actor, task, input or run options'): await Actor.call_task('some-task', run_name='scrape-eu') assert apify_client_async_patcher.calls['task']['start'] == [] @@ -489,7 +673,7 @@ async def test_named_start_forwards_max_items_to_resurrect( apify_client_async_patcher.patch('run', 'resurrect', return_value=make_run('old-run', 'RUNNING')) async with Actor: - await record_child_run('scrape-eu', 'old-run') + await record_child_run('scrape-eu', 'old-run', run_options={'max_items': 10}) with pytest.warns(FutureWarning, match='max_items') as warnings: await Actor.start('some-actor', run_name='scrape-eu', max_items=10) @@ -570,7 +754,7 @@ async def test_named_start_rejects_name_recorded_with_other_input( async with Actor: await record_child_run('scrape-eu', 'old-run', run_input={'since': '2025-01-01'}) - with pytest.raises(ValueError, match='already used for a different Actor, task or input'): + with pytest.raises(ValueError, match='already used for a different Actor, task, input or run options'): await Actor.start('some-actor', {'since': '2026-01-01'}, run_name='scrape-eu') assert apify_client_async_patcher.calls['actor']['start'] == [] @@ -605,7 +789,6 @@ async def test_named_start_replaces_run_recorded_for_other_request( 'new-run', 'READY', run_input={'since': '2026-01-01'}, - history=[{'runId': 'old-run', 'status': 'FAILED', 'startedAt': STARTED_AT}], ) } @@ -676,13 +859,28 @@ async def test_named_call_records_finished_status(apify_client_async_patcher: Ap assert stored == {'scrape-eu': stored_record('new-run', 'FAILED')} -async def test_child_runs_is_empty_without_named_runs(apify_client_async_patcher: ApifyClientAsyncPatcher) -> None: - """`Actor.child_runs` is empty when no run was started with a `run_name`.""" - apify_client_async_patcher.patch('actor', 'start', return_value=make_run('new-run', 'READY')) +async def test_child_runs_is_empty_without_child_runs() -> None: + """`Actor.child_runs` is empty when no child run was started.""" + async with Actor: + assert Actor.child_runs == {} + + +async def test_child_runs_keys_unnamed_run_by_implicit_key(apify_client_async_patcher: ApifyClientAsyncPatcher) -> None: + """A run started without a name shows up in `Actor.child_runs` under its implicit key, next to the named runs.""" + run_ids = iter(['unnamed-run', 'named-run']) + apify_client_async_patcher.patch( + 'actor', 'start', replacement_method=AsyncMock(side_effect=lambda *_, **__: make_run(next(run_ids), 'READY')) + ) async with Actor: await Actor.start('some-actor') - assert Actor.child_runs == {} + await Actor.start('some-actor', run_name='scrape-eu') + child_runs = Actor.child_runs + + assert {key: run_client.resource_id for key, run_client in child_runs.items()} == { + implicit_key(): 'unnamed-run', + 'scrape-eu': 'named-run', + } def test_child_runs_requires_initialized_actor() -> None: @@ -699,7 +897,10 @@ async def test_child_runs_includes_runs_recorded_before_init() -> None: CHILD_RUNS_KEY, { 'scrape-eu': stored_record('eu-run', 'RUNNING'), - 'scrape-us': stored_record('us-run', 'FAILED', actor_id=None, task_id='some-task'), + 'scrape-us': stored_record( + 'us-run', 'FAILED', run_name='scrape-us', actor_id=None, task_id='some-task' + ), + implicit_key(): stored_unnamed_record('unnamed-run', 'RUNNING'), }, ) @@ -709,6 +910,7 @@ async def test_child_runs_includes_runs_recorded_before_init() -> None: assert {name: run_client.resource_id for name, run_client in child_runs.items()} == { 'scrape-eu': 'eu-run', 'scrape-us': 'us-run', + implicit_key(): 'unnamed-run', } @@ -753,6 +955,28 @@ def recording_new_client(self: _ActorType, **kwargs: Any) -> ApifyClientAsync: assert child_runs['recorded']._http_client is default_http_client +async def test_child_runs_uses_client_of_unnamed_start( + apify_client_async_patcher: ApifyClientAsyncPatcher, monkeypatch: pytest.MonkeyPatch +) -> None: + """A run started without a name with a custom token gets a client with that token in `Actor.child_runs`.""" + apify_client_async_patcher.patch('actor', 'start', return_value=make_run('custom-run', 'READY')) + new_client = _ActorType.new_client + clients_by_token: dict[str | None, ApifyClientAsync] = {} + + def recording_new_client(self: _ActorType, **kwargs: Any) -> ApifyClientAsync: + client = new_client(self, **kwargs) + clients_by_token[kwargs.get('token')] = client + return client + + monkeypatch.setattr(_ActorType, 'new_client', recording_new_client) + + async with Actor: + await Actor.start('some-actor', token='custom-token') + child_runs = Actor.child_runs + + assert child_runs[implicit_key()]._http_client is clients_by_token['custom-token']._http_client + + async def test_child_runs_keeps_client_of_name_after_rejected_reuse( apify_client_async_patcher: ApifyClientAsyncPatcher, ) -> None: @@ -762,7 +986,7 @@ async def test_child_runs_keeps_client_of_name_after_rejected_reuse( async with Actor: await Actor.start('some-actor', {'since': '2025-01-01'}, run_name='scrape-eu') - with pytest.raises(ValueError, match='already used for a different Actor, task or input'): + with pytest.raises(ValueError, match='already used for a different Actor, task, input or run options'): await Actor.start('some-actor', {'since': '2026-01-01'}, run_name='scrape-eu', token='other-token') child_runs = Actor.child_runs default_http_client = Actor.apify_client._http_client @@ -880,7 +1104,7 @@ async def abort_run(run_client: Any, *_args: Any, **_kwargs: Any) -> Run: await kvs.set_value( CHILD_RUNS_KEY, { - name: stored_record(run_id, 'RUNNING', abort_with_parent=marked) + name: stored_record(run_id, 'RUNNING', run_name=name, abort_with_parent=marked) for name, run_id, marked in [ ('running', 'running-run', True), ('ready', 'ready-run', True), @@ -913,7 +1137,9 @@ async def test_aborting_event_skips_a_child_run_that_no_longer_exists( async with Actor: kvs = await Actor.open_key_value_store() - await kvs.set_value(CHILD_RUNS_KEY, {'gone': stored_record('gone-run', 'RUNNING', abort_with_parent=True)}) + await kvs.set_value( + CHILD_RUNS_KEY, {'gone': stored_record('gone-run', 'RUNNING', run_name='gone', abort_with_parent=True)} + ) await Actor._child_run_registry.load() apify_event_manager.emit(event=Event.ABORTING, event_data=EventAbortingData()) await apify_event_manager.wait_for_all_listeners_to_complete() @@ -940,7 +1166,10 @@ async def abort_run(run_client: Any, *_args: Any, **_kwargs: Any) -> Run: kvs = await Actor.open_key_value_store() await kvs.set_value( CHILD_RUNS_KEY, - {name: stored_record(f'{name}-run', 'RUNNING', abort_with_parent=True) for name in ['broken', 'healthy']}, + { + name: stored_record(f'{name}-run', 'RUNNING', run_name=name, abort_with_parent=True) + for name in ['broken', 'healthy'] + }, ) await Actor._child_run_registry.load() apify_event_manager.emit(event=Event.ABORTING, event_data=EventAbortingData()) @@ -987,7 +1216,7 @@ async def test_rejected_named_start_keeps_the_client_used_to_abort() -> None: kvs = await Actor.open_key_value_store() await kvs.set_value(CHILD_RUNS_KEY, {'scrape-eu': stored_record('old-run', 'RUNNING', abort_with_parent=True)}) registry = ChildRunRegistry(Actor.open_key_value_store) - with pytest.raises(ValueError, match='already used for a different Actor, task or input'): + with pytest.raises(ValueError, match='already used for a different Actor, task, input or run options'): await registry.find_or_start( 'scrape-eu', actor_id='other-actor', diff --git a/tests/unit/actor/test_actor_helpers.py b/tests/unit/actor/test_actor_helpers.py index 2d740e4b..30f142e9 100644 --- a/tests/unit/actor/test_actor_helpers.py +++ b/tests/unit/actor/test_actor_helpers.py @@ -6,7 +6,7 @@ from datetime import UTC, datetime, timedelta from decimal import Decimal from typing import TYPE_CHECKING -from unittest.mock import AsyncMock, Mock, patch +from unittest.mock import AsyncMock, MagicMock, Mock, patch import pytest @@ -106,26 +106,32 @@ async def test_new_client_uses_configured_api_public_base_url(monkeypatch: pytes async def test_call_actor(apify_client_async_patcher: ApifyClientAsyncPatcher, fake_actor_run: Run) -> None: - apify_client_async_patcher.patch('actor', 'call', return_value=fake_actor_run) + """`Actor.call` starts the Actor through the client's `actor.start` and waits for the run.""" + apify_client_async_patcher.patch('actor', 'start', return_value=fake_actor_run) + apify_client_async_patcher.patch('run', 'wait_for_finish', return_value=fake_actor_run) actor_id = 'some-actor-id' async with Actor: - await Actor.call(actor_id) + await Actor.call(actor_id, logger=None) - assert len(apify_client_async_patcher.calls['actor']['call']) == 1 + assert len(apify_client_async_patcher.calls['actor']['start']) == 1 # The first argument is ActorClientAsync, which was called, let's check its id. - assert apify_client_async_patcher.calls['actor']['call'][0][0][0].resource_id == actor_id + assert apify_client_async_patcher.calls['actor']['start'][0][0][0].resource_id == actor_id + assert len(apify_client_async_patcher.calls['run']['wait_for_finish']) == 1 async def test_call_actor_task(apify_client_async_patcher: ApifyClientAsyncPatcher, fake_actor_run: Run) -> None: - apify_client_async_patcher.patch('task', 'call', return_value=fake_actor_run) + """`Actor.call_task` starts the task through the client's `task.start` and waits for the run.""" + apify_client_async_patcher.patch('task', 'start', return_value=fake_actor_run) + apify_client_async_patcher.patch('run', 'wait_for_finish', return_value=fake_actor_run) task_id = 'some-task-id' async with Actor: await Actor.call_task(task_id) - assert len(apify_client_async_patcher.calls['task']['call']) == 1 - assert apify_client_async_patcher.calls['task']['call'][0][0][0].resource_id == task_id + assert len(apify_client_async_patcher.calls['task']['start']) == 1 + assert apify_client_async_patcher.calls['task']['start'][0][0][0].resource_id == task_id + assert len(apify_client_async_patcher.calls['run']['wait_for_finish']) == 1 async def test_start_actor_task(apify_client_async_patcher: ApifyClientAsyncPatcher, fake_actor_run: Run) -> None: @@ -154,13 +160,20 @@ async def test_start_actor(apify_client_async_patcher: ApifyClientAsyncPatcher, assert apify_client_async_patcher.calls['actor']['start'][0][0][0].resource_id == actor_id +def patch_run_wait(apify_client_async_patcher: ApifyClientAsyncPatcher, run: Run) -> None: + """Make `Actor.call` and `Actor.call_task` get `run` back from the wait, without streaming its log.""" + apify_client_async_patcher.patch('run', 'wait_for_finish', return_value=run) + apify_client_async_patcher.patch('run', 'get_status_message_watcher', return_value=MagicMock()) + apify_client_async_patcher.patch('run', 'get_streamed_log', return_value=MagicMock()) + + @pytest.mark.parametrize( ('client_resource', 'client_method', 'sdk_method'), [ pytest.param('actor', 'start', 'start', id='start'), - pytest.param('actor', 'call', 'call', id='call'), + pytest.param('actor', 'start', 'call', id='call'), pytest.param('task', 'start', 'start_task', id='start_task'), - pytest.param('task', 'call', 'call_task', id='call_task'), + pytest.param('task', 'start', 'call_task', id='call_task'), ], ) async def test_max_items_forwarded_to_client( @@ -172,6 +185,7 @@ async def test_max_items_forwarded_to_client( ) -> None: """`max_items` passed to any of the run-starting helpers reaches the API client.""" apify_client_async_patcher.patch(client_resource, client_method, return_value=fake_actor_run) + patch_run_wait(apify_client_async_patcher, fake_actor_run) async with Actor: with pytest.warns(FutureWarning): @@ -184,9 +198,9 @@ async def test_max_items_forwarded_to_client( ('client_resource', 'client_method', 'sdk_method'), [ pytest.param('actor', 'start', 'start', id='start'), - pytest.param('actor', 'call', 'call', id='call'), + pytest.param('actor', 'start', 'call', id='call'), pytest.param('task', 'start', 'start_task', id='start_task'), - pytest.param('task', 'call', 'call_task', id='call_task'), + pytest.param('task', 'start', 'call_task', id='call_task'), pytest.param('run', 'resurrect', 'resurrect', id='resurrect'), ], ) @@ -199,6 +213,7 @@ async def test_max_items_warns_deprecated( ) -> None: """Passing `max_items` emits a `FutureWarning` that recommends `max_total_charge_usd` and points at the caller.""" apify_client_async_patcher.patch(client_resource, client_method, return_value=fake_actor_run) + patch_run_wait(apify_client_async_patcher, fake_actor_run) async with Actor: with pytest.warns(FutureWarning, match='`max_total_charge_usd`') as record: @@ -212,9 +227,9 @@ async def test_max_items_warns_deprecated( ('client_resource', 'client_method', 'sdk_method'), [ pytest.param('actor', 'start', 'start', id='start'), - pytest.param('actor', 'call', 'call', id='call'), + pytest.param('actor', 'start', 'call', id='call'), pytest.param('task', 'start', 'start_task', id='start_task'), - pytest.param('task', 'call', 'call_task', id='call_task'), + pytest.param('task', 'start', 'call_task', id='call_task'), pytest.param('run', 'resurrect', 'resurrect', id='resurrect'), ], ) @@ -227,6 +242,7 @@ async def test_no_max_items_warning_without_max_items( ) -> None: """The `max_items` deprecation warning is not emitted when `max_items` is not passed.""" apify_client_async_patcher.patch(client_resource, client_method, return_value=fake_actor_run) + patch_run_wait(apify_client_async_patcher, fake_actor_run) async with Actor: with warnings.catch_warnings(): @@ -238,9 +254,9 @@ async def test_no_max_items_warning_without_max_items( ('client_resource', 'client_method', 'sdk_method'), [ pytest.param('actor', 'start', 'start', id='start'), - pytest.param('actor', 'call', 'call', id='call'), + pytest.param('actor', 'start', 'call', id='call'), pytest.param('task', 'start', 'start_task', id='start_task'), - pytest.param('task', 'call', 'call_task', id='call_task'), + pytest.param('task', 'start', 'call_task', id='call_task'), ], ) async def test_max_total_charge_usd_forwarded_to_client( @@ -252,6 +268,7 @@ async def test_max_total_charge_usd_forwarded_to_client( ) -> None: """`max_total_charge_usd` passed to any of the run-starting helpers reaches the API client.""" apify_client_async_patcher.patch(client_resource, client_method, return_value=fake_actor_run) + patch_run_wait(apify_client_async_patcher, fake_actor_run) async with Actor: await getattr(Actor, sdk_method)('some-id', max_total_charge_usd=Decimal('2.5')) @@ -497,9 +514,9 @@ async def listener(_data: object) -> None: _ACTOR_REMOTE_METHODS = [ pytest.param('actor', 'start', 'start', 'some-actor-id', id='start'), - pytest.param('actor', 'call', 'call', 'some-actor-id', id='call'), + pytest.param('actor', 'start', 'call', 'some-actor-id', id='call'), pytest.param('task', 'start', 'start_task', 'some-task-id', id='start_task'), - pytest.param('task', 'call', 'call_task', 'some-task-id', id='call_task'), + pytest.param('task', 'start', 'call_task', 'some-task-id', id='call_task'), ] @@ -515,6 +532,7 @@ async def test_remote_method_with_webhooks( ) -> None: """Test that start/call/start_task/call_task correctly serialize webhooks.""" apify_client_async_patcher.patch(client_resource, client_method, return_value=fake_actor_run) + patch_run_wait(apify_client_async_patcher, fake_actor_run) async with Actor: actor_method = getattr(Actor, actor_method_name) @@ -542,6 +560,7 @@ async def test_remote_method_with_timedelta_timeout( ) -> None: """Test that start/call/start_task/call_task accept a timedelta timeout.""" apify_client_async_patcher.patch(client_resource, client_method, return_value=fake_actor_run) + patch_run_wait(apify_client_async_patcher, fake_actor_run) async with Actor: actor_method = getattr(Actor, actor_method_name) @@ -565,6 +584,7 @@ async def test_remote_method_with_invalid_timeout( ) -> None: """Test that start/call/start_task/call_task raise ValueError for invalid timeout.""" apify_client_async_patcher.patch(client_resource, client_method, return_value=fake_actor_run) + patch_run_wait(apify_client_async_patcher, fake_actor_run) async with Actor: actor_method = getattr(Actor, actor_method_name) @@ -619,45 +639,55 @@ async def test_get_remaining_time_returns_positive_when_timeout_in_future() -> N assert result <= timedelta(minutes=5) -@pytest.mark.parametrize('method_name', ['start', 'call']) +@pytest.mark.parametrize( + 'method_name', + [ + pytest.param('start', id='start'), + pytest.param('call', id='call'), + ], +) async def test_actor_start_and_call_skipped_when_no_inherited_time_remains( apify_client_async_patcher: ApifyClientAsyncPatcher, + fake_actor_run: Run, method_name: str, ) -> None: """Test that Actor.start/Actor.call with `timeout='inherit'` is skipped when the run is past its timeout.""" - apify_client_async_patcher.patch('actor', method_name, return_value=Mock()) + apify_client_async_patcher.patch('actor', 'start', return_value=fake_actor_run) + apify_client_async_patcher.patch('run', 'wait_for_finish', return_value=fake_actor_run) + kwargs = {'logger': None} if method_name == 'call' else {} async with Actor: Actor.configuration.is_at_home = True Actor.configuration.timeout_at = datetime.now(tz=UTC) - timedelta(minutes=5) - await getattr(Actor, method_name)('some-actor-id', timeout='inherit') + await getattr(Actor, method_name)('some-actor-id', timeout='inherit', **kwargs) - assert len(apify_client_async_patcher.calls['actor'][method_name]) == 1 - assert apify_client_async_patcher.calls['actor'][method_name][0][1]['run_timeout'] == timedelta(seconds=1) + assert len(apify_client_async_patcher.calls['actor']['start']) == 1 + assert apify_client_async_patcher.calls['actor']['start'][0][1]['run_timeout'] == timedelta(seconds=1) @pytest.mark.parametrize( - ('client_method', 'actor_method_name'), + 'actor_method_name', [ - pytest.param('start', 'start_task', id='start_task'), - pytest.param('call', 'call_task', id='call_task'), + pytest.param('start_task', id='start task'), + pytest.param('call_task', id='call task'), ], ) async def test_actor_task_methods_skipped_when_no_inherited_time_remains( apify_client_async_patcher: ApifyClientAsyncPatcher, - client_method: str, + fake_actor_run: Run, actor_method_name: str, ) -> None: """Test that start_task/call_task with `timeout='inherit'` is skipped when the run is past its timeout.""" - apify_client_async_patcher.patch('task', client_method, return_value=Mock()) + apify_client_async_patcher.patch('task', 'start', return_value=fake_actor_run) + apify_client_async_patcher.patch('run', 'wait_for_finish', return_value=fake_actor_run) async with Actor: Actor.configuration.is_at_home = True Actor.configuration.timeout_at = datetime.now(tz=UTC) - timedelta(minutes=5) await getattr(Actor, actor_method_name)('some-task-id', timeout='inherit') - assert len(apify_client_async_patcher.calls['task'][client_method]) == 1 - assert apify_client_async_patcher.calls['task'][client_method][0][1]['run_timeout'] == timedelta(seconds=1) + assert len(apify_client_async_patcher.calls['task']['start']) == 1 + assert apify_client_async_patcher.calls['task']['start'][0][1]['run_timeout'] == timedelta(seconds=1) async def test_reboot_runs_all_listeners_even_when_one_fails(