Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .rules.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ Don't introduce a shared Pydantic base model to centralize `model_config`. Each

- **`Actor`** (`_actor.py`) is a `lazy-object-proxy` wrapping `_ActorType`. It works both as a class (`Actor.is_at_home()`) and as a context manager (`async with Actor:`); on `__aenter__` the proxy's `__wrapped__` is replaced with the active `_ActorType` instance.
- **`Configuration`** (`_configuration.py`) extends Crawlee's `Configuration` and is populated from `APIFY_*` environment variables.
- **Named child runs** (`_child_runs.py`) — runs started with `run_name` are persisted in the default key-value store so they can be reattached after a migration or resurrection; exposed through `Actor.child_runs`.
- **Child runs** (`_child_runs.py`) — runs are persisted in the default key-value store under their `run_name`, or without one under an implicit key (a hash of the Actor or task, input and run options, so identical calls reuse one run), so they can be reattached after a migration or resurrection. All of them are exposed through `Actor.child_runs`.
- **Storage clients** (`storage_clients/`) implement Crawlee's storage client interface: `ApifyStorageClient` talks to the Apify API (separate sub-clients for single vs. shared request queues), `FileSystemStorageClient` adds Apify-specific key-value store behavior, and `SmartApifyStorageClient` delegates to the cloud client on the platform and to the local one elsewhere.
- **Events** — `ApifyEventManager` receives platform events over a WebSocket.
- **Errors** — `apify.errors` only re-exports the exceptions from `apify_client.errors`; the SDK defines no error classes of its own, and API calls raise the client exceptions unchanged.
Expand Down
211 changes: 108 additions & 103 deletions src/apify/_actor.py
Original file line number Diff line number Diff line change
Expand Up @@ -433,11 +433,11 @@ def apify_client(self) -> ApifyClientAsync:
@property
@_ensure_context
def child_runs(self) -> dict[str, RunClientAsync]:
"""Clients for the named child runs of this Actor run, keyed by the run name.
"""Clients for the child runs of this Actor run, keyed by the run name, or by an implicit key without one.

Every run started by `Actor.start`, `Actor.call`, `Actor.start_task` or `Actor.call_task` with a `run_name` is
included, even one started before a migration or resurrection of this Actor run. Runs started without a
`run_name` are not tracked. Each client points to the current run under its name:
Every run started by `Actor.start`, `Actor.call`, `Actor.start_task` or `Actor.call_task` is included, even one
started before a migration or resurrection of this Actor run. The client of a named run points to the current
run under its name:

```python
run = await Actor.child_runs['my-child'].wait_for_finish()
Expand Down Expand Up @@ -1103,7 +1103,11 @@ async def start(
to the recorded run. A `SUCCEEDED` run is returned as is, an `ABORTED` or `TIMED-OUT` one is
resurrected, and a new run is started only when nothing is recorded under the name, or the recorded
run `FAILED` or no longer exists. Using `run_name` to attach to a run of a different Actor, task or
with a different input raises a `ValueError`.
with a different input or run options raises a `ValueError`. A run started without `run_name` is
recorded under an implicit key, a hash of the Actor or task, the input and the run options, and the
same rules apply to it. So an identical call without `run_name` reuses that run, also before a
migration. To start a new run for an identical call, e.g. when polling, pass a `run_name` that differs
between the calls.
abort_with_parent: If true, the child run is gracefully aborted when this Actor run is gracefully
aborted. It requires `run_name`, and the value is recorded under it, replacing the one from an earlier
call. Once this Actor run is being gracefully aborted, a call with it raises a `RuntimeError` and
Expand Down Expand Up @@ -1136,21 +1140,21 @@ async def start(
webhooks=to_client_representations(webhooks),
)

if run_name is None:
return await start_run()

run, _ = await self._find_or_start_child_run(
run_name,
actor_id=actor_id,
run_input=run_input,
client=client,
start_run=start_run,
content_type=content_type,
force_permission_level=force_permission_level,
build=build,
max_items=max_items,
max_total_charge_usd=max_total_charge_usd,
restart_on_error=restart_on_error,
memory_mbytes=memory_mbytes,
timeout=timeout,
webhooks=webhooks,
abort_with_parent=abort_with_parent,
)
return run
Expand Down Expand Up @@ -1304,7 +1308,11 @@ async def call(
to the recorded run. A `SUCCEEDED` run is returned as is, an `ABORTED` or `TIMED-OUT` one is
resurrected, and a new run is started only when nothing is recorded under the name, or the recorded
run `FAILED` or no longer exists. Using `run_name` to attach to a run of a different Actor, task or
with a different input raises a `ValueError`.
with a different input or run options raises a `ValueError`. A run started without `run_name` is
recorded under an implicit key, a hash of the Actor or task, the input and the run options, and the
same rules apply to it. So an identical call without `run_name` reuses that run, also before a
migration. To start a new run for an identical call, e.g. when polling, pass a `run_name` that differs
between the calls.
abort_with_parent: If true, the child run is gracefully aborted when this Actor run is gracefully
aborted. It requires `run_name`, and the value is recorded under it, replacing the one from an earlier
call. Once this Actor run is being gracefully aborted, a call with it raises a `RuntimeError` and
Expand All @@ -1323,53 +1331,41 @@ async def call(
client = self.new_client(token=token) if token else self.apify_client

actor_client = client.actor(actor_id)
start_run = partial(
actor_client.start,
run_input=run_input,
content_type=content_type,
build=build,
max_items=max_items,
max_total_charge_usd=max_total_charge_usd,
restart_on_error=restart_on_error,
memory_mbytes=memory_mbytes,
run_timeout=self._resolve_run_timeout(timeout),
force_permission_level=force_permission_level,
webhooks=to_client_representations(webhooks),
)

if run_name is None:
run = await actor_client.call(
run_input=run_input,
content_type=content_type,
build=build,
max_items=max_items,
max_total_charge_usd=max_total_charge_usd,
restart_on_error=restart_on_error,
memory_mbytes=memory_mbytes,
run_timeout=self._resolve_run_timeout(timeout),
force_permission_level=force_permission_level,
webhooks=to_client_representations(webhooks),
wait_duration=wait,
logger=logger,
)
else:
started_run, is_new = await self._find_or_start_child_run(
run_name,
actor_id=actor_id,
run_input=run_input,
client=client,
start_run=partial(
actor_client.start,
run_input=run_input,
content_type=content_type,
build=build,
max_items=max_items,
max_total_charge_usd=max_total_charge_usd,
restart_on_error=restart_on_error,
memory_mbytes=memory_mbytes,
run_timeout=self._resolve_run_timeout(timeout),
force_permission_level=force_permission_level,
webhooks=to_client_representations(webhooks),
),
build=build,
max_items=max_items,
max_total_charge_usd=max_total_charge_usd,
restart_on_error=restart_on_error,
memory_mbytes=memory_mbytes,
timeout=timeout,
abort_with_parent=abort_with_parent,
)
# The earlier attempt of this call already streamed the log of a reattached or resurrected run.
run = await self._wait_for_child_run(
run_name, client.run(started_run.id), started_run, wait=wait, logger=logger, from_start=is_new
)
started_run, is_new = await self._find_or_start_child_run(
run_name,
actor_id=actor_id,
run_input=run_input,
client=client,
start_run=start_run,
content_type=content_type,
force_permission_level=force_permission_level,
build=build,
max_items=max_items,
max_total_charge_usd=max_total_charge_usd,
restart_on_error=restart_on_error,
memory_mbytes=memory_mbytes,
timeout=timeout,
webhooks=webhooks,
abort_with_parent=abort_with_parent,
)
# The earlier attempt of this call already streamed the log of a reattached or resurrected run.
run = await self._wait_for_child_run(
client.run(started_run.id), started_run, wait=wait, logger=logger, from_start=is_new
)

if run is None:
raise RuntimeError(f'Failed to call Actor with ID "{actor_id}".')
Expand All @@ -1378,26 +1374,41 @@ async def call(

async def _find_or_start_child_run(
self,
name: str,
name: str | None,
*,
actor_id: str | None = None,
task_id: str | None = None,
run_input: Any,
client: ApifyClientAsync,
start_run: Callable[[], Awaitable[Run]],
content_type: str | None = None,
force_permission_level: ActorPermissionLevel | None = None,
build: str | None,
max_items: int | None,
max_total_charge_usd: Decimal | None,
restart_on_error: bool | None,
memory_mbytes: int | None,
timeout: timedelta | Literal['inherit'] | None,
webhooks: list[Webhook] | None,
abort_with_parent: bool,
) -> tuple[Run, bool]:
run_options = {
'content_type': content_type,
'build': build,
'max_items': max_items,
'max_total_charge_usd': max_total_charge_usd,
'restart_on_error': restart_on_error,
'memory_mbytes': memory_mbytes,
'timeout': timeout,
'force_permission_level': force_permission_level,
'webhooks': [asdict(webhook) for webhook in webhooks] if webhooks else None,
}
return await self._child_run_registry.find_or_start(
name,
actor_id=actor_id,
task_id=task_id,
run_input=run_input,
run_options=run_options,
client=client,
start_run=start_run,
resurrect_run=lambda run_id: client.run(run_id).resurrect(
Expand All @@ -1416,7 +1427,6 @@ async def _abort_child_runs(self) -> None:

async def _wait_for_child_run(
self,
name: str,
run_client: RunClientAsync,
run: Run,
*,
Expand All @@ -1438,7 +1448,7 @@ async def _wait_for_child_run(
finished_run = await run_client.wait_for_finish(wait_duration=wait)

if finished_run is not None:
await self._child_run_registry.update(name, finished_run)
await self._child_run_registry.update(finished_run)
return finished_run

@_ensure_context
Expand Down Expand Up @@ -1491,7 +1501,11 @@ async def start_task(
to the recorded run. A `SUCCEEDED` run is returned as is, an `ABORTED` or `TIMED-OUT` one is
resurrected, and a new run is started only when nothing is recorded under the name, or the recorded
run `FAILED` or no longer exists. Using `run_name` to attach to a run of a different Actor, task or
with a different input raises a `ValueError`.
with a different input or run options raises a `ValueError`. A run started without `run_name` is
recorded under an implicit key, a hash of the Actor or task, the input and the run options, and the
same rules apply to it. So an identical call without `run_name` reuses that run, also before a
migration. To start a new run for an identical call, e.g. when polling, pass a `run_name` that differs
between the calls.
abort_with_parent: If true, the child run is gracefully aborted when this Actor run is gracefully
aborted. It requires `run_name`, and the value is recorded under it, replacing the one from an earlier
call. Once this Actor run is being gracefully aborted, a call with it raises a `RuntimeError` and
Expand Down Expand Up @@ -1521,9 +1535,6 @@ async def start_task(
webhooks=to_client_representations(webhooks),
)

if run_name is None:
return await start_run()

run, _ = await self._find_or_start_child_run(
run_name,
task_id=task_id,
Expand All @@ -1536,6 +1547,7 @@ async def start_task(
restart_on_error=restart_on_error,
memory_mbytes=memory_mbytes,
timeout=timeout,
webhooks=webhooks,
abort_with_parent=abort_with_parent,
)
return run
Expand Down Expand Up @@ -1591,7 +1603,11 @@ async def call_task(
to the recorded run. A `SUCCEEDED` run is returned as is, an `ABORTED` or `TIMED-OUT` one is
resurrected, and a new run is started only when nothing is recorded under the name, or the recorded
run `FAILED` or no longer exists. Using `run_name` to attach to a run of a different Actor, task or
with a different input raises a `ValueError`.
with a different input or run options raises a `ValueError`. A run started without `run_name` is
recorded under an implicit key, a hash of the Actor or task, the input and the run options, and the
same rules apply to it. So an identical call without `run_name` reuses that run, also before a
migration. To start a new run for an identical call, e.g. when polling, pass a `run_name` that differs
between the calls.
abort_with_parent: If true, the child run is gracefully aborted when this Actor run is gracefully
aborted. It requires `run_name`, and the value is recorded under it, replacing the one from an earlier
call. Once this Actor run is being gracefully aborted, a call with it raises a `RuntimeError` and
Expand All @@ -1610,47 +1626,36 @@ async def call_task(
client = self.new_client(token=token) if token else self.apify_client

task_client = client.task(task_id)
start_run = partial(
task_client.start,
task_input=task_input,
build=build,
max_items=max_items,
max_total_charge_usd=max_total_charge_usd,
restart_on_error=restart_on_error,
memory_mbytes=memory_mbytes,
run_timeout=self._resolve_run_timeout(timeout),
webhooks=to_client_representations(webhooks),
)

if run_name is None:
run = await task_client.call(
task_input=task_input,
build=build,
max_items=max_items,
max_total_charge_usd=max_total_charge_usd,
restart_on_error=restart_on_error,
memory_mbytes=memory_mbytes,
run_timeout=self._resolve_run_timeout(timeout),
webhooks=to_client_representations(webhooks),
wait_duration=wait,
)
else:
started_run, _ = await self._find_or_start_child_run(
run_name,
task_id=task_id,
run_input=task_input,
client=client,
start_run=partial(
task_client.start,
task_input=task_input,
build=build,
max_items=max_items,
max_total_charge_usd=max_total_charge_usd,
restart_on_error=restart_on_error,
memory_mbytes=memory_mbytes,
run_timeout=self._resolve_run_timeout(timeout),
webhooks=to_client_representations(webhooks),
),
build=build,
max_items=max_items,
max_total_charge_usd=max_total_charge_usd,
restart_on_error=restart_on_error,
memory_mbytes=memory_mbytes,
timeout=timeout,
abort_with_parent=abort_with_parent,
)
run = await self._wait_for_child_run(
run_name, client.run(started_run.id), started_run, wait=wait, logger=None, from_start=False
)
started_run, _ = await self._find_or_start_child_run(
run_name,
task_id=task_id,
run_input=task_input,
client=client,
start_run=start_run,
build=build,
max_items=max_items,
max_total_charge_usd=max_total_charge_usd,
restart_on_error=restart_on_error,
memory_mbytes=memory_mbytes,
timeout=timeout,
webhooks=webhooks,
abort_with_parent=abort_with_parent,
)
run = await self._wait_for_child_run(
client.run(started_run.id), started_run, wait=wait, logger=None, from_start=False
)

if run is None:
raise RuntimeError(f'Failed to call Task with ID "{task_id}".')
Expand Down
Loading
Loading