From 5baddead6da035ad66d2d05efe638eea87f2af59 Mon Sep 17 00:00:00 2001 From: Donatas Kasparavicius Date: Wed, 22 Jul 2026 10:23:08 +0300 Subject: [PATCH 01/17] Wire up request-schema validation in scrape/scrape_async - scrape() and scrape_async() now accept ScrapeRequest | Mapping[str, Any] - Added _to_payload() helper: model_dump(by_alias=True) for Pydantic models, dict() for mappings - Added _validate() method using jsonschema, raises decodo.errors.ValidationError on failure - scrape_batch() accepts BatchRequest | Mapping[str, Any] but skips validation (mirrors TS) - Constructor default changed to BundledSchema.shared (mirrors TS) - Removed now-unnecessary type: ignore[arg-type] comments in tests Co-Authored-By: Claude Opus 4.8 (1M context) --- src/decodo/api/web_scraping_api.py | 46 ++++++++++++++++++++++++------ tests/test_web_scraping_api.py | 6 ++-- 2 files changed, 40 insertions(+), 12 deletions(-) diff --git a/src/decodo/api/web_scraping_api.py b/src/decodo/api/web_scraping_api.py index 5711759..e8e86e8 100644 --- a/src/decodo/api/web_scraping_api.py +++ b/src/decodo/api/web_scraping_api.py @@ -1,9 +1,15 @@ from __future__ import annotations -from typing import cast +from collections.abc import Mapping +from typing import Any, cast -from decodo.generated.targets import ScrapeRequest +import jsonschema +from pydantic import BaseModel + +import decodo.errors +from decodo.generated.targets import BatchRequest, ScrapeRequest from decodo.http import HttpClient +from decodo.schema.bundled_schema import BundledSchema from decodo.schema.types import DecodoSchema from decodo.types.responses import ( AsyncTaskResponse, @@ -14,19 +20,41 @@ ) +def _to_payload(params: ScrapeRequest | BatchRequest | Mapping[str, Any]) -> dict[str, Any]: + if isinstance(params, BaseModel): + return params.model_dump(by_alias=True, exclude_none=True, mode="json") + return dict(params) + + class WebScrapingApi: - def __init__(self, http: HttpClient, schema: DecodoSchema | None = None) -> None: + def __init__(self, http: HttpClient, schema: DecodoSchema = BundledSchema.shared) -> None: self._http = http self._schema = schema - def scrape(self, params: ScrapeRequest) -> SyncResponse: - return cast(SyncResponse, self._http.post("/v2/scrape", params.model_dump(exclude_none=True, mode="json"))) + def _validate(self, payload: dict[str, Any]) -> None: + if self._schema is None: + return + schema = self._schema.get_request_schema(payload.get("target")) # type: ignore[arg-type] + if not schema: + return + try: + jsonschema.validate(payload, schema) + except jsonschema.ValidationError as e: + raise decodo.errors.ValidationError(str(e)) from e + + def scrape(self, params: ScrapeRequest | Mapping[str, Any]) -> SyncResponse: + payload = _to_payload(params) + self._validate(payload) + return cast(SyncResponse, self._http.post("/v2/scrape", payload)) - def scrape_async(self, params: ScrapeRequest) -> AsyncTaskResponse: - return cast(AsyncTaskResponse, self._http.post("/v3/task", params.model_dump(exclude_none=True, mode="json"))) + def scrape_async(self, params: ScrapeRequest | Mapping[str, Any]) -> AsyncTaskResponse: + payload = _to_payload(params) + self._validate(payload) + return cast(AsyncTaskResponse, self._http.post("/v3/task", payload)) - def scrape_batch(self, params: ScrapeRequest) -> BatchResponse: - return cast(BatchResponse, self._http.post("/v3/task/batch", params.model_dump(exclude_none=True, mode="json"))) + def scrape_batch(self, params: BatchRequest | Mapping[str, Any]) -> BatchResponse: + payload = _to_payload(params) + return cast(BatchResponse, self._http.post("/v3/task/batch", payload)) def get_status(self, task_id: str) -> TaskMetadata: return cast(TaskMetadata, self._http.get(f"/v3/task/{task_id}")) diff --git a/tests/test_web_scraping_api.py b/tests/test_web_scraping_api.py index 6b03139..2e5832a 100644 --- a/tests/test_web_scraping_api.py +++ b/tests/test_web_scraping_api.py @@ -53,7 +53,7 @@ def test_throws_validation_error_before_http_when_schema_rejects(self) -> None: api = WebScrapingApi(http, _StrictSchema()) with pytest.raises(ValidationError): - api.scrape({"target": "google_search", "query": ""}) # type: ignore[arg-type] + api.scrape({"target": "google_search", "query": ""}) http.post.assert_not_called() @@ -61,7 +61,7 @@ def test_calls_http_when_params_pass_schema_validation(self) -> None: http = _make_http_mock() api = WebScrapingApi(http, _StrictSchema()) - api.scrape({"target": "google_search", "query": "coffee"}) # type: ignore[arg-type] + api.scrape({"target": "google_search", "query": "coffee"}) http.post.assert_called_once() @@ -69,6 +69,6 @@ def test_validates_bundled_google_search_payloads(self) -> None: http = _make_http_mock() api = WebScrapingApi(http, BundledSchema.shared) - api.scrape({"target": "google_search", "query": "coffee"}) # type: ignore[arg-type] + api.scrape({"target": "google_search", "query": "coffee"}) http.post.assert_called_once() From bf16f35a57a16571e7e2d5b7fd882e0804a977fd Mon Sep 17 00:00:00 2001 From: Donatas Kasparavicius Date: Wed, 22 Jul 2026 10:23:15 +0300 Subject: [PATCH 02/17] Codegen: emit target_meta as plain dicts, add BatchRequest batch params - generate_targets.py: removed generated TargetMeta pydantic class; target_meta is now emitted as dict[str, dict[str, Any]] (plain dict literals, matching TS behavior) - For each target, generate a *BatchParams class identical to *Params except url/query are typed list[str] | None instead of str | None - BatchRequest is now a proper discriminated union of all *BatchParams classes (was wrongly aliased to ScrapeRequest) - Regenerated src/decodo/generated/targets.py and request_schemas.py from IR Co-Authored-By: Claude Opus 4.8 (1M context) --- .../web_scraping_api/generate_targets.py | 67 +- src/decodo/generated/request_schemas.py | 2 +- src/decodo/generated/targets.py | 1724 ++++++++++++++--- 3 files changed, 1506 insertions(+), 287 deletions(-) diff --git a/src/decodo/codegen/web_scraping_api/generate_targets.py b/src/decodo/codegen/web_scraping_api/generate_targets.py index b76168a..0c8be62 100644 --- a/src/decodo/codegen/web_scraping_api/generate_targets.py +++ b/src/decodo/codegen/web_scraping_api/generate_targets.py @@ -81,11 +81,21 @@ def _get_targets_file_contents(api: WebScrapingApiIR) -> str: "target": "TargetStoreParams", } + BATCH_CLASS_NAME_OVERRIDES: dict[str, str] = { + "target": "TargetStoreBatchParams", + } + def _class_name(target_key: str) -> str: if target_key in CLASS_NAME_OVERRIDES: return CLASS_NAME_OVERRIDES[target_key] return f"{to_pascal_case(target_key)}Params" + def _batch_class_name(target_key: str) -> str: + if target_key in BATCH_CLASS_NAME_OVERRIDES: + return BATCH_CLASS_NAME_OVERRIDES[target_key] + return f"{to_pascal_case(target_key)}BatchParams" + + # Emit per-target sync Params classes for target_key, target in api["targets"].items(): type_name = _class_name(target_key) member = to_enum_member_name(target_key) @@ -100,37 +110,66 @@ def _class_name(target_key: str) -> str: python_type = _json_schema_type_to_python(param_schema) field_name = _sanitize_field_name(param_key) if field_name != param_key: - lines.append(f" {field_name}: {python_type} | None = pydantic.Field(None, alias={json.dumps(param_key)})") # noqa: E501 + lines.append( + f" {field_name}: {python_type} | None = pydantic.Field(None, alias={json.dumps(param_key)})" + ) # noqa: E501 else: lines.append(f" {field_name}: {python_type} | None = None") lines.append("") - lines.append("class TargetMeta(pydantic.BaseModel):") - lines.append(" group: str") - lines.append(" response_format: str") - lines.append(" parameters: list[str]") - lines.append("") + # Emit per-target batch Params classes (url/query become list[str]) + for target_key, target in api["targets"].items(): + batch_type_name = _batch_class_name(target_key) + member = to_enum_member_name(target_key) + properties = target["parameter_schema"].get("properties", {}) + + lines.append(f"class {batch_type_name}(pydantic.BaseModel):") + lines.append(" model_config = pydantic.ConfigDict(populate_by_name=True)") + lines.append(f" target: Literal[Target.{member}] = Target.{member}") + params = {k: v for k, v in properties.items() if k != "target"} + if params: + for param_key, param_schema in params.items(): + field_name = _sanitize_field_name(param_key) + if param_key in ("url", "query"): + python_type = "list[str]" + else: + python_type = _json_schema_type_to_python(param_schema) + if field_name != param_key: + lines.append( + f" {field_name}: {python_type} | None = pydantic.Field(None, alias={json.dumps(param_key)})" + ) # noqa: E501 + else: + lines.append(f" {field_name}: {python_type} | None = None") + lines.append("") - lines.append("target_meta: dict[str, TargetMeta] = {") + # target_meta as plain dict-of-dicts (no TargetMeta class here) + lines.append("target_meta: dict[str, dict[str, Any]] = {") for target_key, target in api["targets"].items(): member = to_enum_member_name(target_key) param_keys = _get_target_parameter_keys(target["parameter_schema"]) params_list = ", ".join(json.dumps(p) for p in param_keys) - lines.append(f" Target.{member}.value: TargetMeta(") - lines.append(f" group={json.dumps(target['group'])},") - lines.append(f" response_format={json.dumps(target['response_format'])},") - lines.append(f" parameters=[{params_list}],") - lines.append(" ),") + lines.append(f" Target.{member}.value: {{") + lines.append(f' "group": {json.dumps(target["group"])},') + lines.append(f' "response_format": {json.dumps(target["response_format"])},') + lines.append(f' "parameters": [{params_list}],') + lines.append(" },") lines.append("}") lines.append("") - # Discriminated union for ScrapeRequest / BatchRequest + # Discriminated union for ScrapeRequest union_parts = " | ".join(_class_name(k) for k in target_keys) lines.append("ScrapeRequest = Annotated[") lines.append(f" Union[{union_parts}],") lines.append(" pydantic.Field(discriminator='target'),") lines.append("]") - lines.append("BatchRequest = ScrapeRequest") + lines.append("") + + # Discriminated union for BatchRequest + batch_union_parts = " | ".join(_batch_class_name(k) for k in target_keys) + lines.append("BatchRequest = Annotated[") + lines.append(f" Union[{batch_union_parts}],") + lines.append(" pydantic.Field(discriminator='target'),") + lines.append("]") lines.append("") return "\n".join(lines) diff --git a/src/decodo/generated/request_schemas.py b/src/decodo/generated/request_schemas.py index 72fe3ea..25ca96e 100644 --- a/src/decodo/generated/request_schemas.py +++ b/src/decodo/generated/request_schemas.py @@ -1,4 +1,4 @@ -# Auto-generated — do not edit +# Auto-generated by src/decodo/codegen/web_scraping_api/generate_parameter_schemas.py — do not edit from __future__ import annotations request_json_schemas: dict[str, dict] = { diff --git a/src/decodo/generated/targets.py b/src/decodo/generated/targets.py index 37efb44..7354c0c 100644 --- a/src/decodo/generated/targets.py +++ b/src/decodo/generated/targets.py @@ -2,7 +2,7 @@ from __future__ import annotations from enum import Enum -from typing import Annotated, Any, Literal +from typing import Annotated, Any, Literal, Union import pydantic @@ -61,13 +61,16 @@ class Target(str, Enum): YoutubeSubtitles = "youtube_subtitles" YoutubeChannel = "youtube_channel" + targets: list[str] = [t.value for t in Target] + class UniversalEcommerceParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.UniversalEcommerce] = Target.UniversalEcommerce callback_url: str | None = None + class GoogleSearchParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.GoogleSearch] = Target.GoogleSearch @@ -89,6 +92,7 @@ class GoogleSearchParams(pydantic.BaseModel): page_count: float | None = None callback_url: str | None = None + class GoogleTravelHotelsParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.GoogleTravelHotels] = Target.GoogleTravelHotels @@ -105,6 +109,7 @@ class GoogleTravelHotelsParams(pydantic.BaseModel): markdown: bool | None = None callback_url: str | None = None + class GoogleTrendsExploreParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.GoogleTrendsExplore] = Target.GoogleTrendsExplore @@ -116,6 +121,7 @@ class GoogleTrendsExploreParams(pydantic.BaseModel): date_end: str | None = None callback_url: str | None = None + class GoogleShoppingSearchParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.GoogleShoppingSearch] = Target.GoogleShoppingSearch @@ -132,6 +138,7 @@ class GoogleShoppingSearchParams(pydantic.BaseModel): markdown: bool | None = None callback_url: str | None = None + class GoogleShoppingProductParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.GoogleShoppingProduct] = Target.GoogleShoppingProduct @@ -148,6 +155,7 @@ class GoogleShoppingProductParams(pydantic.BaseModel): xhr: bool | None = None callback_url: str | None = None + class GoogleParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.Google] = Target.Google @@ -162,6 +170,7 @@ class GoogleParams(pydantic.BaseModel): page_count: float | None = None callback_url: str | None = None + class GoogleSuggestParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.GoogleSuggest] = Target.GoogleSuggest @@ -172,6 +181,7 @@ class GoogleSuggestParams(pydantic.BaseModel): session_id: str | None = None callback_url: str | None = None + class GoogleMapsParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.GoogleMaps] = Target.GoogleMaps @@ -189,6 +199,7 @@ class GoogleMapsParams(pydantic.BaseModel): markdown: bool | None = None callback_url: str | None = None + class GoogleAiModeParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.GoogleAiMode] = Target.GoogleAiMode @@ -201,6 +212,7 @@ class GoogleAiModeParams(pydantic.BaseModel): xhr: bool | None = None callback_url: str | None = None + class GoogleAdsParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.GoogleAds] = Target.GoogleAds @@ -221,6 +233,7 @@ class GoogleAdsParams(pydantic.BaseModel): page_count: float | None = None callback_url: str | None = None + class GoogleLensParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.GoogleLens] = Target.GoogleLens @@ -231,6 +244,7 @@ class GoogleLensParams(pydantic.BaseModel): markdown: bool | None = None callback_url: str | None = None + class BingSearchParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.BingSearch] = Target.BingSearch @@ -248,6 +262,7 @@ class BingSearchParams(pydantic.BaseModel): xhr: bool | None = None callback_url: str | None = None + class BingParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.Bing] = Target.Bing @@ -263,6 +278,7 @@ class BingParams(pydantic.BaseModel): xhr: bool | None = None callback_url: str | None = None + class YoutubeTranscriptParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.YoutubeTranscript] = Target.YoutubeTranscript @@ -271,6 +287,7 @@ class YoutubeTranscriptParams(pydantic.BaseModel): transcript_origin: str | None = None callback_url: str | None = None + class AmazonProductParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.AmazonProduct] = Target.AmazonProduct @@ -287,6 +304,7 @@ class AmazonProductParams(pydantic.BaseModel): xhr: bool | None = None callback_url: str | None = None + class AmazonPricingParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.AmazonPricing] = Target.AmazonPricing @@ -303,6 +321,7 @@ class AmazonPricingParams(pydantic.BaseModel): xhr: bool | None = None callback_url: str | None = None + class AmazonSearchParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.AmazonSearch] = Target.AmazonSearch @@ -322,6 +341,7 @@ class AmazonSearchParams(pydantic.BaseModel): xhr: bool | None = None callback_url: str | None = None + class AmazonSellersParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.AmazonSellers] = Target.AmazonSellers @@ -336,6 +356,7 @@ class AmazonSellersParams(pydantic.BaseModel): xhr: bool | None = None callback_url: str | None = None + class AmazonBestsellersParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.AmazonBestsellers] = Target.AmazonBestsellers @@ -352,6 +373,7 @@ class AmazonBestsellersParams(pydantic.BaseModel): xhr: bool | None = None callback_url: str | None = None + class AmazonParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.Amazon] = Target.Amazon @@ -365,6 +387,7 @@ class AmazonParams(pydantic.BaseModel): xhr: bool | None = None callback_url: str | None = None + class EcommerceParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.Ecommerce] = Target.Ecommerce @@ -377,6 +400,7 @@ class EcommerceParams(pydantic.BaseModel): parser_type: str | None = None callback_url: str | None = None + class WalmartProductParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.WalmartProduct] = Target.WalmartProduct @@ -390,6 +414,7 @@ class WalmartProductParams(pydantic.BaseModel): delivery_zip: str | None = None callback_url: str | None = None + class WalmartSearchParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.WalmartSearch] = Target.WalmartSearch @@ -402,6 +427,7 @@ class WalmartSearchParams(pydantic.BaseModel): delivery_zip: str | None = None callback_url: str | None = None + class WalmartParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.Walmart] = Target.Walmart @@ -415,6 +441,7 @@ class WalmartParams(pydantic.BaseModel): xhr: bool | None = None callback_url: str | None = None + class TargetProductParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.TargetProduct] = Target.TargetProduct @@ -429,6 +456,7 @@ class TargetProductParams(pydantic.BaseModel): delivery_zip: str | None = None callback_url: str | None = None + class TargetSearchParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.TargetSearch] = Target.TargetSearch @@ -443,6 +471,7 @@ class TargetSearchParams(pydantic.BaseModel): markdown: bool | None = None callback_url: str | None = None + class TargetStoreParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.Target] = Target.Target @@ -454,6 +483,7 @@ class TargetStoreParams(pydantic.BaseModel): target_store_id: str | None = None callback_url: str | None = None + class LowesSearchParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.LowesSearch] = Target.LowesSearch @@ -467,6 +497,7 @@ class LowesSearchParams(pydantic.BaseModel): delivery_today_tomorrow: bool | None = None callback_url: str | None = None + class UniversalParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.Universal] = Target.Universal @@ -488,6 +519,7 @@ class UniversalParams(pydantic.BaseModel): markdown: bool | None = None callback_url: str | None = None + class ChatgptParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.Chatgpt] = Target.Chatgpt @@ -500,6 +532,7 @@ class ChatgptParams(pydantic.BaseModel): xhr: bool | None = None callback_url: str | None = None + class PerplexityParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.Perplexity] = Target.Perplexity @@ -511,6 +544,7 @@ class PerplexityParams(pydantic.BaseModel): xhr: bool | None = None callback_url: str | None = None + class GeminiParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.Gemini] = Target.Gemini @@ -520,6 +554,7 @@ class GeminiParams(pydantic.BaseModel): xhr: bool | None = None callback_url: str | None = None + class BbbParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.Bbb] = Target.Bbb @@ -531,6 +566,7 @@ class BbbParams(pydantic.BaseModel): xhr: bool | None = None callback_url: str | None = None + class AutotraderParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.Autotrader] = Target.Autotrader @@ -542,6 +578,7 @@ class AutotraderParams(pydantic.BaseModel): xhr: bool | None = None callback_url: str | None = None + class MobileParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.Mobile] = Target.Mobile @@ -553,6 +590,7 @@ class MobileParams(pydantic.BaseModel): xhr: bool | None = None callback_url: str | None = None + class AirbnbParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.Airbnb] = Target.Airbnb @@ -564,6 +602,7 @@ class AirbnbParams(pydantic.BaseModel): xhr: bool | None = None callback_url: str | None = None + class AppleAppStoreParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.AppleAppStore] = Target.AppleAppStore @@ -575,12 +614,14 @@ class AppleAppStoreParams(pydantic.BaseModel): xhr: bool | None = None callback_url: str | None = None + class InstagramGraphqlProfileParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.InstagramGraphqlProfile] = Target.InstagramGraphqlProfile query: str | None = None callback_url: str | None = None + class TiktokPostParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.TiktokPost] = Target.TiktokPost @@ -588,6 +629,7 @@ class TiktokPostParams(pydantic.BaseModel): xhr: bool | None = None callback_url: str | None = None + class TiktokShopSearchParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.TiktokShopSearch] = Target.TiktokShopSearch @@ -598,6 +640,7 @@ class TiktokShopSearchParams(pydantic.BaseModel): country: str | None = None callback_url: str | None = None + class TiktokShopProductParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.TiktokShopProduct] = Target.TiktokShopProduct @@ -609,6 +652,7 @@ class TiktokShopProductParams(pydantic.BaseModel): country: str | None = None callback_url: str | None = None + class TiktokParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.Tiktok] = Target.Tiktok @@ -617,6 +661,7 @@ class TiktokParams(pydantic.BaseModel): user_agent_type: str | None = None callback_url: str | None = None + class RedditPostParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.RedditPost] = Target.RedditPost @@ -625,6 +670,7 @@ class RedditPostParams(pydantic.BaseModel): geo: str | None = None callback_url: str | None = None + class RedditSubredditParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.RedditSubreddit] = Target.RedditSubreddit @@ -633,6 +679,7 @@ class RedditSubredditParams(pydantic.BaseModel): geo: str | None = None callback_url: str | None = None + class RedditUserParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.RedditUser] = Target.RedditUser @@ -642,6 +689,7 @@ class RedditUserParams(pydantic.BaseModel): sort: str | None = None callback_url: str | None = None + class YoutubeVideoParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.YoutubeVideo] = Target.YoutubeVideo @@ -649,12 +697,14 @@ class YoutubeVideoParams(pydantic.BaseModel): geo: str | None = None callback_url: str | None = None + class YoutubeMetadataParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.YoutubeMetadata] = Target.YoutubeMetadata query: str | None = None callback_url: str | None = None + class YoutubeSearchParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.YoutubeSearch] = Target.YoutubeSearch @@ -676,6 +726,7 @@ class YoutubeSearchParams(pydantic.BaseModel): subtitles: bool | None = None callback_url: str | None = None + class YoutubeSearchMaxParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.YoutubeSearchMax] = Target.YoutubeSearchMax @@ -698,6 +749,7 @@ class YoutubeSearchMaxParams(pydantic.BaseModel): markdown: bool | None = None callback_url: str | None = None + class YoutubeSubtitlesParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.YoutubeSubtitles] = Target.YoutubeSubtitles @@ -706,6 +758,7 @@ class YoutubeSubtitlesParams(pydantic.BaseModel): subtitle_origin: str | None = None callback_url: str | None = None + class YoutubeChannelParams(pydantic.BaseModel): model_config = pydantic.ConfigDict(populate_by_name=True) target: Literal[Target.YoutubeChannel] = Target.YoutubeChannel @@ -715,276 +768,1403 @@ class YoutubeChannelParams(pydantic.BaseModel): markdown: bool | None = None callback_url: str | None = None -class TargetMeta(pydantic.BaseModel): - group: str - response_format: str - parameters: list[str] - -target_meta: dict[str, TargetMeta] = { - Target.UniversalEcommerce.value: TargetMeta( - group="None", - response_format="html", - parameters=["callback_url"], - ), - Target.GoogleSearch.value: TargetMeta( - group="Google", - response_format="json", - parameters=["query", "headless", "locale", "geo", "device_type", "page_from", "google_results_language", "google_tbm", "google_tbs", "parse", "google_nfpr", "google_safe_search", "session_id", "xhr", "markdown", "page_count", "callback_url"], - ), - Target.GoogleTravelHotels.value: TargetMeta( - group="Google", - response_format="html", - parameters=["query", "headless", "locale", "device_type", "page_from", "date_range", "stars", "adults", "children", "session_id", "markdown", "callback_url"], - ), - Target.GoogleTrendsExplore.value: TargetMeta( - group="Google", - response_format="json", - parameters=["query", "geo", "device_type", "search_type", "date_start", "date_end", "callback_url"], - ), - Target.GoogleShoppingSearch.value: TargetMeta( - group="Google", - response_format="json", - parameters=["query", "headless", "locale", "geo", "device_type", "page_from", "google_tbs", "parse", "session_id", "google_results_language", "markdown", "callback_url"], - ), - Target.GoogleShoppingProduct.value: TargetMeta( - group="Google", - response_format="json", - parameters=["query", "headless", "locale", "geo", "device_type", "page_from", "parse", "session_id", "google_results_language", "markdown", "xhr", "callback_url"], - ), - Target.Google.value: TargetMeta( - group="Google", - response_format="json", - parameters=["url", "headless", "locale", "device_type", "parse", "session_id", "markdown", "xhr", "page_count", "callback_url"], - ), - Target.GoogleSuggest.value: TargetMeta( - group="Google", - response_format="json", - parameters=["query", "device_type", "geo", "locale", "session_id", "callback_url"], - ), - Target.GoogleMaps.value: TargetMeta( - group="Google", - response_format="html", - parameters=["query", "headless", "geo", "locale", "page_from", "device_type", "session_id", "google_results_language", "google_nfpr", "hotel_occupancy", "date_range", "markdown", "callback_url"], - ), - Target.GoogleAiMode.value: TargetMeta( - group="AI Tools", - response_format="json", - parameters=["query", "geo", "parse", "device_type", "session_id", "markdown", "xhr", "callback_url"], - ), - Target.GoogleAds.value: TargetMeta( - group="Google", - response_format="json", - parameters=["query", "headless", "locale", "geo", "device_type", "page_from", "google_results_language", "google_tbm", "google_tbs", "parse", "google_nfpr", "session_id", "markdown", "xhr", "page_count", "callback_url"], - ), - Target.GoogleLens.value: TargetMeta( - group="Google", - response_format="json", - parameters=["query", "headless", "parse", "device_type", "markdown", "callback_url"], - ), - Target.BingSearch.value: TargetMeta( - group="Bing", - response_format="json", - parameters=["query", "headless", "locale", "geo", "domain", "device_type", "page_from", "parse", "page_count", "session_id", "markdown", "xhr", "callback_url"], - ), - Target.Bing.value: TargetMeta( - group="Bing", - response_format="json", - parameters=["url", "headless", "locale", "geo", "device_type", "page_from", "parse", "session_id", "markdown", "xhr", "callback_url"], - ), - Target.YoutubeTranscript.value: TargetMeta( - group="YouTube", - response_format="json", - parameters=["query", "language_code", "transcript_origin", "callback_url"], - ), - Target.AmazonProduct.value: TargetMeta( - group="Amazon", - response_format="json", - parameters=["query", "headless", "domain", "device_type", "parse", "autoselect_variant", "geo", "session_id", "currency", "markdown", "xhr", "callback_url"], - ), - Target.AmazonPricing.value: TargetMeta( - group="Amazon", - response_format="json", - parameters=["query", "headless", "domain", "device_type", "page_from", "parse", "geo", "session_id", "currency", "markdown", "xhr", "callback_url"], - ), - Target.AmazonSearch.value: TargetMeta( - group="Amazon", - response_format="json", - parameters=["query", "headless", "domain", "device_type", "page_from", "category", "merchant", "parse", "geo", "session_id", "sort_by", "currency", "markdown", "xhr", "callback_url"], - ), - Target.AmazonSellers.value: TargetMeta( - group="Amazon", - response_format="json", - parameters=["query", "headless", "locale", "domain", "device_type", "geo", "parse", "markdown", "xhr", "callback_url"], - ), - Target.AmazonBestsellers.value: TargetMeta( - group="Amazon", - response_format="json", - parameters=["query", "domain", "device_type", "geo", "page_from", "category", "parse", "session_id", "currency", "markdown", "xhr", "callback_url"], - ), - Target.Amazon.value: TargetMeta( - group="Amazon", - response_format="json", - parameters=["url", "headless", "device_type", "parse", "geo", "session_id", "markdown", "xhr", "callback_url"], - ), - Target.Ecommerce.value: TargetMeta( - group="Other eCommerce", - response_format="json", - parameters=["url", "headless", "locale", "geo", "device_type", "parse", "parser_type", "callback_url"], - ), - Target.WalmartProduct.value: TargetMeta( - group="Walmart", - response_format="html", - parameters=["product_id", "headless", "parse", "xhr", "markdown", "fulfillment_type", "walmart_store_id", "delivery_zip", "callback_url"], - ), - Target.WalmartSearch.value: TargetMeta( - group="Walmart", - response_format="json", - parameters=["query", "headless", "parse", "markdown", "fulfillment_type", "walmart_store_id", "delivery_zip", "callback_url"], - ), - Target.Walmart.value: TargetMeta( - group="Walmart", - response_format="html", - parameters=["url", "headless", "locale", "geo", "device_type", "store_id", "markdown", "xhr", "callback_url"], - ), - Target.TargetProduct.value: TargetMeta( - group="Target", - response_format="json", - parameters=["product_id", "headless", "parse", "device_type", "markdown", "xhr", "delivery_type", "target_store_id", "delivery_zip", "callback_url"], - ), - Target.TargetSearch.value: TargetMeta( - group="Target", - response_format="json", - parameters=["query", "headless", "parse", "device_type", "delivery_type", "target_store_id", "delivery_zip", "xhr", "markdown", "callback_url"], - ), - Target.Target.value: TargetMeta( - group="Target", - response_format="html", - parameters=["url", "headless", "device_type", "xhr", "delivery_zip", "target_store_id", "callback_url"], - ), - Target.LowesSearch.value: TargetMeta( - group="Lowe's", - response_format="json", - parameters=["query", "lowes_store_id", "headless", "delivery_zip", "user_agent_type", "free_delivery", "pickup_today", "delivery_today_tomorrow", "callback_url"], - ), - Target.Universal.value: TargetMeta( - group="Universal", - response_format="html", - parameters=["url", "payload", "proxy_pool", "http_method", "headless", "geo", "locale", "device_type", "session_id", "successful_status_codes", "headers", "cookies", "force_headers", "force_cookies", "xhr", "markdown", "callback_url"], - ), - Target.Chatgpt.value: TargetMeta( - group="AI Tools", - response_format="json", - parameters=["prompt", "search", "parse", "geo", "device_type", "markdown", "xhr", "callback_url"], - ), - Target.Perplexity.value: TargetMeta( - group="AI Tools", - response_format="json", - parameters=["prompt", "parse", "geo", "device_type", "markdown", "xhr", "callback_url"], - ), - Target.Gemini.value: TargetMeta( - group="AI Tools", - response_format="json", - parameters=["prompt", "parse", "geo", "xhr", "callback_url"], - ), - Target.Bbb.value: TargetMeta( - group="Business Reviews", - response_format="html", - parameters=["url", "headless", "geo", "device_type", "markdown", "xhr", "callback_url"], - ), - Target.Autotrader.value: TargetMeta( - group="Marketplace", - response_format="html", - parameters=["url", "headless", "geo", "device_type", "markdown", "xhr", "callback_url"], - ), - Target.Mobile.value: TargetMeta( - group="Marketplace", - response_format="html", - parameters=["url", "headless", "geo", "device_type", "markdown", "xhr", "callback_url"], - ), - Target.Airbnb.value: TargetMeta( - group="Travel", - response_format="html", - parameters=["url", "headless", "geo", "device_type", "markdown", "xhr", "callback_url"], - ), - Target.AppleAppStore.value: TargetMeta( - group="Marketplace", - response_format="html", - parameters=["url", "headless", "geo", "device_type", "markdown", "xhr", "callback_url"], - ), - Target.InstagramGraphqlProfile.value: TargetMeta( - group="Instagram", - response_format="json", - parameters=["query", "callback_url"], - ), - Target.TiktokPost.value: TargetMeta( - group="TikTok", - response_format="html", - parameters=["url", "xhr", "callback_url"], - ), - Target.TiktokShopSearch.value: TargetMeta( - group="TikTok", - response_format="html", - parameters=["query", "headless", "device_type", "markdown", "country", "callback_url"], - ), - Target.TiktokShopProduct.value: TargetMeta( - group="TikTok", - response_format="html", - parameters=["product_id", "headless", "device_type", "xhr", "markdown", "country", "callback_url"], - ), - Target.Tiktok.value: TargetMeta( - group="TikTok", - response_format="html", - parameters=["url", "headless", "user_agent_type", "callback_url"], - ), - Target.RedditPost.value: TargetMeta( - group="Reddit", - response_format="json", - parameters=["url", "locale", "geo", "callback_url"], - ), - Target.RedditSubreddit.value: TargetMeta( - group="Reddit", - response_format="json", - parameters=["url", "locale", "geo", "callback_url"], - ), - Target.RedditUser.value: TargetMeta( - group="Reddit", - response_format="json", - parameters=["url", "locale", "geo", "sort", "callback_url"], - ), - Target.YoutubeVideo.value: TargetMeta( - group="None", - response_format="json", - parameters=["query", "geo", "callback_url"], - ), - Target.YoutubeMetadata.value: TargetMeta( - group="YouTube", - response_format="json", - parameters=["query", "callback_url"], - ), - Target.YoutubeSearch.value: TargetMeta( - group="YouTube", - response_format="json", - parameters=["360", "query", "upload_date", "type", "duration", "video_sort_by", "3d", "4k", "creative_commons", "hd", "hdr", "vr180", "live", "location", "purchased", "subtitles", "callback_url"], - ), - Target.YoutubeSearchMax.value: TargetMeta( - group="YouTube", - response_format="json", - parameters=["360", "query", "upload_date", "type", "duration", "video_sort_by", "3d", "4k", "creative_commons", "hd", "hdr", "vr180", "live", "location", "purchased", "subtitles", "markdown", "callback_url"], - ), - Target.YoutubeSubtitles.value: TargetMeta( - group="YouTube", - response_format="json", - parameters=["query", "language_code", "subtitle_origin", "callback_url"], - ), - Target.YoutubeChannel.value: TargetMeta( - group="YouTube", - response_format="json", - parameters=["query", "parse", "limit", "markdown", "callback_url"], - ), -} -ScrapeRequest = Annotated[ - UniversalEcommerceParams | GoogleSearchParams | GoogleTravelHotelsParams | GoogleTrendsExploreParams | GoogleShoppingSearchParams | GoogleShoppingProductParams | GoogleParams | GoogleSuggestParams | GoogleMapsParams | GoogleAiModeParams | GoogleAdsParams | GoogleLensParams | BingSearchParams | BingParams | YoutubeTranscriptParams | AmazonProductParams | AmazonPricingParams | AmazonSearchParams | AmazonSellersParams | AmazonBestsellersParams | AmazonParams | EcommerceParams | WalmartProductParams | WalmartSearchParams | WalmartParams | TargetProductParams | TargetSearchParams | TargetStoreParams | LowesSearchParams | UniversalParams | ChatgptParams | PerplexityParams | GeminiParams | BbbParams | AutotraderParams | MobileParams | AirbnbParams | AppleAppStoreParams | InstagramGraphqlProfileParams | TiktokPostParams | TiktokShopSearchParams | TiktokShopProductParams | TiktokParams | RedditPostParams | RedditSubredditParams | RedditUserParams | YoutubeVideoParams | YoutubeMetadataParams | YoutubeSearchParams | YoutubeSearchMaxParams | YoutubeSubtitlesParams | YoutubeChannelParams, - pydantic.Field(discriminator='target'), +class UniversalEcommerceBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.UniversalEcommerce] = Target.UniversalEcommerce + callback_url: str | None = None + + +class GoogleSearchBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.GoogleSearch] = Target.GoogleSearch + query: list[str] | None = None + headless: Literal["html", "png"] | None = None + locale: str | None = None + geo: str | None = None + device_type: str | None = None + page_from: float | None = None + google_results_language: str | None = None + google_tbm: str | None = None + google_tbs: str | None = None + parse: bool | None = None + google_nfpr: bool | None = None + google_safe_search: bool | None = None + session_id: str | None = None + xhr: bool | None = None + markdown: bool | None = None + page_count: float | None = None + callback_url: str | None = None + + +class GoogleTravelHotelsBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.GoogleTravelHotels] = Target.GoogleTravelHotels + query: list[str] | None = None + headless: Literal["html", "png"] | None = None + locale: str | None = None + device_type: str | None = None + page_from: float | None = None + date_range: str | None = None + stars: float | None = None + adults: float | None = None + children: float | None = None + session_id: str | None = None + markdown: bool | None = None + callback_url: str | None = None + + +class GoogleTrendsExploreBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.GoogleTrendsExplore] = Target.GoogleTrendsExplore + query: list[str] | None = None + geo: str | None = None + device_type: str | None = None + search_type: str | None = None + date_start: str | None = None + date_end: str | None = None + callback_url: str | None = None + + +class GoogleShoppingSearchBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.GoogleShoppingSearch] = Target.GoogleShoppingSearch + query: list[str] | None = None + headless: Literal["html", "png"] | None = None + locale: str | None = None + geo: str | None = None + device_type: str | None = None + page_from: float | None = None + google_tbs: str | None = None + parse: bool | None = None + session_id: str | None = None + google_results_language: str | None = None + markdown: bool | None = None + callback_url: str | None = None + + +class GoogleShoppingProductBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.GoogleShoppingProduct] = Target.GoogleShoppingProduct + query: list[str] | None = None + headless: Literal["html", "png"] | None = None + locale: str | None = None + geo: str | None = None + device_type: str | None = None + page_from: float | None = None + parse: bool | None = None + session_id: str | None = None + google_results_language: str | None = None + markdown: bool | None = None + xhr: bool | None = None + callback_url: str | None = None + + +class GoogleBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.Google] = Target.Google + url: list[str] | None = None + headless: Literal["html", "png"] | None = None + locale: str | None = None + device_type: str | None = None + parse: bool | None = None + session_id: str | None = None + markdown: bool | None = None + xhr: bool | None = None + page_count: float | None = None + callback_url: str | None = None + + +class GoogleSuggestBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.GoogleSuggest] = Target.GoogleSuggest + query: list[str] | None = None + device_type: str | None = None + geo: str | None = None + locale: str | None = None + session_id: str | None = None + callback_url: str | None = None + + +class GoogleMapsBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.GoogleMaps] = Target.GoogleMaps + query: list[str] | None = None + headless: Literal["html", "png"] | None = None + geo: str | None = None + locale: str | None = None + page_from: float | None = None + device_type: str | None = None + session_id: str | None = None + google_results_language: str | None = None + google_nfpr: bool | None = None + hotel_occupancy: str | None = None + date_range: str | None = None + markdown: bool | None = None + callback_url: str | None = None + + +class GoogleAiModeBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.GoogleAiMode] = Target.GoogleAiMode + query: list[str] | None = None + geo: str | None = None + parse: bool | None = None + device_type: str | None = None + session_id: str | None = None + markdown: bool | None = None + xhr: bool | None = None + callback_url: str | None = None + + +class GoogleAdsBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.GoogleAds] = Target.GoogleAds + query: list[str] | None = None + headless: Literal["html", "png"] | None = None + locale: str | None = None + geo: str | None = None + device_type: str | None = None + page_from: float | None = None + google_results_language: str | None = None + google_tbm: str | None = None + google_tbs: str | None = None + parse: bool | None = None + google_nfpr: bool | None = None + session_id: str | None = None + markdown: bool | None = None + xhr: bool | None = None + page_count: float | None = None + callback_url: str | None = None + + +class GoogleLensBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.GoogleLens] = Target.GoogleLens + query: list[str] | None = None + headless: Literal["html", "png"] | None = None + parse: bool | None = None + device_type: str | None = None + markdown: bool | None = None + callback_url: str | None = None + + +class BingSearchBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.BingSearch] = Target.BingSearch + query: list[str] | None = None + headless: Literal["html", "png"] | None = None + locale: str | None = None + geo: str | None = None + domain: str | None = None + device_type: str | None = None + page_from: float | None = None + parse: bool | None = None + page_count: float | None = None + session_id: str | None = None + markdown: bool | None = None + xhr: bool | None = None + callback_url: str | None = None + + +class BingBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.Bing] = Target.Bing + url: list[str] | None = None + headless: Literal["html", "png"] | None = None + locale: str | None = None + geo: str | None = None + device_type: str | None = None + page_from: float | None = None + parse: bool | None = None + session_id: str | None = None + markdown: bool | None = None + xhr: bool | None = None + callback_url: str | None = None + + +class YoutubeTranscriptBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.YoutubeTranscript] = Target.YoutubeTranscript + query: list[str] | None = None + language_code: str | None = None + transcript_origin: str | None = None + callback_url: str | None = None + + +class AmazonProductBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.AmazonProduct] = Target.AmazonProduct + query: list[str] | None = None + headless: Literal["html", "png"] | None = None + domain: str | None = None + device_type: str | None = None + parse: bool | None = None + autoselect_variant: bool | None = None + geo: str | None = None + session_id: str | None = None + currency: str | None = None + markdown: bool | None = None + xhr: bool | None = None + callback_url: str | None = None + + +class AmazonPricingBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.AmazonPricing] = Target.AmazonPricing + query: list[str] | None = None + headless: Literal["html", "png"] | None = None + domain: str | None = None + device_type: str | None = None + page_from: float | None = None + parse: bool | None = None + geo: str | None = None + session_id: str | None = None + currency: str | None = None + markdown: bool | None = None + xhr: bool | None = None + callback_url: str | None = None + + +class AmazonSearchBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.AmazonSearch] = Target.AmazonSearch + query: list[str] | None = None + headless: Literal["html", "png"] | None = None + domain: str | None = None + device_type: str | None = None + page_from: float | None = None + category: str | None = None + merchant: str | None = None + parse: bool | None = None + geo: str | None = None + session_id: str | None = None + sort_by: str | None = None + currency: str | None = None + markdown: bool | None = None + xhr: bool | None = None + callback_url: str | None = None + + +class AmazonSellersBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.AmazonSellers] = Target.AmazonSellers + query: list[str] | None = None + headless: Literal["html", "png"] | None = None + locale: str | None = None + domain: str | None = None + device_type: str | None = None + geo: str | None = None + parse: bool | None = None + markdown: bool | None = None + xhr: bool | None = None + callback_url: str | None = None + + +class AmazonBestsellersBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.AmazonBestsellers] = Target.AmazonBestsellers + query: list[str] | None = None + domain: str | None = None + device_type: str | None = None + geo: str | None = None + page_from: float | None = None + category: str | None = None + parse: bool | None = None + session_id: str | None = None + currency: str | None = None + markdown: bool | None = None + xhr: bool | None = None + callback_url: str | None = None + + +class AmazonBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.Amazon] = Target.Amazon + url: list[str] | None = None + headless: Literal["html", "png"] | None = None + device_type: str | None = None + parse: bool | None = None + geo: str | None = None + session_id: str | None = None + markdown: bool | None = None + xhr: bool | None = None + callback_url: str | None = None + + +class EcommerceBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.Ecommerce] = Target.Ecommerce + url: list[str] | None = None + headless: Literal["html", "png"] | None = None + locale: str | None = None + geo: str | None = None + device_type: str | None = None + parse: bool | None = None + parser_type: str | None = None + callback_url: str | None = None + + +class WalmartProductBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.WalmartProduct] = Target.WalmartProduct + product_id: str | None = None + headless: Literal["html", "png"] | None = None + parse: bool | None = None + xhr: bool | None = None + markdown: bool | None = None + fulfillment_type: str | None = None + walmart_store_id: str | None = None + delivery_zip: str | None = None + callback_url: str | None = None + + +class WalmartSearchBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.WalmartSearch] = Target.WalmartSearch + query: list[str] | None = None + headless: Literal["html", "png"] | None = None + parse: bool | None = None + markdown: bool | None = None + fulfillment_type: str | None = None + walmart_store_id: str | None = None + delivery_zip: str | None = None + callback_url: str | None = None + + +class WalmartBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.Walmart] = Target.Walmart + url: list[str] | None = None + headless: Literal["html", "png"] | None = None + locale: str | None = None + geo: str | None = None + device_type: str | None = None + store_id: str | None = None + markdown: bool | None = None + xhr: bool | None = None + callback_url: str | None = None + + +class TargetProductBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.TargetProduct] = Target.TargetProduct + product_id: str | None = None + headless: Literal["html", "png"] | None = None + parse: bool | None = None + device_type: str | None = None + markdown: bool | None = None + xhr: bool | None = None + delivery_type: str | None = None + target_store_id: str | None = None + delivery_zip: str | None = None + callback_url: str | None = None + + +class TargetSearchBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.TargetSearch] = Target.TargetSearch + query: list[str] | None = None + headless: Literal["html", "png"] | None = None + parse: bool | None = None + device_type: str | None = None + delivery_type: str | None = None + target_store_id: str | None = None + delivery_zip: str | None = None + xhr: bool | None = None + markdown: bool | None = None + callback_url: str | None = None + + +class TargetStoreBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.Target] = Target.Target + url: list[str] | None = None + headless: Literal["html", "png"] | None = None + device_type: str | None = None + xhr: bool | None = None + delivery_zip: str | None = None + target_store_id: str | None = None + callback_url: str | None = None + + +class LowesSearchBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.LowesSearch] = Target.LowesSearch + query: list[str] | None = None + lowes_store_id: str | None = None + headless: Literal["html", "png"] | None = None + delivery_zip: str | None = None + user_agent_type: str | None = None + free_delivery: bool | None = None + pickup_today: bool | None = None + delivery_today_tomorrow: bool | None = None + callback_url: str | None = None + + +class UniversalBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.Universal] = Target.Universal + url: list[str] | None = None + payload: str | None = None + proxy_pool: Literal["standard", "premium"] | None = None + http_method: str | None = None + headless: Literal["html", "png"] | None = None + geo: str | None = None + locale: str | None = None + device_type: str | None = None + session_id: str | None = None + successful_status_codes: list[Any] | None = None + headers: Any | None = None + cookies: Any | None = None + force_headers: bool | None = None + force_cookies: bool | None = None + xhr: bool | None = None + markdown: bool | None = None + callback_url: str | None = None + + +class ChatgptBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.Chatgpt] = Target.Chatgpt + prompt: str | None = None + search: bool | None = None + parse: bool | None = None + geo: str | None = None + device_type: str | None = None + markdown: bool | None = None + xhr: bool | None = None + callback_url: str | None = None + + +class PerplexityBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.Perplexity] = Target.Perplexity + prompt: str | None = None + parse: bool | None = None + geo: str | None = None + device_type: str | None = None + markdown: bool | None = None + xhr: bool | None = None + callback_url: str | None = None + + +class GeminiBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.Gemini] = Target.Gemini + prompt: str | None = None + parse: bool | None = None + geo: str | None = None + xhr: bool | None = None + callback_url: str | None = None + + +class BbbBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.Bbb] = Target.Bbb + url: list[str] | None = None + headless: Literal["html", "png"] | None = None + geo: str | None = None + device_type: str | None = None + markdown: bool | None = None + xhr: bool | None = None + callback_url: str | None = None + + +class AutotraderBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.Autotrader] = Target.Autotrader + url: list[str] | None = None + headless: Literal["html", "png"] | None = None + geo: str | None = None + device_type: str | None = None + markdown: bool | None = None + xhr: bool | None = None + callback_url: str | None = None + + +class MobileBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.Mobile] = Target.Mobile + url: list[str] | None = None + headless: Literal["html", "png"] | None = None + geo: str | None = None + device_type: str | None = None + markdown: bool | None = None + xhr: bool | None = None + callback_url: str | None = None + + +class AirbnbBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.Airbnb] = Target.Airbnb + url: list[str] | None = None + headless: Literal["html", "png"] | None = None + geo: str | None = None + device_type: str | None = None + markdown: bool | None = None + xhr: bool | None = None + callback_url: str | None = None + + +class AppleAppStoreBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.AppleAppStore] = Target.AppleAppStore + url: list[str] | None = None + headless: Literal["html", "png"] | None = None + geo: str | None = None + device_type: str | None = None + markdown: bool | None = None + xhr: bool | None = None + callback_url: str | None = None + + +class InstagramGraphqlProfileBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.InstagramGraphqlProfile] = Target.InstagramGraphqlProfile + query: list[str] | None = None + callback_url: str | None = None + + +class TiktokPostBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.TiktokPost] = Target.TiktokPost + url: list[str] | None = None + xhr: bool | None = None + callback_url: str | None = None + + +class TiktokShopSearchBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.TiktokShopSearch] = Target.TiktokShopSearch + query: list[str] | None = None + headless: Literal["html", "png"] | None = None + device_type: str | None = None + markdown: bool | None = None + country: str | None = None + callback_url: str | None = None + + +class TiktokShopProductBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.TiktokShopProduct] = Target.TiktokShopProduct + product_id: str | None = None + headless: Literal["html", "png"] | None = None + device_type: str | None = None + xhr: bool | None = None + markdown: bool | None = None + country: str | None = None + callback_url: str | None = None + + +class TiktokBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.Tiktok] = Target.Tiktok + url: list[str] | None = None + headless: Literal["html", "png"] | None = None + user_agent_type: str | None = None + callback_url: str | None = None + + +class RedditPostBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.RedditPost] = Target.RedditPost + url: list[str] | None = None + locale: str | None = None + geo: str | None = None + callback_url: str | None = None + + +class RedditSubredditBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.RedditSubreddit] = Target.RedditSubreddit + url: list[str] | None = None + locale: str | None = None + geo: str | None = None + callback_url: str | None = None + + +class RedditUserBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.RedditUser] = Target.RedditUser + url: list[str] | None = None + locale: str | None = None + geo: str | None = None + sort: str | None = None + callback_url: str | None = None + + +class YoutubeVideoBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.YoutubeVideo] = Target.YoutubeVideo + query: list[str] | None = None + geo: str | None = None + callback_url: str | None = None + + +class YoutubeMetadataBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.YoutubeMetadata] = Target.YoutubeMetadata + query: list[str] | None = None + callback_url: str | None = None + + +class YoutubeSearchBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.YoutubeSearch] = Target.YoutubeSearch + f360: bool | None = pydantic.Field(None, alias="360") + query: list[str] | None = None + upload_date: str | None = None + type: str | None = None + duration: str | None = None + video_sort_by: str | None = None + f3d: bool | None = pydantic.Field(None, alias="3d") + f4k: bool | None = pydantic.Field(None, alias="4k") + creative_commons: bool | None = None + hd: bool | None = None + hdr: bool | None = None + vr180: bool | None = None + live: bool | None = None + location: bool | None = None + purchased: bool | None = None + subtitles: bool | None = None + callback_url: str | None = None + + +class YoutubeSearchMaxBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.YoutubeSearchMax] = Target.YoutubeSearchMax + f360: bool | None = pydantic.Field(None, alias="360") + query: list[str] | None = None + upload_date: str | None = None + type: str | None = None + duration: str | None = None + video_sort_by: str | None = None + f3d: bool | None = pydantic.Field(None, alias="3d") + f4k: bool | None = pydantic.Field(None, alias="4k") + creative_commons: bool | None = None + hd: bool | None = None + hdr: bool | None = None + vr180: bool | None = None + live: bool | None = None + location: bool | None = None + purchased: bool | None = None + subtitles: bool | None = None + markdown: bool | None = None + callback_url: str | None = None + + +class YoutubeSubtitlesBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.YoutubeSubtitles] = Target.YoutubeSubtitles + query: list[str] | None = None + language_code: str | None = None + subtitle_origin: str | None = None + callback_url: str | None = None + + +class YoutubeChannelBatchParams(pydantic.BaseModel): + model_config = pydantic.ConfigDict(populate_by_name=True) + target: Literal[Target.YoutubeChannel] = Target.YoutubeChannel + query: list[str] | None = None + parse: bool | None = None + limit: float | None = None + markdown: bool | None = None + callback_url: str | None = None + + +target_meta: dict[str, dict[str, Any]] = { + Target.UniversalEcommerce.value: { + "group": "None", + "response_format": "html", + "parameters": ["callback_url"], + }, + Target.GoogleSearch.value: { + "group": "Google", + "response_format": "json", + "parameters": [ + "query", + "headless", + "locale", + "geo", + "device_type", + "page_from", + "google_results_language", + "google_tbm", + "google_tbs", + "parse", + "google_nfpr", + "google_safe_search", + "session_id", + "xhr", + "markdown", + "page_count", + "callback_url", + ], + }, + Target.GoogleTravelHotels.value: { + "group": "Google", + "response_format": "html", + "parameters": [ + "query", + "headless", + "locale", + "device_type", + "page_from", + "date_range", + "stars", + "adults", + "children", + "session_id", + "markdown", + "callback_url", + ], + }, + Target.GoogleTrendsExplore.value: { + "group": "Google", + "response_format": "json", + "parameters": ["query", "geo", "device_type", "search_type", "date_start", "date_end", "callback_url"], + }, + Target.GoogleShoppingSearch.value: { + "group": "Google", + "response_format": "json", + "parameters": [ + "query", + "headless", + "locale", + "geo", + "device_type", + "page_from", + "google_tbs", + "parse", + "session_id", + "google_results_language", + "markdown", + "callback_url", + ], + }, + Target.GoogleShoppingProduct.value: { + "group": "Google", + "response_format": "json", + "parameters": [ + "query", + "headless", + "locale", + "geo", + "device_type", + "page_from", + "parse", + "session_id", + "google_results_language", + "markdown", + "xhr", + "callback_url", + ], + }, + Target.Google.value: { + "group": "Google", + "response_format": "json", + "parameters": [ + "url", + "headless", + "locale", + "device_type", + "parse", + "session_id", + "markdown", + "xhr", + "page_count", + "callback_url", + ], + }, + Target.GoogleSuggest.value: { + "group": "Google", + "response_format": "json", + "parameters": ["query", "device_type", "geo", "locale", "session_id", "callback_url"], + }, + Target.GoogleMaps.value: { + "group": "Google", + "response_format": "html", + "parameters": [ + "query", + "headless", + "geo", + "locale", + "page_from", + "device_type", + "session_id", + "google_results_language", + "google_nfpr", + "hotel_occupancy", + "date_range", + "markdown", + "callback_url", + ], + }, + Target.GoogleAiMode.value: { + "group": "AI Tools", + "response_format": "json", + "parameters": ["query", "geo", "parse", "device_type", "session_id", "markdown", "xhr", "callback_url"], + }, + Target.GoogleAds.value: { + "group": "Google", + "response_format": "json", + "parameters": [ + "query", + "headless", + "locale", + "geo", + "device_type", + "page_from", + "google_results_language", + "google_tbm", + "google_tbs", + "parse", + "google_nfpr", + "session_id", + "markdown", + "xhr", + "page_count", + "callback_url", + ], + }, + Target.GoogleLens.value: { + "group": "Google", + "response_format": "json", + "parameters": ["query", "headless", "parse", "device_type", "markdown", "callback_url"], + }, + Target.BingSearch.value: { + "group": "Bing", + "response_format": "json", + "parameters": [ + "query", + "headless", + "locale", + "geo", + "domain", + "device_type", + "page_from", + "parse", + "page_count", + "session_id", + "markdown", + "xhr", + "callback_url", + ], + }, + Target.Bing.value: { + "group": "Bing", + "response_format": "json", + "parameters": [ + "url", + "headless", + "locale", + "geo", + "device_type", + "page_from", + "parse", + "session_id", + "markdown", + "xhr", + "callback_url", + ], + }, + Target.YoutubeTranscript.value: { + "group": "YouTube", + "response_format": "json", + "parameters": ["query", "language_code", "transcript_origin", "callback_url"], + }, + Target.AmazonProduct.value: { + "group": "Amazon", + "response_format": "json", + "parameters": [ + "query", + "headless", + "domain", + "device_type", + "parse", + "autoselect_variant", + "geo", + "session_id", + "currency", + "markdown", + "xhr", + "callback_url", + ], + }, + Target.AmazonPricing.value: { + "group": "Amazon", + "response_format": "json", + "parameters": [ + "query", + "headless", + "domain", + "device_type", + "page_from", + "parse", + "geo", + "session_id", + "currency", + "markdown", + "xhr", + "callback_url", + ], + }, + Target.AmazonSearch.value: { + "group": "Amazon", + "response_format": "json", + "parameters": [ + "query", + "headless", + "domain", + "device_type", + "page_from", + "category", + "merchant", + "parse", + "geo", + "session_id", + "sort_by", + "currency", + "markdown", + "xhr", + "callback_url", + ], + }, + Target.AmazonSellers.value: { + "group": "Amazon", + "response_format": "json", + "parameters": [ + "query", + "headless", + "locale", + "domain", + "device_type", + "geo", + "parse", + "markdown", + "xhr", + "callback_url", + ], + }, + Target.AmazonBestsellers.value: { + "group": "Amazon", + "response_format": "json", + "parameters": [ + "query", + "domain", + "device_type", + "geo", + "page_from", + "category", + "parse", + "session_id", + "currency", + "markdown", + "xhr", + "callback_url", + ], + }, + Target.Amazon.value: { + "group": "Amazon", + "response_format": "json", + "parameters": [ + "url", + "headless", + "device_type", + "parse", + "geo", + "session_id", + "markdown", + "xhr", + "callback_url", + ], + }, + Target.Ecommerce.value: { + "group": "Other eCommerce", + "response_format": "json", + "parameters": ["url", "headless", "locale", "geo", "device_type", "parse", "parser_type", "callback_url"], + }, + Target.WalmartProduct.value: { + "group": "Walmart", + "response_format": "html", + "parameters": [ + "product_id", + "headless", + "parse", + "xhr", + "markdown", + "fulfillment_type", + "walmart_store_id", + "delivery_zip", + "callback_url", + ], + }, + Target.WalmartSearch.value: { + "group": "Walmart", + "response_format": "json", + "parameters": [ + "query", + "headless", + "parse", + "markdown", + "fulfillment_type", + "walmart_store_id", + "delivery_zip", + "callback_url", + ], + }, + Target.Walmart.value: { + "group": "Walmart", + "response_format": "html", + "parameters": [ + "url", + "headless", + "locale", + "geo", + "device_type", + "store_id", + "markdown", + "xhr", + "callback_url", + ], + }, + Target.TargetProduct.value: { + "group": "Target", + "response_format": "json", + "parameters": [ + "product_id", + "headless", + "parse", + "device_type", + "markdown", + "xhr", + "delivery_type", + "target_store_id", + "delivery_zip", + "callback_url", + ], + }, + Target.TargetSearch.value: { + "group": "Target", + "response_format": "json", + "parameters": [ + "query", + "headless", + "parse", + "device_type", + "delivery_type", + "target_store_id", + "delivery_zip", + "xhr", + "markdown", + "callback_url", + ], + }, + Target.Target.value: { + "group": "Target", + "response_format": "html", + "parameters": ["url", "headless", "device_type", "xhr", "delivery_zip", "target_store_id", "callback_url"], + }, + Target.LowesSearch.value: { + "group": "Lowe's", + "response_format": "json", + "parameters": [ + "query", + "lowes_store_id", + "headless", + "delivery_zip", + "user_agent_type", + "free_delivery", + "pickup_today", + "delivery_today_tomorrow", + "callback_url", + ], + }, + Target.Universal.value: { + "group": "Universal", + "response_format": "html", + "parameters": [ + "url", + "payload", + "proxy_pool", + "http_method", + "headless", + "geo", + "locale", + "device_type", + "session_id", + "successful_status_codes", + "headers", + "cookies", + "force_headers", + "force_cookies", + "xhr", + "markdown", + "callback_url", + ], + }, + Target.Chatgpt.value: { + "group": "AI Tools", + "response_format": "json", + "parameters": ["prompt", "search", "parse", "geo", "device_type", "markdown", "xhr", "callback_url"], + }, + Target.Perplexity.value: { + "group": "AI Tools", + "response_format": "json", + "parameters": ["prompt", "parse", "geo", "device_type", "markdown", "xhr", "callback_url"], + }, + Target.Gemini.value: { + "group": "AI Tools", + "response_format": "json", + "parameters": ["prompt", "parse", "geo", "xhr", "callback_url"], + }, + Target.Bbb.value: { + "group": "Business Reviews", + "response_format": "html", + "parameters": ["url", "headless", "geo", "device_type", "markdown", "xhr", "callback_url"], + }, + Target.Autotrader.value: { + "group": "Marketplace", + "response_format": "html", + "parameters": ["url", "headless", "geo", "device_type", "markdown", "xhr", "callback_url"], + }, + Target.Mobile.value: { + "group": "Marketplace", + "response_format": "html", + "parameters": ["url", "headless", "geo", "device_type", "markdown", "xhr", "callback_url"], + }, + Target.Airbnb.value: { + "group": "Travel", + "response_format": "html", + "parameters": ["url", "headless", "geo", "device_type", "markdown", "xhr", "callback_url"], + }, + Target.AppleAppStore.value: { + "group": "Marketplace", + "response_format": "html", + "parameters": ["url", "headless", "geo", "device_type", "markdown", "xhr", "callback_url"], + }, + Target.InstagramGraphqlProfile.value: { + "group": "Instagram", + "response_format": "json", + "parameters": ["query", "callback_url"], + }, + Target.TiktokPost.value: { + "group": "TikTok", + "response_format": "html", + "parameters": ["url", "xhr", "callback_url"], + }, + Target.TiktokShopSearch.value: { + "group": "TikTok", + "response_format": "html", + "parameters": ["query", "headless", "device_type", "markdown", "country", "callback_url"], + }, + Target.TiktokShopProduct.value: { + "group": "TikTok", + "response_format": "html", + "parameters": ["product_id", "headless", "device_type", "xhr", "markdown", "country", "callback_url"], + }, + Target.Tiktok.value: { + "group": "TikTok", + "response_format": "html", + "parameters": ["url", "headless", "user_agent_type", "callback_url"], + }, + Target.RedditPost.value: { + "group": "Reddit", + "response_format": "json", + "parameters": ["url", "locale", "geo", "callback_url"], + }, + Target.RedditSubreddit.value: { + "group": "Reddit", + "response_format": "json", + "parameters": ["url", "locale", "geo", "callback_url"], + }, + Target.RedditUser.value: { + "group": "Reddit", + "response_format": "json", + "parameters": ["url", "locale", "geo", "sort", "callback_url"], + }, + Target.YoutubeVideo.value: { + "group": "None", + "response_format": "json", + "parameters": ["query", "geo", "callback_url"], + }, + Target.YoutubeMetadata.value: { + "group": "YouTube", + "response_format": "json", + "parameters": ["query", "callback_url"], + }, + Target.YoutubeSearch.value: { + "group": "YouTube", + "response_format": "json", + "parameters": [ + "360", + "query", + "upload_date", + "type", + "duration", + "video_sort_by", + "3d", + "4k", + "creative_commons", + "hd", + "hdr", + "vr180", + "live", + "location", + "purchased", + "subtitles", + "callback_url", + ], + }, + Target.YoutubeSearchMax.value: { + "group": "YouTube", + "response_format": "json", + "parameters": [ + "360", + "query", + "upload_date", + "type", + "duration", + "video_sort_by", + "3d", + "4k", + "creative_commons", + "hd", + "hdr", + "vr180", + "live", + "location", + "purchased", + "subtitles", + "markdown", + "callback_url", + ], + }, + Target.YoutubeSubtitles.value: { + "group": "YouTube", + "response_format": "json", + "parameters": ["query", "language_code", "subtitle_origin", "callback_url"], + }, + Target.YoutubeChannel.value: { + "group": "YouTube", + "response_format": "json", + "parameters": ["query", "parse", "limit", "markdown", "callback_url"], + }, +} + +ScrapeRequest = Annotated[ + Union[ + UniversalEcommerceParams + | GoogleSearchParams + | GoogleTravelHotelsParams + | GoogleTrendsExploreParams + | GoogleShoppingSearchParams + | GoogleShoppingProductParams + | GoogleParams + | GoogleSuggestParams + | GoogleMapsParams + | GoogleAiModeParams + | GoogleAdsParams + | GoogleLensParams + | BingSearchParams + | BingParams + | YoutubeTranscriptParams + | AmazonProductParams + | AmazonPricingParams + | AmazonSearchParams + | AmazonSellersParams + | AmazonBestsellersParams + | AmazonParams + | EcommerceParams + | WalmartProductParams + | WalmartSearchParams + | WalmartParams + | TargetProductParams + | TargetSearchParams + | TargetStoreParams + | LowesSearchParams + | UniversalParams + | ChatgptParams + | PerplexityParams + | GeminiParams + | BbbParams + | AutotraderParams + | MobileParams + | AirbnbParams + | AppleAppStoreParams + | InstagramGraphqlProfileParams + | TiktokPostParams + | TiktokShopSearchParams + | TiktokShopProductParams + | TiktokParams + | RedditPostParams + | RedditSubredditParams + | RedditUserParams + | YoutubeVideoParams + | YoutubeMetadataParams + | YoutubeSearchParams + | YoutubeSearchMaxParams + | YoutubeSubtitlesParams + | YoutubeChannelParams + ], + pydantic.Field(discriminator="target"), +] + +BatchRequest = Annotated[ + Union[ + UniversalEcommerceBatchParams + | GoogleSearchBatchParams + | GoogleTravelHotelsBatchParams + | GoogleTrendsExploreBatchParams + | GoogleShoppingSearchBatchParams + | GoogleShoppingProductBatchParams + | GoogleBatchParams + | GoogleSuggestBatchParams + | GoogleMapsBatchParams + | GoogleAiModeBatchParams + | GoogleAdsBatchParams + | GoogleLensBatchParams + | BingSearchBatchParams + | BingBatchParams + | YoutubeTranscriptBatchParams + | AmazonProductBatchParams + | AmazonPricingBatchParams + | AmazonSearchBatchParams + | AmazonSellersBatchParams + | AmazonBestsellersBatchParams + | AmazonBatchParams + | EcommerceBatchParams + | WalmartProductBatchParams + | WalmartSearchBatchParams + | WalmartBatchParams + | TargetProductBatchParams + | TargetSearchBatchParams + | TargetStoreBatchParams + | LowesSearchBatchParams + | UniversalBatchParams + | ChatgptBatchParams + | PerplexityBatchParams + | GeminiBatchParams + | BbbBatchParams + | AutotraderBatchParams + | MobileBatchParams + | AirbnbBatchParams + | AppleAppStoreBatchParams + | InstagramGraphqlProfileBatchParams + | TiktokPostBatchParams + | TiktokShopSearchBatchParams + | TiktokShopProductBatchParams + | TiktokBatchParams + | RedditPostBatchParams + | RedditSubredditBatchParams + | RedditUserBatchParams + | YoutubeVideoBatchParams + | YoutubeMetadataBatchParams + | YoutubeSearchBatchParams + | YoutubeSearchMaxBatchParams + | YoutubeSubtitlesBatchParams + | YoutubeChannelBatchParams + ], + pydantic.Field(discriminator="target"), ] -BatchRequest = ScrapeRequest From 1b021b0273b49e73bed71e0f383e88fff910ae61 Mon Sep 17 00:00:00 2001 From: Donatas Kasparavicius Date: Wed, 22 Jul 2026 10:23:21 +0300 Subject: [PATCH 03/17] Fix __init__ exports: drop phantom TargetTargetParams, add batch params - Removed TargetTargetParams from __all__ (never existed, caused ImportError) - Added all 52 *BatchParams classes and BatchRequest to imports and __all__ - GoogleSearchBatchParams and BatchRequest now publicly accessible for examples Co-Authored-By: Claude Opus 4.8 (1M context) --- src/decodo/__init__.py | 107 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 106 insertions(+), 1 deletion(-) diff --git a/src/decodo/__init__.py b/src/decodo/__init__.py index a9bf751..04b4621 100644 --- a/src/decodo/__init__.py +++ b/src/decodo/__init__.py @@ -9,59 +9,112 @@ ) from .generated.parameters import ParameterMeta, parameter_meta from .generated.targets import ( + AirbnbBatchParams, AirbnbParams, + AmazonBatchParams, + AmazonBestsellersBatchParams, AmazonBestsellersParams, AmazonParams, + AmazonPricingBatchParams, AmazonPricingParams, + AmazonProductBatchParams, AmazonProductParams, + AmazonSearchBatchParams, AmazonSearchParams, + AmazonSellersBatchParams, AmazonSellersParams, + AppleAppStoreBatchParams, AppleAppStoreParams, + AutotraderBatchParams, AutotraderParams, + BatchRequest, + BbbBatchParams, BbbParams, + BingBatchParams, BingParams, + BingSearchBatchParams, BingSearchParams, + ChatgptBatchParams, ChatgptParams, + EcommerceBatchParams, EcommerceParams, + GeminiBatchParams, GeminiParams, + GoogleAdsBatchParams, GoogleAdsParams, + GoogleAiModeBatchParams, GoogleAiModeParams, + GoogleBatchParams, + GoogleLensBatchParams, GoogleLensParams, + GoogleMapsBatchParams, GoogleMapsParams, GoogleParams, + GoogleSearchBatchParams, GoogleSearchParams, + GoogleShoppingProductBatchParams, GoogleShoppingProductParams, + GoogleShoppingSearchBatchParams, GoogleShoppingSearchParams, + GoogleSuggestBatchParams, GoogleSuggestParams, + GoogleTravelHotelsBatchParams, GoogleTravelHotelsParams, + GoogleTrendsExploreBatchParams, GoogleTrendsExploreParams, + InstagramGraphqlProfileBatchParams, InstagramGraphqlProfileParams, + LowesSearchBatchParams, LowesSearchParams, + MobileBatchParams, MobileParams, + PerplexityBatchParams, PerplexityParams, + RedditPostBatchParams, RedditPostParams, + RedditSubredditBatchParams, RedditSubredditParams, + RedditUserBatchParams, RedditUserParams, ScrapeRequest, Target, + TargetProductBatchParams, TargetProductParams, + TargetSearchBatchParams, TargetSearchParams, + TargetStoreBatchParams, TargetStoreParams, + TiktokBatchParams, TiktokParams, + TiktokPostBatchParams, TiktokPostParams, + TiktokShopProductBatchParams, TiktokShopProductParams, + TiktokShopSearchBatchParams, TiktokShopSearchParams, + UniversalBatchParams, + UniversalEcommerceBatchParams, UniversalEcommerceParams, UniversalParams, + WalmartBatchParams, WalmartParams, + WalmartProductBatchParams, WalmartProductParams, + WalmartSearchBatchParams, WalmartSearchParams, + YoutubeChannelBatchParams, YoutubeChannelParams, + YoutubeMetadataBatchParams, YoutubeMetadataParams, + YoutubeSearchBatchParams, + YoutubeSearchMaxBatchParams, YoutubeSearchMaxParams, YoutubeSearchParams, + YoutubeSubtitlesBatchParams, YoutubeSubtitlesParams, + YoutubeTranscriptBatchParams, YoutubeTranscriptParams, + YoutubeVideoBatchParams, YoutubeVideoParams, target_meta, targets, @@ -92,6 +145,7 @@ "target_meta", "targets", "ScrapeRequest", + "BatchRequest", "UniversalEcommerceParams", "GoogleSearchParams", "GoogleTravelHotelsParams", @@ -120,7 +174,6 @@ "TargetProductParams", "TargetSearchParams", "TargetStoreParams", - "TargetTargetParams", "LowesSearchParams", "UniversalParams", "ChatgptParams", @@ -145,6 +198,58 @@ "YoutubeSearchMaxParams", "YoutubeSubtitlesParams", "YoutubeChannelParams", + "UniversalEcommerceBatchParams", + "GoogleSearchBatchParams", + "GoogleTravelHotelsBatchParams", + "GoogleTrendsExploreBatchParams", + "GoogleShoppingSearchBatchParams", + "GoogleShoppingProductBatchParams", + "GoogleBatchParams", + "GoogleSuggestBatchParams", + "GoogleMapsBatchParams", + "GoogleAiModeBatchParams", + "GoogleAdsBatchParams", + "GoogleLensBatchParams", + "BingSearchBatchParams", + "BingBatchParams", + "YoutubeTranscriptBatchParams", + "AmazonProductBatchParams", + "AmazonPricingBatchParams", + "AmazonSearchBatchParams", + "AmazonSellersBatchParams", + "AmazonBestsellersBatchParams", + "AmazonBatchParams", + "EcommerceBatchParams", + "WalmartProductBatchParams", + "WalmartSearchBatchParams", + "WalmartBatchParams", + "TargetProductBatchParams", + "TargetSearchBatchParams", + "TargetStoreBatchParams", + "LowesSearchBatchParams", + "UniversalBatchParams", + "ChatgptBatchParams", + "PerplexityBatchParams", + "GeminiBatchParams", + "BbbBatchParams", + "AutotraderBatchParams", + "MobileBatchParams", + "AirbnbBatchParams", + "AppleAppStoreBatchParams", + "InstagramGraphqlProfileBatchParams", + "TiktokPostBatchParams", + "TiktokShopSearchBatchParams", + "TiktokShopProductBatchParams", + "TiktokBatchParams", + "RedditPostBatchParams", + "RedditSubredditBatchParams", + "RedditUserBatchParams", + "YoutubeVideoBatchParams", + "YoutubeMetadataBatchParams", + "YoutubeSearchBatchParams", + "YoutubeSearchMaxBatchParams", + "YoutubeSubtitlesBatchParams", + "YoutubeChannelBatchParams", "ParameterMeta", "parameter_meta", "SyncResponse", From 058051bc6781997d9f22772732d291281d116d6d Mon Sep 17 00:00:00 2001 From: Donatas Kasparavicius Date: Wed, 22 Jul 2026 10:23:29 +0300 Subject: [PATCH 04/17] Add py.typed marker and jsonschema runtime dep; remove codegen console script - src/decodo/py.typed: empty marker file so mypy and PEP 561 consumers find types - pyproject.toml: added jsonschema>=4.0 to runtime dependencies (scrape() uses it) - pyproject.toml: added types-jsonschema>=4.0 to dev deps for mypy - pyproject.toml: added [tool.setuptools.package-data] to ship py.typed in wheel - pyproject.toml: removed [project.scripts] decodo-codegen entry (footgun: runs full network fetch + file overwrite on any invocation including --help; maintainers run via python -m decodo.codegen.codegen instead) - pyproject.toml: added ^build/ to mypy exclude to avoid duplicate-module error Co-Authored-By: Claude Opus 4.8 (1M context) --- pyproject.toml | 9 ++++++--- src/decodo/py.typed | 0 2 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 src/decodo/py.typed diff --git a/pyproject.toml b/pyproject.toml index 1b1beeb..3805ae8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ requires-python = ">=3.12" dependencies = [ "httpx>=0.27.0", "pydantic>=2.0", + "jsonschema>=4.0", ] [project.urls] @@ -30,6 +31,7 @@ dev = [ "pytest-mock>=3.14.0", "ruff>=0.4.0", "mypy>=1.10.0", + "types-jsonschema>=4.0", ] [build-system] @@ -39,6 +41,9 @@ build-backend = "setuptools.build_meta" [tool.setuptools.packages.find] where = ["src"] +[tool.setuptools.package-data] +decodo = ["py.typed"] + [tool.ruff] line-length = 120 src = ["src"] @@ -50,10 +55,8 @@ select = ["E", "F", "I", "UP"] [tool.mypy] strict = true packages = ["decodo"] -exclude = "src/decodo/generated|^examples/" +exclude = "src/decodo/generated|^examples/|^build/" [tool.pytest.ini_options] testpaths = ["tests"] -[project.scripts] -decodo-codegen = "decodo.codegen.codegen:main" diff --git a/src/decodo/py.typed b/src/decodo/py.typed new file mode 100644 index 0000000..e69de29 From 2d51a985fa025edb61de13df9324c700227bf032 Mon Sep 17 00:00:00 2001 From: Donatas Kasparavicius Date: Wed, 22 Jul 2026 10:23:35 +0300 Subject: [PATCH 05/17] CI: run pytest; fix publish workflow checkout permission - .github/workflows/test.yml: new workflow running pytest on push+PR, Python 3.12, installs .[dev]; mirrors lint.yml style - .github/workflows/worklfow.yml: added contents: read permission so actions/checkout can access the private repo (id-token: write alone zeros contents, causing 'Repository not found' on checkout) Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/test.yml | 23 +++++++++++++++++++++++ .github/workflows/worklfow.yml | 1 + 2 files changed, 24 insertions(+) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..14cfdf3 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,23 @@ +name: Test + +on: + push: + branches: + - main + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install dependencies + run: pip install -e ".[dev]" + + - name: Pytest + run: pytest diff --git a/.github/workflows/worklfow.yml b/.github/workflows/worklfow.yml index e43c20b..b9a12fd 100644 --- a/.github/workflows/worklfow.yml +++ b/.github/workflows/worklfow.yml @@ -12,6 +12,7 @@ jobs: environment: pypi permissions: id-token: write + contents: read steps: - uses: actions/checkout@v4 From 7da9204abda008f506d28c89cc20a51347bea27b Mon Sep 17 00:00:00 2001 From: Donatas Kasparavicius Date: Wed, 22 Jul 2026 10:23:45 +0300 Subject: [PATCH 06/17] Fix README snippets and broken batch example README.md: - All DecodoClient(...) calls now use DecodoConfig(web_scraping_api=WebScrapingApiConfig(token=...)) - Quick start and all API snippets use typed Params objects (GoogleSearchParams etc.) - Batch snippet uses GoogleSearchBatchParams with query as list - Error handling snippet fixed to use typed params - Documented that token is base64-encoded user:password from dashboard - Added git install fallback (pip install from GitHub) until PyPI publish - Removed git conflict marker line (>>>>>>> 03b68da) examples/web_scraping_api/batch/google_search_batch.py: - Import GoogleSearchBatchParams instead of GoogleSearchParams - Removed duplicate print('Polling for results...') before the loop Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 103 +++++++++++------- .../batch/google_search_batch.py | 16 ++- 2 files changed, 68 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index 5dd0e52..25ff89c 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,12 @@ Instead of manually constructing HTTP requests and validating payloads, you can pip install decodo-sdk ``` +Until the package is published to PyPI, install directly from the repository (requires access): + +```bash +pip install "git+https://github.com/Decodo/sdk-python.git" +``` + ## Quick start Create a new project: @@ -55,22 +61,26 @@ pip install decodo-sdk touch main.py ``` -Get a Web Scraping API basic authentication token from the [Decodo dashboard](https://dashboard.decodo.com/welcome) and use it in the following example: +Get your Web Scraping API token from the [Decodo dashboard](https://dashboard.decodo.com/welcome). The token is the base64-encoded `user:password` value from the Basic Auth credentials shown in the dashboard. ```python # main.py -from decodo import DecodoClient, Target +from decodo import DecodoClient, DecodoConfig, GoogleSearchParams, Target, WebScrapingApiConfig client = DecodoClient( - web_scraping_api={"token": ""} + DecodoConfig( + web_scraping_api=WebScrapingApiConfig(token=""), + ) ) -result = client.web_scraping_api.scrape({ - "target": Target.GoogleSearch, - "query": "coffee shops", - "geo": "United States", - "parse": True, -}) +result = client.web_scraping_api.scrape( + GoogleSearchParams( + target=Target.GoogleSearch, + query="coffee shops", + geo="United States", + parse=True, + ) +) print(result) ``` @@ -162,39 +172,41 @@ python main.py ## Configuration ```python -from decodo import DecodoClient +from decodo import DecodoClient, DecodoConfig, WebScrapingApiConfig client = DecodoClient( - web_scraping_api={"token": ""}, - timeout_ms=120_000, # optional, request timeout in ms (default: 180000) + DecodoConfig( + web_scraping_api=WebScrapingApiConfig(token=""), + timeout_ms=120_000, # optional, request timeout in ms (default: 180000) + ) ) ``` | Parameter | Description | | --- | --- | -| `token` | Web Scraping API basic authentication token | +| `token` | Web Scraping API basic auth token - the base64-encoded `user:password` string from the Decodo dashboard | | `timeout_ms` | Request timeout in milliseconds (default: 180000) | ## Web Scraping API Access the API via `client.web_scraping_api`. -The snippets below assume you have already imported `Target` (and `DecodoClient` where a client is constructed), for example: - -```python -from decodo import DecodoClient, Target -``` +The snippets below assume you have already constructed a client. See [Configuration](#configuration) for how to build one. ### Sync scrape Waits for the scraping result before returning: ```python -result = client.web_scraping_api.scrape({ - "target": Target.AmazonProduct, - "query": "B09H74FXNW", - "parse": True, -}) +from decodo import AmazonProductParams, Target + +result = client.web_scraping_api.scrape( + AmazonProductParams( + target=Target.AmazonProduct, + query="B09H74FXNW", + parse=True, + ) +) ``` ### Async scrape @@ -202,11 +214,15 @@ result = client.web_scraping_api.scrape({ Creates a scraping task and returns immediately. Poll separately for task status and results: ```python -task = client.web_scraping_api.scrape_async({ - "target": Target.GoogleSearch, - "query": "laptop reviews", - "parse": True, -}) +from decodo import GoogleSearchParams, Target + +task = client.web_scraping_api.scrape_async( + GoogleSearchParams( + target=Target.GoogleSearch, + query="laptop reviews", + parse=True, + ) +) meta = client.web_scraping_api.get_status(task["id"]) print(meta["status"]) # 'pending' | 'done' | 'faulted' @@ -219,11 +235,15 @@ results = client.web_scraping_api.get_results(task["id"]) Send multiple queries or URLs in a single request: ```python -batch = client.web_scraping_api.scrape_batch({ - "target": Target.GoogleSearch, - "query": ["coffee", "tea", "juice"], - "parse": True, -}) +from decodo import GoogleSearchBatchParams, Target + +batch = client.web_scraping_api.scrape_batch( + GoogleSearchBatchParams( + target=Target.GoogleSearch, + query=["coffee", "tea", "juice"], + parse=True, + ) +) coffee_task_id = batch["queries"][0]["id"] @@ -305,18 +325,18 @@ from decodo import ( Target, ) +from decodo import GoogleSearchParams + try: - client.web_scraping_api.scrape({ - "target": Target.GoogleSearch, - "query": "test", - "parse": True, - }) + client.web_scraping_api.scrape( + GoogleSearchParams(target=Target.GoogleSearch, query="test", parse=True) + ) except AuthenticationError: - pass # 401/403 — bad credentials + pass # 401/403 - bad credentials except RateLimitError: - pass # 429 — too many requests + pass # 429 - too many requests except ValidationError as err: - print(err.errors) # 422 — invalid parameters + print(err.errors) # 422 - invalid parameters except TimeoutError: pass # request timed out ``` @@ -338,4 +358,3 @@ Build scraping workflows with the Decodo Web Scraping API: ## License Released under the [MIT License](https://github.com/Decodo/Decodo/blob/master/LICENSE). ->>>>>>> 03b68da (Initial sdk setup) diff --git a/examples/web_scraping_api/batch/google_search_batch.py b/examples/web_scraping_api/batch/google_search_batch.py index d40bf67..1006790 100644 --- a/examples/web_scraping_api/batch/google_search_batch.py +++ b/examples/web_scraping_api/batch/google_search_batch.py @@ -5,7 +5,7 @@ from decodo import ( DecodoClient, DecodoConfig, - GoogleSearchParams, + GoogleSearchBatchParams, Target, WebScrapingApiConfig, ) @@ -19,25 +19,23 @@ ) metadata = client.web_scraping_api.scrape_batch( - GoogleSearchParams( + GoogleSearchBatchParams( target=Target.GoogleSearch, - query=['shoes', 'laptop'], + query=["shoes", "laptop"], parse=True, ) ) -print('Polling for results...') - while True: - print('Polling for results...') - queries = metadata.get('queries') or [] + print("Polling for results...") + queries = metadata.get("queries") or [] if not queries: break - first_task_id = queries[0].get('id') + first_task_id = queries[0].get("id") if not first_task_id: break results = client.web_scraping_api.get_results(first_task_id) if results: - print(json.dumps(results['results'][0]['content'], indent=2)) + print(json.dumps(results["results"][0]["content"], indent=2)) break time.sleep(3) From 8bb8e4abc4b6eb2c0daff7420e61672d19ebdac5 Mon Sep 17 00:00:00 2001 From: julka Date: Mon, 27 Jul 2026 13:29:50 +0300 Subject: [PATCH 07/17] Return type generation from json --- .gitignore | 3 + README.md | 11 +- pyproject.toml | 5 + src/decodo/__init__.py | 227 +- src/decodo/api/web_scraping_api.py | 6 +- src/decodo/codegen/codegen.py | 20 +- .../generate_parameter_schemas.py | 44 - .../web_scraping_api/generate_targets.py | 46 +- src/decodo/codegen/web_scraping_api/shared.py | 1 + src/decodo/generated/__init__.py | 0 src/decodo/generated/parameters.py | 93 - src/decodo/generated/request_schemas.py | 57 - src/decodo/generated/targets.py | 2170 ----------------- src/decodo/schema/bundled_schema.py | 44 +- src/decodo/targets.py | 63 + src/decodo/types/__init__.py | 5 +- src/decodo/types/requests.py | 8 +- tests/conftest.py | 8 + 18 files changed, 312 insertions(+), 2499 deletions(-) delete mode 100644 src/decodo/codegen/web_scraping_api/generate_parameter_schemas.py delete mode 100644 src/decodo/generated/__init__.py delete mode 100644 src/decodo/generated/parameters.py delete mode 100644 src/decodo/generated/request_schemas.py delete mode 100644 src/decodo/generated/targets.py create mode 100644 src/decodo/targets.py diff --git a/.gitignore b/.gitignore index 9ee3aa6..be9a556 100644 --- a/.gitignore +++ b/.gitignore @@ -33,5 +33,8 @@ htmlcov/ # Inputs (fetched at codegen time) inputs/decodo.ir.json +# Generated types (run `python -m decodo.codegen.codegen` to regenerate) +src/decodo/generated/ + # OS .DS_Store diff --git a/README.md b/README.md index 25ff89c..0d1f117 100644 --- a/README.md +++ b/README.md @@ -42,12 +42,19 @@ Instead of manually constructing HTTP requests and validating payloads, you can pip install decodo-sdk ``` -Until the package is published to PyPI, install directly from the repository (requires access): + +## Generate types + +After installing, run the type generator to create typed target parameters: ```bash -pip install "git+https://github.com/Decodo/sdk-python.git" +python -m decodo.codegen.codegen ``` +This fetches the latest API schema from the Decodo registry and writes typed classes to the `generated/` directory inside the package. The generated files are not included in the repository — you control when to update them. + +Re-run this command whenever Decodo publishes an updated schema to pick up new targets or changed parameters. + ## Quick start Create a new project: diff --git a/pyproject.toml b/pyproject.toml index 3805ae8..c72ebd7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,11 @@ strict = true packages = ["decodo"] exclude = "src/decodo/generated|^examples/|^build/" +[[tool.mypy.overrides]] +module = "decodo.generated.*" +ignore_missing_imports = true +ignore_errors = true + [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/src/decodo/__init__.py b/src/decodo/__init__.py index 04b4621..ed94623 100644 --- a/src/decodo/__init__.py +++ b/src/decodo/__init__.py @@ -7,118 +7,121 @@ TimeoutError, ValidationError, ) -from .generated.parameters import ParameterMeta, parameter_meta -from .generated.targets import ( - AirbnbBatchParams, - AirbnbParams, - AmazonBatchParams, - AmazonBestsellersBatchParams, - AmazonBestsellersParams, - AmazonParams, - AmazonPricingBatchParams, - AmazonPricingParams, - AmazonProductBatchParams, - AmazonProductParams, - AmazonSearchBatchParams, - AmazonSearchParams, - AmazonSellersBatchParams, - AmazonSellersParams, - AppleAppStoreBatchParams, - AppleAppStoreParams, - AutotraderBatchParams, - AutotraderParams, - BatchRequest, - BbbBatchParams, - BbbParams, - BingBatchParams, - BingParams, - BingSearchBatchParams, - BingSearchParams, - ChatgptBatchParams, - ChatgptParams, - EcommerceBatchParams, - EcommerceParams, - GeminiBatchParams, - GeminiParams, - GoogleAdsBatchParams, - GoogleAdsParams, - GoogleAiModeBatchParams, - GoogleAiModeParams, - GoogleBatchParams, - GoogleLensBatchParams, - GoogleLensParams, - GoogleMapsBatchParams, - GoogleMapsParams, - GoogleParams, - GoogleSearchBatchParams, - GoogleSearchParams, - GoogleShoppingProductBatchParams, - GoogleShoppingProductParams, - GoogleShoppingSearchBatchParams, - GoogleShoppingSearchParams, - GoogleSuggestBatchParams, - GoogleSuggestParams, - GoogleTravelHotelsBatchParams, - GoogleTravelHotelsParams, - GoogleTrendsExploreBatchParams, - GoogleTrendsExploreParams, - InstagramGraphqlProfileBatchParams, - InstagramGraphqlProfileParams, - LowesSearchBatchParams, - LowesSearchParams, - MobileBatchParams, - MobileParams, - PerplexityBatchParams, - PerplexityParams, - RedditPostBatchParams, - RedditPostParams, - RedditSubredditBatchParams, - RedditSubredditParams, - RedditUserBatchParams, - RedditUserParams, - ScrapeRequest, - Target, - TargetProductBatchParams, - TargetProductParams, - TargetSearchBatchParams, - TargetSearchParams, - TargetStoreBatchParams, - TargetStoreParams, - TiktokBatchParams, - TiktokParams, - TiktokPostBatchParams, - TiktokPostParams, - TiktokShopProductBatchParams, - TiktokShopProductParams, - TiktokShopSearchBatchParams, - TiktokShopSearchParams, - UniversalBatchParams, - UniversalEcommerceBatchParams, - UniversalEcommerceParams, - UniversalParams, - WalmartBatchParams, - WalmartParams, - WalmartProductBatchParams, - WalmartProductParams, - WalmartSearchBatchParams, - WalmartSearchParams, - YoutubeChannelBatchParams, - YoutubeChannelParams, - YoutubeMetadataBatchParams, - YoutubeMetadataParams, - YoutubeSearchBatchParams, - YoutubeSearchMaxBatchParams, - YoutubeSearchMaxParams, - YoutubeSearchParams, - YoutubeSubtitlesBatchParams, - YoutubeSubtitlesParams, - YoutubeTranscriptBatchParams, - YoutubeTranscriptParams, - YoutubeVideoBatchParams, - YoutubeVideoParams, - target_meta, - targets, -) +from .targets import Target, targets + +try: + from .generated.parameters import ParameterMeta, parameter_meta + from .generated.targets import ( + AirbnbBatchParams, + AirbnbParams, + AmazonBatchParams, + AmazonBestsellersBatchParams, + AmazonBestsellersParams, + AmazonParams, + AmazonPricingBatchParams, + AmazonPricingParams, + AmazonProductBatchParams, + AmazonProductParams, + AmazonSearchBatchParams, + AmazonSearchParams, + AmazonSellersBatchParams, + AmazonSellersParams, + AppleAppStoreBatchParams, + AppleAppStoreParams, + AutotraderBatchParams, + AutotraderParams, + BatchRequest, + BbbBatchParams, + BbbParams, + BingBatchParams, + BingParams, + BingSearchBatchParams, + BingSearchParams, + ChatgptBatchParams, + ChatgptParams, + EcommerceBatchParams, + EcommerceParams, + GeminiBatchParams, + GeminiParams, + GoogleAdsBatchParams, + GoogleAdsParams, + GoogleAiModeBatchParams, + GoogleAiModeParams, + GoogleBatchParams, + GoogleLensBatchParams, + GoogleLensParams, + GoogleMapsBatchParams, + GoogleMapsParams, + GoogleParams, + GoogleSearchBatchParams, + GoogleSearchParams, + GoogleShoppingProductBatchParams, + GoogleShoppingProductParams, + GoogleShoppingSearchBatchParams, + GoogleShoppingSearchParams, + GoogleSuggestBatchParams, + GoogleSuggestParams, + GoogleTravelHotelsBatchParams, + GoogleTravelHotelsParams, + GoogleTrendsExploreBatchParams, + GoogleTrendsExploreParams, + InstagramGraphqlProfileBatchParams, + InstagramGraphqlProfileParams, + LowesSearchBatchParams, + LowesSearchParams, + MobileBatchParams, + MobileParams, + PerplexityBatchParams, + PerplexityParams, + RedditPostBatchParams, + RedditPostParams, + RedditSubredditBatchParams, + RedditSubredditParams, + RedditUserBatchParams, + RedditUserParams, + ScrapeRequest, + TargetProductBatchParams, + TargetProductParams, + TargetSearchBatchParams, + TargetSearchParams, + TargetStoreBatchParams, + TargetStoreParams, + TiktokBatchParams, + TiktokParams, + TiktokPostBatchParams, + TiktokPostParams, + TiktokShopProductBatchParams, + TiktokShopProductParams, + TiktokShopSearchBatchParams, + TiktokShopSearchParams, + UniversalBatchParams, + UniversalEcommerceBatchParams, + UniversalEcommerceParams, + UniversalParams, + WalmartBatchParams, + WalmartParams, + WalmartProductBatchParams, + WalmartProductParams, + WalmartSearchBatchParams, + WalmartSearchParams, + YoutubeChannelBatchParams, + YoutubeChannelParams, + YoutubeMetadataBatchParams, + YoutubeMetadataParams, + YoutubeSearchBatchParams, + YoutubeSearchMaxBatchParams, + YoutubeSearchMaxParams, + YoutubeSearchParams, + YoutubeSubtitlesBatchParams, + YoutubeSubtitlesParams, + YoutubeTranscriptBatchParams, + YoutubeTranscriptParams, + YoutubeVideoBatchParams, + YoutubeVideoParams, + target_meta, + ) +except ImportError: + pass from .schema.bundled_schema import BundledSchema from .schema.remote_schema import RemoteSchema from .schema.types import DecodoSchema, RemoteSchemaLoadOptions diff --git a/src/decodo/api/web_scraping_api.py b/src/decodo/api/web_scraping_api.py index e8e86e8..86d5698 100644 --- a/src/decodo/api/web_scraping_api.py +++ b/src/decodo/api/web_scraping_api.py @@ -1,14 +1,16 @@ from __future__ import annotations from collections.abc import Mapping -from typing import Any, cast +from typing import TYPE_CHECKING, Any, cast import jsonschema from pydantic import BaseModel import decodo.errors -from decodo.generated.targets import BatchRequest, ScrapeRequest from decodo.http import HttpClient + +if TYPE_CHECKING: + from decodo.generated.targets import BatchRequest, ScrapeRequest from decodo.schema.bundled_schema import BundledSchema from decodo.schema.types import DecodoSchema from decodo.types.responses import ( diff --git a/src/decodo/codegen/codegen.py b/src/decodo/codegen/codegen.py index 698ab9a..ed06c4a 100644 --- a/src/decodo/codegen/codegen.py +++ b/src/decodo/codegen/codegen.py @@ -1,14 +1,28 @@ from __future__ import annotations -from .web_scraping_api.generate_parameter_schemas import generate_parameter_schemas_file +import os +import shutil + from .web_scraping_api.generate_parameters import generate_parameters_file -from .web_scraping_api.generate_targets import generate_targets_file +from .web_scraping_api.generate_targets import generate_targets_enum_file, generate_targets_file +from .web_scraping_api.shared import local_ir_path, out_dir def main() -> None: + os.makedirs(out_dir, exist_ok=True) + init_path = os.path.join(out_dir, "__init__.py") + if not os.path.exists(init_path): + open(init_path, "w").close() + generate_parameters_file() + generate_targets_enum_file() generate_targets_file() - generate_parameter_schemas_file() + + # Copy the downloaded IR JSON into generated/ so BundledSchema can load + # schemas directly at runtime without a generated Python file. + ir_out_path = os.path.join(out_dir, "decodo.ir.json") + shutil.copy2(local_ir_path, ir_out_path) + print(f"Saved IR JSON to {ir_out_path}") if __name__ == "__main__": diff --git a/src/decodo/codegen/web_scraping_api/generate_parameter_schemas.py b/src/decodo/codegen/web_scraping_api/generate_parameter_schemas.py deleted file mode 100644 index 25e7913..0000000 --- a/src/decodo/codegen/web_scraping_api/generate_parameter_schemas.py +++ /dev/null @@ -1,44 +0,0 @@ -from __future__ import annotations - -import json -import os - -from .shared import ( - fetch_intermediate_representation, - local_ir_path, - out_dir, -) -from .types import WebScrapingApiIR - - -def _render_generated_module(api: WebScrapingApiIR) -> str: - lines: list[str] = [] - - lines.append("# Auto-generated by src/decodo/codegen/web_scraping_api/generate_parameter_schemas.py — do not edit") - lines.append("from __future__ import annotations") - lines.append("") - lines.append("request_json_schemas: dict[str, dict] = {") - - for target_key, target in api["targets"].items(): - # Use repr() so booleans/None are valid Python literals (True/False/None) - schema_repr = repr(target["parameter_schema"]) - lines.append(f" {json.dumps(target_key)}: {schema_repr},") - - lines.append("}") - lines.append("") - - return "\n".join(lines) - - -def generate_parameter_schemas_file() -> None: - ir = fetch_intermediate_representation() - api = ir["apis"]["webScrapingApi"] - - os.makedirs(out_dir, exist_ok=True) - out_path = os.path.join(out_dir, "request_schemas.py") - with open(out_path, "w", encoding="utf-8") as f: - f.write(_render_generated_module(api)) - - target_count = len(api["targets"]) - print(f"Generated request schemas from {local_ir_path}:") - print(f" {out_path} ({target_count} targets)") diff --git a/src/decodo/codegen/web_scraping_api/generate_targets.py b/src/decodo/codegen/web_scraping_api/generate_targets.py index 0c8be62..07d0da3 100644 --- a/src/decodo/codegen/web_scraping_api/generate_targets.py +++ b/src/decodo/codegen/web_scraping_api/generate_targets.py @@ -8,6 +8,7 @@ fetch_intermediate_representation, local_ir_path, out_dir, + targets_enum_path, to_enum_member_name, to_pascal_case, ) @@ -50,21 +51,20 @@ def _get_target_parameter_keys(parameter_schema: dict[str, Any]) -> list[str]: return [k for k in properties if k != "target"] -def _get_targets_file_contents(api: WebScrapingApiIR) -> str: +def _get_targets_enum_file_contents(api: WebScrapingApiIR) -> str: lines: list[str] = [] - lines.append("# Auto-generated by src/decodo/codegen/web_scraping_api/generate_targets.py — do not edit") + lines.append("# Minimal target enum — committed to source.") + lines.append("# Run `python -m decodo.codegen.codegen` to regenerate when the IR schema changes.") lines.append("from __future__ import annotations") lines.append("") - lines.append("from enum import Enum") - lines.append("from typing import Annotated, Any, Literal, Union") + lines.append("from enum import StrEnum") lines.append("") - lines.append("import pydantic") lines.append("") target_keys = list(api["targets"].keys()) - lines.append("class Target(str, Enum):") + lines.append("class Target(StrEnum):") if target_keys: for key in target_keys: member = to_enum_member_name(key) @@ -72,10 +72,29 @@ def _get_targets_file_contents(api: WebScrapingApiIR) -> str: else: lines.append(" pass") lines.append("") + lines.append("") lines.append("targets: list[str] = [t.value for t in Target]") lines.append("") + return "\n".join(lines) + + +def _get_targets_file_contents(api: WebScrapingApiIR) -> str: + lines: list[str] = [] + + lines.append("# Auto-generated by src/decodo/codegen/web_scraping_api/generate_targets.py — do not edit") + lines.append("from __future__ import annotations") + lines.append("") + lines.append("from typing import Annotated, Any, Literal, Union") + lines.append("") + lines.append("import pydantic") + lines.append("") + lines.append("from decodo.targets import Target") + lines.append("") + + target_keys = list(api["targets"].keys()) + # Override class names for target keys that conflict with reserved names CLASS_NAME_OVERRIDES: dict[str, str] = { "target": "TargetStoreParams", @@ -175,6 +194,19 @@ def _batch_class_name(target_key: str) -> str: return "\n".join(lines) +def generate_targets_enum_file() -> None: + ir = fetch_intermediate_representation() + api = ir["apis"]["webScrapingApi"] + file_contents = _get_targets_enum_file_contents(api) + + with open(targets_enum_path, "w", encoding="utf-8") as f: + f.write(file_contents) + + target_count = len(api["targets"]) + print(f"Generated Target enum from {local_ir_path}:") + print(f" {targets_enum_path} ({target_count} targets)") + + def generate_targets_file() -> None: ir = fetch_intermediate_representation() api = ir["apis"]["webScrapingApi"] @@ -186,5 +218,5 @@ def generate_targets_file() -> None: f.write(file_contents) target_count = len(api["targets"]) - print(f"Generated targets from {local_ir_path}:") + print(f"Generated target Params from {local_ir_path}:") print(f" {out_path} ({target_count} targets)") diff --git a/src/decodo/codegen/web_scraping_api/shared.py b/src/decodo/codegen/web_scraping_api/shared.py index 20f3dfa..d0c27ff 100644 --- a/src/decodo/codegen/web_scraping_api/shared.py +++ b/src/decodo/codegen/web_scraping_api/shared.py @@ -17,6 +17,7 @@ local_ir_path = str((_THIS_DIR / "../../../.." / "inputs" / "decodo.ir.json").resolve()) out_dir = str((_THIS_DIR / "../../generated").resolve()) +targets_enum_path = str((_THIS_DIR / "../../targets.py").resolve()) def to_pascal_case(s: str) -> str: diff --git a/src/decodo/generated/__init__.py b/src/decodo/generated/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/decodo/generated/parameters.py b/src/decodo/generated/parameters.py deleted file mode 100644 index 9f801e5..0000000 --- a/src/decodo/generated/parameters.py +++ /dev/null @@ -1,93 +0,0 @@ -# Auto-generated by src/decodo/codegen/web_scraping_api/generate_parameters.py — do not edit -from __future__ import annotations - -from typing import Any, TypedDict - - -class ParameterMeta(TypedDict, total=False): - type: str - max_length: int - minimum: float - maximum: float - enum: list[Any] - items: dict[str, Any] - - -parameter_meta: dict[str, ParameterMeta] = { - "callback_url": ParameterMeta(type="string"), - "query": ParameterMeta(type="string", max_length=2048), - "headless": ParameterMeta(type="string", enum=["html", "png"]), - "locale": ParameterMeta(type="string"), - "geo": ParameterMeta(type="string"), - "device_type": ParameterMeta(type="string"), - "page_from": ParameterMeta(type="number", minimum=0, maximum=100), - "google_results_language": ParameterMeta(type="string"), - "google_tbm": ParameterMeta(type="string"), - "google_tbs": ParameterMeta(type="string"), - "parse": ParameterMeta(type="boolean"), - "google_nfpr": ParameterMeta(type="boolean"), - "google_safe_search": ParameterMeta(type="boolean"), - "session_id": ParameterMeta(type="string"), - "xhr": ParameterMeta(type="boolean"), - "markdown": ParameterMeta(type="boolean"), - "page_count": ParameterMeta(type="number", minimum=1, maximum=10), - "date_range": ParameterMeta(type="string"), - "stars": ParameterMeta(type="number"), - "adults": ParameterMeta(type="number", minimum=1, maximum=3), - "children": ParameterMeta(type="number", minimum=1, maximum=3), - "search_type": ParameterMeta(type="string"), - "date_start": ParameterMeta(type="string"), - "date_end": ParameterMeta(type="string"), - "url": ParameterMeta(type="string", max_length=2056), - "hotel_occupancy": ParameterMeta(type="string"), - "domain": ParameterMeta(type="string"), - "language_code": ParameterMeta(type="string"), - "transcript_origin": ParameterMeta(type="string"), - "autoselect_variant": ParameterMeta(type="boolean"), - "currency": ParameterMeta(type="string"), - "category": ParameterMeta(type="string"), - "merchant": ParameterMeta(type="string"), - "sort_by": ParameterMeta(type="string"), - "parser_type": ParameterMeta(type="string"), - "product_id": ParameterMeta(type="string"), - "fulfillment_type": ParameterMeta(type="string"), - "walmart_store_id": ParameterMeta(type="string"), - "delivery_zip": ParameterMeta(type="string"), - "store_id": ParameterMeta(type="string"), - "delivery_type": ParameterMeta(type="string"), - "target_store_id": ParameterMeta(type="string"), - "lowes_store_id": ParameterMeta(type="string"), - "user_agent_type": ParameterMeta(type="string"), - "free_delivery": ParameterMeta(type="boolean"), - "pickup_today": ParameterMeta(type="boolean"), - "delivery_today_tomorrow": ParameterMeta(type="boolean"), - "payload": ParameterMeta(type="string"), - "proxy_pool": ParameterMeta(type="string", enum=["standard", "premium"]), - "http_method": ParameterMeta(type="string"), - "successful_status_codes": ParameterMeta(type="array", items={'type': "number"}), - "headers": ParameterMeta(type="object"), - "cookies": ParameterMeta(type="object"), - "force_headers": ParameterMeta(type="boolean"), - "force_cookies": ParameterMeta(type="boolean"), - "prompt": ParameterMeta(type="string", max_length=8192), - "search": ParameterMeta(type="boolean"), - "country": ParameterMeta(type="string"), - "sort": ParameterMeta(type="string"), - "360": ParameterMeta(type="boolean"), - "upload_date": ParameterMeta(type="string"), - "type": ParameterMeta(type="string"), - "duration": ParameterMeta(type="string"), - "video_sort_by": ParameterMeta(type="string"), - "3d": ParameterMeta(type="boolean"), - "4k": ParameterMeta(type="boolean"), - "creative_commons": ParameterMeta(type="boolean"), - "hd": ParameterMeta(type="boolean"), - "hdr": ParameterMeta(type="boolean"), - "vr180": ParameterMeta(type="boolean"), - "live": ParameterMeta(type="boolean"), - "location": ParameterMeta(type="boolean"), - "purchased": ParameterMeta(type="boolean"), - "subtitles": ParameterMeta(type="boolean"), - "subtitle_origin": ParameterMeta(type="string"), - "limit": ParameterMeta(type="number", minimum=0, maximum=100), -} diff --git a/src/decodo/generated/request_schemas.py b/src/decodo/generated/request_schemas.py deleted file mode 100644 index 25ca96e..0000000 --- a/src/decodo/generated/request_schemas.py +++ /dev/null @@ -1,57 +0,0 @@ -# Auto-generated by src/decodo/codegen/web_scraping_api/generate_parameter_schemas.py — do not edit -from __future__ import annotations - -request_json_schemas: dict[str, dict] = { - "universal_ecommerce": {'type': 'object', 'properties': {'target': {'const': 'universal_ecommerce'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "google_search": {'type': 'object', 'properties': {'target': {'const': 'google_search'}, 'query': {'type': 'string', 'maxLength': 2048}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'locale': {'type': 'string'}, 'geo': {'type': 'string'}, 'device_type': {'type': 'string'}, 'page_from': {'type': 'number', 'minimum': 0, 'maximum': 100}, 'google_results_language': {'type': 'string'}, 'google_tbm': {'type': 'string'}, 'google_tbs': {'type': 'string'}, 'parse': {'type': 'boolean'}, 'google_nfpr': {'type': 'boolean'}, 'google_safe_search': {'type': 'boolean'}, 'session_id': {'type': 'string'}, 'xhr': {'type': 'boolean'}, 'markdown': {'type': 'boolean'}, 'page_count': {'type': 'number', 'minimum': 1, 'maximum': 10}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "google_travel_hotels": {'type': 'object', 'properties': {'target': {'const': 'google_travel_hotels'}, 'query': {'type': 'string', 'maxLength': 2048}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'locale': {'type': 'string'}, 'device_type': {'type': 'string'}, 'page_from': {'type': 'number', 'minimum': 0, 'maximum': 100}, 'date_range': {'type': 'string'}, 'stars': {'type': 'number'}, 'adults': {'type': 'number', 'minimum': 1, 'maximum': 3}, 'children': {'type': 'number', 'minimum': 1, 'maximum': 3}, 'session_id': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "google_trends_explore": {'type': 'object', 'properties': {'target': {'const': 'google_trends_explore'}, 'query': {'type': 'string', 'maxLength': 2048}, 'geo': {'type': 'string'}, 'device_type': {'type': 'string'}, 'search_type': {'type': 'string'}, 'date_start': {'type': 'string'}, 'date_end': {'type': 'string'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "google_shopping_search": {'type': 'object', 'properties': {'target': {'const': 'google_shopping_search'}, 'query': {'type': 'string', 'maxLength': 2048}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'locale': {'type': 'string'}, 'geo': {'type': 'string'}, 'device_type': {'type': 'string'}, 'page_from': {'type': 'number', 'minimum': 0, 'maximum': 100}, 'google_tbs': {'type': 'string'}, 'parse': {'type': 'boolean'}, 'session_id': {'type': 'string'}, 'google_results_language': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "google_shopping_product": {'type': 'object', 'properties': {'target': {'const': 'google_shopping_product'}, 'query': {'type': 'string', 'maxLength': 2048}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'locale': {'type': 'string'}, 'geo': {'type': 'string'}, 'device_type': {'type': 'string'}, 'page_from': {'type': 'number', 'minimum': 0, 'maximum': 100}, 'parse': {'type': 'boolean'}, 'session_id': {'type': 'string'}, 'google_results_language': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'xhr': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "google": {'type': 'object', 'properties': {'target': {'const': 'google'}, 'url': {'type': 'string', 'maxLength': 2056}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'locale': {'type': 'string'}, 'device_type': {'type': 'string'}, 'parse': {'type': 'boolean'}, 'session_id': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'xhr': {'type': 'boolean'}, 'page_count': {'type': 'number', 'minimum': 1, 'maximum': 10}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "google_suggest": {'type': 'object', 'properties': {'target': {'const': 'google_suggest'}, 'query': {'type': 'string', 'maxLength': 2048}, 'device_type': {'type': 'string'}, 'geo': {'type': 'string'}, 'locale': {'type': 'string'}, 'session_id': {'type': 'string'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "google_maps": {'type': 'object', 'properties': {'target': {'const': 'google_maps'}, 'query': {'type': 'string', 'maxLength': 2048}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'geo': {'type': 'string'}, 'locale': {'type': 'string'}, 'page_from': {'type': 'number', 'minimum': 0, 'maximum': 100}, 'device_type': {'type': 'string'}, 'session_id': {'type': 'string'}, 'google_results_language': {'type': 'string'}, 'google_nfpr': {'type': 'boolean'}, 'hotel_occupancy': {'type': 'string'}, 'date_range': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "google_ai_mode": {'type': 'object', 'properties': {'target': {'const': 'google_ai_mode'}, 'query': {'type': 'string', 'maxLength': 2048}, 'geo': {'type': 'string'}, 'parse': {'type': 'boolean'}, 'device_type': {'type': 'string'}, 'session_id': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'xhr': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "google_ads": {'type': 'object', 'properties': {'target': {'const': 'google_ads'}, 'query': {'type': 'string', 'maxLength': 2048}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'locale': {'type': 'string'}, 'geo': {'type': 'string'}, 'device_type': {'type': 'string'}, 'page_from': {'type': 'number', 'minimum': 0, 'maximum': 100}, 'google_results_language': {'type': 'string'}, 'google_tbm': {'type': 'string'}, 'google_tbs': {'type': 'string'}, 'parse': {'type': 'boolean'}, 'google_nfpr': {'type': 'boolean'}, 'session_id': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'xhr': {'type': 'boolean'}, 'page_count': {'type': 'number', 'minimum': 1, 'maximum': 10}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "google_lens": {'type': 'object', 'properties': {'target': {'const': 'google_lens'}, 'query': {'type': 'string', 'maxLength': 2048}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'parse': {'type': 'boolean'}, 'device_type': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "bing_search": {'type': 'object', 'properties': {'target': {'const': 'bing_search'}, 'query': {'type': 'string', 'maxLength': 2048}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'locale': {'type': 'string'}, 'geo': {'type': 'string'}, 'domain': {'type': 'string'}, 'device_type': {'type': 'string'}, 'page_from': {'type': 'number', 'minimum': 0, 'maximum': 100}, 'parse': {'type': 'boolean'}, 'page_count': {'type': 'number', 'minimum': 1, 'maximum': 10}, 'session_id': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'xhr': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "bing": {'type': 'object', 'properties': {'target': {'const': 'bing'}, 'url': {'type': 'string', 'maxLength': 2056}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'locale': {'type': 'string'}, 'geo': {'type': 'string'}, 'device_type': {'type': 'string'}, 'page_from': {'type': 'number', 'minimum': 0, 'maximum': 100}, 'parse': {'type': 'boolean'}, 'session_id': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'xhr': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "youtube_transcript": {'type': 'object', 'properties': {'target': {'const': 'youtube_transcript'}, 'query': {'type': 'string', 'maxLength': 2048}, 'language_code': {'type': 'string'}, 'transcript_origin': {'type': 'string'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "amazon_product": {'type': 'object', 'properties': {'target': {'const': 'amazon_product'}, 'query': {'type': 'string', 'maxLength': 2048}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'domain': {'type': 'string'}, 'device_type': {'type': 'string'}, 'parse': {'type': 'boolean'}, 'autoselect_variant': {'type': 'boolean'}, 'geo': {'type': 'string'}, 'session_id': {'type': 'string'}, 'currency': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'xhr': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "amazon_pricing": {'type': 'object', 'properties': {'target': {'const': 'amazon_pricing'}, 'query': {'type': 'string', 'maxLength': 2048}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'domain': {'type': 'string'}, 'device_type': {'type': 'string'}, 'page_from': {'type': 'number', 'minimum': 0, 'maximum': 100}, 'parse': {'type': 'boolean'}, 'geo': {'type': 'string'}, 'session_id': {'type': 'string'}, 'currency': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'xhr': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "amazon_search": {'type': 'object', 'properties': {'target': {'const': 'amazon_search'}, 'query': {'type': 'string', 'maxLength': 2048}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'domain': {'type': 'string'}, 'device_type': {'type': 'string'}, 'page_from': {'type': 'number', 'minimum': 0, 'maximum': 100}, 'category': {'type': 'string'}, 'merchant': {'type': 'string'}, 'parse': {'type': 'boolean'}, 'geo': {'type': 'string'}, 'session_id': {'type': 'string'}, 'sort_by': {'type': 'string'}, 'currency': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'xhr': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "amazon_sellers": {'type': 'object', 'properties': {'target': {'const': 'amazon_sellers'}, 'query': {'type': 'string', 'maxLength': 2048}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'locale': {'type': 'string'}, 'domain': {'type': 'string'}, 'device_type': {'type': 'string'}, 'geo': {'type': 'string'}, 'parse': {'type': 'boolean'}, 'markdown': {'type': 'boolean'}, 'xhr': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "amazon_bestsellers": {'type': 'object', 'properties': {'target': {'const': 'amazon_bestsellers'}, 'query': {'type': 'string', 'maxLength': 2048}, 'domain': {'type': 'string'}, 'device_type': {'type': 'string'}, 'geo': {'type': 'string'}, 'page_from': {'type': 'number', 'minimum': 0, 'maximum': 100}, 'category': {'type': 'string'}, 'parse': {'type': 'boolean'}, 'session_id': {'type': 'string'}, 'currency': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'xhr': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "amazon": {'type': 'object', 'properties': {'target': {'const': 'amazon'}, 'url': {'type': 'string', 'maxLength': 2056}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'device_type': {'type': 'string'}, 'parse': {'type': 'boolean'}, 'geo': {'type': 'string'}, 'session_id': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'xhr': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "ecommerce": {'type': 'object', 'properties': {'target': {'const': 'ecommerce'}, 'url': {'type': 'string', 'maxLength': 2056}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'locale': {'type': 'string'}, 'geo': {'type': 'string'}, 'device_type': {'type': 'string'}, 'parse': {'type': 'boolean'}, 'parser_type': {'type': 'string'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "walmart_product": {'type': 'object', 'properties': {'target': {'const': 'walmart_product'}, 'product_id': {'type': 'string'}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'parse': {'type': 'boolean'}, 'xhr': {'type': 'boolean'}, 'markdown': {'type': 'boolean'}, 'fulfillment_type': {'type': 'string'}, 'walmart_store_id': {'type': 'string'}, 'delivery_zip': {'type': 'string'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "walmart_search": {'type': 'object', 'properties': {'target': {'const': 'walmart_search'}, 'query': {'type': 'string', 'maxLength': 2048}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'parse': {'type': 'boolean'}, 'markdown': {'type': 'boolean'}, 'fulfillment_type': {'type': 'string'}, 'walmart_store_id': {'type': 'string'}, 'delivery_zip': {'type': 'string'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "walmart": {'type': 'object', 'properties': {'target': {'const': 'walmart'}, 'url': {'type': 'string', 'maxLength': 2056}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'locale': {'type': 'string'}, 'geo': {'type': 'string'}, 'device_type': {'type': 'string'}, 'store_id': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'xhr': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "target_product": {'type': 'object', 'properties': {'target': {'const': 'target_product'}, 'product_id': {'type': 'string'}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'parse': {'type': 'boolean'}, 'device_type': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'xhr': {'type': 'boolean'}, 'delivery_type': {'type': 'string'}, 'target_store_id': {'type': 'string'}, 'delivery_zip': {'type': 'string'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "target_search": {'type': 'object', 'properties': {'target': {'const': 'target_search'}, 'query': {'type': 'string', 'maxLength': 2048}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'parse': {'type': 'boolean'}, 'device_type': {'type': 'string'}, 'delivery_type': {'type': 'string'}, 'target_store_id': {'type': 'string'}, 'delivery_zip': {'type': 'string'}, 'xhr': {'type': 'boolean'}, 'markdown': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "target": {'type': 'object', 'properties': {'target': {'const': 'target'}, 'url': {'type': 'string', 'maxLength': 2056}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'device_type': {'type': 'string'}, 'xhr': {'type': 'boolean'}, 'delivery_zip': {'type': 'string'}, 'target_store_id': {'type': 'string'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "lowes_search": {'type': 'object', 'properties': {'target': {'const': 'lowes_search'}, 'query': {'type': 'string', 'maxLength': 2048}, 'lowes_store_id': {'type': 'string'}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'delivery_zip': {'type': 'string'}, 'user_agent_type': {'type': 'string'}, 'free_delivery': {'type': 'boolean'}, 'pickup_today': {'type': 'boolean'}, 'delivery_today_tomorrow': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "universal": {'type': 'object', 'properties': {'target': {'const': 'universal'}, 'url': {'type': 'string', 'maxLength': 2056}, 'payload': {'type': 'string'}, 'proxy_pool': {'type': 'string', 'enum': ['standard', 'premium']}, 'http_method': {'type': 'string'}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'geo': {'type': 'string'}, 'locale': {'type': 'string'}, 'device_type': {'type': 'string'}, 'session_id': {'type': 'string'}, 'successful_status_codes': {'type': 'array', 'items': {'type': 'number'}}, 'headers': {'type': 'object'}, 'cookies': {'type': 'object'}, 'force_headers': {'type': 'boolean'}, 'force_cookies': {'type': 'boolean'}, 'xhr': {'type': 'boolean'}, 'markdown': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "chatgpt": {'type': 'object', 'properties': {'target': {'const': 'chatgpt'}, 'prompt': {'type': 'string', 'maxLength': 8192}, 'search': {'type': 'boolean'}, 'parse': {'type': 'boolean'}, 'geo': {'type': 'string'}, 'device_type': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'xhr': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "perplexity": {'type': 'object', 'properties': {'target': {'const': 'perplexity'}, 'prompt': {'type': 'string', 'maxLength': 8192}, 'parse': {'type': 'boolean'}, 'geo': {'type': 'string'}, 'device_type': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'xhr': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "gemini": {'type': 'object', 'properties': {'target': {'const': 'gemini'}, 'prompt': {'type': 'string', 'maxLength': 8192}, 'parse': {'type': 'boolean'}, 'geo': {'type': 'string'}, 'xhr': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "bbb": {'type': 'object', 'properties': {'target': {'const': 'bbb'}, 'url': {'type': 'string', 'maxLength': 2056}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'geo': {'type': 'string'}, 'device_type': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'xhr': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "autotrader": {'type': 'object', 'properties': {'target': {'const': 'autotrader'}, 'url': {'type': 'string', 'maxLength': 2056}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'geo': {'type': 'string'}, 'device_type': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'xhr': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "mobile": {'type': 'object', 'properties': {'target': {'const': 'mobile'}, 'url': {'type': 'string', 'maxLength': 2056}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'geo': {'type': 'string'}, 'device_type': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'xhr': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "airbnb": {'type': 'object', 'properties': {'target': {'const': 'airbnb'}, 'url': {'type': 'string', 'maxLength': 2056}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'geo': {'type': 'string'}, 'device_type': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'xhr': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "apple_app_store": {'type': 'object', 'properties': {'target': {'const': 'apple_app_store'}, 'url': {'type': 'string', 'maxLength': 2056}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'geo': {'type': 'string'}, 'device_type': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'xhr': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "instagram_graphql_profile": {'type': 'object', 'properties': {'target': {'const': 'instagram_graphql_profile'}, 'query': {'type': 'string', 'maxLength': 2048}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "tiktok_post": {'type': 'object', 'properties': {'target': {'const': 'tiktok_post'}, 'url': {'type': 'string', 'maxLength': 2056}, 'xhr': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "tiktok_shop_search": {'type': 'object', 'properties': {'target': {'const': 'tiktok_shop_search'}, 'query': {'type': 'string', 'maxLength': 2048}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'device_type': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'country': {'type': 'string'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "tiktok_shop_product": {'type': 'object', 'properties': {'target': {'const': 'tiktok_shop_product'}, 'product_id': {'type': 'string'}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'device_type': {'type': 'string'}, 'xhr': {'type': 'boolean'}, 'markdown': {'type': 'boolean'}, 'country': {'type': 'string'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "tiktok": {'type': 'object', 'properties': {'target': {'const': 'tiktok'}, 'url': {'type': 'string', 'maxLength': 2056}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'user_agent_type': {'type': 'string'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "reddit_post": {'type': 'object', 'properties': {'target': {'const': 'reddit_post'}, 'url': {'type': 'string', 'maxLength': 2056}, 'locale': {'type': 'string'}, 'geo': {'type': 'string'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "reddit_subreddit": {'type': 'object', 'properties': {'target': {'const': 'reddit_subreddit'}, 'url': {'type': 'string', 'maxLength': 2056}, 'locale': {'type': 'string'}, 'geo': {'type': 'string'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "reddit_user": {'type': 'object', 'properties': {'target': {'const': 'reddit_user'}, 'url': {'type': 'string', 'maxLength': 2056}, 'locale': {'type': 'string'}, 'geo': {'type': 'string'}, 'sort': {'type': 'string'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "youtube_video": {'type': 'object', 'properties': {'target': {'const': 'youtube_video'}, 'query': {'type': 'string', 'maxLength': 2048}, 'geo': {'type': 'string'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "youtube_metadata": {'type': 'object', 'properties': {'target': {'const': 'youtube_metadata'}, 'query': {'type': 'string', 'maxLength': 2048}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "youtube_search": {'type': 'object', 'properties': {'360': {'type': 'boolean'}, 'target': {'const': 'youtube_search'}, 'query': {'type': 'string', 'maxLength': 2048}, 'upload_date': {'type': 'string'}, 'type': {'type': 'string'}, 'duration': {'type': 'string'}, 'video_sort_by': {'type': 'string'}, '3d': {'type': 'boolean'}, '4k': {'type': 'boolean'}, 'creative_commons': {'type': 'boolean'}, 'hd': {'type': 'boolean'}, 'hdr': {'type': 'boolean'}, 'vr180': {'type': 'boolean'}, 'live': {'type': 'boolean'}, 'location': {'type': 'boolean'}, 'purchased': {'type': 'boolean'}, 'subtitles': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "youtube_search_max": {'type': 'object', 'properties': {'360': {'type': 'boolean'}, 'target': {'const': 'youtube_search_max'}, 'query': {'type': 'string', 'maxLength': 2048}, 'upload_date': {'type': 'string'}, 'type': {'type': 'string'}, 'duration': {'type': 'string'}, 'video_sort_by': {'type': 'string'}, '3d': {'type': 'boolean'}, '4k': {'type': 'boolean'}, 'creative_commons': {'type': 'boolean'}, 'hd': {'type': 'boolean'}, 'hdr': {'type': 'boolean'}, 'vr180': {'type': 'boolean'}, 'live': {'type': 'boolean'}, 'location': {'type': 'boolean'}, 'purchased': {'type': 'boolean'}, 'subtitles': {'type': 'boolean'}, 'markdown': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "youtube_subtitles": {'type': 'object', 'properties': {'target': {'const': 'youtube_subtitles'}, 'query': {'type': 'string', 'maxLength': 2048}, 'language_code': {'type': 'string'}, 'subtitle_origin': {'type': 'string'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "youtube_channel": {'type': 'object', 'properties': {'target': {'const': 'youtube_channel'}, 'query': {'type': 'string', 'maxLength': 2048}, 'parse': {'type': 'boolean'}, 'limit': {'type': 'number', 'minimum': 0, 'maximum': 100}, 'markdown': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, -} diff --git a/src/decodo/generated/targets.py b/src/decodo/generated/targets.py deleted file mode 100644 index 7354c0c..0000000 --- a/src/decodo/generated/targets.py +++ /dev/null @@ -1,2170 +0,0 @@ -# Auto-generated by src/decodo/codegen/web_scraping_api/generate_targets.py — do not edit -from __future__ import annotations - -from enum import Enum -from typing import Annotated, Any, Literal, Union - -import pydantic - - -class Target(str, Enum): - UniversalEcommerce = "universal_ecommerce" - GoogleSearch = "google_search" - GoogleTravelHotels = "google_travel_hotels" - GoogleTrendsExplore = "google_trends_explore" - GoogleShoppingSearch = "google_shopping_search" - GoogleShoppingProduct = "google_shopping_product" - Google = "google" - GoogleSuggest = "google_suggest" - GoogleMaps = "google_maps" - GoogleAiMode = "google_ai_mode" - GoogleAds = "google_ads" - GoogleLens = "google_lens" - BingSearch = "bing_search" - Bing = "bing" - YoutubeTranscript = "youtube_transcript" - AmazonProduct = "amazon_product" - AmazonPricing = "amazon_pricing" - AmazonSearch = "amazon_search" - AmazonSellers = "amazon_sellers" - AmazonBestsellers = "amazon_bestsellers" - Amazon = "amazon" - Ecommerce = "ecommerce" - WalmartProduct = "walmart_product" - WalmartSearch = "walmart_search" - Walmart = "walmart" - TargetProduct = "target_product" - TargetSearch = "target_search" - Target = "target" - LowesSearch = "lowes_search" - Universal = "universal" - Chatgpt = "chatgpt" - Perplexity = "perplexity" - Gemini = "gemini" - Bbb = "bbb" - Autotrader = "autotrader" - Mobile = "mobile" - Airbnb = "airbnb" - AppleAppStore = "apple_app_store" - InstagramGraphqlProfile = "instagram_graphql_profile" - TiktokPost = "tiktok_post" - TiktokShopSearch = "tiktok_shop_search" - TiktokShopProduct = "tiktok_shop_product" - Tiktok = "tiktok" - RedditPost = "reddit_post" - RedditSubreddit = "reddit_subreddit" - RedditUser = "reddit_user" - YoutubeVideo = "youtube_video" - YoutubeMetadata = "youtube_metadata" - YoutubeSearch = "youtube_search" - YoutubeSearchMax = "youtube_search_max" - YoutubeSubtitles = "youtube_subtitles" - YoutubeChannel = "youtube_channel" - - -targets: list[str] = [t.value for t in Target] - - -class UniversalEcommerceParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.UniversalEcommerce] = Target.UniversalEcommerce - callback_url: str | None = None - - -class GoogleSearchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.GoogleSearch] = Target.GoogleSearch - query: str | None = None - headless: Literal["html", "png"] | None = None - locale: str | None = None - geo: str | None = None - device_type: str | None = None - page_from: float | None = None - google_results_language: str | None = None - google_tbm: str | None = None - google_tbs: str | None = None - parse: bool | None = None - google_nfpr: bool | None = None - google_safe_search: bool | None = None - session_id: str | None = None - xhr: bool | None = None - markdown: bool | None = None - page_count: float | None = None - callback_url: str | None = None - - -class GoogleTravelHotelsParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.GoogleTravelHotels] = Target.GoogleTravelHotels - query: str | None = None - headless: Literal["html", "png"] | None = None - locale: str | None = None - device_type: str | None = None - page_from: float | None = None - date_range: str | None = None - stars: float | None = None - adults: float | None = None - children: float | None = None - session_id: str | None = None - markdown: bool | None = None - callback_url: str | None = None - - -class GoogleTrendsExploreParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.GoogleTrendsExplore] = Target.GoogleTrendsExplore - query: str | None = None - geo: str | None = None - device_type: str | None = None - search_type: str | None = None - date_start: str | None = None - date_end: str | None = None - callback_url: str | None = None - - -class GoogleShoppingSearchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.GoogleShoppingSearch] = Target.GoogleShoppingSearch - query: str | None = None - headless: Literal["html", "png"] | None = None - locale: str | None = None - geo: str | None = None - device_type: str | None = None - page_from: float | None = None - google_tbs: str | None = None - parse: bool | None = None - session_id: str | None = None - google_results_language: str | None = None - markdown: bool | None = None - callback_url: str | None = None - - -class GoogleShoppingProductParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.GoogleShoppingProduct] = Target.GoogleShoppingProduct - query: str | None = None - headless: Literal["html", "png"] | None = None - locale: str | None = None - geo: str | None = None - device_type: str | None = None - page_from: float | None = None - parse: bool | None = None - session_id: str | None = None - google_results_language: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - - -class GoogleParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.Google] = Target.Google - url: str | None = None - headless: Literal["html", "png"] | None = None - locale: str | None = None - device_type: str | None = None - parse: bool | None = None - session_id: str | None = None - markdown: bool | None = None - xhr: bool | None = None - page_count: float | None = None - callback_url: str | None = None - - -class GoogleSuggestParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.GoogleSuggest] = Target.GoogleSuggest - query: str | None = None - device_type: str | None = None - geo: str | None = None - locale: str | None = None - session_id: str | None = None - callback_url: str | None = None - - -class GoogleMapsParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.GoogleMaps] = Target.GoogleMaps - query: str | None = None - headless: Literal["html", "png"] | None = None - geo: str | None = None - locale: str | None = None - page_from: float | None = None - device_type: str | None = None - session_id: str | None = None - google_results_language: str | None = None - google_nfpr: bool | None = None - hotel_occupancy: str | None = None - date_range: str | None = None - markdown: bool | None = None - callback_url: str | None = None - - -class GoogleAiModeParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.GoogleAiMode] = Target.GoogleAiMode - query: str | None = None - geo: str | None = None - parse: bool | None = None - device_type: str | None = None - session_id: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - - -class GoogleAdsParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.GoogleAds] = Target.GoogleAds - query: str | None = None - headless: Literal["html", "png"] | None = None - locale: str | None = None - geo: str | None = None - device_type: str | None = None - page_from: float | None = None - google_results_language: str | None = None - google_tbm: str | None = None - google_tbs: str | None = None - parse: bool | None = None - google_nfpr: bool | None = None - session_id: str | None = None - markdown: bool | None = None - xhr: bool | None = None - page_count: float | None = None - callback_url: str | None = None - - -class GoogleLensParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.GoogleLens] = Target.GoogleLens - query: str | None = None - headless: Literal["html", "png"] | None = None - parse: bool | None = None - device_type: str | None = None - markdown: bool | None = None - callback_url: str | None = None - - -class BingSearchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.BingSearch] = Target.BingSearch - query: str | None = None - headless: Literal["html", "png"] | None = None - locale: str | None = None - geo: str | None = None - domain: str | None = None - device_type: str | None = None - page_from: float | None = None - parse: bool | None = None - page_count: float | None = None - session_id: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - - -class BingParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.Bing] = Target.Bing - url: str | None = None - headless: Literal["html", "png"] | None = None - locale: str | None = None - geo: str | None = None - device_type: str | None = None - page_from: float | None = None - parse: bool | None = None - session_id: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - - -class YoutubeTranscriptParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.YoutubeTranscript] = Target.YoutubeTranscript - query: str | None = None - language_code: str | None = None - transcript_origin: str | None = None - callback_url: str | None = None - - -class AmazonProductParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.AmazonProduct] = Target.AmazonProduct - query: str | None = None - headless: Literal["html", "png"] | None = None - domain: str | None = None - device_type: str | None = None - parse: bool | None = None - autoselect_variant: bool | None = None - geo: str | None = None - session_id: str | None = None - currency: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - - -class AmazonPricingParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.AmazonPricing] = Target.AmazonPricing - query: str | None = None - headless: Literal["html", "png"] | None = None - domain: str | None = None - device_type: str | None = None - page_from: float | None = None - parse: bool | None = None - geo: str | None = None - session_id: str | None = None - currency: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - - -class AmazonSearchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.AmazonSearch] = Target.AmazonSearch - query: str | None = None - headless: Literal["html", "png"] | None = None - domain: str | None = None - device_type: str | None = None - page_from: float | None = None - category: str | None = None - merchant: str | None = None - parse: bool | None = None - geo: str | None = None - session_id: str | None = None - sort_by: str | None = None - currency: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - - -class AmazonSellersParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.AmazonSellers] = Target.AmazonSellers - query: str | None = None - headless: Literal["html", "png"] | None = None - locale: str | None = None - domain: str | None = None - device_type: str | None = None - geo: str | None = None - parse: bool | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - - -class AmazonBestsellersParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.AmazonBestsellers] = Target.AmazonBestsellers - query: str | None = None - domain: str | None = None - device_type: str | None = None - geo: str | None = None - page_from: float | None = None - category: str | None = None - parse: bool | None = None - session_id: str | None = None - currency: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - - -class AmazonParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.Amazon] = Target.Amazon - url: str | None = None - headless: Literal["html", "png"] | None = None - device_type: str | None = None - parse: bool | None = None - geo: str | None = None - session_id: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - - -class EcommerceParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.Ecommerce] = Target.Ecommerce - url: str | None = None - headless: Literal["html", "png"] | None = None - locale: str | None = None - geo: str | None = None - device_type: str | None = None - parse: bool | None = None - parser_type: str | None = None - callback_url: str | None = None - - -class WalmartProductParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.WalmartProduct] = Target.WalmartProduct - product_id: str | None = None - headless: Literal["html", "png"] | None = None - parse: bool | None = None - xhr: bool | None = None - markdown: bool | None = None - fulfillment_type: str | None = None - walmart_store_id: str | None = None - delivery_zip: str | None = None - callback_url: str | None = None - - -class WalmartSearchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.WalmartSearch] = Target.WalmartSearch - query: str | None = None - headless: Literal["html", "png"] | None = None - parse: bool | None = None - markdown: bool | None = None - fulfillment_type: str | None = None - walmart_store_id: str | None = None - delivery_zip: str | None = None - callback_url: str | None = None - - -class WalmartParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.Walmart] = Target.Walmart - url: str | None = None - headless: Literal["html", "png"] | None = None - locale: str | None = None - geo: str | None = None - device_type: str | None = None - store_id: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - - -class TargetProductParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.TargetProduct] = Target.TargetProduct - product_id: str | None = None - headless: Literal["html", "png"] | None = None - parse: bool | None = None - device_type: str | None = None - markdown: bool | None = None - xhr: bool | None = None - delivery_type: str | None = None - target_store_id: str | None = None - delivery_zip: str | None = None - callback_url: str | None = None - - -class TargetSearchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.TargetSearch] = Target.TargetSearch - query: str | None = None - headless: Literal["html", "png"] | None = None - parse: bool | None = None - device_type: str | None = None - delivery_type: str | None = None - target_store_id: str | None = None - delivery_zip: str | None = None - xhr: bool | None = None - markdown: bool | None = None - callback_url: str | None = None - - -class TargetStoreParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.Target] = Target.Target - url: str | None = None - headless: Literal["html", "png"] | None = None - device_type: str | None = None - xhr: bool | None = None - delivery_zip: str | None = None - target_store_id: str | None = None - callback_url: str | None = None - - -class LowesSearchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.LowesSearch] = Target.LowesSearch - query: str | None = None - lowes_store_id: str | None = None - headless: Literal["html", "png"] | None = None - delivery_zip: str | None = None - user_agent_type: str | None = None - free_delivery: bool | None = None - pickup_today: bool | None = None - delivery_today_tomorrow: bool | None = None - callback_url: str | None = None - - -class UniversalParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.Universal] = Target.Universal - url: str | None = None - payload: str | None = None - proxy_pool: Literal["standard", "premium"] | None = None - http_method: str | None = None - headless: Literal["html", "png"] | None = None - geo: str | None = None - locale: str | None = None - device_type: str | None = None - session_id: str | None = None - successful_status_codes: list[Any] | None = None - headers: Any | None = None - cookies: Any | None = None - force_headers: bool | None = None - force_cookies: bool | None = None - xhr: bool | None = None - markdown: bool | None = None - callback_url: str | None = None - - -class ChatgptParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.Chatgpt] = Target.Chatgpt - prompt: str | None = None - search: bool | None = None - parse: bool | None = None - geo: str | None = None - device_type: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - - -class PerplexityParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.Perplexity] = Target.Perplexity - prompt: str | None = None - parse: bool | None = None - geo: str | None = None - device_type: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - - -class GeminiParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.Gemini] = Target.Gemini - prompt: str | None = None - parse: bool | None = None - geo: str | None = None - xhr: bool | None = None - callback_url: str | None = None - - -class BbbParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.Bbb] = Target.Bbb - url: str | None = None - headless: Literal["html", "png"] | None = None - geo: str | None = None - device_type: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - - -class AutotraderParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.Autotrader] = Target.Autotrader - url: str | None = None - headless: Literal["html", "png"] | None = None - geo: str | None = None - device_type: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - - -class MobileParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.Mobile] = Target.Mobile - url: str | None = None - headless: Literal["html", "png"] | None = None - geo: str | None = None - device_type: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - - -class AirbnbParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.Airbnb] = Target.Airbnb - url: str | None = None - headless: Literal["html", "png"] | None = None - geo: str | None = None - device_type: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - - -class AppleAppStoreParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.AppleAppStore] = Target.AppleAppStore - url: str | None = None - headless: Literal["html", "png"] | None = None - geo: str | None = None - device_type: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - - -class InstagramGraphqlProfileParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.InstagramGraphqlProfile] = Target.InstagramGraphqlProfile - query: str | None = None - callback_url: str | None = None - - -class TiktokPostParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.TiktokPost] = Target.TiktokPost - url: str | None = None - xhr: bool | None = None - callback_url: str | None = None - - -class TiktokShopSearchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.TiktokShopSearch] = Target.TiktokShopSearch - query: str | None = None - headless: Literal["html", "png"] | None = None - device_type: str | None = None - markdown: bool | None = None - country: str | None = None - callback_url: str | None = None - - -class TiktokShopProductParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.TiktokShopProduct] = Target.TiktokShopProduct - product_id: str | None = None - headless: Literal["html", "png"] | None = None - device_type: str | None = None - xhr: bool | None = None - markdown: bool | None = None - country: str | None = None - callback_url: str | None = None - - -class TiktokParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.Tiktok] = Target.Tiktok - url: str | None = None - headless: Literal["html", "png"] | None = None - user_agent_type: str | None = None - callback_url: str | None = None - - -class RedditPostParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.RedditPost] = Target.RedditPost - url: str | None = None - locale: str | None = None - geo: str | None = None - callback_url: str | None = None - - -class RedditSubredditParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.RedditSubreddit] = Target.RedditSubreddit - url: str | None = None - locale: str | None = None - geo: str | None = None - callback_url: str | None = None - - -class RedditUserParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.RedditUser] = Target.RedditUser - url: str | None = None - locale: str | None = None - geo: str | None = None - sort: str | None = None - callback_url: str | None = None - - -class YoutubeVideoParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.YoutubeVideo] = Target.YoutubeVideo - query: str | None = None - geo: str | None = None - callback_url: str | None = None - - -class YoutubeMetadataParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.YoutubeMetadata] = Target.YoutubeMetadata - query: str | None = None - callback_url: str | None = None - - -class YoutubeSearchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.YoutubeSearch] = Target.YoutubeSearch - f360: bool | None = pydantic.Field(None, alias="360") - query: str | None = None - upload_date: str | None = None - type: str | None = None - duration: str | None = None - video_sort_by: str | None = None - f3d: bool | None = pydantic.Field(None, alias="3d") - f4k: bool | None = pydantic.Field(None, alias="4k") - creative_commons: bool | None = None - hd: bool | None = None - hdr: bool | None = None - vr180: bool | None = None - live: bool | None = None - location: bool | None = None - purchased: bool | None = None - subtitles: bool | None = None - callback_url: str | None = None - - -class YoutubeSearchMaxParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.YoutubeSearchMax] = Target.YoutubeSearchMax - f360: bool | None = pydantic.Field(None, alias="360") - query: str | None = None - upload_date: str | None = None - type: str | None = None - duration: str | None = None - video_sort_by: str | None = None - f3d: bool | None = pydantic.Field(None, alias="3d") - f4k: bool | None = pydantic.Field(None, alias="4k") - creative_commons: bool | None = None - hd: bool | None = None - hdr: bool | None = None - vr180: bool | None = None - live: bool | None = None - location: bool | None = None - purchased: bool | None = None - subtitles: bool | None = None - markdown: bool | None = None - callback_url: str | None = None - - -class YoutubeSubtitlesParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.YoutubeSubtitles] = Target.YoutubeSubtitles - query: str | None = None - language_code: str | None = None - subtitle_origin: str | None = None - callback_url: str | None = None - - -class YoutubeChannelParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.YoutubeChannel] = Target.YoutubeChannel - query: str | None = None - parse: bool | None = None - limit: float | None = None - markdown: bool | None = None - callback_url: str | None = None - - -class UniversalEcommerceBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.UniversalEcommerce] = Target.UniversalEcommerce - callback_url: str | None = None - - -class GoogleSearchBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.GoogleSearch] = Target.GoogleSearch - query: list[str] | None = None - headless: Literal["html", "png"] | None = None - locale: str | None = None - geo: str | None = None - device_type: str | None = None - page_from: float | None = None - google_results_language: str | None = None - google_tbm: str | None = None - google_tbs: str | None = None - parse: bool | None = None - google_nfpr: bool | None = None - google_safe_search: bool | None = None - session_id: str | None = None - xhr: bool | None = None - markdown: bool | None = None - page_count: float | None = None - callback_url: str | None = None - - -class GoogleTravelHotelsBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.GoogleTravelHotels] = Target.GoogleTravelHotels - query: list[str] | None = None - headless: Literal["html", "png"] | None = None - locale: str | None = None - device_type: str | None = None - page_from: float | None = None - date_range: str | None = None - stars: float | None = None - adults: float | None = None - children: float | None = None - session_id: str | None = None - markdown: bool | None = None - callback_url: str | None = None - - -class GoogleTrendsExploreBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.GoogleTrendsExplore] = Target.GoogleTrendsExplore - query: list[str] | None = None - geo: str | None = None - device_type: str | None = None - search_type: str | None = None - date_start: str | None = None - date_end: str | None = None - callback_url: str | None = None - - -class GoogleShoppingSearchBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.GoogleShoppingSearch] = Target.GoogleShoppingSearch - query: list[str] | None = None - headless: Literal["html", "png"] | None = None - locale: str | None = None - geo: str | None = None - device_type: str | None = None - page_from: float | None = None - google_tbs: str | None = None - parse: bool | None = None - session_id: str | None = None - google_results_language: str | None = None - markdown: bool | None = None - callback_url: str | None = None - - -class GoogleShoppingProductBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.GoogleShoppingProduct] = Target.GoogleShoppingProduct - query: list[str] | None = None - headless: Literal["html", "png"] | None = None - locale: str | None = None - geo: str | None = None - device_type: str | None = None - page_from: float | None = None - parse: bool | None = None - session_id: str | None = None - google_results_language: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - - -class GoogleBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.Google] = Target.Google - url: list[str] | None = None - headless: Literal["html", "png"] | None = None - locale: str | None = None - device_type: str | None = None - parse: bool | None = None - session_id: str | None = None - markdown: bool | None = None - xhr: bool | None = None - page_count: float | None = None - callback_url: str | None = None - - -class GoogleSuggestBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.GoogleSuggest] = Target.GoogleSuggest - query: list[str] | None = None - device_type: str | None = None - geo: str | None = None - locale: str | None = None - session_id: str | None = None - callback_url: str | None = None - - -class GoogleMapsBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.GoogleMaps] = Target.GoogleMaps - query: list[str] | None = None - headless: Literal["html", "png"] | None = None - geo: str | None = None - locale: str | None = None - page_from: float | None = None - device_type: str | None = None - session_id: str | None = None - google_results_language: str | None = None - google_nfpr: bool | None = None - hotel_occupancy: str | None = None - date_range: str | None = None - markdown: bool | None = None - callback_url: str | None = None - - -class GoogleAiModeBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.GoogleAiMode] = Target.GoogleAiMode - query: list[str] | None = None - geo: str | None = None - parse: bool | None = None - device_type: str | None = None - session_id: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - - -class GoogleAdsBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.GoogleAds] = Target.GoogleAds - query: list[str] | None = None - headless: Literal["html", "png"] | None = None - locale: str | None = None - geo: str | None = None - device_type: str | None = None - page_from: float | None = None - google_results_language: str | None = None - google_tbm: str | None = None - google_tbs: str | None = None - parse: bool | None = None - google_nfpr: bool | None = None - session_id: str | None = None - markdown: bool | None = None - xhr: bool | None = None - page_count: float | None = None - callback_url: str | None = None - - -class GoogleLensBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.GoogleLens] = Target.GoogleLens - query: list[str] | None = None - headless: Literal["html", "png"] | None = None - parse: bool | None = None - device_type: str | None = None - markdown: bool | None = None - callback_url: str | None = None - - -class BingSearchBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.BingSearch] = Target.BingSearch - query: list[str] | None = None - headless: Literal["html", "png"] | None = None - locale: str | None = None - geo: str | None = None - domain: str | None = None - device_type: str | None = None - page_from: float | None = None - parse: bool | None = None - page_count: float | None = None - session_id: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - - -class BingBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.Bing] = Target.Bing - url: list[str] | None = None - headless: Literal["html", "png"] | None = None - locale: str | None = None - geo: str | None = None - device_type: str | None = None - page_from: float | None = None - parse: bool | None = None - session_id: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - - -class YoutubeTranscriptBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.YoutubeTranscript] = Target.YoutubeTranscript - query: list[str] | None = None - language_code: str | None = None - transcript_origin: str | None = None - callback_url: str | None = None - - -class AmazonProductBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.AmazonProduct] = Target.AmazonProduct - query: list[str] | None = None - headless: Literal["html", "png"] | None = None - domain: str | None = None - device_type: str | None = None - parse: bool | None = None - autoselect_variant: bool | None = None - geo: str | None = None - session_id: str | None = None - currency: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - - -class AmazonPricingBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.AmazonPricing] = Target.AmazonPricing - query: list[str] | None = None - headless: Literal["html", "png"] | None = None - domain: str | None = None - device_type: str | None = None - page_from: float | None = None - parse: bool | None = None - geo: str | None = None - session_id: str | None = None - currency: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - - -class AmazonSearchBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.AmazonSearch] = Target.AmazonSearch - query: list[str] | None = None - headless: Literal["html", "png"] | None = None - domain: str | None = None - device_type: str | None = None - page_from: float | None = None - category: str | None = None - merchant: str | None = None - parse: bool | None = None - geo: str | None = None - session_id: str | None = None - sort_by: str | None = None - currency: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - - -class AmazonSellersBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.AmazonSellers] = Target.AmazonSellers - query: list[str] | None = None - headless: Literal["html", "png"] | None = None - locale: str | None = None - domain: str | None = None - device_type: str | None = None - geo: str | None = None - parse: bool | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - - -class AmazonBestsellersBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.AmazonBestsellers] = Target.AmazonBestsellers - query: list[str] | None = None - domain: str | None = None - device_type: str | None = None - geo: str | None = None - page_from: float | None = None - category: str | None = None - parse: bool | None = None - session_id: str | None = None - currency: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - - -class AmazonBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.Amazon] = Target.Amazon - url: list[str] | None = None - headless: Literal["html", "png"] | None = None - device_type: str | None = None - parse: bool | None = None - geo: str | None = None - session_id: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - - -class EcommerceBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.Ecommerce] = Target.Ecommerce - url: list[str] | None = None - headless: Literal["html", "png"] | None = None - locale: str | None = None - geo: str | None = None - device_type: str | None = None - parse: bool | None = None - parser_type: str | None = None - callback_url: str | None = None - - -class WalmartProductBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.WalmartProduct] = Target.WalmartProduct - product_id: str | None = None - headless: Literal["html", "png"] | None = None - parse: bool | None = None - xhr: bool | None = None - markdown: bool | None = None - fulfillment_type: str | None = None - walmart_store_id: str | None = None - delivery_zip: str | None = None - callback_url: str | None = None - - -class WalmartSearchBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.WalmartSearch] = Target.WalmartSearch - query: list[str] | None = None - headless: Literal["html", "png"] | None = None - parse: bool | None = None - markdown: bool | None = None - fulfillment_type: str | None = None - walmart_store_id: str | None = None - delivery_zip: str | None = None - callback_url: str | None = None - - -class WalmartBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.Walmart] = Target.Walmart - url: list[str] | None = None - headless: Literal["html", "png"] | None = None - locale: str | None = None - geo: str | None = None - device_type: str | None = None - store_id: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - - -class TargetProductBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.TargetProduct] = Target.TargetProduct - product_id: str | None = None - headless: Literal["html", "png"] | None = None - parse: bool | None = None - device_type: str | None = None - markdown: bool | None = None - xhr: bool | None = None - delivery_type: str | None = None - target_store_id: str | None = None - delivery_zip: str | None = None - callback_url: str | None = None - - -class TargetSearchBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.TargetSearch] = Target.TargetSearch - query: list[str] | None = None - headless: Literal["html", "png"] | None = None - parse: bool | None = None - device_type: str | None = None - delivery_type: str | None = None - target_store_id: str | None = None - delivery_zip: str | None = None - xhr: bool | None = None - markdown: bool | None = None - callback_url: str | None = None - - -class TargetStoreBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.Target] = Target.Target - url: list[str] | None = None - headless: Literal["html", "png"] | None = None - device_type: str | None = None - xhr: bool | None = None - delivery_zip: str | None = None - target_store_id: str | None = None - callback_url: str | None = None - - -class LowesSearchBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.LowesSearch] = Target.LowesSearch - query: list[str] | None = None - lowes_store_id: str | None = None - headless: Literal["html", "png"] | None = None - delivery_zip: str | None = None - user_agent_type: str | None = None - free_delivery: bool | None = None - pickup_today: bool | None = None - delivery_today_tomorrow: bool | None = None - callback_url: str | None = None - - -class UniversalBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.Universal] = Target.Universal - url: list[str] | None = None - payload: str | None = None - proxy_pool: Literal["standard", "premium"] | None = None - http_method: str | None = None - headless: Literal["html", "png"] | None = None - geo: str | None = None - locale: str | None = None - device_type: str | None = None - session_id: str | None = None - successful_status_codes: list[Any] | None = None - headers: Any | None = None - cookies: Any | None = None - force_headers: bool | None = None - force_cookies: bool | None = None - xhr: bool | None = None - markdown: bool | None = None - callback_url: str | None = None - - -class ChatgptBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.Chatgpt] = Target.Chatgpt - prompt: str | None = None - search: bool | None = None - parse: bool | None = None - geo: str | None = None - device_type: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - - -class PerplexityBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.Perplexity] = Target.Perplexity - prompt: str | None = None - parse: bool | None = None - geo: str | None = None - device_type: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - - -class GeminiBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.Gemini] = Target.Gemini - prompt: str | None = None - parse: bool | None = None - geo: str | None = None - xhr: bool | None = None - callback_url: str | None = None - - -class BbbBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.Bbb] = Target.Bbb - url: list[str] | None = None - headless: Literal["html", "png"] | None = None - geo: str | None = None - device_type: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - - -class AutotraderBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.Autotrader] = Target.Autotrader - url: list[str] | None = None - headless: Literal["html", "png"] | None = None - geo: str | None = None - device_type: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - - -class MobileBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.Mobile] = Target.Mobile - url: list[str] | None = None - headless: Literal["html", "png"] | None = None - geo: str | None = None - device_type: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - - -class AirbnbBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.Airbnb] = Target.Airbnb - url: list[str] | None = None - headless: Literal["html", "png"] | None = None - geo: str | None = None - device_type: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - - -class AppleAppStoreBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.AppleAppStore] = Target.AppleAppStore - url: list[str] | None = None - headless: Literal["html", "png"] | None = None - geo: str | None = None - device_type: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - - -class InstagramGraphqlProfileBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.InstagramGraphqlProfile] = Target.InstagramGraphqlProfile - query: list[str] | None = None - callback_url: str | None = None - - -class TiktokPostBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.TiktokPost] = Target.TiktokPost - url: list[str] | None = None - xhr: bool | None = None - callback_url: str | None = None - - -class TiktokShopSearchBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.TiktokShopSearch] = Target.TiktokShopSearch - query: list[str] | None = None - headless: Literal["html", "png"] | None = None - device_type: str | None = None - markdown: bool | None = None - country: str | None = None - callback_url: str | None = None - - -class TiktokShopProductBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.TiktokShopProduct] = Target.TiktokShopProduct - product_id: str | None = None - headless: Literal["html", "png"] | None = None - device_type: str | None = None - xhr: bool | None = None - markdown: bool | None = None - country: str | None = None - callback_url: str | None = None - - -class TiktokBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.Tiktok] = Target.Tiktok - url: list[str] | None = None - headless: Literal["html", "png"] | None = None - user_agent_type: str | None = None - callback_url: str | None = None - - -class RedditPostBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.RedditPost] = Target.RedditPost - url: list[str] | None = None - locale: str | None = None - geo: str | None = None - callback_url: str | None = None - - -class RedditSubredditBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.RedditSubreddit] = Target.RedditSubreddit - url: list[str] | None = None - locale: str | None = None - geo: str | None = None - callback_url: str | None = None - - -class RedditUserBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.RedditUser] = Target.RedditUser - url: list[str] | None = None - locale: str | None = None - geo: str | None = None - sort: str | None = None - callback_url: str | None = None - - -class YoutubeVideoBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.YoutubeVideo] = Target.YoutubeVideo - query: list[str] | None = None - geo: str | None = None - callback_url: str | None = None - - -class YoutubeMetadataBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.YoutubeMetadata] = Target.YoutubeMetadata - query: list[str] | None = None - callback_url: str | None = None - - -class YoutubeSearchBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.YoutubeSearch] = Target.YoutubeSearch - f360: bool | None = pydantic.Field(None, alias="360") - query: list[str] | None = None - upload_date: str | None = None - type: str | None = None - duration: str | None = None - video_sort_by: str | None = None - f3d: bool | None = pydantic.Field(None, alias="3d") - f4k: bool | None = pydantic.Field(None, alias="4k") - creative_commons: bool | None = None - hd: bool | None = None - hdr: bool | None = None - vr180: bool | None = None - live: bool | None = None - location: bool | None = None - purchased: bool | None = None - subtitles: bool | None = None - callback_url: str | None = None - - -class YoutubeSearchMaxBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.YoutubeSearchMax] = Target.YoutubeSearchMax - f360: bool | None = pydantic.Field(None, alias="360") - query: list[str] | None = None - upload_date: str | None = None - type: str | None = None - duration: str | None = None - video_sort_by: str | None = None - f3d: bool | None = pydantic.Field(None, alias="3d") - f4k: bool | None = pydantic.Field(None, alias="4k") - creative_commons: bool | None = None - hd: bool | None = None - hdr: bool | None = None - vr180: bool | None = None - live: bool | None = None - location: bool | None = None - purchased: bool | None = None - subtitles: bool | None = None - markdown: bool | None = None - callback_url: str | None = None - - -class YoutubeSubtitlesBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.YoutubeSubtitles] = Target.YoutubeSubtitles - query: list[str] | None = None - language_code: str | None = None - subtitle_origin: str | None = None - callback_url: str | None = None - - -class YoutubeChannelBatchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.YoutubeChannel] = Target.YoutubeChannel - query: list[str] | None = None - parse: bool | None = None - limit: float | None = None - markdown: bool | None = None - callback_url: str | None = None - - -target_meta: dict[str, dict[str, Any]] = { - Target.UniversalEcommerce.value: { - "group": "None", - "response_format": "html", - "parameters": ["callback_url"], - }, - Target.GoogleSearch.value: { - "group": "Google", - "response_format": "json", - "parameters": [ - "query", - "headless", - "locale", - "geo", - "device_type", - "page_from", - "google_results_language", - "google_tbm", - "google_tbs", - "parse", - "google_nfpr", - "google_safe_search", - "session_id", - "xhr", - "markdown", - "page_count", - "callback_url", - ], - }, - Target.GoogleTravelHotels.value: { - "group": "Google", - "response_format": "html", - "parameters": [ - "query", - "headless", - "locale", - "device_type", - "page_from", - "date_range", - "stars", - "adults", - "children", - "session_id", - "markdown", - "callback_url", - ], - }, - Target.GoogleTrendsExplore.value: { - "group": "Google", - "response_format": "json", - "parameters": ["query", "geo", "device_type", "search_type", "date_start", "date_end", "callback_url"], - }, - Target.GoogleShoppingSearch.value: { - "group": "Google", - "response_format": "json", - "parameters": [ - "query", - "headless", - "locale", - "geo", - "device_type", - "page_from", - "google_tbs", - "parse", - "session_id", - "google_results_language", - "markdown", - "callback_url", - ], - }, - Target.GoogleShoppingProduct.value: { - "group": "Google", - "response_format": "json", - "parameters": [ - "query", - "headless", - "locale", - "geo", - "device_type", - "page_from", - "parse", - "session_id", - "google_results_language", - "markdown", - "xhr", - "callback_url", - ], - }, - Target.Google.value: { - "group": "Google", - "response_format": "json", - "parameters": [ - "url", - "headless", - "locale", - "device_type", - "parse", - "session_id", - "markdown", - "xhr", - "page_count", - "callback_url", - ], - }, - Target.GoogleSuggest.value: { - "group": "Google", - "response_format": "json", - "parameters": ["query", "device_type", "geo", "locale", "session_id", "callback_url"], - }, - Target.GoogleMaps.value: { - "group": "Google", - "response_format": "html", - "parameters": [ - "query", - "headless", - "geo", - "locale", - "page_from", - "device_type", - "session_id", - "google_results_language", - "google_nfpr", - "hotel_occupancy", - "date_range", - "markdown", - "callback_url", - ], - }, - Target.GoogleAiMode.value: { - "group": "AI Tools", - "response_format": "json", - "parameters": ["query", "geo", "parse", "device_type", "session_id", "markdown", "xhr", "callback_url"], - }, - Target.GoogleAds.value: { - "group": "Google", - "response_format": "json", - "parameters": [ - "query", - "headless", - "locale", - "geo", - "device_type", - "page_from", - "google_results_language", - "google_tbm", - "google_tbs", - "parse", - "google_nfpr", - "session_id", - "markdown", - "xhr", - "page_count", - "callback_url", - ], - }, - Target.GoogleLens.value: { - "group": "Google", - "response_format": "json", - "parameters": ["query", "headless", "parse", "device_type", "markdown", "callback_url"], - }, - Target.BingSearch.value: { - "group": "Bing", - "response_format": "json", - "parameters": [ - "query", - "headless", - "locale", - "geo", - "domain", - "device_type", - "page_from", - "parse", - "page_count", - "session_id", - "markdown", - "xhr", - "callback_url", - ], - }, - Target.Bing.value: { - "group": "Bing", - "response_format": "json", - "parameters": [ - "url", - "headless", - "locale", - "geo", - "device_type", - "page_from", - "parse", - "session_id", - "markdown", - "xhr", - "callback_url", - ], - }, - Target.YoutubeTranscript.value: { - "group": "YouTube", - "response_format": "json", - "parameters": ["query", "language_code", "transcript_origin", "callback_url"], - }, - Target.AmazonProduct.value: { - "group": "Amazon", - "response_format": "json", - "parameters": [ - "query", - "headless", - "domain", - "device_type", - "parse", - "autoselect_variant", - "geo", - "session_id", - "currency", - "markdown", - "xhr", - "callback_url", - ], - }, - Target.AmazonPricing.value: { - "group": "Amazon", - "response_format": "json", - "parameters": [ - "query", - "headless", - "domain", - "device_type", - "page_from", - "parse", - "geo", - "session_id", - "currency", - "markdown", - "xhr", - "callback_url", - ], - }, - Target.AmazonSearch.value: { - "group": "Amazon", - "response_format": "json", - "parameters": [ - "query", - "headless", - "domain", - "device_type", - "page_from", - "category", - "merchant", - "parse", - "geo", - "session_id", - "sort_by", - "currency", - "markdown", - "xhr", - "callback_url", - ], - }, - Target.AmazonSellers.value: { - "group": "Amazon", - "response_format": "json", - "parameters": [ - "query", - "headless", - "locale", - "domain", - "device_type", - "geo", - "parse", - "markdown", - "xhr", - "callback_url", - ], - }, - Target.AmazonBestsellers.value: { - "group": "Amazon", - "response_format": "json", - "parameters": [ - "query", - "domain", - "device_type", - "geo", - "page_from", - "category", - "parse", - "session_id", - "currency", - "markdown", - "xhr", - "callback_url", - ], - }, - Target.Amazon.value: { - "group": "Amazon", - "response_format": "json", - "parameters": [ - "url", - "headless", - "device_type", - "parse", - "geo", - "session_id", - "markdown", - "xhr", - "callback_url", - ], - }, - Target.Ecommerce.value: { - "group": "Other eCommerce", - "response_format": "json", - "parameters": ["url", "headless", "locale", "geo", "device_type", "parse", "parser_type", "callback_url"], - }, - Target.WalmartProduct.value: { - "group": "Walmart", - "response_format": "html", - "parameters": [ - "product_id", - "headless", - "parse", - "xhr", - "markdown", - "fulfillment_type", - "walmart_store_id", - "delivery_zip", - "callback_url", - ], - }, - Target.WalmartSearch.value: { - "group": "Walmart", - "response_format": "json", - "parameters": [ - "query", - "headless", - "parse", - "markdown", - "fulfillment_type", - "walmart_store_id", - "delivery_zip", - "callback_url", - ], - }, - Target.Walmart.value: { - "group": "Walmart", - "response_format": "html", - "parameters": [ - "url", - "headless", - "locale", - "geo", - "device_type", - "store_id", - "markdown", - "xhr", - "callback_url", - ], - }, - Target.TargetProduct.value: { - "group": "Target", - "response_format": "json", - "parameters": [ - "product_id", - "headless", - "parse", - "device_type", - "markdown", - "xhr", - "delivery_type", - "target_store_id", - "delivery_zip", - "callback_url", - ], - }, - Target.TargetSearch.value: { - "group": "Target", - "response_format": "json", - "parameters": [ - "query", - "headless", - "parse", - "device_type", - "delivery_type", - "target_store_id", - "delivery_zip", - "xhr", - "markdown", - "callback_url", - ], - }, - Target.Target.value: { - "group": "Target", - "response_format": "html", - "parameters": ["url", "headless", "device_type", "xhr", "delivery_zip", "target_store_id", "callback_url"], - }, - Target.LowesSearch.value: { - "group": "Lowe's", - "response_format": "json", - "parameters": [ - "query", - "lowes_store_id", - "headless", - "delivery_zip", - "user_agent_type", - "free_delivery", - "pickup_today", - "delivery_today_tomorrow", - "callback_url", - ], - }, - Target.Universal.value: { - "group": "Universal", - "response_format": "html", - "parameters": [ - "url", - "payload", - "proxy_pool", - "http_method", - "headless", - "geo", - "locale", - "device_type", - "session_id", - "successful_status_codes", - "headers", - "cookies", - "force_headers", - "force_cookies", - "xhr", - "markdown", - "callback_url", - ], - }, - Target.Chatgpt.value: { - "group": "AI Tools", - "response_format": "json", - "parameters": ["prompt", "search", "parse", "geo", "device_type", "markdown", "xhr", "callback_url"], - }, - Target.Perplexity.value: { - "group": "AI Tools", - "response_format": "json", - "parameters": ["prompt", "parse", "geo", "device_type", "markdown", "xhr", "callback_url"], - }, - Target.Gemini.value: { - "group": "AI Tools", - "response_format": "json", - "parameters": ["prompt", "parse", "geo", "xhr", "callback_url"], - }, - Target.Bbb.value: { - "group": "Business Reviews", - "response_format": "html", - "parameters": ["url", "headless", "geo", "device_type", "markdown", "xhr", "callback_url"], - }, - Target.Autotrader.value: { - "group": "Marketplace", - "response_format": "html", - "parameters": ["url", "headless", "geo", "device_type", "markdown", "xhr", "callback_url"], - }, - Target.Mobile.value: { - "group": "Marketplace", - "response_format": "html", - "parameters": ["url", "headless", "geo", "device_type", "markdown", "xhr", "callback_url"], - }, - Target.Airbnb.value: { - "group": "Travel", - "response_format": "html", - "parameters": ["url", "headless", "geo", "device_type", "markdown", "xhr", "callback_url"], - }, - Target.AppleAppStore.value: { - "group": "Marketplace", - "response_format": "html", - "parameters": ["url", "headless", "geo", "device_type", "markdown", "xhr", "callback_url"], - }, - Target.InstagramGraphqlProfile.value: { - "group": "Instagram", - "response_format": "json", - "parameters": ["query", "callback_url"], - }, - Target.TiktokPost.value: { - "group": "TikTok", - "response_format": "html", - "parameters": ["url", "xhr", "callback_url"], - }, - Target.TiktokShopSearch.value: { - "group": "TikTok", - "response_format": "html", - "parameters": ["query", "headless", "device_type", "markdown", "country", "callback_url"], - }, - Target.TiktokShopProduct.value: { - "group": "TikTok", - "response_format": "html", - "parameters": ["product_id", "headless", "device_type", "xhr", "markdown", "country", "callback_url"], - }, - Target.Tiktok.value: { - "group": "TikTok", - "response_format": "html", - "parameters": ["url", "headless", "user_agent_type", "callback_url"], - }, - Target.RedditPost.value: { - "group": "Reddit", - "response_format": "json", - "parameters": ["url", "locale", "geo", "callback_url"], - }, - Target.RedditSubreddit.value: { - "group": "Reddit", - "response_format": "json", - "parameters": ["url", "locale", "geo", "callback_url"], - }, - Target.RedditUser.value: { - "group": "Reddit", - "response_format": "json", - "parameters": ["url", "locale", "geo", "sort", "callback_url"], - }, - Target.YoutubeVideo.value: { - "group": "None", - "response_format": "json", - "parameters": ["query", "geo", "callback_url"], - }, - Target.YoutubeMetadata.value: { - "group": "YouTube", - "response_format": "json", - "parameters": ["query", "callback_url"], - }, - Target.YoutubeSearch.value: { - "group": "YouTube", - "response_format": "json", - "parameters": [ - "360", - "query", - "upload_date", - "type", - "duration", - "video_sort_by", - "3d", - "4k", - "creative_commons", - "hd", - "hdr", - "vr180", - "live", - "location", - "purchased", - "subtitles", - "callback_url", - ], - }, - Target.YoutubeSearchMax.value: { - "group": "YouTube", - "response_format": "json", - "parameters": [ - "360", - "query", - "upload_date", - "type", - "duration", - "video_sort_by", - "3d", - "4k", - "creative_commons", - "hd", - "hdr", - "vr180", - "live", - "location", - "purchased", - "subtitles", - "markdown", - "callback_url", - ], - }, - Target.YoutubeSubtitles.value: { - "group": "YouTube", - "response_format": "json", - "parameters": ["query", "language_code", "subtitle_origin", "callback_url"], - }, - Target.YoutubeChannel.value: { - "group": "YouTube", - "response_format": "json", - "parameters": ["query", "parse", "limit", "markdown", "callback_url"], - }, -} - -ScrapeRequest = Annotated[ - Union[ - UniversalEcommerceParams - | GoogleSearchParams - | GoogleTravelHotelsParams - | GoogleTrendsExploreParams - | GoogleShoppingSearchParams - | GoogleShoppingProductParams - | GoogleParams - | GoogleSuggestParams - | GoogleMapsParams - | GoogleAiModeParams - | GoogleAdsParams - | GoogleLensParams - | BingSearchParams - | BingParams - | YoutubeTranscriptParams - | AmazonProductParams - | AmazonPricingParams - | AmazonSearchParams - | AmazonSellersParams - | AmazonBestsellersParams - | AmazonParams - | EcommerceParams - | WalmartProductParams - | WalmartSearchParams - | WalmartParams - | TargetProductParams - | TargetSearchParams - | TargetStoreParams - | LowesSearchParams - | UniversalParams - | ChatgptParams - | PerplexityParams - | GeminiParams - | BbbParams - | AutotraderParams - | MobileParams - | AirbnbParams - | AppleAppStoreParams - | InstagramGraphqlProfileParams - | TiktokPostParams - | TiktokShopSearchParams - | TiktokShopProductParams - | TiktokParams - | RedditPostParams - | RedditSubredditParams - | RedditUserParams - | YoutubeVideoParams - | YoutubeMetadataParams - | YoutubeSearchParams - | YoutubeSearchMaxParams - | YoutubeSubtitlesParams - | YoutubeChannelParams - ], - pydantic.Field(discriminator="target"), -] - -BatchRequest = Annotated[ - Union[ - UniversalEcommerceBatchParams - | GoogleSearchBatchParams - | GoogleTravelHotelsBatchParams - | GoogleTrendsExploreBatchParams - | GoogleShoppingSearchBatchParams - | GoogleShoppingProductBatchParams - | GoogleBatchParams - | GoogleSuggestBatchParams - | GoogleMapsBatchParams - | GoogleAiModeBatchParams - | GoogleAdsBatchParams - | GoogleLensBatchParams - | BingSearchBatchParams - | BingBatchParams - | YoutubeTranscriptBatchParams - | AmazonProductBatchParams - | AmazonPricingBatchParams - | AmazonSearchBatchParams - | AmazonSellersBatchParams - | AmazonBestsellersBatchParams - | AmazonBatchParams - | EcommerceBatchParams - | WalmartProductBatchParams - | WalmartSearchBatchParams - | WalmartBatchParams - | TargetProductBatchParams - | TargetSearchBatchParams - | TargetStoreBatchParams - | LowesSearchBatchParams - | UniversalBatchParams - | ChatgptBatchParams - | PerplexityBatchParams - | GeminiBatchParams - | BbbBatchParams - | AutotraderBatchParams - | MobileBatchParams - | AirbnbBatchParams - | AppleAppStoreBatchParams - | InstagramGraphqlProfileBatchParams - | TiktokPostBatchParams - | TiktokShopSearchBatchParams - | TiktokShopProductBatchParams - | TiktokBatchParams - | RedditPostBatchParams - | RedditSubredditBatchParams - | RedditUserBatchParams - | YoutubeVideoBatchParams - | YoutubeMetadataBatchParams - | YoutubeSearchBatchParams - | YoutubeSearchMaxBatchParams - | YoutubeSubtitlesBatchParams - | YoutubeChannelBatchParams - ], - pydantic.Field(discriminator="target"), -] diff --git a/src/decodo/schema/bundled_schema.py b/src/decodo/schema/bundled_schema.py index b139239..08e3343 100644 --- a/src/decodo/schema/bundled_schema.py +++ b/src/decodo/schema/bundled_schema.py @@ -1,27 +1,61 @@ from __future__ import annotations +import json +from pathlib import Path from typing import Any, ClassVar, cast -from decodo.generated.request_schemas import request_json_schemas -from decodo.generated.targets import target_meta, targets +from decodo.targets import targets +from .build_target_meta import build_target_meta from .types import DecodoSchema, TargetMeta +def _load_target_meta() -> dict[str, Any] | None: + try: + from decodo.generated.targets import target_meta # noqa: PLC0415 + return cast(dict[str, Any], target_meta) + except ImportError: + return None + +_target_meta = _load_target_meta() + +_IR_JSON_PATH = Path(__file__).parent.parent / "generated" / "decodo.ir.json" + + +def _load_ir_json() -> dict[str, Any] | None: + try: + with open(_IR_JSON_PATH, encoding="utf-8") as f: + return json.load(f) # type: ignore[no-any-return] + except FileNotFoundError: + return None + + class BundledSchema: shared: ClassVar[BundledSchema] + def __init__(self) -> None: + ir = _load_ir_json() + if ir is not None: + api_targets = ir["apis"]["webScrapingApi"]["targets"] + self._request_schemas: dict[str, dict[str, Any]] = { + k: v["parameter_schema"] for k, v in api_targets.items() + } + self._target_meta = _target_meta if _target_meta is not None else build_target_meta(api_targets) + else: + self._request_schemas = {} + self._target_meta = _target_meta or {} + def get_request_schema(self, target: str) -> dict[str, Any] | None: - return request_json_schemas.get(target) + return self._request_schemas.get(target) def list_targets(self) -> list[str]: return list(targets) def get_target_meta(self, target: str) -> TargetMeta | None: - return cast(TargetMeta, target_meta.get(target)) + return cast(TargetMeta, self._target_meta.get(target)) def get_target_parameter_schema(self, target: str) -> dict[str, Any] | None: - return request_json_schemas.get(target) + return self._request_schemas.get(target) def get_shared_parameters(self) -> dict[str, Any]: return {} diff --git a/src/decodo/targets.py b/src/decodo/targets.py new file mode 100644 index 0000000..75412e4 --- /dev/null +++ b/src/decodo/targets.py @@ -0,0 +1,63 @@ +# Minimal target enum — committed to source. +# Run `python -m decodo.codegen.codegen` to regenerate when the IR schema changes. +from __future__ import annotations + +from enum import StrEnum + + +class Target(StrEnum): + UniversalEcommerce = "universal_ecommerce" + GoogleSearch = "google_search" + GoogleTravelHotels = "google_travel_hotels" + GoogleTrendsExplore = "google_trends_explore" + GoogleShoppingSearch = "google_shopping_search" + GoogleShoppingProduct = "google_shopping_product" + Google = "google" + GoogleSuggest = "google_suggest" + GoogleMaps = "google_maps" + GoogleAiMode = "google_ai_mode" + GoogleAds = "google_ads" + GoogleLens = "google_lens" + BingSearch = "bing_search" + Bing = "bing" + YoutubeTranscript = "youtube_transcript" + AmazonProduct = "amazon_product" + AmazonPricing = "amazon_pricing" + AmazonSearch = "amazon_search" + AmazonSellers = "amazon_sellers" + AmazonBestsellers = "amazon_bestsellers" + Amazon = "amazon" + Ecommerce = "ecommerce" + WalmartProduct = "walmart_product" + WalmartSearch = "walmart_search" + Walmart = "walmart" + TargetProduct = "target_product" + TargetSearch = "target_search" + Target = "target" + LowesSearch = "lowes_search" + Universal = "universal" + Chatgpt = "chatgpt" + Perplexity = "perplexity" + Gemini = "gemini" + Bbb = "bbb" + Autotrader = "autotrader" + Mobile = "mobile" + Airbnb = "airbnb" + AppleAppStore = "apple_app_store" + InstagramGraphqlProfile = "instagram_graphql_profile" + TiktokPost = "tiktok_post" + TiktokShopSearch = "tiktok_shop_search" + TiktokShopProduct = "tiktok_shop_product" + Tiktok = "tiktok" + RedditPost = "reddit_post" + RedditSubreddit = "reddit_subreddit" + RedditUser = "reddit_user" + YoutubeVideo = "youtube_video" + YoutubeMetadata = "youtube_metadata" + YoutubeSearch = "youtube_search" + YoutubeSearchMax = "youtube_search_max" + YoutubeSubtitles = "youtube_subtitles" + YoutubeChannel = "youtube_channel" + + +targets: list[str] = [t.value for t in Target] diff --git a/src/decodo/types/__init__.py b/src/decodo/types/__init__.py index 3d568bb..4991a2c 100644 --- a/src/decodo/types/__init__.py +++ b/src/decodo/types/__init__.py @@ -1,4 +1,7 @@ -from .requests import BatchRequest, ScrapeRequest +try: + from .requests import BatchRequest, ScrapeRequest +except ImportError: + pass from .responses import ( AsyncTaskResponse, BatchResponse, diff --git a/src/decodo/types/requests.py b/src/decodo/types/requests.py index f5fcccd..2ab1cc6 100644 --- a/src/decodo/types/requests.py +++ b/src/decodo/types/requests.py @@ -1,5 +1,7 @@ from __future__ import annotations -from decodo.generated.targets import BatchRequest, ScrapeRequest - -__all__ = ["ScrapeRequest", "BatchRequest"] +try: + from decodo.generated.targets import BatchRequest, ScrapeRequest + __all__ = ["ScrapeRequest", "BatchRequest"] +except ImportError: + __all__ = [] diff --git a/tests/conftest.py b/tests/conftest.py index d704d57..45319bf 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,9 +6,17 @@ import pytest +import decodo.schema.bundled_schema as _bundled_schema_mod + MINIMAL_IR_PATH = Path(__file__).parent / "schema" / "fixtures" / "minimal_ir.json" +@pytest.fixture(autouse=True, scope="session") +def _patch_bundled_schema_ir_path() -> None: + _bundled_schema_mod._IR_JSON_PATH = MINIMAL_IR_PATH + _bundled_schema_mod.BundledSchema.shared = _bundled_schema_mod.BundledSchema() + + @pytest.fixture def minimal_ir() -> dict[str, Any]: with open(MINIMAL_IR_PATH, encoding="utf-8") as f: From ceee0a4bdc4bc8be0693359bea62dc64babb4467 Mon Sep 17 00:00:00 2001 From: julka Date: Thu, 30 Jul 2026 13:56:57 +0300 Subject: [PATCH 08/17] Add message if imports missing, run codegen --- src/decodo/__init__.py | 124 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 124 insertions(+) diff --git a/src/decodo/__init__.py b/src/decodo/__init__.py index ed94623..41db65b 100644 --- a/src/decodo/__init__.py +++ b/src/decodo/__init__.py @@ -135,6 +135,130 @@ TaskStatus, ) +_CODEGEN_NAMES: frozenset[str] = frozenset( + [ + "AirbnbBatchParams", + "AirbnbParams", + "AmazonBatchParams", + "AmazonBestsellersBatchParams", + "AmazonBestsellersParams", + "AmazonParams", + "AmazonPricingBatchParams", + "AmazonPricingParams", + "AmazonProductBatchParams", + "AmazonProductParams", + "AmazonSearchBatchParams", + "AmazonSearchParams", + "AmazonSellersBatchParams", + "AmazonSellersParams", + "AppleAppStoreBatchParams", + "AppleAppStoreParams", + "AutotraderBatchParams", + "AutotraderParams", + "BatchRequest", + "BbbBatchParams", + "BbbParams", + "BingBatchParams", + "BingParams", + "BingSearchBatchParams", + "BingSearchParams", + "ChatgptBatchParams", + "ChatgptParams", + "EcommerceBatchParams", + "EcommerceParams", + "GeminiBatchParams", + "GeminiParams", + "GoogleAdsBatchParams", + "GoogleAdsParams", + "GoogleAiModeBatchParams", + "GoogleAiModeParams", + "GoogleBatchParams", + "GoogleLensBatchParams", + "GoogleLensParams", + "GoogleMapsBatchParams", + "GoogleMapsParams", + "GoogleParams", + "GoogleSearchBatchParams", + "GoogleSearchParams", + "GoogleShoppingProductBatchParams", + "GoogleShoppingProductParams", + "GoogleShoppingSearchBatchParams", + "GoogleShoppingSearchParams", + "GoogleSuggestBatchParams", + "GoogleSuggestParams", + "GoogleTravelHotelsBatchParams", + "GoogleTravelHotelsParams", + "GoogleTrendsExploreBatchParams", + "GoogleTrendsExploreParams", + "InstagramGraphqlProfileBatchParams", + "InstagramGraphqlProfileParams", + "LowesSearchBatchParams", + "LowesSearchParams", + "MobileBatchParams", + "MobileParams", + "ParameterMeta", + "PerplexityBatchParams", + "PerplexityParams", + "RedditPostBatchParams", + "RedditPostParams", + "RedditSubredditBatchParams", + "RedditSubredditParams", + "RedditUserBatchParams", + "RedditUserParams", + "ScrapeRequest", + "TargetProductBatchParams", + "TargetProductParams", + "TargetSearchBatchParams", + "TargetSearchParams", + "TargetStoreBatchParams", + "TargetStoreParams", + "TiktokBatchParams", + "TiktokParams", + "TiktokPostBatchParams", + "TiktokPostParams", + "TiktokShopProductBatchParams", + "TiktokShopProductParams", + "TiktokShopSearchBatchParams", + "TiktokShopSearchParams", + "UniversalBatchParams", + "UniversalEcommerceBatchParams", + "UniversalEcommerceParams", + "UniversalParams", + "WalmartBatchParams", + "WalmartParams", + "WalmartProductBatchParams", + "WalmartProductParams", + "WalmartSearchBatchParams", + "WalmartSearchParams", + "YoutubeChannelBatchParams", + "YoutubeChannelParams", + "YoutubeMetadataBatchParams", + "YoutubeMetadataParams", + "YoutubeSearchBatchParams", + "YoutubeSearchMaxBatchParams", + "YoutubeSearchMaxParams", + "YoutubeSearchParams", + "YoutubeSubtitlesBatchParams", + "YoutubeSubtitlesParams", + "YoutubeTranscriptBatchParams", + "YoutubeTranscriptParams", + "YoutubeVideoBatchParams", + "YoutubeVideoParams", + "parameter_meta", + "target_meta", + ] +) + + +def __getattr__(name: str) -> object: + if name in _CODEGEN_NAMES: + raise ImportError( + f"'{name}' requires generated types. " + "Run: python -m decodo.codegen.codegen" + ) + raise AttributeError(f"module 'decodo' has no attribute {name!r}") + + __all__ = [ "DecodoClient", "DecodoConfig", From de7195c3df03343459ff3be6e11caed1ea9cd8f1 Mon Sep 17 00:00:00 2001 From: julka Date: Thu, 30 Jul 2026 13:57:05 +0300 Subject: [PATCH 09/17] Add codegen in ci/cd --- .github/workflows/test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 14cfdf3..e7e55c7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,5 +19,8 @@ jobs: - name: Install dependencies run: pip install -e ".[dev]" + - name: Generate types + run: python -m decodo.codegen.codegen + - name: Pytest run: pytest From 8437192365d9573b36c824024088a01d4b164ea3 Mon Sep 17 00:00:00 2001 From: julka Date: Thu, 30 Jul 2026 14:35:23 +0300 Subject: [PATCH 10/17] Update readme --- README.md | 108 ++++++++++++++-------------- examples/web_scraping_api/README.md | 25 +++++++ 2 files changed, 79 insertions(+), 54 deletions(-) create mode 100644 examples/web_scraping_api/README.md diff --git a/README.md b/README.md index 0d1f117..8e8fcf6 100644 --- a/README.md +++ b/README.md @@ -42,19 +42,6 @@ Instead of manually constructing HTTP requests and validating payloads, you can pip install decodo-sdk ``` - -## Generate types - -After installing, run the type generator to create typed target parameters: - -```bash -python -m decodo.codegen.codegen -``` - -This fetches the latest API schema from the Decodo registry and writes typed classes to the `generated/` directory inside the package. The generated files are not included in the repository — you control when to update them. - -Re-run this command whenever Decodo publishes an updated schema to pick up new targets or changed parameters. - ## Quick start Create a new project: @@ -72,6 +59,40 @@ Get your Web Scraping API token from the [Decodo dashboard](https://dashboard.de ```python # main.py +from decodo import DecodoClient, DecodoConfig, WebScrapingApiConfig + +client = DecodoClient( + DecodoConfig( + web_scraping_api=WebScrapingApiConfig(token=""), + ) +) + +result = client.web_scraping_api.scrape({ + "target": "google_search", + "query": "coffee shops", + "geo": "United States", + "parse": True, +}) +print(result) +``` + +Run the script: + +``` +python main.py +``` + +### With typed parameters (optional) + +For IDE autocomplete and parameter validation, run the type generator after installing: + +```bash +python -m decodo.codegen.codegen +``` + +Then use typed parameter classes instead of dicts: + +```python from decodo import DecodoClient, DecodoConfig, GoogleSearchParams, Target, WebScrapingApiConfig client = DecodoClient( @@ -91,12 +112,6 @@ result = client.web_scraping_api.scrape( print(result) ``` -Run the script: - -``` -python main.py -``` -
Example response @@ -205,15 +220,11 @@ The snippets below assume you have already constructed a client. See [Configurat Waits for the scraping result before returning: ```python -from decodo import AmazonProductParams, Target - -result = client.web_scraping_api.scrape( - AmazonProductParams( - target=Target.AmazonProduct, - query="B09H74FXNW", - parse=True, - ) -) +result = client.web_scraping_api.scrape({ + "target": "amazon_product", + "query": "B09H74FXNW", + "parse": True, +}) ``` ### Async scrape @@ -221,15 +232,11 @@ result = client.web_scraping_api.scrape( Creates a scraping task and returns immediately. Poll separately for task status and results: ```python -from decodo import GoogleSearchParams, Target - -task = client.web_scraping_api.scrape_async( - GoogleSearchParams( - target=Target.GoogleSearch, - query="laptop reviews", - parse=True, - ) -) +task = client.web_scraping_api.scrape_async({ + "target": "google_search", + "query": "laptop reviews", + "parse": True, +}) meta = client.web_scraping_api.get_status(task["id"]) print(meta["status"]) # 'pending' | 'done' | 'faulted' @@ -242,15 +249,11 @@ results = client.web_scraping_api.get_results(task["id"]) Send multiple queries or URLs in a single request: ```python -from decodo import GoogleSearchBatchParams, Target - -batch = client.web_scraping_api.scrape_batch( - GoogleSearchBatchParams( - target=Target.GoogleSearch, - query=["coffee", "tea", "juice"], - parse=True, - ) -) +batch = client.web_scraping_api.scrape_batch({ + "target": "google_search", + "query": ["coffee", "tea", "juice"], + "parse": True, +}) coffee_task_id = batch["queries"][0]["id"] @@ -323,21 +326,18 @@ The SDK raises typed errors that map to API error codes: ```python from decodo import ( - DecodoClient, - DecodoError, AuthenticationError, RateLimitError, ValidationError, TimeoutError, - Target, ) -from decodo import GoogleSearchParams - try: - client.web_scraping_api.scrape( - GoogleSearchParams(target=Target.GoogleSearch, query="test", parse=True) - ) + client.web_scraping_api.scrape({ + "target": "google_search", + "query": "test", + "parse": True, + }) except AuthenticationError: pass # 401/403 - bad credentials except RateLimitError: diff --git a/examples/web_scraping_api/README.md b/examples/web_scraping_api/README.md new file mode 100644 index 0000000..c2ce1c5 --- /dev/null +++ b/examples/web_scraping_api/README.md @@ -0,0 +1,25 @@ +# Examples + +These examples use typed parameter classes (`GoogleSearchParams`, `AmazonProductParams`, etc.) which require running the type generator first: + +```bash +python -m decodo.codegen.codegen +``` + +To run any example without the generator, replace the typed params with a plain dict: + +```python +# instead of: +result = client.web_scraping_api.scrape( + GoogleSearchParams(target=Target.GoogleSearch, query="coffee shops", parse=True) +) + +# use: +result = client.web_scraping_api.scrape({ + "target": "google_search", + "query": "coffee shops", + "parse": True, +}) +``` + +See the [root README](../../README.md) for the full list of target strings. From e4b4da4a9a88adfdf83a17baf10e2aa97ed13248 Mon Sep 17 00:00:00 2001 From: julka Date: Thu, 30 Jul 2026 14:42:24 +0300 Subject: [PATCH 11/17] Add codegen --- README.md | 4 +- src/decodo/codegen/codegen.py | 50 +++++++++++++++++-- .../web_scraping_api/generate_parameters.py | 7 +-- .../web_scraping_api/generate_targets.py | 7 +-- src/decodo/schema/bundled_schema.py | 7 +++ 5 files changed, 63 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 8e8fcf6..56204ac 100644 --- a/README.md +++ b/README.md @@ -82,9 +82,9 @@ Run the script: python main.py ``` -### With typed parameters (optional) +### With typed parameters (recommended) -For IDE autocomplete and parameter validation, run the type generator after installing: +For IDE autocomplete and parameter validation, run the type generator once after installing. Dict-based usage works without this step, but typed classes require it: ```bash python -m decodo.codegen.codegen diff --git a/src/decodo/codegen/codegen.py b/src/decodo/codegen/codegen.py index ed06c4a..0f9e7c9 100644 --- a/src/decodo/codegen/codegen.py +++ b/src/decodo/codegen/codegen.py @@ -1,22 +1,64 @@ from __future__ import annotations +import argparse import os import shutil +import sys +from pathlib import Path from .web_scraping_api.generate_parameters import generate_parameters_file from .web_scraping_api.generate_targets import generate_targets_enum_file, generate_targets_file -from .web_scraping_api.shared import local_ir_path, out_dir +from .web_scraping_api.shared import local_ir_path +from .web_scraping_api.shared import out_dir as _default_out_dir + + +def _in_site_packages() -> bool: + return "site-packages" in str(Path(__file__).resolve()) def main() -> None: + parser = argparse.ArgumentParser( + prog="python -m decodo.codegen.codegen", + description="Generate typed parameter classes from the Decodo IR schema.", + ) + parser.add_argument( + "--out-dir", + metavar="PATH", + default=None, + help=( + "Directory to write generated files into. " + "Defaults to the package's built-in generated/ directory " + "(editable installs only). Required for non-editable installs." + ), + ) + args = parser.parse_args() + + if args.out_dir is not None: + out_dir = str(Path(args.out_dir).resolve()) + elif _in_site_packages(): + print( + "error: running from a non-editable install.\n" + "Generated files would be written into site-packages and silently reverted\n" + "on the next `pip install --upgrade`.\n" + "Use --out-dir to specify a project-local output directory.", + file=sys.stderr, + ) + sys.exit(1) + else: + out_dir = _default_out_dir + os.makedirs(out_dir, exist_ok=True) init_path = os.path.join(out_dir, "__init__.py") if not os.path.exists(init_path): open(init_path, "w").close() - generate_parameters_file() - generate_targets_enum_file() - generate_targets_file() + generate_parameters_file(out_dir) + # The Target enum lives in decodo/targets.py (committed source). Only regenerate + # it when we have write access to the source tree (editable install). Non-editable + # installs already ship the enum baked into the package. + if not _in_site_packages(): + generate_targets_enum_file() + generate_targets_file(out_dir) # Copy the downloaded IR JSON into generated/ so BundledSchema can load # schemas directly at runtime without a generated Python file. diff --git a/src/decodo/codegen/web_scraping_api/generate_parameters.py b/src/decodo/codegen/web_scraping_api/generate_parameters.py index 9307e07..3555c94 100644 --- a/src/decodo/codegen/web_scraping_api/generate_parameters.py +++ b/src/decodo/codegen/web_scraping_api/generate_parameters.py @@ -108,13 +108,14 @@ def get_parameters_file(api: WebScrapingApiIR) -> str: return "\n".join(lines) -def generate_parameters_file() -> None: +def generate_parameters_file(dest_dir: str | None = None) -> None: ir = fetch_intermediate_representation() api = ir["apis"]["webScrapingApi"] file_contents = get_parameters_file(api) - os.makedirs(out_dir, exist_ok=True) - out_path = os.path.join(out_dir, "parameters.py") + target = dest_dir if dest_dir is not None else out_dir + os.makedirs(target, exist_ok=True) + out_path = os.path.join(target, "parameters.py") with open(out_path, "w", encoding="utf-8") as f: f.write(file_contents) diff --git a/src/decodo/codegen/web_scraping_api/generate_targets.py b/src/decodo/codegen/web_scraping_api/generate_targets.py index 07d0da3..8b93c47 100644 --- a/src/decodo/codegen/web_scraping_api/generate_targets.py +++ b/src/decodo/codegen/web_scraping_api/generate_targets.py @@ -207,13 +207,14 @@ def generate_targets_enum_file() -> None: print(f" {targets_enum_path} ({target_count} targets)") -def generate_targets_file() -> None: +def generate_targets_file(dest_dir: str | None = None) -> None: ir = fetch_intermediate_representation() api = ir["apis"]["webScrapingApi"] file_contents = _get_targets_file_contents(api) - os.makedirs(out_dir, exist_ok=True) - out_path = os.path.join(out_dir, "targets.py") + target = dest_dir if dest_dir is not None else out_dir + os.makedirs(target, exist_ok=True) + out_path = os.path.join(target, "targets.py") with open(out_path, "w", encoding="utf-8") as f: f.write(file_contents) diff --git a/src/decodo/schema/bundled_schema.py b/src/decodo/schema/bundled_schema.py index 08e3343..441bcae 100644 --- a/src/decodo/schema/bundled_schema.py +++ b/src/decodo/schema/bundled_schema.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import warnings from pathlib import Path from typing import Any, ClassVar, cast @@ -42,6 +43,12 @@ def __init__(self) -> None: } self._target_meta = _target_meta if _target_meta is not None else build_target_meta(api_targets) else: + warnings.warn( + "Decodo IR schema not found — payload validation is disabled. " + "Run: python -m decodo.codegen.codegen", + RuntimeWarning, + stacklevel=2, + ) self._request_schemas = {} self._target_meta = _target_meta or {} From 3f9aad18e5c8315359411079516b3c541695de32 Mon Sep 17 00:00:00 2001 From: julka Date: Fri, 31 Jul 2026 09:56:46 +0300 Subject: [PATCH 12/17] Update readme about outdir --- README.md | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 56204ac..524914f 100644 --- a/README.md +++ b/README.md @@ -84,16 +84,19 @@ python main.py ### With typed parameters (recommended) -For IDE autocomplete and parameter validation, run the type generator once after installing. Dict-based usage works without this step, but typed classes require it: +For IDE autocomplete and parameter validation, run the type generator once after installing. Dict-based usage works without this step, but typed classes require it. + +**Installed via pip (standard install):** Generated files cannot be written into site-packages, so specify a local output directory: ```bash -python -m decodo.codegen.codegen +python -m decodo.codegen.codegen --out-dir ./decodo_generated ``` -Then use typed parameter classes instead of dicts: +Then import directly from that directory: ```python -from decodo import DecodoClient, DecodoConfig, GoogleSearchParams, Target, WebScrapingApiConfig +from decodo_generated.targets import GoogleSearchParams +from decodo import DecodoClient, DecodoConfig, Target, WebScrapingApiConfig client = DecodoClient( DecodoConfig( @@ -112,6 +115,14 @@ result = client.web_scraping_api.scrape( print(result) ``` +> **Note:** The directory name you pass to `--out-dir` becomes the import namespace. Using `./decodo_generated` means `from decodo_generated.targets import ...`. You can choose any name, but keep it consistent across your project. + +**Editable / source install (`pip install -e .`):** You can omit `--out-dir` and generated files will be written into the package directly, making `from decodo import GoogleSearchParams` work: + +```bash +python -m decodo.codegen.codegen +``` +
Example response From 0a188f75c65dd9890bfb17dc02831ab0be4e5d02 Mon Sep 17 00:00:00 2001 From: julka Date: Fri, 31 Jul 2026 09:58:40 +0300 Subject: [PATCH 13/17] Update codegen dynamic imports --- src/decodo/__init__.py | 142 ++++++----------------------------------- 1 file changed, 19 insertions(+), 123 deletions(-) diff --git a/src/decodo/__init__.py b/src/decodo/__init__.py index 41db65b..f1098cd 100644 --- a/src/decodo/__init__.py +++ b/src/decodo/__init__.py @@ -7,11 +7,24 @@ TimeoutError, ValidationError, ) +from .schema.bundled_schema import BundledSchema +from .schema.remote_schema import RemoteSchema +from .schema.types import DecodoSchema, RemoteSchemaLoadOptions from .targets import Target, targets +from .types.responses import ( + AsyncTaskResponse, + BatchResponse, + ResultEntry, + SyncResponse, + TaskMetadata, + TaskResultsResponse, + TaskStatus, +) +_codegen_available = False try: - from .generated.parameters import ParameterMeta, parameter_meta - from .generated.targets import ( + from .generated.parameters import ParameterMeta, parameter_meta # noqa: F401 + from .generated.targets import ( # noqa: F401 AirbnbBatchParams, AirbnbParams, AmazonBatchParams, @@ -120,20 +133,9 @@ YoutubeVideoParams, target_meta, ) + _codegen_available = True except ImportError: pass -from .schema.bundled_schema import BundledSchema -from .schema.remote_schema import RemoteSchema -from .schema.types import DecodoSchema, RemoteSchemaLoadOptions -from .types.responses import ( - AsyncTaskResponse, - BatchResponse, - ResultEntry, - SyncResponse, - TaskMetadata, - TaskResultsResponse, - TaskStatus, -) _CODEGEN_NAMES: frozenset[str] = frozenset( [ @@ -269,116 +271,7 @@ def __getattr__(name: str) -> object: "DecodoSchema", "RemoteSchemaLoadOptions", "Target", - "target_meta", "targets", - "ScrapeRequest", - "BatchRequest", - "UniversalEcommerceParams", - "GoogleSearchParams", - "GoogleTravelHotelsParams", - "GoogleTrendsExploreParams", - "GoogleShoppingSearchParams", - "GoogleShoppingProductParams", - "GoogleParams", - "GoogleSuggestParams", - "GoogleMapsParams", - "GoogleAiModeParams", - "GoogleAdsParams", - "GoogleLensParams", - "BingSearchParams", - "BingParams", - "YoutubeTranscriptParams", - "AmazonProductParams", - "AmazonPricingParams", - "AmazonSearchParams", - "AmazonSellersParams", - "AmazonBestsellersParams", - "AmazonParams", - "EcommerceParams", - "WalmartProductParams", - "WalmartSearchParams", - "WalmartParams", - "TargetProductParams", - "TargetSearchParams", - "TargetStoreParams", - "LowesSearchParams", - "UniversalParams", - "ChatgptParams", - "PerplexityParams", - "GeminiParams", - "BbbParams", - "AutotraderParams", - "MobileParams", - "AirbnbParams", - "AppleAppStoreParams", - "InstagramGraphqlProfileParams", - "TiktokPostParams", - "TiktokShopSearchParams", - "TiktokShopProductParams", - "TiktokParams", - "RedditPostParams", - "RedditSubredditParams", - "RedditUserParams", - "YoutubeVideoParams", - "YoutubeMetadataParams", - "YoutubeSearchParams", - "YoutubeSearchMaxParams", - "YoutubeSubtitlesParams", - "YoutubeChannelParams", - "UniversalEcommerceBatchParams", - "GoogleSearchBatchParams", - "GoogleTravelHotelsBatchParams", - "GoogleTrendsExploreBatchParams", - "GoogleShoppingSearchBatchParams", - "GoogleShoppingProductBatchParams", - "GoogleBatchParams", - "GoogleSuggestBatchParams", - "GoogleMapsBatchParams", - "GoogleAiModeBatchParams", - "GoogleAdsBatchParams", - "GoogleLensBatchParams", - "BingSearchBatchParams", - "BingBatchParams", - "YoutubeTranscriptBatchParams", - "AmazonProductBatchParams", - "AmazonPricingBatchParams", - "AmazonSearchBatchParams", - "AmazonSellersBatchParams", - "AmazonBestsellersBatchParams", - "AmazonBatchParams", - "EcommerceBatchParams", - "WalmartProductBatchParams", - "WalmartSearchBatchParams", - "WalmartBatchParams", - "TargetProductBatchParams", - "TargetSearchBatchParams", - "TargetStoreBatchParams", - "LowesSearchBatchParams", - "UniversalBatchParams", - "ChatgptBatchParams", - "PerplexityBatchParams", - "GeminiBatchParams", - "BbbBatchParams", - "AutotraderBatchParams", - "MobileBatchParams", - "AirbnbBatchParams", - "AppleAppStoreBatchParams", - "InstagramGraphqlProfileBatchParams", - "TiktokPostBatchParams", - "TiktokShopSearchBatchParams", - "TiktokShopProductBatchParams", - "TiktokBatchParams", - "RedditPostBatchParams", - "RedditSubredditBatchParams", - "RedditUserBatchParams", - "YoutubeVideoBatchParams", - "YoutubeMetadataBatchParams", - "YoutubeSearchBatchParams", - "YoutubeSearchMaxBatchParams", - "YoutubeSubtitlesBatchParams", - "YoutubeChannelBatchParams", - "ParameterMeta", - "parameter_meta", "SyncResponse", "AsyncTaskResponse", "BatchResponse", @@ -392,3 +285,6 @@ def __getattr__(name: str) -> object: "ValidationError", "TimeoutError", ] + +if _codegen_available: + __all__ += list(_CODEGEN_NAMES) From cd989a35b8ca901d365b14c86c5511f3b229e228 Mon Sep 17 00:00:00 2001 From: julka Date: Fri, 31 Jul 2026 10:28:56 +0300 Subject: [PATCH 14/17] Update target validation --- README.md | 32 ++++++++++++++++++++++++++++++ inputs/README.md | 2 +- src/decodo/api/web_scraping_api.py | 2 ++ 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 524914f..92b49d2 100644 --- a/README.md +++ b/README.md @@ -282,9 +282,14 @@ Each target accepts one primary input parameter (`url`, `query`, `product_id`, o | `Target.GoogleSearch` | Google Search results for a query | `{"target": Target.GoogleSearch, "query": "coffee shops"}` | | `Target.GoogleMaps` | Google Maps search results | `{"target": Target.GoogleMaps, "query": "coffee shops brooklyn"}` | | `Target.GoogleShoppingSearch` | Google Shopping search results | `{"target": Target.GoogleShoppingSearch, "query": "laptop"}` | +| `Target.GoogleShoppingProduct` | Google Shopping product page | `{"target": Target.GoogleShoppingProduct, "query": "B09H74FXNW"}` | | `Target.GoogleSuggest` | Google Autocomplete suggestions | `{"target": Target.GoogleSuggest, "query": "coffee"}` | | `Target.GoogleLens` | Google Lens reverse image search | `{"target": Target.GoogleLens, "query": "https://example.com/cat.jpg"}` | +| `Target.GoogleTravelHotels` | Google Travel hotel listings | `{"target": Target.GoogleTravelHotels, "query": "hotels in paris"}` | +| `Target.GoogleTrendsExplore` | Google Trends explore data | `{"target": Target.GoogleTrendsExplore, "query": "coffee"}` | +| `Target.GoogleAds` | Google Ads results for a query | `{"target": Target.GoogleAds, "query": "laptop"}` | | `Target.BingSearch` | Bing Search results | `{"target": Target.BingSearch, "query": "electric vehicles"}` | +| `Target.Bing` | Raw Bing URL scraping | `{"target": Target.Bing, "url": "https://bing.com/search?q=laptop"}` | ### eCommerce @@ -293,8 +298,15 @@ Each target accepts one primary input parameter (`url`, `query`, `product_id`, o | `Target.AmazonProduct` | Amazon product detail page by ASIN | `{"target": Target.AmazonProduct, "query": "B09H74FXNW"}` | | `Target.AmazonSearch` | Amazon search results | `{"target": Target.AmazonSearch, "query": "laptop"}` | | `Target.AmazonPricing` | Amazon pricing and offers | `{"target": Target.AmazonPricing, "query": "B09H74FXNW"}` | +| `Target.AmazonSellers` | Amazon seller listings | `{"target": Target.AmazonSellers, "query": "B09H74FXNW"}` | +| `Target.AmazonBestsellers` | Amazon bestsellers by category | `{"target": Target.AmazonBestsellers, "query": "electronics"}` | | `Target.WalmartProduct` | Walmart product page by product ID | `{"target": Target.WalmartProduct, "product_id": "15296401808"}` | +| `Target.WalmartSearch` | Walmart search results | `{"target": Target.WalmartSearch, "query": "laptop"}` | +| `Target.Walmart` | Raw Walmart URL scraping | `{"target": Target.Walmart, "url": "https://walmart.com/ip/15296401808"}` | | `Target.TargetProduct` | Target.com product page by product ID | `{"target": Target.TargetProduct, "product_id": "92186007"}` | +| `Target.TargetSearch` | Target.com search results | `{"target": Target.TargetSearch, "query": "laptop"}` | +| `Target.Target` | Raw Target.com URL scraping | `{"target": Target.Target, "url": "https://target.com/p/-/A-92186007"}` | +| `Target.LowesSearch` | Lowe's search results | `{"target": Target.LowesSearch, "query": "drill"}` | | `Target.Ecommerce` | Generic eCommerce page with parser | `{"target": Target.Ecommerce, "url": "https://example.com/product/123"}` | ### Social media @@ -303,9 +315,19 @@ Each target accepts one primary input parameter (`url`, `query`, `product_id`, o | --- | --- | --- | | `Target.RedditPost` | Reddit post by URL | `{"target": Target.RedditPost, "url": "https://reddit.com/r/nba/..."}` | | `Target.RedditSubreddit` | Reddit subreddit by URL | `{"target": Target.RedditSubreddit, "url": "https://reddit.com/r/nba/"}` | +| `Target.RedditUser` | Reddit user profile by URL | `{"target": Target.RedditUser, "url": "https://reddit.com/user/example/"}` | | `Target.YoutubeVideo` | YouTube video by ID | `{"target": Target.YoutubeVideo, "query": "dFu9aKJoqGg"}` | | `Target.YoutubeSearch` | YouTube search results | `{"target": Target.YoutubeSearch, "query": "ambient music"}` | +| `Target.YoutubeSearchMax` | YouTube search results (extended) | `{"target": Target.YoutubeSearchMax, "query": "ambient music"}` | +| `Target.YoutubeMetadata` | YouTube video metadata by ID | `{"target": Target.YoutubeMetadata, "query": "dFu9aKJoqGg"}` | +| `Target.YoutubeTranscript` | YouTube video transcript by ID | `{"target": Target.YoutubeTranscript, "query": "dFu9aKJoqGg"}` | +| `Target.YoutubeSubtitles` | YouTube video subtitles by ID | `{"target": Target.YoutubeSubtitles, "query": "dFu9aKJoqGg"}` | +| `Target.YoutubeChannel` | YouTube channel by URL | `{"target": Target.YoutubeChannel, "url": "https://youtube.com/@mkbhd"}` | | `Target.TiktokPost` | TikTok post by URL | `{"target": Target.TiktokPost, "url": "https://www.tiktok.com/@nba/video/..."}` | +| `Target.TiktokShopSearch` | TikTok Shop search results | `{"target": Target.TiktokShopSearch, "query": "wireless earbuds"}` | +| `Target.TiktokShopProduct` | TikTok Shop product page | `{"target": Target.TiktokShopProduct, "url": "https://www.tiktok.com/view/product/..."}` | +| `Target.Tiktok` | Raw TikTok URL scraping | `{"target": Target.Tiktok, "url": "https://www.tiktok.com/@nba"}` | +| `Target.InstagramGraphqlProfile` | Instagram profile via GraphQL | `{"target": Target.InstagramGraphqlProfile, "query": "nba"}` | ### AI tools @@ -316,6 +338,16 @@ Each target accepts one primary input parameter (`url`, `query`, `product_id`, o | `Target.Gemini` | Gemini response for a prompt | `{"target": Target.Gemini, "prompt": "What are the top three dog breeds?"}` | | `Target.GoogleAiMode` | Google AI Mode response | `{"target": Target.GoogleAiMode, "query": "What are the top three dog breeds?"}` | +### Other + +| Target | Description | Example | +| --- | --- | --- | +| `Target.Bbb` | Better Business Bureau listing by URL | `{"target": Target.Bbb, "url": "https://bbb.org/us/ny/new-york/..."}` | +| `Target.Autotrader` | Autotrader listing by URL | `{"target": Target.Autotrader, "url": "https://autotrader.com/cars-for-sale/..."}` | +| `Target.Mobile` | Mobile.de listing by URL | `{"target": Target.Mobile, "url": "https://mobile.de/auto/..."}` | +| `Target.Airbnb` | Airbnb listing by URL | `{"target": Target.Airbnb, "url": "https://airbnb.com/rooms/12345"}` | +| `Target.AppleAppStore` | Apple App Store app by URL | `{"target": Target.AppleAppStore, "url": "https://apps.apple.com/app/id12345"}` | + ### Universal scraping | Target | Description | Example | diff --git a/inputs/README.md b/inputs/README.md index 7e4331e..9c1610f 100644 --- a/inputs/README.md +++ b/inputs/README.md @@ -3,4 +3,4 @@ This directory contains files fetched at codegen time and is excluded from version control. - `decodo.ir.json` — the Intermediate Representation (IR) fetched from the Decodo GCS bucket. - Run `decodo-codegen` (or `python -m decodo.codegen.codegen`) to populate it. + Run `python -m decodo.codegen.codegen` to populate it. diff --git a/src/decodo/api/web_scraping_api.py b/src/decodo/api/web_scraping_api.py index 86d5698..fc06273 100644 --- a/src/decodo/api/web_scraping_api.py +++ b/src/decodo/api/web_scraping_api.py @@ -36,6 +36,8 @@ def __init__(self, http: HttpClient, schema: DecodoSchema = BundledSchema.shared def _validate(self, payload: dict[str, Any]) -> None: if self._schema is None: return + if "target" not in payload: + raise decodo.errors.ValidationError("missing required field 'target'") schema = self._schema.get_request_schema(payload.get("target")) # type: ignore[arg-type] if not schema: return From 5c399879408a776adbd763f54460af4d1f15c5f5 Mon Sep 17 00:00:00 2001 From: julka Date: Mon, 3 Aug 2026 09:28:38 +0300 Subject: [PATCH 15/17] Remove friction from pip installs for codegen and bubundled schema issues after install --- .github/workflows/worklfow.yml | 6 ++++++ README.md | 27 +++++++++++++-------------- pyproject.toml | 2 +- src/decodo/__init__.py | 4 +++- src/decodo/schema/bundled_schema.py | 27 +++++++++++++++++++++++---- 5 files changed, 46 insertions(+), 20 deletions(-) diff --git a/.github/workflows/worklfow.yml b/.github/workflows/worklfow.yml index b9a12fd..7b1f677 100644 --- a/.github/workflows/worklfow.yml +++ b/.github/workflows/worklfow.yml @@ -20,6 +20,12 @@ jobs: with: python-version: "3.12" + - name: Generate types + run: | + pip install httpx pydantic jsonschema + pip install -e . + python -m decodo.codegen.codegen + - name: Build run: | pip install build diff --git a/README.md b/README.md index 92b49d2..7d2dcf6 100644 --- a/README.md +++ b/README.md @@ -84,19 +84,10 @@ python main.py ### With typed parameters (recommended) -For IDE autocomplete and parameter validation, run the type generator once after installing. Dict-based usage works without this step, but typed classes require it. - -**Installed via pip (standard install):** Generated files cannot be written into site-packages, so specify a local output directory: - -```bash -python -m decodo.codegen.codegen --out-dir ./decodo_generated -``` - -Then import directly from that directory: +Typed parameter classes are included in the package — no extra steps needed after `pip install decodo-sdk`: ```python -from decodo_generated.targets import GoogleSearchParams -from decodo import DecodoClient, DecodoConfig, Target, WebScrapingApiConfig +from decodo import DecodoClient, DecodoConfig, GoogleSearchParams, Target, WebScrapingApiConfig client = DecodoClient( DecodoConfig( @@ -115,14 +106,22 @@ result = client.web_scraping_api.scrape( print(result) ``` -> **Note:** The directory name you pass to `--out-dir` becomes the import namespace. Using `./decodo_generated` means `from decodo_generated.targets import ...`. You can choose any name, but keep it consistent across your project. +### Updating types to a newer schema -**Editable / source install (`pip install -e .`):** You can omit `--out-dir` and generated files will be written into the package directly, making `from decodo import GoogleSearchParams` work: +The types bundled in the package reflect the schema at release time. To update them to the latest schema without waiting for a new release, run the type generator: ```bash -python -m decodo.codegen.codegen +python -m decodo.codegen.codegen --out-dir ./decodo_generated ``` +Then import from that directory instead: + +```python +from decodo_generated.targets import GoogleSearchParams +``` + +> The directory name passed to `--out-dir` becomes the import namespace. `./decodo_generated` → `from decodo_generated.targets import ...`. You can use any name, but keep it consistent across your project. +
Example response diff --git a/pyproject.toml b/pyproject.toml index c72ebd7..b6a7582 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,7 @@ build-backend = "setuptools.build_meta" where = ["src"] [tool.setuptools.package-data] -decodo = ["py.typed"] +decodo = ["py.typed", "generated/decodo.ir.json"] [tool.ruff] line-length = 120 diff --git a/src/decodo/__init__.py b/src/decodo/__init__.py index f1098cd..dc042a4 100644 --- a/src/decodo/__init__.py +++ b/src/decodo/__init__.py @@ -256,7 +256,9 @@ def __getattr__(name: str) -> object: if name in _CODEGEN_NAMES: raise ImportError( f"'{name}' requires generated types. " - "Run: python -m decodo.codegen.codegen" + "Editable install: run python -m decodo.codegen.codegen, then use 'from decodo import ...'. " + "Pip install: run python -m decodo.codegen.codegen --out-dir ./decodo_generated, " + "then use 'from decodo_generated.targets import ...'." ) raise AttributeError(f"module 'decodo' has no attribute {name!r}") diff --git a/src/decodo/schema/bundled_schema.py b/src/decodo/schema/bundled_schema.py index 441bcae..368f455 100644 --- a/src/decodo/schema/bundled_schema.py +++ b/src/decodo/schema/bundled_schema.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import os import warnings from pathlib import Path from typing import Any, ClassVar, cast @@ -20,14 +21,31 @@ def _load_target_meta() -> dict[str, Any] | None: _target_meta = _load_target_meta() -_IR_JSON_PATH = Path(__file__).parent.parent / "generated" / "decodo.ir.json" +_BUNDLED_IR_PATH = Path(__file__).parent.parent / "generated" / "decodo.ir.json" + + +def _find_ir_path() -> Path | None: + env = os.environ.get("DECODO_IR_PATH") + if env: + p = Path(env) + if p.is_file(): + return p + local = Path.cwd() / "decodo_generated" / "decodo.ir.json" + if local.is_file(): + return local + if _BUNDLED_IR_PATH.is_file(): + return _BUNDLED_IR_PATH + return None def _load_ir_json() -> dict[str, Any] | None: + path = _find_ir_path() + if path is None: + return None try: - with open(_IR_JSON_PATH, encoding="utf-8") as f: + with open(path, encoding="utf-8") as f: return json.load(f) # type: ignore[no-any-return] - except FileNotFoundError: + except (OSError, json.JSONDecodeError): return None @@ -45,7 +63,8 @@ def __init__(self) -> None: else: warnings.warn( "Decodo IR schema not found — payload validation is disabled. " - "Run: python -m decodo.codegen.codegen", + "Run: python -m decodo.codegen.codegen (editable install) or " + "python -m decodo.codegen.codegen --out-dir ./decodo_generated (pip install).", RuntimeWarning, stacklevel=2, ) From 5db0ad54f0cc92179edbb23e57cca8ee386790d0 Mon Sep 17 00:00:00 2001 From: julka Date: Mon, 3 Aug 2026 10:04:32 +0300 Subject: [PATCH 16/17] Fix client side confusing warning error --- src/decodo/api/web_scraping_api.py | 6 ++++++ src/decodo/codegen/web_scraping_api/generate_targets.py | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/decodo/api/web_scraping_api.py b/src/decodo/api/web_scraping_api.py index fc06273..ef6ca5a 100644 --- a/src/decodo/api/web_scraping_api.py +++ b/src/decodo/api/web_scraping_api.py @@ -40,6 +40,12 @@ def _validate(self, payload: dict[str, Any]) -> None: raise decodo.errors.ValidationError("missing required field 'target'") schema = self._schema.get_request_schema(payload.get("target")) # type: ignore[arg-type] if not schema: + target = payload["target"] + valid = self._schema.list_targets() + if target not in valid: + raise decodo.errors.ValidationError( + f"unknown target {target!r}. Valid targets: {', '.join(sorted(valid))}" + ) return try: jsonschema.validate(payload, schema) diff --git a/src/decodo/codegen/web_scraping_api/generate_targets.py b/src/decodo/codegen/web_scraping_api/generate_targets.py index 8b93c47..a64683f 100644 --- a/src/decodo/codegen/web_scraping_api/generate_targets.py +++ b/src/decodo/codegen/web_scraping_api/generate_targets.py @@ -121,7 +121,7 @@ def _batch_class_name(target_key: str) -> str: properties: dict[str, Any] = target["parameter_schema"].get("properties", {}) lines.append(f"class {type_name}(pydantic.BaseModel):") - lines.append(" model_config = pydantic.ConfigDict(populate_by_name=True)") + lines.append(" model_config = pydantic.ConfigDict(populate_by_name=True, extra='forbid')") lines.append(f" target: Literal[Target.{member}] = Target.{member}") params = {k: v for k, v in properties.items() if k != "target"} if params: @@ -143,7 +143,7 @@ def _batch_class_name(target_key: str) -> str: properties = target["parameter_schema"].get("properties", {}) lines.append(f"class {batch_type_name}(pydantic.BaseModel):") - lines.append(" model_config = pydantic.ConfigDict(populate_by_name=True)") + lines.append(" model_config = pydantic.ConfigDict(populate_by_name=True, extra='forbid')") lines.append(f" target: Literal[Target.{member}] = Target.{member}") params = {k: v for k, v in properties.items() if k != "target"} if params: From bacbb75d570d8c546353855fc7fdc3c25e49886a Mon Sep 17 00:00:00 2001 From: julka Date: Mon, 3 Aug 2026 10:26:49 +0300 Subject: [PATCH 17/17] Fix mypy ir path issue --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 45319bf..c553613 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,7 +13,7 @@ @pytest.fixture(autouse=True, scope="session") def _patch_bundled_schema_ir_path() -> None: - _bundled_schema_mod._IR_JSON_PATH = MINIMAL_IR_PATH + _bundled_schema_mod._BUNDLED_IR_PATH = MINIMAL_IR_PATH _bundled_schema_mod.BundledSchema.shared = _bundled_schema_mod.BundledSchema()