diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..e7e55c7 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,26 @@ +name: Test + +on: + push: + branches: + - main + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install dependencies + run: pip install -e ".[dev]" + + - name: Generate types + run: python -m decodo.codegen.codegen + + - name: Pytest + run: pytest diff --git a/.github/workflows/worklfow.yml b/.github/workflows/worklfow.yml index e43c20b..7b1f677 100644 --- a/.github/workflows/worklfow.yml +++ b/.github/workflows/worklfow.yml @@ -12,6 +12,7 @@ jobs: environment: pypi permissions: id-token: write + contents: read steps: - uses: actions/checkout@v4 @@ -19,6 +20,12 @@ jobs: with: python-version: "3.12" + - name: Generate types + run: | + pip install httpx pydantic jsonschema + pip install -e . + python -m decodo.codegen.codegen + - name: Build run: | pip install build diff --git a/.gitignore b/.gitignore index 9ee3aa6..be9a556 100644 --- a/.gitignore +++ b/.gitignore @@ -33,5 +33,8 @@ htmlcov/ # Inputs (fetched at codegen time) inputs/decodo.ir.json +# Generated types (run `python -m decodo.codegen.codegen` to regenerate) +src/decodo/generated/ + # OS .DS_Store diff --git a/README.md b/README.md index 5dd0e52..7d2dcf6 100644 --- a/README.md +++ b/README.md @@ -55,18 +55,20 @@ pip install decodo-sdk touch main.py ``` -Get a Web Scraping API basic authentication token from the [Decodo dashboard](https://dashboard.decodo.com/welcome) and use it in the following example: +Get your Web Scraping API token from the [Decodo dashboard](https://dashboard.decodo.com/welcome). The token is the base64-encoded `user:password` value from the Basic Auth credentials shown in the dashboard. ```python # main.py -from decodo import DecodoClient, Target +from decodo import DecodoClient, DecodoConfig, WebScrapingApiConfig client = DecodoClient( - web_scraping_api={"token": ""} + DecodoConfig( + web_scraping_api=WebScrapingApiConfig(token=""), + ) ) result = client.web_scraping_api.scrape({ - "target": Target.GoogleSearch, + "target": "google_search", "query": "coffee shops", "geo": "United States", "parse": True, @@ -80,6 +82,46 @@ Run the script: python main.py ``` +### With typed parameters (recommended) + +Typed parameter classes are included in the package — no extra steps needed after `pip install decodo-sdk`: + +```python +from decodo import DecodoClient, DecodoConfig, GoogleSearchParams, Target, WebScrapingApiConfig + +client = DecodoClient( + DecodoConfig( + web_scraping_api=WebScrapingApiConfig(token=""), + ) +) + +result = client.web_scraping_api.scrape( + GoogleSearchParams( + target=Target.GoogleSearch, + query="coffee shops", + geo="United States", + parse=True, + ) +) +print(result) +``` + +### Updating types to a newer schema + +The types bundled in the package reflect the schema at release time. To update them to the latest schema without waiting for a new release, run the type generator: + +```bash +python -m decodo.codegen.codegen --out-dir ./decodo_generated +``` + +Then import from that directory instead: + +```python +from decodo_generated.targets import GoogleSearchParams +``` + +> The directory name passed to `--out-dir` becomes the import namespace. `./decodo_generated` → `from decodo_generated.targets import ...`. You can use any name, but keep it consistent across your project. +
Example response @@ -162,28 +204,26 @@ python main.py ## Configuration ```python -from decodo import DecodoClient +from decodo import DecodoClient, DecodoConfig, WebScrapingApiConfig client = DecodoClient( - web_scraping_api={"token": ""}, - timeout_ms=120_000, # optional, request timeout in ms (default: 180000) + DecodoConfig( + web_scraping_api=WebScrapingApiConfig(token=""), + timeout_ms=120_000, # optional, request timeout in ms (default: 180000) + ) ) ``` | Parameter | Description | | --- | --- | -| `token` | Web Scraping API basic authentication token | +| `token` | Web Scraping API basic auth token - the base64-encoded `user:password` string from the Decodo dashboard | | `timeout_ms` | Request timeout in milliseconds (default: 180000) | ## Web Scraping API Access the API via `client.web_scraping_api`. -The snippets below assume you have already imported `Target` (and `DecodoClient` where a client is constructed), for example: - -```python -from decodo import DecodoClient, Target -``` +The snippets below assume you have already constructed a client. See [Configuration](#configuration) for how to build one. ### Sync scrape @@ -191,7 +231,7 @@ Waits for the scraping result before returning: ```python result = client.web_scraping_api.scrape({ - "target": Target.AmazonProduct, + "target": "amazon_product", "query": "B09H74FXNW", "parse": True, }) @@ -203,7 +243,7 @@ Creates a scraping task and returns immediately. Poll separately for task status ```python task = client.web_scraping_api.scrape_async({ - "target": Target.GoogleSearch, + "target": "google_search", "query": "laptop reviews", "parse": True, }) @@ -220,7 +260,7 @@ Send multiple queries or URLs in a single request: ```python batch = client.web_scraping_api.scrape_batch({ - "target": Target.GoogleSearch, + "target": "google_search", "query": ["coffee", "tea", "juice"], "parse": True, }) @@ -241,9 +281,14 @@ Each target accepts one primary input parameter (`url`, `query`, `product_id`, o | `Target.GoogleSearch` | Google Search results for a query | `{"target": Target.GoogleSearch, "query": "coffee shops"}` | | `Target.GoogleMaps` | Google Maps search results | `{"target": Target.GoogleMaps, "query": "coffee shops brooklyn"}` | | `Target.GoogleShoppingSearch` | Google Shopping search results | `{"target": Target.GoogleShoppingSearch, "query": "laptop"}` | +| `Target.GoogleShoppingProduct` | Google Shopping product page | `{"target": Target.GoogleShoppingProduct, "query": "B09H74FXNW"}` | | `Target.GoogleSuggest` | Google Autocomplete suggestions | `{"target": Target.GoogleSuggest, "query": "coffee"}` | | `Target.GoogleLens` | Google Lens reverse image search | `{"target": Target.GoogleLens, "query": "https://example.com/cat.jpg"}` | +| `Target.GoogleTravelHotels` | Google Travel hotel listings | `{"target": Target.GoogleTravelHotels, "query": "hotels in paris"}` | +| `Target.GoogleTrendsExplore` | Google Trends explore data | `{"target": Target.GoogleTrendsExplore, "query": "coffee"}` | +| `Target.GoogleAds` | Google Ads results for a query | `{"target": Target.GoogleAds, "query": "laptop"}` | | `Target.BingSearch` | Bing Search results | `{"target": Target.BingSearch, "query": "electric vehicles"}` | +| `Target.Bing` | Raw Bing URL scraping | `{"target": Target.Bing, "url": "https://bing.com/search?q=laptop"}` | ### eCommerce @@ -252,8 +297,15 @@ Each target accepts one primary input parameter (`url`, `query`, `product_id`, o | `Target.AmazonProduct` | Amazon product detail page by ASIN | `{"target": Target.AmazonProduct, "query": "B09H74FXNW"}` | | `Target.AmazonSearch` | Amazon search results | `{"target": Target.AmazonSearch, "query": "laptop"}` | | `Target.AmazonPricing` | Amazon pricing and offers | `{"target": Target.AmazonPricing, "query": "B09H74FXNW"}` | +| `Target.AmazonSellers` | Amazon seller listings | `{"target": Target.AmazonSellers, "query": "B09H74FXNW"}` | +| `Target.AmazonBestsellers` | Amazon bestsellers by category | `{"target": Target.AmazonBestsellers, "query": "electronics"}` | | `Target.WalmartProduct` | Walmart product page by product ID | `{"target": Target.WalmartProduct, "product_id": "15296401808"}` | +| `Target.WalmartSearch` | Walmart search results | `{"target": Target.WalmartSearch, "query": "laptop"}` | +| `Target.Walmart` | Raw Walmart URL scraping | `{"target": Target.Walmart, "url": "https://walmart.com/ip/15296401808"}` | | `Target.TargetProduct` | Target.com product page by product ID | `{"target": Target.TargetProduct, "product_id": "92186007"}` | +| `Target.TargetSearch` | Target.com search results | `{"target": Target.TargetSearch, "query": "laptop"}` | +| `Target.Target` | Raw Target.com URL scraping | `{"target": Target.Target, "url": "https://target.com/p/-/A-92186007"}` | +| `Target.LowesSearch` | Lowe's search results | `{"target": Target.LowesSearch, "query": "drill"}` | | `Target.Ecommerce` | Generic eCommerce page with parser | `{"target": Target.Ecommerce, "url": "https://example.com/product/123"}` | ### Social media @@ -262,9 +314,19 @@ Each target accepts one primary input parameter (`url`, `query`, `product_id`, o | --- | --- | --- | | `Target.RedditPost` | Reddit post by URL | `{"target": Target.RedditPost, "url": "https://reddit.com/r/nba/..."}` | | `Target.RedditSubreddit` | Reddit subreddit by URL | `{"target": Target.RedditSubreddit, "url": "https://reddit.com/r/nba/"}` | +| `Target.RedditUser` | Reddit user profile by URL | `{"target": Target.RedditUser, "url": "https://reddit.com/user/example/"}` | | `Target.YoutubeVideo` | YouTube video by ID | `{"target": Target.YoutubeVideo, "query": "dFu9aKJoqGg"}` | | `Target.YoutubeSearch` | YouTube search results | `{"target": Target.YoutubeSearch, "query": "ambient music"}` | +| `Target.YoutubeSearchMax` | YouTube search results (extended) | `{"target": Target.YoutubeSearchMax, "query": "ambient music"}` | +| `Target.YoutubeMetadata` | YouTube video metadata by ID | `{"target": Target.YoutubeMetadata, "query": "dFu9aKJoqGg"}` | +| `Target.YoutubeTranscript` | YouTube video transcript by ID | `{"target": Target.YoutubeTranscript, "query": "dFu9aKJoqGg"}` | +| `Target.YoutubeSubtitles` | YouTube video subtitles by ID | `{"target": Target.YoutubeSubtitles, "query": "dFu9aKJoqGg"}` | +| `Target.YoutubeChannel` | YouTube channel by URL | `{"target": Target.YoutubeChannel, "url": "https://youtube.com/@mkbhd"}` | | `Target.TiktokPost` | TikTok post by URL | `{"target": Target.TiktokPost, "url": "https://www.tiktok.com/@nba/video/..."}` | +| `Target.TiktokShopSearch` | TikTok Shop search results | `{"target": Target.TiktokShopSearch, "query": "wireless earbuds"}` | +| `Target.TiktokShopProduct` | TikTok Shop product page | `{"target": Target.TiktokShopProduct, "url": "https://www.tiktok.com/view/product/..."}` | +| `Target.Tiktok` | Raw TikTok URL scraping | `{"target": Target.Tiktok, "url": "https://www.tiktok.com/@nba"}` | +| `Target.InstagramGraphqlProfile` | Instagram profile via GraphQL | `{"target": Target.InstagramGraphqlProfile, "query": "nba"}` | ### AI tools @@ -275,6 +337,16 @@ Each target accepts one primary input parameter (`url`, `query`, `product_id`, o | `Target.Gemini` | Gemini response for a prompt | `{"target": Target.Gemini, "prompt": "What are the top three dog breeds?"}` | | `Target.GoogleAiMode` | Google AI Mode response | `{"target": Target.GoogleAiMode, "query": "What are the top three dog breeds?"}` | +### Other + +| Target | Description | Example | +| --- | --- | --- | +| `Target.Bbb` | Better Business Bureau listing by URL | `{"target": Target.Bbb, "url": "https://bbb.org/us/ny/new-york/..."}` | +| `Target.Autotrader` | Autotrader listing by URL | `{"target": Target.Autotrader, "url": "https://autotrader.com/cars-for-sale/..."}` | +| `Target.Mobile` | Mobile.de listing by URL | `{"target": Target.Mobile, "url": "https://mobile.de/auto/..."}` | +| `Target.Airbnb` | Airbnb listing by URL | `{"target": Target.Airbnb, "url": "https://airbnb.com/rooms/12345"}` | +| `Target.AppleAppStore` | Apple App Store app by URL | `{"target": Target.AppleAppStore, "url": "https://apps.apple.com/app/id12345"}` | + ### Universal scraping | Target | Description | Example | @@ -296,27 +368,24 @@ The SDK raises typed errors that map to API error codes: ```python from decodo import ( - DecodoClient, - DecodoError, AuthenticationError, RateLimitError, ValidationError, TimeoutError, - Target, ) try: client.web_scraping_api.scrape({ - "target": Target.GoogleSearch, + "target": "google_search", "query": "test", "parse": True, }) except AuthenticationError: - pass # 401/403 — bad credentials + pass # 401/403 - bad credentials except RateLimitError: - pass # 429 — too many requests + pass # 429 - too many requests except ValidationError as err: - print(err.errors) # 422 — invalid parameters + print(err.errors) # 422 - invalid parameters except TimeoutError: pass # request timed out ``` @@ -338,4 +407,3 @@ Build scraping workflows with the Decodo Web Scraping API: ## License Released under the [MIT License](https://github.com/Decodo/Decodo/blob/master/LICENSE). ->>>>>>> 03b68da (Initial sdk setup) diff --git a/examples/web_scraping_api/README.md b/examples/web_scraping_api/README.md new file mode 100644 index 0000000..c2ce1c5 --- /dev/null +++ b/examples/web_scraping_api/README.md @@ -0,0 +1,25 @@ +# Examples + +These examples use typed parameter classes (`GoogleSearchParams`, `AmazonProductParams`, etc.) which require running the type generator first: + +```bash +python -m decodo.codegen.codegen +``` + +To run any example without the generator, replace the typed params with a plain dict: + +```python +# instead of: +result = client.web_scraping_api.scrape( + GoogleSearchParams(target=Target.GoogleSearch, query="coffee shops", parse=True) +) + +# use: +result = client.web_scraping_api.scrape({ + "target": "google_search", + "query": "coffee shops", + "parse": True, +}) +``` + +See the [root README](../../README.md) for the full list of target strings. diff --git a/examples/web_scraping_api/batch/google_search_batch.py b/examples/web_scraping_api/batch/google_search_batch.py index d40bf67..1006790 100644 --- a/examples/web_scraping_api/batch/google_search_batch.py +++ b/examples/web_scraping_api/batch/google_search_batch.py @@ -5,7 +5,7 @@ from decodo import ( DecodoClient, DecodoConfig, - GoogleSearchParams, + GoogleSearchBatchParams, Target, WebScrapingApiConfig, ) @@ -19,25 +19,23 @@ ) metadata = client.web_scraping_api.scrape_batch( - GoogleSearchParams( + GoogleSearchBatchParams( target=Target.GoogleSearch, - query=['shoes', 'laptop'], + query=["shoes", "laptop"], parse=True, ) ) -print('Polling for results...') - while True: - print('Polling for results...') - queries = metadata.get('queries') or [] + print("Polling for results...") + queries = metadata.get("queries") or [] if not queries: break - first_task_id = queries[0].get('id') + first_task_id = queries[0].get("id") if not first_task_id: break results = client.web_scraping_api.get_results(first_task_id) if results: - print(json.dumps(results['results'][0]['content'], indent=2)) + print(json.dumps(results["results"][0]["content"], indent=2)) break time.sleep(3) diff --git a/inputs/README.md b/inputs/README.md index 7e4331e..9c1610f 100644 --- a/inputs/README.md +++ b/inputs/README.md @@ -3,4 +3,4 @@ This directory contains files fetched at codegen time and is excluded from version control. - `decodo.ir.json` — the Intermediate Representation (IR) fetched from the Decodo GCS bucket. - Run `decodo-codegen` (or `python -m decodo.codegen.codegen`) to populate it. + Run `python -m decodo.codegen.codegen` to populate it. diff --git a/pyproject.toml b/pyproject.toml index 1b1beeb..b6a7582 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ requires-python = ">=3.12" dependencies = [ "httpx>=0.27.0", "pydantic>=2.0", + "jsonschema>=4.0", ] [project.urls] @@ -30,6 +31,7 @@ dev = [ "pytest-mock>=3.14.0", "ruff>=0.4.0", "mypy>=1.10.0", + "types-jsonschema>=4.0", ] [build-system] @@ -39,6 +41,9 @@ build-backend = "setuptools.build_meta" [tool.setuptools.packages.find] where = ["src"] +[tool.setuptools.package-data] +decodo = ["py.typed", "generated/decodo.ir.json"] + [tool.ruff] line-length = 120 src = ["src"] @@ -50,10 +55,13 @@ select = ["E", "F", "I", "UP"] [tool.mypy] strict = true packages = ["decodo"] -exclude = "src/decodo/generated|^examples/" +exclude = "src/decodo/generated|^examples/|^build/" + +[[tool.mypy.overrides]] +module = "decodo.generated.*" +ignore_missing_imports = true +ignore_errors = true [tool.pytest.ini_options] testpaths = ["tests"] -[project.scripts] -decodo-codegen = "decodo.codegen.codegen:main" diff --git a/src/decodo/__init__.py b/src/decodo/__init__.py index a9bf751..dc042a4 100644 --- a/src/decodo/__init__.py +++ b/src/decodo/__init__.py @@ -7,68 +7,10 @@ TimeoutError, ValidationError, ) -from .generated.parameters import ParameterMeta, parameter_meta -from .generated.targets import ( - AirbnbParams, - AmazonBestsellersParams, - AmazonParams, - AmazonPricingParams, - AmazonProductParams, - AmazonSearchParams, - AmazonSellersParams, - AppleAppStoreParams, - AutotraderParams, - BbbParams, - BingParams, - BingSearchParams, - ChatgptParams, - EcommerceParams, - GeminiParams, - GoogleAdsParams, - GoogleAiModeParams, - GoogleLensParams, - GoogleMapsParams, - GoogleParams, - GoogleSearchParams, - GoogleShoppingProductParams, - GoogleShoppingSearchParams, - GoogleSuggestParams, - GoogleTravelHotelsParams, - GoogleTrendsExploreParams, - InstagramGraphqlProfileParams, - LowesSearchParams, - MobileParams, - PerplexityParams, - RedditPostParams, - RedditSubredditParams, - RedditUserParams, - ScrapeRequest, - Target, - TargetProductParams, - TargetSearchParams, - TargetStoreParams, - TiktokParams, - TiktokPostParams, - TiktokShopProductParams, - TiktokShopSearchParams, - UniversalEcommerceParams, - UniversalParams, - WalmartParams, - WalmartProductParams, - WalmartSearchParams, - YoutubeChannelParams, - YoutubeMetadataParams, - YoutubeSearchMaxParams, - YoutubeSearchParams, - YoutubeSubtitlesParams, - YoutubeTranscriptParams, - YoutubeVideoParams, - target_meta, - targets, -) from .schema.bundled_schema import BundledSchema from .schema.remote_schema import RemoteSchema from .schema.types import DecodoSchema, RemoteSchemaLoadOptions +from .targets import Target, targets from .types.responses import ( AsyncTaskResponse, BatchResponse, @@ -79,6 +21,248 @@ TaskStatus, ) +_codegen_available = False +try: + from .generated.parameters import ParameterMeta, parameter_meta # noqa: F401 + from .generated.targets import ( # noqa: F401 + AirbnbBatchParams, + AirbnbParams, + AmazonBatchParams, + AmazonBestsellersBatchParams, + AmazonBestsellersParams, + AmazonParams, + AmazonPricingBatchParams, + AmazonPricingParams, + AmazonProductBatchParams, + AmazonProductParams, + AmazonSearchBatchParams, + AmazonSearchParams, + AmazonSellersBatchParams, + AmazonSellersParams, + AppleAppStoreBatchParams, + AppleAppStoreParams, + AutotraderBatchParams, + AutotraderParams, + BatchRequest, + BbbBatchParams, + BbbParams, + BingBatchParams, + BingParams, + BingSearchBatchParams, + BingSearchParams, + ChatgptBatchParams, + ChatgptParams, + EcommerceBatchParams, + EcommerceParams, + GeminiBatchParams, + GeminiParams, + GoogleAdsBatchParams, + GoogleAdsParams, + GoogleAiModeBatchParams, + GoogleAiModeParams, + GoogleBatchParams, + GoogleLensBatchParams, + GoogleLensParams, + GoogleMapsBatchParams, + GoogleMapsParams, + GoogleParams, + GoogleSearchBatchParams, + GoogleSearchParams, + GoogleShoppingProductBatchParams, + GoogleShoppingProductParams, + GoogleShoppingSearchBatchParams, + GoogleShoppingSearchParams, + GoogleSuggestBatchParams, + GoogleSuggestParams, + GoogleTravelHotelsBatchParams, + GoogleTravelHotelsParams, + GoogleTrendsExploreBatchParams, + GoogleTrendsExploreParams, + InstagramGraphqlProfileBatchParams, + InstagramGraphqlProfileParams, + LowesSearchBatchParams, + LowesSearchParams, + MobileBatchParams, + MobileParams, + PerplexityBatchParams, + PerplexityParams, + RedditPostBatchParams, + RedditPostParams, + RedditSubredditBatchParams, + RedditSubredditParams, + RedditUserBatchParams, + RedditUserParams, + ScrapeRequest, + TargetProductBatchParams, + TargetProductParams, + TargetSearchBatchParams, + TargetSearchParams, + TargetStoreBatchParams, + TargetStoreParams, + TiktokBatchParams, + TiktokParams, + TiktokPostBatchParams, + TiktokPostParams, + TiktokShopProductBatchParams, + TiktokShopProductParams, + TiktokShopSearchBatchParams, + TiktokShopSearchParams, + UniversalBatchParams, + UniversalEcommerceBatchParams, + UniversalEcommerceParams, + UniversalParams, + WalmartBatchParams, + WalmartParams, + WalmartProductBatchParams, + WalmartProductParams, + WalmartSearchBatchParams, + WalmartSearchParams, + YoutubeChannelBatchParams, + YoutubeChannelParams, + YoutubeMetadataBatchParams, + YoutubeMetadataParams, + YoutubeSearchBatchParams, + YoutubeSearchMaxBatchParams, + YoutubeSearchMaxParams, + YoutubeSearchParams, + YoutubeSubtitlesBatchParams, + YoutubeSubtitlesParams, + YoutubeTranscriptBatchParams, + YoutubeTranscriptParams, + YoutubeVideoBatchParams, + YoutubeVideoParams, + target_meta, + ) + _codegen_available = True +except ImportError: + pass + +_CODEGEN_NAMES: frozenset[str] = frozenset( + [ + "AirbnbBatchParams", + "AirbnbParams", + "AmazonBatchParams", + "AmazonBestsellersBatchParams", + "AmazonBestsellersParams", + "AmazonParams", + "AmazonPricingBatchParams", + "AmazonPricingParams", + "AmazonProductBatchParams", + "AmazonProductParams", + "AmazonSearchBatchParams", + "AmazonSearchParams", + "AmazonSellersBatchParams", + "AmazonSellersParams", + "AppleAppStoreBatchParams", + "AppleAppStoreParams", + "AutotraderBatchParams", + "AutotraderParams", + "BatchRequest", + "BbbBatchParams", + "BbbParams", + "BingBatchParams", + "BingParams", + "BingSearchBatchParams", + "BingSearchParams", + "ChatgptBatchParams", + "ChatgptParams", + "EcommerceBatchParams", + "EcommerceParams", + "GeminiBatchParams", + "GeminiParams", + "GoogleAdsBatchParams", + "GoogleAdsParams", + "GoogleAiModeBatchParams", + "GoogleAiModeParams", + "GoogleBatchParams", + "GoogleLensBatchParams", + "GoogleLensParams", + "GoogleMapsBatchParams", + "GoogleMapsParams", + "GoogleParams", + "GoogleSearchBatchParams", + "GoogleSearchParams", + "GoogleShoppingProductBatchParams", + "GoogleShoppingProductParams", + "GoogleShoppingSearchBatchParams", + "GoogleShoppingSearchParams", + "GoogleSuggestBatchParams", + "GoogleSuggestParams", + "GoogleTravelHotelsBatchParams", + "GoogleTravelHotelsParams", + "GoogleTrendsExploreBatchParams", + "GoogleTrendsExploreParams", + "InstagramGraphqlProfileBatchParams", + "InstagramGraphqlProfileParams", + "LowesSearchBatchParams", + "LowesSearchParams", + "MobileBatchParams", + "MobileParams", + "ParameterMeta", + "PerplexityBatchParams", + "PerplexityParams", + "RedditPostBatchParams", + "RedditPostParams", + "RedditSubredditBatchParams", + "RedditSubredditParams", + "RedditUserBatchParams", + "RedditUserParams", + "ScrapeRequest", + "TargetProductBatchParams", + "TargetProductParams", + "TargetSearchBatchParams", + "TargetSearchParams", + "TargetStoreBatchParams", + "TargetStoreParams", + "TiktokBatchParams", + "TiktokParams", + "TiktokPostBatchParams", + "TiktokPostParams", + "TiktokShopProductBatchParams", + "TiktokShopProductParams", + "TiktokShopSearchBatchParams", + "TiktokShopSearchParams", + "UniversalBatchParams", + "UniversalEcommerceBatchParams", + "UniversalEcommerceParams", + "UniversalParams", + "WalmartBatchParams", + "WalmartParams", + "WalmartProductBatchParams", + "WalmartProductParams", + "WalmartSearchBatchParams", + "WalmartSearchParams", + "YoutubeChannelBatchParams", + "YoutubeChannelParams", + "YoutubeMetadataBatchParams", + "YoutubeMetadataParams", + "YoutubeSearchBatchParams", + "YoutubeSearchMaxBatchParams", + "YoutubeSearchMaxParams", + "YoutubeSearchParams", + "YoutubeSubtitlesBatchParams", + "YoutubeSubtitlesParams", + "YoutubeTranscriptBatchParams", + "YoutubeTranscriptParams", + "YoutubeVideoBatchParams", + "YoutubeVideoParams", + "parameter_meta", + "target_meta", + ] +) + + +def __getattr__(name: str) -> object: + if name in _CODEGEN_NAMES: + raise ImportError( + f"'{name}' requires generated types. " + "Editable install: run python -m decodo.codegen.codegen, then use 'from decodo import ...'. " + "Pip install: run python -m decodo.codegen.codegen --out-dir ./decodo_generated, " + "then use 'from decodo_generated.targets import ...'." + ) + raise AttributeError(f"module 'decodo' has no attribute {name!r}") + + __all__ = [ "DecodoClient", "DecodoConfig", @@ -89,64 +273,7 @@ "DecodoSchema", "RemoteSchemaLoadOptions", "Target", - "target_meta", "targets", - "ScrapeRequest", - "UniversalEcommerceParams", - "GoogleSearchParams", - "GoogleTravelHotelsParams", - "GoogleTrendsExploreParams", - "GoogleShoppingSearchParams", - "GoogleShoppingProductParams", - "GoogleParams", - "GoogleSuggestParams", - "GoogleMapsParams", - "GoogleAiModeParams", - "GoogleAdsParams", - "GoogleLensParams", - "BingSearchParams", - "BingParams", - "YoutubeTranscriptParams", - "AmazonProductParams", - "AmazonPricingParams", - "AmazonSearchParams", - "AmazonSellersParams", - "AmazonBestsellersParams", - "AmazonParams", - "EcommerceParams", - "WalmartProductParams", - "WalmartSearchParams", - "WalmartParams", - "TargetProductParams", - "TargetSearchParams", - "TargetStoreParams", - "TargetTargetParams", - "LowesSearchParams", - "UniversalParams", - "ChatgptParams", - "PerplexityParams", - "GeminiParams", - "BbbParams", - "AutotraderParams", - "MobileParams", - "AirbnbParams", - "AppleAppStoreParams", - "InstagramGraphqlProfileParams", - "TiktokPostParams", - "TiktokShopSearchParams", - "TiktokShopProductParams", - "TiktokParams", - "RedditPostParams", - "RedditSubredditParams", - "RedditUserParams", - "YoutubeVideoParams", - "YoutubeMetadataParams", - "YoutubeSearchParams", - "YoutubeSearchMaxParams", - "YoutubeSubtitlesParams", - "YoutubeChannelParams", - "ParameterMeta", - "parameter_meta", "SyncResponse", "AsyncTaskResponse", "BatchResponse", @@ -160,3 +287,6 @@ "ValidationError", "TimeoutError", ] + +if _codegen_available: + __all__ += list(_CODEGEN_NAMES) diff --git a/src/decodo/api/web_scraping_api.py b/src/decodo/api/web_scraping_api.py index 5711759..ef6ca5a 100644 --- a/src/decodo/api/web_scraping_api.py +++ b/src/decodo/api/web_scraping_api.py @@ -1,9 +1,17 @@ from __future__ import annotations -from typing import cast +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, cast -from decodo.generated.targets import ScrapeRequest +import jsonschema +from pydantic import BaseModel + +import decodo.errors from decodo.http import HttpClient + +if TYPE_CHECKING: + from decodo.generated.targets import BatchRequest, ScrapeRequest +from decodo.schema.bundled_schema import BundledSchema from decodo.schema.types import DecodoSchema from decodo.types.responses import ( AsyncTaskResponse, @@ -14,19 +22,49 @@ ) +def _to_payload(params: ScrapeRequest | BatchRequest | Mapping[str, Any]) -> dict[str, Any]: + if isinstance(params, BaseModel): + return params.model_dump(by_alias=True, exclude_none=True, mode="json") + return dict(params) + + class WebScrapingApi: - def __init__(self, http: HttpClient, schema: DecodoSchema | None = None) -> None: + def __init__(self, http: HttpClient, schema: DecodoSchema = BundledSchema.shared) -> None: self._http = http self._schema = schema - def scrape(self, params: ScrapeRequest) -> SyncResponse: - return cast(SyncResponse, self._http.post("/v2/scrape", params.model_dump(exclude_none=True, mode="json"))) + def _validate(self, payload: dict[str, Any]) -> None: + if self._schema is None: + return + if "target" not in payload: + raise decodo.errors.ValidationError("missing required field 'target'") + schema = self._schema.get_request_schema(payload.get("target")) # type: ignore[arg-type] + if not schema: + target = payload["target"] + valid = self._schema.list_targets() + if target not in valid: + raise decodo.errors.ValidationError( + f"unknown target {target!r}. Valid targets: {', '.join(sorted(valid))}" + ) + return + try: + jsonschema.validate(payload, schema) + except jsonschema.ValidationError as e: + raise decodo.errors.ValidationError(str(e)) from e + + def scrape(self, params: ScrapeRequest | Mapping[str, Any]) -> SyncResponse: + payload = _to_payload(params) + self._validate(payload) + return cast(SyncResponse, self._http.post("/v2/scrape", payload)) - def scrape_async(self, params: ScrapeRequest) -> AsyncTaskResponse: - return cast(AsyncTaskResponse, self._http.post("/v3/task", params.model_dump(exclude_none=True, mode="json"))) + def scrape_async(self, params: ScrapeRequest | Mapping[str, Any]) -> AsyncTaskResponse: + payload = _to_payload(params) + self._validate(payload) + return cast(AsyncTaskResponse, self._http.post("/v3/task", payload)) - def scrape_batch(self, params: ScrapeRequest) -> BatchResponse: - return cast(BatchResponse, self._http.post("/v3/task/batch", params.model_dump(exclude_none=True, mode="json"))) + def scrape_batch(self, params: BatchRequest | Mapping[str, Any]) -> BatchResponse: + payload = _to_payload(params) + return cast(BatchResponse, self._http.post("/v3/task/batch", payload)) def get_status(self, task_id: str) -> TaskMetadata: return cast(TaskMetadata, self._http.get(f"/v3/task/{task_id}")) diff --git a/src/decodo/codegen/codegen.py b/src/decodo/codegen/codegen.py index 698ab9a..0f9e7c9 100644 --- a/src/decodo/codegen/codegen.py +++ b/src/decodo/codegen/codegen.py @@ -1,14 +1,70 @@ from __future__ import annotations -from .web_scraping_api.generate_parameter_schemas import generate_parameter_schemas_file +import argparse +import os +import shutil +import sys +from pathlib import Path + from .web_scraping_api.generate_parameters import generate_parameters_file -from .web_scraping_api.generate_targets import generate_targets_file +from .web_scraping_api.generate_targets import generate_targets_enum_file, generate_targets_file +from .web_scraping_api.shared import local_ir_path +from .web_scraping_api.shared import out_dir as _default_out_dir + + +def _in_site_packages() -> bool: + return "site-packages" in str(Path(__file__).resolve()) def main() -> None: - generate_parameters_file() - generate_targets_file() - generate_parameter_schemas_file() + parser = argparse.ArgumentParser( + prog="python -m decodo.codegen.codegen", + description="Generate typed parameter classes from the Decodo IR schema.", + ) + parser.add_argument( + "--out-dir", + metavar="PATH", + default=None, + help=( + "Directory to write generated files into. " + "Defaults to the package's built-in generated/ directory " + "(editable installs only). Required for non-editable installs." + ), + ) + args = parser.parse_args() + + if args.out_dir is not None: + out_dir = str(Path(args.out_dir).resolve()) + elif _in_site_packages(): + print( + "error: running from a non-editable install.\n" + "Generated files would be written into site-packages and silently reverted\n" + "on the next `pip install --upgrade`.\n" + "Use --out-dir to specify a project-local output directory.", + file=sys.stderr, + ) + sys.exit(1) + else: + out_dir = _default_out_dir + + os.makedirs(out_dir, exist_ok=True) + init_path = os.path.join(out_dir, "__init__.py") + if not os.path.exists(init_path): + open(init_path, "w").close() + + generate_parameters_file(out_dir) + # The Target enum lives in decodo/targets.py (committed source). Only regenerate + # it when we have write access to the source tree (editable install). Non-editable + # installs already ship the enum baked into the package. + if not _in_site_packages(): + generate_targets_enum_file() + generate_targets_file(out_dir) + + # Copy the downloaded IR JSON into generated/ so BundledSchema can load + # schemas directly at runtime without a generated Python file. + ir_out_path = os.path.join(out_dir, "decodo.ir.json") + shutil.copy2(local_ir_path, ir_out_path) + print(f"Saved IR JSON to {ir_out_path}") if __name__ == "__main__": diff --git a/src/decodo/codegen/web_scraping_api/generate_parameter_schemas.py b/src/decodo/codegen/web_scraping_api/generate_parameter_schemas.py deleted file mode 100644 index 25e7913..0000000 --- a/src/decodo/codegen/web_scraping_api/generate_parameter_schemas.py +++ /dev/null @@ -1,44 +0,0 @@ -from __future__ import annotations - -import json -import os - -from .shared import ( - fetch_intermediate_representation, - local_ir_path, - out_dir, -) -from .types import WebScrapingApiIR - - -def _render_generated_module(api: WebScrapingApiIR) -> str: - lines: list[str] = [] - - lines.append("# Auto-generated by src/decodo/codegen/web_scraping_api/generate_parameter_schemas.py — do not edit") - lines.append("from __future__ import annotations") - lines.append("") - lines.append("request_json_schemas: dict[str, dict] = {") - - for target_key, target in api["targets"].items(): - # Use repr() so booleans/None are valid Python literals (True/False/None) - schema_repr = repr(target["parameter_schema"]) - lines.append(f" {json.dumps(target_key)}: {schema_repr},") - - lines.append("}") - lines.append("") - - return "\n".join(lines) - - -def generate_parameter_schemas_file() -> None: - ir = fetch_intermediate_representation() - api = ir["apis"]["webScrapingApi"] - - os.makedirs(out_dir, exist_ok=True) - out_path = os.path.join(out_dir, "request_schemas.py") - with open(out_path, "w", encoding="utf-8") as f: - f.write(_render_generated_module(api)) - - target_count = len(api["targets"]) - print(f"Generated request schemas from {local_ir_path}:") - print(f" {out_path} ({target_count} targets)") diff --git a/src/decodo/codegen/web_scraping_api/generate_parameters.py b/src/decodo/codegen/web_scraping_api/generate_parameters.py index 9307e07..3555c94 100644 --- a/src/decodo/codegen/web_scraping_api/generate_parameters.py +++ b/src/decodo/codegen/web_scraping_api/generate_parameters.py @@ -108,13 +108,14 @@ def get_parameters_file(api: WebScrapingApiIR) -> str: return "\n".join(lines) -def generate_parameters_file() -> None: +def generate_parameters_file(dest_dir: str | None = None) -> None: ir = fetch_intermediate_representation() api = ir["apis"]["webScrapingApi"] file_contents = get_parameters_file(api) - os.makedirs(out_dir, exist_ok=True) - out_path = os.path.join(out_dir, "parameters.py") + target = dest_dir if dest_dir is not None else out_dir + os.makedirs(target, exist_ok=True) + out_path = os.path.join(target, "parameters.py") with open(out_path, "w", encoding="utf-8") as f: f.write(file_contents) diff --git a/src/decodo/codegen/web_scraping_api/generate_targets.py b/src/decodo/codegen/web_scraping_api/generate_targets.py index b76168a..a64683f 100644 --- a/src/decodo/codegen/web_scraping_api/generate_targets.py +++ b/src/decodo/codegen/web_scraping_api/generate_targets.py @@ -8,6 +8,7 @@ fetch_intermediate_representation, local_ir_path, out_dir, + targets_enum_path, to_enum_member_name, to_pascal_case, ) @@ -50,21 +51,20 @@ def _get_target_parameter_keys(parameter_schema: dict[str, Any]) -> list[str]: return [k for k in properties if k != "target"] -def _get_targets_file_contents(api: WebScrapingApiIR) -> str: +def _get_targets_enum_file_contents(api: WebScrapingApiIR) -> str: lines: list[str] = [] - lines.append("# Auto-generated by src/decodo/codegen/web_scraping_api/generate_targets.py — do not edit") + lines.append("# Minimal target enum — committed to source.") + lines.append("# Run `python -m decodo.codegen.codegen` to regenerate when the IR schema changes.") lines.append("from __future__ import annotations") lines.append("") - lines.append("from enum import Enum") - lines.append("from typing import Annotated, Any, Literal, Union") + lines.append("from enum import StrEnum") lines.append("") - lines.append("import pydantic") lines.append("") target_keys = list(api["targets"].keys()) - lines.append("class Target(str, Enum):") + lines.append("class Target(StrEnum):") if target_keys: for key in target_keys: member = to_enum_member_name(key) @@ -72,27 +72,56 @@ def _get_targets_file_contents(api: WebScrapingApiIR) -> str: else: lines.append(" pass") lines.append("") + lines.append("") lines.append("targets: list[str] = [t.value for t in Target]") lines.append("") + return "\n".join(lines) + + +def _get_targets_file_contents(api: WebScrapingApiIR) -> str: + lines: list[str] = [] + + lines.append("# Auto-generated by src/decodo/codegen/web_scraping_api/generate_targets.py — do not edit") + lines.append("from __future__ import annotations") + lines.append("") + lines.append("from typing import Annotated, Any, Literal, Union") + lines.append("") + lines.append("import pydantic") + lines.append("") + lines.append("from decodo.targets import Target") + lines.append("") + + target_keys = list(api["targets"].keys()) + # Override class names for target keys that conflict with reserved names CLASS_NAME_OVERRIDES: dict[str, str] = { "target": "TargetStoreParams", } + BATCH_CLASS_NAME_OVERRIDES: dict[str, str] = { + "target": "TargetStoreBatchParams", + } + def _class_name(target_key: str) -> str: if target_key in CLASS_NAME_OVERRIDES: return CLASS_NAME_OVERRIDES[target_key] return f"{to_pascal_case(target_key)}Params" + def _batch_class_name(target_key: str) -> str: + if target_key in BATCH_CLASS_NAME_OVERRIDES: + return BATCH_CLASS_NAME_OVERRIDES[target_key] + return f"{to_pascal_case(target_key)}BatchParams" + + # Emit per-target sync Params classes for target_key, target in api["targets"].items(): type_name = _class_name(target_key) member = to_enum_member_name(target_key) properties: dict[str, Any] = target["parameter_schema"].get("properties", {}) lines.append(f"class {type_name}(pydantic.BaseModel):") - lines.append(" model_config = pydantic.ConfigDict(populate_by_name=True)") + lines.append(" model_config = pydantic.ConfigDict(populate_by_name=True, extra='forbid')") lines.append(f" target: Literal[Target.{member}] = Target.{member}") params = {k: v for k, v in properties.items() if k != "target"} if params: @@ -100,52 +129,95 @@ def _class_name(target_key: str) -> str: python_type = _json_schema_type_to_python(param_schema) field_name = _sanitize_field_name(param_key) if field_name != param_key: - lines.append(f" {field_name}: {python_type} | None = pydantic.Field(None, alias={json.dumps(param_key)})") # noqa: E501 + lines.append( + f" {field_name}: {python_type} | None = pydantic.Field(None, alias={json.dumps(param_key)})" + ) # noqa: E501 else: lines.append(f" {field_name}: {python_type} | None = None") lines.append("") - lines.append("class TargetMeta(pydantic.BaseModel):") - lines.append(" group: str") - lines.append(" response_format: str") - lines.append(" parameters: list[str]") - lines.append("") + # Emit per-target batch Params classes (url/query become list[str]) + for target_key, target in api["targets"].items(): + batch_type_name = _batch_class_name(target_key) + member = to_enum_member_name(target_key) + properties = target["parameter_schema"].get("properties", {}) + + lines.append(f"class {batch_type_name}(pydantic.BaseModel):") + lines.append(" model_config = pydantic.ConfigDict(populate_by_name=True, extra='forbid')") + lines.append(f" target: Literal[Target.{member}] = Target.{member}") + params = {k: v for k, v in properties.items() if k != "target"} + if params: + for param_key, param_schema in params.items(): + field_name = _sanitize_field_name(param_key) + if param_key in ("url", "query"): + python_type = "list[str]" + else: + python_type = _json_schema_type_to_python(param_schema) + if field_name != param_key: + lines.append( + f" {field_name}: {python_type} | None = pydantic.Field(None, alias={json.dumps(param_key)})" + ) # noqa: E501 + else: + lines.append(f" {field_name}: {python_type} | None = None") + lines.append("") - lines.append("target_meta: dict[str, TargetMeta] = {") + # target_meta as plain dict-of-dicts (no TargetMeta class here) + lines.append("target_meta: dict[str, dict[str, Any]] = {") for target_key, target in api["targets"].items(): member = to_enum_member_name(target_key) param_keys = _get_target_parameter_keys(target["parameter_schema"]) params_list = ", ".join(json.dumps(p) for p in param_keys) - lines.append(f" Target.{member}.value: TargetMeta(") - lines.append(f" group={json.dumps(target['group'])},") - lines.append(f" response_format={json.dumps(target['response_format'])},") - lines.append(f" parameters=[{params_list}],") - lines.append(" ),") + lines.append(f" Target.{member}.value: {{") + lines.append(f' "group": {json.dumps(target["group"])},') + lines.append(f' "response_format": {json.dumps(target["response_format"])},') + lines.append(f' "parameters": [{params_list}],') + lines.append(" },") lines.append("}") lines.append("") - # Discriminated union for ScrapeRequest / BatchRequest + # Discriminated union for ScrapeRequest union_parts = " | ".join(_class_name(k) for k in target_keys) lines.append("ScrapeRequest = Annotated[") lines.append(f" Union[{union_parts}],") lines.append(" pydantic.Field(discriminator='target'),") lines.append("]") - lines.append("BatchRequest = ScrapeRequest") + lines.append("") + + # Discriminated union for BatchRequest + batch_union_parts = " | ".join(_batch_class_name(k) for k in target_keys) + lines.append("BatchRequest = Annotated[") + lines.append(f" Union[{batch_union_parts}],") + lines.append(" pydantic.Field(discriminator='target'),") + lines.append("]") lines.append("") return "\n".join(lines) -def generate_targets_file() -> None: +def generate_targets_enum_file() -> None: + ir = fetch_intermediate_representation() + api = ir["apis"]["webScrapingApi"] + file_contents = _get_targets_enum_file_contents(api) + + with open(targets_enum_path, "w", encoding="utf-8") as f: + f.write(file_contents) + + target_count = len(api["targets"]) + print(f"Generated Target enum from {local_ir_path}:") + print(f" {targets_enum_path} ({target_count} targets)") + + +def generate_targets_file(dest_dir: str | None = None) -> None: ir = fetch_intermediate_representation() api = ir["apis"]["webScrapingApi"] file_contents = _get_targets_file_contents(api) - os.makedirs(out_dir, exist_ok=True) - out_path = os.path.join(out_dir, "targets.py") + target = dest_dir if dest_dir is not None else out_dir + os.makedirs(target, exist_ok=True) + out_path = os.path.join(target, "targets.py") with open(out_path, "w", encoding="utf-8") as f: f.write(file_contents) target_count = len(api["targets"]) - print(f"Generated targets from {local_ir_path}:") + print(f"Generated target Params from {local_ir_path}:") print(f" {out_path} ({target_count} targets)") diff --git a/src/decodo/codegen/web_scraping_api/shared.py b/src/decodo/codegen/web_scraping_api/shared.py index 20f3dfa..d0c27ff 100644 --- a/src/decodo/codegen/web_scraping_api/shared.py +++ b/src/decodo/codegen/web_scraping_api/shared.py @@ -17,6 +17,7 @@ local_ir_path = str((_THIS_DIR / "../../../.." / "inputs" / "decodo.ir.json").resolve()) out_dir = str((_THIS_DIR / "../../generated").resolve()) +targets_enum_path = str((_THIS_DIR / "../../targets.py").resolve()) def to_pascal_case(s: str) -> str: diff --git a/src/decodo/generated/parameters.py b/src/decodo/generated/parameters.py deleted file mode 100644 index 9f801e5..0000000 --- a/src/decodo/generated/parameters.py +++ /dev/null @@ -1,93 +0,0 @@ -# Auto-generated by src/decodo/codegen/web_scraping_api/generate_parameters.py — do not edit -from __future__ import annotations - -from typing import Any, TypedDict - - -class ParameterMeta(TypedDict, total=False): - type: str - max_length: int - minimum: float - maximum: float - enum: list[Any] - items: dict[str, Any] - - -parameter_meta: dict[str, ParameterMeta] = { - "callback_url": ParameterMeta(type="string"), - "query": ParameterMeta(type="string", max_length=2048), - "headless": ParameterMeta(type="string", enum=["html", "png"]), - "locale": ParameterMeta(type="string"), - "geo": ParameterMeta(type="string"), - "device_type": ParameterMeta(type="string"), - "page_from": ParameterMeta(type="number", minimum=0, maximum=100), - "google_results_language": ParameterMeta(type="string"), - "google_tbm": ParameterMeta(type="string"), - "google_tbs": ParameterMeta(type="string"), - "parse": ParameterMeta(type="boolean"), - "google_nfpr": ParameterMeta(type="boolean"), - "google_safe_search": ParameterMeta(type="boolean"), - "session_id": ParameterMeta(type="string"), - "xhr": ParameterMeta(type="boolean"), - "markdown": ParameterMeta(type="boolean"), - "page_count": ParameterMeta(type="number", minimum=1, maximum=10), - "date_range": ParameterMeta(type="string"), - "stars": ParameterMeta(type="number"), - "adults": ParameterMeta(type="number", minimum=1, maximum=3), - "children": ParameterMeta(type="number", minimum=1, maximum=3), - "search_type": ParameterMeta(type="string"), - "date_start": ParameterMeta(type="string"), - "date_end": ParameterMeta(type="string"), - "url": ParameterMeta(type="string", max_length=2056), - "hotel_occupancy": ParameterMeta(type="string"), - "domain": ParameterMeta(type="string"), - "language_code": ParameterMeta(type="string"), - "transcript_origin": ParameterMeta(type="string"), - "autoselect_variant": ParameterMeta(type="boolean"), - "currency": ParameterMeta(type="string"), - "category": ParameterMeta(type="string"), - "merchant": ParameterMeta(type="string"), - "sort_by": ParameterMeta(type="string"), - "parser_type": ParameterMeta(type="string"), - "product_id": ParameterMeta(type="string"), - "fulfillment_type": ParameterMeta(type="string"), - "walmart_store_id": ParameterMeta(type="string"), - "delivery_zip": ParameterMeta(type="string"), - "store_id": ParameterMeta(type="string"), - "delivery_type": ParameterMeta(type="string"), - "target_store_id": ParameterMeta(type="string"), - "lowes_store_id": ParameterMeta(type="string"), - "user_agent_type": ParameterMeta(type="string"), - "free_delivery": ParameterMeta(type="boolean"), - "pickup_today": ParameterMeta(type="boolean"), - "delivery_today_tomorrow": ParameterMeta(type="boolean"), - "payload": ParameterMeta(type="string"), - "proxy_pool": ParameterMeta(type="string", enum=["standard", "premium"]), - "http_method": ParameterMeta(type="string"), - "successful_status_codes": ParameterMeta(type="array", items={'type': "number"}), - "headers": ParameterMeta(type="object"), - "cookies": ParameterMeta(type="object"), - "force_headers": ParameterMeta(type="boolean"), - "force_cookies": ParameterMeta(type="boolean"), - "prompt": ParameterMeta(type="string", max_length=8192), - "search": ParameterMeta(type="boolean"), - "country": ParameterMeta(type="string"), - "sort": ParameterMeta(type="string"), - "360": ParameterMeta(type="boolean"), - "upload_date": ParameterMeta(type="string"), - "type": ParameterMeta(type="string"), - "duration": ParameterMeta(type="string"), - "video_sort_by": ParameterMeta(type="string"), - "3d": ParameterMeta(type="boolean"), - "4k": ParameterMeta(type="boolean"), - "creative_commons": ParameterMeta(type="boolean"), - "hd": ParameterMeta(type="boolean"), - "hdr": ParameterMeta(type="boolean"), - "vr180": ParameterMeta(type="boolean"), - "live": ParameterMeta(type="boolean"), - "location": ParameterMeta(type="boolean"), - "purchased": ParameterMeta(type="boolean"), - "subtitles": ParameterMeta(type="boolean"), - "subtitle_origin": ParameterMeta(type="string"), - "limit": ParameterMeta(type="number", minimum=0, maximum=100), -} diff --git a/src/decodo/generated/request_schemas.py b/src/decodo/generated/request_schemas.py deleted file mode 100644 index 72fe3ea..0000000 --- a/src/decodo/generated/request_schemas.py +++ /dev/null @@ -1,57 +0,0 @@ -# Auto-generated — do not edit -from __future__ import annotations - -request_json_schemas: dict[str, dict] = { - "universal_ecommerce": {'type': 'object', 'properties': {'target': {'const': 'universal_ecommerce'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "google_search": {'type': 'object', 'properties': {'target': {'const': 'google_search'}, 'query': {'type': 'string', 'maxLength': 2048}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'locale': {'type': 'string'}, 'geo': {'type': 'string'}, 'device_type': {'type': 'string'}, 'page_from': {'type': 'number', 'minimum': 0, 'maximum': 100}, 'google_results_language': {'type': 'string'}, 'google_tbm': {'type': 'string'}, 'google_tbs': {'type': 'string'}, 'parse': {'type': 'boolean'}, 'google_nfpr': {'type': 'boolean'}, 'google_safe_search': {'type': 'boolean'}, 'session_id': {'type': 'string'}, 'xhr': {'type': 'boolean'}, 'markdown': {'type': 'boolean'}, 'page_count': {'type': 'number', 'minimum': 1, 'maximum': 10}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "google_travel_hotels": {'type': 'object', 'properties': {'target': {'const': 'google_travel_hotels'}, 'query': {'type': 'string', 'maxLength': 2048}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'locale': {'type': 'string'}, 'device_type': {'type': 'string'}, 'page_from': {'type': 'number', 'minimum': 0, 'maximum': 100}, 'date_range': {'type': 'string'}, 'stars': {'type': 'number'}, 'adults': {'type': 'number', 'minimum': 1, 'maximum': 3}, 'children': {'type': 'number', 'minimum': 1, 'maximum': 3}, 'session_id': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "google_trends_explore": {'type': 'object', 'properties': {'target': {'const': 'google_trends_explore'}, 'query': {'type': 'string', 'maxLength': 2048}, 'geo': {'type': 'string'}, 'device_type': {'type': 'string'}, 'search_type': {'type': 'string'}, 'date_start': {'type': 'string'}, 'date_end': {'type': 'string'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "google_shopping_search": {'type': 'object', 'properties': {'target': {'const': 'google_shopping_search'}, 'query': {'type': 'string', 'maxLength': 2048}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'locale': {'type': 'string'}, 'geo': {'type': 'string'}, 'device_type': {'type': 'string'}, 'page_from': {'type': 'number', 'minimum': 0, 'maximum': 100}, 'google_tbs': {'type': 'string'}, 'parse': {'type': 'boolean'}, 'session_id': {'type': 'string'}, 'google_results_language': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "google_shopping_product": {'type': 'object', 'properties': {'target': {'const': 'google_shopping_product'}, 'query': {'type': 'string', 'maxLength': 2048}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'locale': {'type': 'string'}, 'geo': {'type': 'string'}, 'device_type': {'type': 'string'}, 'page_from': {'type': 'number', 'minimum': 0, 'maximum': 100}, 'parse': {'type': 'boolean'}, 'session_id': {'type': 'string'}, 'google_results_language': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'xhr': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "google": {'type': 'object', 'properties': {'target': {'const': 'google'}, 'url': {'type': 'string', 'maxLength': 2056}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'locale': {'type': 'string'}, 'device_type': {'type': 'string'}, 'parse': {'type': 'boolean'}, 'session_id': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'xhr': {'type': 'boolean'}, 'page_count': {'type': 'number', 'minimum': 1, 'maximum': 10}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "google_suggest": {'type': 'object', 'properties': {'target': {'const': 'google_suggest'}, 'query': {'type': 'string', 'maxLength': 2048}, 'device_type': {'type': 'string'}, 'geo': {'type': 'string'}, 'locale': {'type': 'string'}, 'session_id': {'type': 'string'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "google_maps": {'type': 'object', 'properties': {'target': {'const': 'google_maps'}, 'query': {'type': 'string', 'maxLength': 2048}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'geo': {'type': 'string'}, 'locale': {'type': 'string'}, 'page_from': {'type': 'number', 'minimum': 0, 'maximum': 100}, 'device_type': {'type': 'string'}, 'session_id': {'type': 'string'}, 'google_results_language': {'type': 'string'}, 'google_nfpr': {'type': 'boolean'}, 'hotel_occupancy': {'type': 'string'}, 'date_range': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "google_ai_mode": {'type': 'object', 'properties': {'target': {'const': 'google_ai_mode'}, 'query': {'type': 'string', 'maxLength': 2048}, 'geo': {'type': 'string'}, 'parse': {'type': 'boolean'}, 'device_type': {'type': 'string'}, 'session_id': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'xhr': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "google_ads": {'type': 'object', 'properties': {'target': {'const': 'google_ads'}, 'query': {'type': 'string', 'maxLength': 2048}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'locale': {'type': 'string'}, 'geo': {'type': 'string'}, 'device_type': {'type': 'string'}, 'page_from': {'type': 'number', 'minimum': 0, 'maximum': 100}, 'google_results_language': {'type': 'string'}, 'google_tbm': {'type': 'string'}, 'google_tbs': {'type': 'string'}, 'parse': {'type': 'boolean'}, 'google_nfpr': {'type': 'boolean'}, 'session_id': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'xhr': {'type': 'boolean'}, 'page_count': {'type': 'number', 'minimum': 1, 'maximum': 10}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "google_lens": {'type': 'object', 'properties': {'target': {'const': 'google_lens'}, 'query': {'type': 'string', 'maxLength': 2048}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'parse': {'type': 'boolean'}, 'device_type': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "bing_search": {'type': 'object', 'properties': {'target': {'const': 'bing_search'}, 'query': {'type': 'string', 'maxLength': 2048}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'locale': {'type': 'string'}, 'geo': {'type': 'string'}, 'domain': {'type': 'string'}, 'device_type': {'type': 'string'}, 'page_from': {'type': 'number', 'minimum': 0, 'maximum': 100}, 'parse': {'type': 'boolean'}, 'page_count': {'type': 'number', 'minimum': 1, 'maximum': 10}, 'session_id': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'xhr': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "bing": {'type': 'object', 'properties': {'target': {'const': 'bing'}, 'url': {'type': 'string', 'maxLength': 2056}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'locale': {'type': 'string'}, 'geo': {'type': 'string'}, 'device_type': {'type': 'string'}, 'page_from': {'type': 'number', 'minimum': 0, 'maximum': 100}, 'parse': {'type': 'boolean'}, 'session_id': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'xhr': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "youtube_transcript": {'type': 'object', 'properties': {'target': {'const': 'youtube_transcript'}, 'query': {'type': 'string', 'maxLength': 2048}, 'language_code': {'type': 'string'}, 'transcript_origin': {'type': 'string'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "amazon_product": {'type': 'object', 'properties': {'target': {'const': 'amazon_product'}, 'query': {'type': 'string', 'maxLength': 2048}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'domain': {'type': 'string'}, 'device_type': {'type': 'string'}, 'parse': {'type': 'boolean'}, 'autoselect_variant': {'type': 'boolean'}, 'geo': {'type': 'string'}, 'session_id': {'type': 'string'}, 'currency': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'xhr': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "amazon_pricing": {'type': 'object', 'properties': {'target': {'const': 'amazon_pricing'}, 'query': {'type': 'string', 'maxLength': 2048}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'domain': {'type': 'string'}, 'device_type': {'type': 'string'}, 'page_from': {'type': 'number', 'minimum': 0, 'maximum': 100}, 'parse': {'type': 'boolean'}, 'geo': {'type': 'string'}, 'session_id': {'type': 'string'}, 'currency': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'xhr': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "amazon_search": {'type': 'object', 'properties': {'target': {'const': 'amazon_search'}, 'query': {'type': 'string', 'maxLength': 2048}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'domain': {'type': 'string'}, 'device_type': {'type': 'string'}, 'page_from': {'type': 'number', 'minimum': 0, 'maximum': 100}, 'category': {'type': 'string'}, 'merchant': {'type': 'string'}, 'parse': {'type': 'boolean'}, 'geo': {'type': 'string'}, 'session_id': {'type': 'string'}, 'sort_by': {'type': 'string'}, 'currency': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'xhr': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "amazon_sellers": {'type': 'object', 'properties': {'target': {'const': 'amazon_sellers'}, 'query': {'type': 'string', 'maxLength': 2048}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'locale': {'type': 'string'}, 'domain': {'type': 'string'}, 'device_type': {'type': 'string'}, 'geo': {'type': 'string'}, 'parse': {'type': 'boolean'}, 'markdown': {'type': 'boolean'}, 'xhr': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "amazon_bestsellers": {'type': 'object', 'properties': {'target': {'const': 'amazon_bestsellers'}, 'query': {'type': 'string', 'maxLength': 2048}, 'domain': {'type': 'string'}, 'device_type': {'type': 'string'}, 'geo': {'type': 'string'}, 'page_from': {'type': 'number', 'minimum': 0, 'maximum': 100}, 'category': {'type': 'string'}, 'parse': {'type': 'boolean'}, 'session_id': {'type': 'string'}, 'currency': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'xhr': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "amazon": {'type': 'object', 'properties': {'target': {'const': 'amazon'}, 'url': {'type': 'string', 'maxLength': 2056}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'device_type': {'type': 'string'}, 'parse': {'type': 'boolean'}, 'geo': {'type': 'string'}, 'session_id': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'xhr': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "ecommerce": {'type': 'object', 'properties': {'target': {'const': 'ecommerce'}, 'url': {'type': 'string', 'maxLength': 2056}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'locale': {'type': 'string'}, 'geo': {'type': 'string'}, 'device_type': {'type': 'string'}, 'parse': {'type': 'boolean'}, 'parser_type': {'type': 'string'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "walmart_product": {'type': 'object', 'properties': {'target': {'const': 'walmart_product'}, 'product_id': {'type': 'string'}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'parse': {'type': 'boolean'}, 'xhr': {'type': 'boolean'}, 'markdown': {'type': 'boolean'}, 'fulfillment_type': {'type': 'string'}, 'walmart_store_id': {'type': 'string'}, 'delivery_zip': {'type': 'string'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "walmart_search": {'type': 'object', 'properties': {'target': {'const': 'walmart_search'}, 'query': {'type': 'string', 'maxLength': 2048}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'parse': {'type': 'boolean'}, 'markdown': {'type': 'boolean'}, 'fulfillment_type': {'type': 'string'}, 'walmart_store_id': {'type': 'string'}, 'delivery_zip': {'type': 'string'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "walmart": {'type': 'object', 'properties': {'target': {'const': 'walmart'}, 'url': {'type': 'string', 'maxLength': 2056}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'locale': {'type': 'string'}, 'geo': {'type': 'string'}, 'device_type': {'type': 'string'}, 'store_id': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'xhr': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "target_product": {'type': 'object', 'properties': {'target': {'const': 'target_product'}, 'product_id': {'type': 'string'}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'parse': {'type': 'boolean'}, 'device_type': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'xhr': {'type': 'boolean'}, 'delivery_type': {'type': 'string'}, 'target_store_id': {'type': 'string'}, 'delivery_zip': {'type': 'string'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "target_search": {'type': 'object', 'properties': {'target': {'const': 'target_search'}, 'query': {'type': 'string', 'maxLength': 2048}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'parse': {'type': 'boolean'}, 'device_type': {'type': 'string'}, 'delivery_type': {'type': 'string'}, 'target_store_id': {'type': 'string'}, 'delivery_zip': {'type': 'string'}, 'xhr': {'type': 'boolean'}, 'markdown': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "target": {'type': 'object', 'properties': {'target': {'const': 'target'}, 'url': {'type': 'string', 'maxLength': 2056}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'device_type': {'type': 'string'}, 'xhr': {'type': 'boolean'}, 'delivery_zip': {'type': 'string'}, 'target_store_id': {'type': 'string'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "lowes_search": {'type': 'object', 'properties': {'target': {'const': 'lowes_search'}, 'query': {'type': 'string', 'maxLength': 2048}, 'lowes_store_id': {'type': 'string'}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'delivery_zip': {'type': 'string'}, 'user_agent_type': {'type': 'string'}, 'free_delivery': {'type': 'boolean'}, 'pickup_today': {'type': 'boolean'}, 'delivery_today_tomorrow': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "universal": {'type': 'object', 'properties': {'target': {'const': 'universal'}, 'url': {'type': 'string', 'maxLength': 2056}, 'payload': {'type': 'string'}, 'proxy_pool': {'type': 'string', 'enum': ['standard', 'premium']}, 'http_method': {'type': 'string'}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'geo': {'type': 'string'}, 'locale': {'type': 'string'}, 'device_type': {'type': 'string'}, 'session_id': {'type': 'string'}, 'successful_status_codes': {'type': 'array', 'items': {'type': 'number'}}, 'headers': {'type': 'object'}, 'cookies': {'type': 'object'}, 'force_headers': {'type': 'boolean'}, 'force_cookies': {'type': 'boolean'}, 'xhr': {'type': 'boolean'}, 'markdown': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "chatgpt": {'type': 'object', 'properties': {'target': {'const': 'chatgpt'}, 'prompt': {'type': 'string', 'maxLength': 8192}, 'search': {'type': 'boolean'}, 'parse': {'type': 'boolean'}, 'geo': {'type': 'string'}, 'device_type': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'xhr': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "perplexity": {'type': 'object', 'properties': {'target': {'const': 'perplexity'}, 'prompt': {'type': 'string', 'maxLength': 8192}, 'parse': {'type': 'boolean'}, 'geo': {'type': 'string'}, 'device_type': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'xhr': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "gemini": {'type': 'object', 'properties': {'target': {'const': 'gemini'}, 'prompt': {'type': 'string', 'maxLength': 8192}, 'parse': {'type': 'boolean'}, 'geo': {'type': 'string'}, 'xhr': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "bbb": {'type': 'object', 'properties': {'target': {'const': 'bbb'}, 'url': {'type': 'string', 'maxLength': 2056}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'geo': {'type': 'string'}, 'device_type': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'xhr': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "autotrader": {'type': 'object', 'properties': {'target': {'const': 'autotrader'}, 'url': {'type': 'string', 'maxLength': 2056}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'geo': {'type': 'string'}, 'device_type': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'xhr': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "mobile": {'type': 'object', 'properties': {'target': {'const': 'mobile'}, 'url': {'type': 'string', 'maxLength': 2056}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'geo': {'type': 'string'}, 'device_type': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'xhr': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "airbnb": {'type': 'object', 'properties': {'target': {'const': 'airbnb'}, 'url': {'type': 'string', 'maxLength': 2056}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'geo': {'type': 'string'}, 'device_type': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'xhr': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "apple_app_store": {'type': 'object', 'properties': {'target': {'const': 'apple_app_store'}, 'url': {'type': 'string', 'maxLength': 2056}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'geo': {'type': 'string'}, 'device_type': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'xhr': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "instagram_graphql_profile": {'type': 'object', 'properties': {'target': {'const': 'instagram_graphql_profile'}, 'query': {'type': 'string', 'maxLength': 2048}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "tiktok_post": {'type': 'object', 'properties': {'target': {'const': 'tiktok_post'}, 'url': {'type': 'string', 'maxLength': 2056}, 'xhr': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "tiktok_shop_search": {'type': 'object', 'properties': {'target': {'const': 'tiktok_shop_search'}, 'query': {'type': 'string', 'maxLength': 2048}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'device_type': {'type': 'string'}, 'markdown': {'type': 'boolean'}, 'country': {'type': 'string'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "tiktok_shop_product": {'type': 'object', 'properties': {'target': {'const': 'tiktok_shop_product'}, 'product_id': {'type': 'string'}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'device_type': {'type': 'string'}, 'xhr': {'type': 'boolean'}, 'markdown': {'type': 'boolean'}, 'country': {'type': 'string'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "tiktok": {'type': 'object', 'properties': {'target': {'const': 'tiktok'}, 'url': {'type': 'string', 'maxLength': 2056}, 'headless': {'type': 'string', 'enum': ['html', 'png']}, 'user_agent_type': {'type': 'string'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "reddit_post": {'type': 'object', 'properties': {'target': {'const': 'reddit_post'}, 'url': {'type': 'string', 'maxLength': 2056}, 'locale': {'type': 'string'}, 'geo': {'type': 'string'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "reddit_subreddit": {'type': 'object', 'properties': {'target': {'const': 'reddit_subreddit'}, 'url': {'type': 'string', 'maxLength': 2056}, 'locale': {'type': 'string'}, 'geo': {'type': 'string'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "reddit_user": {'type': 'object', 'properties': {'target': {'const': 'reddit_user'}, 'url': {'type': 'string', 'maxLength': 2056}, 'locale': {'type': 'string'}, 'geo': {'type': 'string'}, 'sort': {'type': 'string'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "youtube_video": {'type': 'object', 'properties': {'target': {'const': 'youtube_video'}, 'query': {'type': 'string', 'maxLength': 2048}, 'geo': {'type': 'string'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "youtube_metadata": {'type': 'object', 'properties': {'target': {'const': 'youtube_metadata'}, 'query': {'type': 'string', 'maxLength': 2048}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "youtube_search": {'type': 'object', 'properties': {'360': {'type': 'boolean'}, 'target': {'const': 'youtube_search'}, 'query': {'type': 'string', 'maxLength': 2048}, 'upload_date': {'type': 'string'}, 'type': {'type': 'string'}, 'duration': {'type': 'string'}, 'video_sort_by': {'type': 'string'}, '3d': {'type': 'boolean'}, '4k': {'type': 'boolean'}, 'creative_commons': {'type': 'boolean'}, 'hd': {'type': 'boolean'}, 'hdr': {'type': 'boolean'}, 'vr180': {'type': 'boolean'}, 'live': {'type': 'boolean'}, 'location': {'type': 'boolean'}, 'purchased': {'type': 'boolean'}, 'subtitles': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "youtube_search_max": {'type': 'object', 'properties': {'360': {'type': 'boolean'}, 'target': {'const': 'youtube_search_max'}, 'query': {'type': 'string', 'maxLength': 2048}, 'upload_date': {'type': 'string'}, 'type': {'type': 'string'}, 'duration': {'type': 'string'}, 'video_sort_by': {'type': 'string'}, '3d': {'type': 'boolean'}, '4k': {'type': 'boolean'}, 'creative_commons': {'type': 'boolean'}, 'hd': {'type': 'boolean'}, 'hdr': {'type': 'boolean'}, 'vr180': {'type': 'boolean'}, 'live': {'type': 'boolean'}, 'location': {'type': 'boolean'}, 'purchased': {'type': 'boolean'}, 'subtitles': {'type': 'boolean'}, 'markdown': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "youtube_subtitles": {'type': 'object', 'properties': {'target': {'const': 'youtube_subtitles'}, 'query': {'type': 'string', 'maxLength': 2048}, 'language_code': {'type': 'string'}, 'subtitle_origin': {'type': 'string'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, - "youtube_channel": {'type': 'object', 'properties': {'target': {'const': 'youtube_channel'}, 'query': {'type': 'string', 'maxLength': 2048}, 'parse': {'type': 'boolean'}, 'limit': {'type': 'number', 'minimum': 0, 'maximum': 100}, 'markdown': {'type': 'boolean'}, 'callback_url': {'type': 'string'}}, 'required': ['target'], 'additionalProperties': False}, -} diff --git a/src/decodo/generated/targets.py b/src/decodo/generated/targets.py deleted file mode 100644 index 37efb44..0000000 --- a/src/decodo/generated/targets.py +++ /dev/null @@ -1,990 +0,0 @@ -# Auto-generated by src/decodo/codegen/web_scraping_api/generate_targets.py — do not edit -from __future__ import annotations - -from enum import Enum -from typing import Annotated, Any, Literal - -import pydantic - - -class Target(str, Enum): - UniversalEcommerce = "universal_ecommerce" - GoogleSearch = "google_search" - GoogleTravelHotels = "google_travel_hotels" - GoogleTrendsExplore = "google_trends_explore" - GoogleShoppingSearch = "google_shopping_search" - GoogleShoppingProduct = "google_shopping_product" - Google = "google" - GoogleSuggest = "google_suggest" - GoogleMaps = "google_maps" - GoogleAiMode = "google_ai_mode" - GoogleAds = "google_ads" - GoogleLens = "google_lens" - BingSearch = "bing_search" - Bing = "bing" - YoutubeTranscript = "youtube_transcript" - AmazonProduct = "amazon_product" - AmazonPricing = "amazon_pricing" - AmazonSearch = "amazon_search" - AmazonSellers = "amazon_sellers" - AmazonBestsellers = "amazon_bestsellers" - Amazon = "amazon" - Ecommerce = "ecommerce" - WalmartProduct = "walmart_product" - WalmartSearch = "walmart_search" - Walmart = "walmart" - TargetProduct = "target_product" - TargetSearch = "target_search" - Target = "target" - LowesSearch = "lowes_search" - Universal = "universal" - Chatgpt = "chatgpt" - Perplexity = "perplexity" - Gemini = "gemini" - Bbb = "bbb" - Autotrader = "autotrader" - Mobile = "mobile" - Airbnb = "airbnb" - AppleAppStore = "apple_app_store" - InstagramGraphqlProfile = "instagram_graphql_profile" - TiktokPost = "tiktok_post" - TiktokShopSearch = "tiktok_shop_search" - TiktokShopProduct = "tiktok_shop_product" - Tiktok = "tiktok" - RedditPost = "reddit_post" - RedditSubreddit = "reddit_subreddit" - RedditUser = "reddit_user" - YoutubeVideo = "youtube_video" - YoutubeMetadata = "youtube_metadata" - YoutubeSearch = "youtube_search" - YoutubeSearchMax = "youtube_search_max" - YoutubeSubtitles = "youtube_subtitles" - YoutubeChannel = "youtube_channel" - -targets: list[str] = [t.value for t in Target] - -class UniversalEcommerceParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.UniversalEcommerce] = Target.UniversalEcommerce - callback_url: str | None = None - -class GoogleSearchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.GoogleSearch] = Target.GoogleSearch - query: str | None = None - headless: Literal["html", "png"] | None = None - locale: str | None = None - geo: str | None = None - device_type: str | None = None - page_from: float | None = None - google_results_language: str | None = None - google_tbm: str | None = None - google_tbs: str | None = None - parse: bool | None = None - google_nfpr: bool | None = None - google_safe_search: bool | None = None - session_id: str | None = None - xhr: bool | None = None - markdown: bool | None = None - page_count: float | None = None - callback_url: str | None = None - -class GoogleTravelHotelsParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.GoogleTravelHotels] = Target.GoogleTravelHotels - query: str | None = None - headless: Literal["html", "png"] | None = None - locale: str | None = None - device_type: str | None = None - page_from: float | None = None - date_range: str | None = None - stars: float | None = None - adults: float | None = None - children: float | None = None - session_id: str | None = None - markdown: bool | None = None - callback_url: str | None = None - -class GoogleTrendsExploreParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.GoogleTrendsExplore] = Target.GoogleTrendsExplore - query: str | None = None - geo: str | None = None - device_type: str | None = None - search_type: str | None = None - date_start: str | None = None - date_end: str | None = None - callback_url: str | None = None - -class GoogleShoppingSearchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.GoogleShoppingSearch] = Target.GoogleShoppingSearch - query: str | None = None - headless: Literal["html", "png"] | None = None - locale: str | None = None - geo: str | None = None - device_type: str | None = None - page_from: float | None = None - google_tbs: str | None = None - parse: bool | None = None - session_id: str | None = None - google_results_language: str | None = None - markdown: bool | None = None - callback_url: str | None = None - -class GoogleShoppingProductParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.GoogleShoppingProduct] = Target.GoogleShoppingProduct - query: str | None = None - headless: Literal["html", "png"] | None = None - locale: str | None = None - geo: str | None = None - device_type: str | None = None - page_from: float | None = None - parse: bool | None = None - session_id: str | None = None - google_results_language: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - -class GoogleParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.Google] = Target.Google - url: str | None = None - headless: Literal["html", "png"] | None = None - locale: str | None = None - device_type: str | None = None - parse: bool | None = None - session_id: str | None = None - markdown: bool | None = None - xhr: bool | None = None - page_count: float | None = None - callback_url: str | None = None - -class GoogleSuggestParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.GoogleSuggest] = Target.GoogleSuggest - query: str | None = None - device_type: str | None = None - geo: str | None = None - locale: str | None = None - session_id: str | None = None - callback_url: str | None = None - -class GoogleMapsParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.GoogleMaps] = Target.GoogleMaps - query: str | None = None - headless: Literal["html", "png"] | None = None - geo: str | None = None - locale: str | None = None - page_from: float | None = None - device_type: str | None = None - session_id: str | None = None - google_results_language: str | None = None - google_nfpr: bool | None = None - hotel_occupancy: str | None = None - date_range: str | None = None - markdown: bool | None = None - callback_url: str | None = None - -class GoogleAiModeParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.GoogleAiMode] = Target.GoogleAiMode - query: str | None = None - geo: str | None = None - parse: bool | None = None - device_type: str | None = None - session_id: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - -class GoogleAdsParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.GoogleAds] = Target.GoogleAds - query: str | None = None - headless: Literal["html", "png"] | None = None - locale: str | None = None - geo: str | None = None - device_type: str | None = None - page_from: float | None = None - google_results_language: str | None = None - google_tbm: str | None = None - google_tbs: str | None = None - parse: bool | None = None - google_nfpr: bool | None = None - session_id: str | None = None - markdown: bool | None = None - xhr: bool | None = None - page_count: float | None = None - callback_url: str | None = None - -class GoogleLensParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.GoogleLens] = Target.GoogleLens - query: str | None = None - headless: Literal["html", "png"] | None = None - parse: bool | None = None - device_type: str | None = None - markdown: bool | None = None - callback_url: str | None = None - -class BingSearchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.BingSearch] = Target.BingSearch - query: str | None = None - headless: Literal["html", "png"] | None = None - locale: str | None = None - geo: str | None = None - domain: str | None = None - device_type: str | None = None - page_from: float | None = None - parse: bool | None = None - page_count: float | None = None - session_id: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - -class BingParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.Bing] = Target.Bing - url: str | None = None - headless: Literal["html", "png"] | None = None - locale: str | None = None - geo: str | None = None - device_type: str | None = None - page_from: float | None = None - parse: bool | None = None - session_id: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - -class YoutubeTranscriptParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.YoutubeTranscript] = Target.YoutubeTranscript - query: str | None = None - language_code: str | None = None - transcript_origin: str | None = None - callback_url: str | None = None - -class AmazonProductParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.AmazonProduct] = Target.AmazonProduct - query: str | None = None - headless: Literal["html", "png"] | None = None - domain: str | None = None - device_type: str | None = None - parse: bool | None = None - autoselect_variant: bool | None = None - geo: str | None = None - session_id: str | None = None - currency: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - -class AmazonPricingParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.AmazonPricing] = Target.AmazonPricing - query: str | None = None - headless: Literal["html", "png"] | None = None - domain: str | None = None - device_type: str | None = None - page_from: float | None = None - parse: bool | None = None - geo: str | None = None - session_id: str | None = None - currency: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - -class AmazonSearchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.AmazonSearch] = Target.AmazonSearch - query: str | None = None - headless: Literal["html", "png"] | None = None - domain: str | None = None - device_type: str | None = None - page_from: float | None = None - category: str | None = None - merchant: str | None = None - parse: bool | None = None - geo: str | None = None - session_id: str | None = None - sort_by: str | None = None - currency: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - -class AmazonSellersParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.AmazonSellers] = Target.AmazonSellers - query: str | None = None - headless: Literal["html", "png"] | None = None - locale: str | None = None - domain: str | None = None - device_type: str | None = None - geo: str | None = None - parse: bool | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - -class AmazonBestsellersParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.AmazonBestsellers] = Target.AmazonBestsellers - query: str | None = None - domain: str | None = None - device_type: str | None = None - geo: str | None = None - page_from: float | None = None - category: str | None = None - parse: bool | None = None - session_id: str | None = None - currency: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - -class AmazonParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.Amazon] = Target.Amazon - url: str | None = None - headless: Literal["html", "png"] | None = None - device_type: str | None = None - parse: bool | None = None - geo: str | None = None - session_id: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - -class EcommerceParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.Ecommerce] = Target.Ecommerce - url: str | None = None - headless: Literal["html", "png"] | None = None - locale: str | None = None - geo: str | None = None - device_type: str | None = None - parse: bool | None = None - parser_type: str | None = None - callback_url: str | None = None - -class WalmartProductParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.WalmartProduct] = Target.WalmartProduct - product_id: str | None = None - headless: Literal["html", "png"] | None = None - parse: bool | None = None - xhr: bool | None = None - markdown: bool | None = None - fulfillment_type: str | None = None - walmart_store_id: str | None = None - delivery_zip: str | None = None - callback_url: str | None = None - -class WalmartSearchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.WalmartSearch] = Target.WalmartSearch - query: str | None = None - headless: Literal["html", "png"] | None = None - parse: bool | None = None - markdown: bool | None = None - fulfillment_type: str | None = None - walmart_store_id: str | None = None - delivery_zip: str | None = None - callback_url: str | None = None - -class WalmartParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.Walmart] = Target.Walmart - url: str | None = None - headless: Literal["html", "png"] | None = None - locale: str | None = None - geo: str | None = None - device_type: str | None = None - store_id: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - -class TargetProductParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.TargetProduct] = Target.TargetProduct - product_id: str | None = None - headless: Literal["html", "png"] | None = None - parse: bool | None = None - device_type: str | None = None - markdown: bool | None = None - xhr: bool | None = None - delivery_type: str | None = None - target_store_id: str | None = None - delivery_zip: str | None = None - callback_url: str | None = None - -class TargetSearchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.TargetSearch] = Target.TargetSearch - query: str | None = None - headless: Literal["html", "png"] | None = None - parse: bool | None = None - device_type: str | None = None - delivery_type: str | None = None - target_store_id: str | None = None - delivery_zip: str | None = None - xhr: bool | None = None - markdown: bool | None = None - callback_url: str | None = None - -class TargetStoreParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.Target] = Target.Target - url: str | None = None - headless: Literal["html", "png"] | None = None - device_type: str | None = None - xhr: bool | None = None - delivery_zip: str | None = None - target_store_id: str | None = None - callback_url: str | None = None - -class LowesSearchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.LowesSearch] = Target.LowesSearch - query: str | None = None - lowes_store_id: str | None = None - headless: Literal["html", "png"] | None = None - delivery_zip: str | None = None - user_agent_type: str | None = None - free_delivery: bool | None = None - pickup_today: bool | None = None - delivery_today_tomorrow: bool | None = None - callback_url: str | None = None - -class UniversalParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.Universal] = Target.Universal - url: str | None = None - payload: str | None = None - proxy_pool: Literal["standard", "premium"] | None = None - http_method: str | None = None - headless: Literal["html", "png"] | None = None - geo: str | None = None - locale: str | None = None - device_type: str | None = None - session_id: str | None = None - successful_status_codes: list[Any] | None = None - headers: Any | None = None - cookies: Any | None = None - force_headers: bool | None = None - force_cookies: bool | None = None - xhr: bool | None = None - markdown: bool | None = None - callback_url: str | None = None - -class ChatgptParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.Chatgpt] = Target.Chatgpt - prompt: str | None = None - search: bool | None = None - parse: bool | None = None - geo: str | None = None - device_type: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - -class PerplexityParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.Perplexity] = Target.Perplexity - prompt: str | None = None - parse: bool | None = None - geo: str | None = None - device_type: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - -class GeminiParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.Gemini] = Target.Gemini - prompt: str | None = None - parse: bool | None = None - geo: str | None = None - xhr: bool | None = None - callback_url: str | None = None - -class BbbParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.Bbb] = Target.Bbb - url: str | None = None - headless: Literal["html", "png"] | None = None - geo: str | None = None - device_type: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - -class AutotraderParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.Autotrader] = Target.Autotrader - url: str | None = None - headless: Literal["html", "png"] | None = None - geo: str | None = None - device_type: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - -class MobileParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.Mobile] = Target.Mobile - url: str | None = None - headless: Literal["html", "png"] | None = None - geo: str | None = None - device_type: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - -class AirbnbParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.Airbnb] = Target.Airbnb - url: str | None = None - headless: Literal["html", "png"] | None = None - geo: str | None = None - device_type: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - -class AppleAppStoreParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.AppleAppStore] = Target.AppleAppStore - url: str | None = None - headless: Literal["html", "png"] | None = None - geo: str | None = None - device_type: str | None = None - markdown: bool | None = None - xhr: bool | None = None - callback_url: str | None = None - -class InstagramGraphqlProfileParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.InstagramGraphqlProfile] = Target.InstagramGraphqlProfile - query: str | None = None - callback_url: str | None = None - -class TiktokPostParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.TiktokPost] = Target.TiktokPost - url: str | None = None - xhr: bool | None = None - callback_url: str | None = None - -class TiktokShopSearchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.TiktokShopSearch] = Target.TiktokShopSearch - query: str | None = None - headless: Literal["html", "png"] | None = None - device_type: str | None = None - markdown: bool | None = None - country: str | None = None - callback_url: str | None = None - -class TiktokShopProductParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.TiktokShopProduct] = Target.TiktokShopProduct - product_id: str | None = None - headless: Literal["html", "png"] | None = None - device_type: str | None = None - xhr: bool | None = None - markdown: bool | None = None - country: str | None = None - callback_url: str | None = None - -class TiktokParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.Tiktok] = Target.Tiktok - url: str | None = None - headless: Literal["html", "png"] | None = None - user_agent_type: str | None = None - callback_url: str | None = None - -class RedditPostParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.RedditPost] = Target.RedditPost - url: str | None = None - locale: str | None = None - geo: str | None = None - callback_url: str | None = None - -class RedditSubredditParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.RedditSubreddit] = Target.RedditSubreddit - url: str | None = None - locale: str | None = None - geo: str | None = None - callback_url: str | None = None - -class RedditUserParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.RedditUser] = Target.RedditUser - url: str | None = None - locale: str | None = None - geo: str | None = None - sort: str | None = None - callback_url: str | None = None - -class YoutubeVideoParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.YoutubeVideo] = Target.YoutubeVideo - query: str | None = None - geo: str | None = None - callback_url: str | None = None - -class YoutubeMetadataParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.YoutubeMetadata] = Target.YoutubeMetadata - query: str | None = None - callback_url: str | None = None - -class YoutubeSearchParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.YoutubeSearch] = Target.YoutubeSearch - f360: bool | None = pydantic.Field(None, alias="360") - query: str | None = None - upload_date: str | None = None - type: str | None = None - duration: str | None = None - video_sort_by: str | None = None - f3d: bool | None = pydantic.Field(None, alias="3d") - f4k: bool | None = pydantic.Field(None, alias="4k") - creative_commons: bool | None = None - hd: bool | None = None - hdr: bool | None = None - vr180: bool | None = None - live: bool | None = None - location: bool | None = None - purchased: bool | None = None - subtitles: bool | None = None - callback_url: str | None = None - -class YoutubeSearchMaxParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.YoutubeSearchMax] = Target.YoutubeSearchMax - f360: bool | None = pydantic.Field(None, alias="360") - query: str | None = None - upload_date: str | None = None - type: str | None = None - duration: str | None = None - video_sort_by: str | None = None - f3d: bool | None = pydantic.Field(None, alias="3d") - f4k: bool | None = pydantic.Field(None, alias="4k") - creative_commons: bool | None = None - hd: bool | None = None - hdr: bool | None = None - vr180: bool | None = None - live: bool | None = None - location: bool | None = None - purchased: bool | None = None - subtitles: bool | None = None - markdown: bool | None = None - callback_url: str | None = None - -class YoutubeSubtitlesParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.YoutubeSubtitles] = Target.YoutubeSubtitles - query: str | None = None - language_code: str | None = None - subtitle_origin: str | None = None - callback_url: str | None = None - -class YoutubeChannelParams(pydantic.BaseModel): - model_config = pydantic.ConfigDict(populate_by_name=True) - target: Literal[Target.YoutubeChannel] = Target.YoutubeChannel - query: str | None = None - parse: bool | None = None - limit: float | None = None - markdown: bool | None = None - callback_url: str | None = None - -class TargetMeta(pydantic.BaseModel): - group: str - response_format: str - parameters: list[str] - -target_meta: dict[str, TargetMeta] = { - Target.UniversalEcommerce.value: TargetMeta( - group="None", - response_format="html", - parameters=["callback_url"], - ), - Target.GoogleSearch.value: TargetMeta( - group="Google", - response_format="json", - parameters=["query", "headless", "locale", "geo", "device_type", "page_from", "google_results_language", "google_tbm", "google_tbs", "parse", "google_nfpr", "google_safe_search", "session_id", "xhr", "markdown", "page_count", "callback_url"], - ), - Target.GoogleTravelHotels.value: TargetMeta( - group="Google", - response_format="html", - parameters=["query", "headless", "locale", "device_type", "page_from", "date_range", "stars", "adults", "children", "session_id", "markdown", "callback_url"], - ), - Target.GoogleTrendsExplore.value: TargetMeta( - group="Google", - response_format="json", - parameters=["query", "geo", "device_type", "search_type", "date_start", "date_end", "callback_url"], - ), - Target.GoogleShoppingSearch.value: TargetMeta( - group="Google", - response_format="json", - parameters=["query", "headless", "locale", "geo", "device_type", "page_from", "google_tbs", "parse", "session_id", "google_results_language", "markdown", "callback_url"], - ), - Target.GoogleShoppingProduct.value: TargetMeta( - group="Google", - response_format="json", - parameters=["query", "headless", "locale", "geo", "device_type", "page_from", "parse", "session_id", "google_results_language", "markdown", "xhr", "callback_url"], - ), - Target.Google.value: TargetMeta( - group="Google", - response_format="json", - parameters=["url", "headless", "locale", "device_type", "parse", "session_id", "markdown", "xhr", "page_count", "callback_url"], - ), - Target.GoogleSuggest.value: TargetMeta( - group="Google", - response_format="json", - parameters=["query", "device_type", "geo", "locale", "session_id", "callback_url"], - ), - Target.GoogleMaps.value: TargetMeta( - group="Google", - response_format="html", - parameters=["query", "headless", "geo", "locale", "page_from", "device_type", "session_id", "google_results_language", "google_nfpr", "hotel_occupancy", "date_range", "markdown", "callback_url"], - ), - Target.GoogleAiMode.value: TargetMeta( - group="AI Tools", - response_format="json", - parameters=["query", "geo", "parse", "device_type", "session_id", "markdown", "xhr", "callback_url"], - ), - Target.GoogleAds.value: TargetMeta( - group="Google", - response_format="json", - parameters=["query", "headless", "locale", "geo", "device_type", "page_from", "google_results_language", "google_tbm", "google_tbs", "parse", "google_nfpr", "session_id", "markdown", "xhr", "page_count", "callback_url"], - ), - Target.GoogleLens.value: TargetMeta( - group="Google", - response_format="json", - parameters=["query", "headless", "parse", "device_type", "markdown", "callback_url"], - ), - Target.BingSearch.value: TargetMeta( - group="Bing", - response_format="json", - parameters=["query", "headless", "locale", "geo", "domain", "device_type", "page_from", "parse", "page_count", "session_id", "markdown", "xhr", "callback_url"], - ), - Target.Bing.value: TargetMeta( - group="Bing", - response_format="json", - parameters=["url", "headless", "locale", "geo", "device_type", "page_from", "parse", "session_id", "markdown", "xhr", "callback_url"], - ), - Target.YoutubeTranscript.value: TargetMeta( - group="YouTube", - response_format="json", - parameters=["query", "language_code", "transcript_origin", "callback_url"], - ), - Target.AmazonProduct.value: TargetMeta( - group="Amazon", - response_format="json", - parameters=["query", "headless", "domain", "device_type", "parse", "autoselect_variant", "geo", "session_id", "currency", "markdown", "xhr", "callback_url"], - ), - Target.AmazonPricing.value: TargetMeta( - group="Amazon", - response_format="json", - parameters=["query", "headless", "domain", "device_type", "page_from", "parse", "geo", "session_id", "currency", "markdown", "xhr", "callback_url"], - ), - Target.AmazonSearch.value: TargetMeta( - group="Amazon", - response_format="json", - parameters=["query", "headless", "domain", "device_type", "page_from", "category", "merchant", "parse", "geo", "session_id", "sort_by", "currency", "markdown", "xhr", "callback_url"], - ), - Target.AmazonSellers.value: TargetMeta( - group="Amazon", - response_format="json", - parameters=["query", "headless", "locale", "domain", "device_type", "geo", "parse", "markdown", "xhr", "callback_url"], - ), - Target.AmazonBestsellers.value: TargetMeta( - group="Amazon", - response_format="json", - parameters=["query", "domain", "device_type", "geo", "page_from", "category", "parse", "session_id", "currency", "markdown", "xhr", "callback_url"], - ), - Target.Amazon.value: TargetMeta( - group="Amazon", - response_format="json", - parameters=["url", "headless", "device_type", "parse", "geo", "session_id", "markdown", "xhr", "callback_url"], - ), - Target.Ecommerce.value: TargetMeta( - group="Other eCommerce", - response_format="json", - parameters=["url", "headless", "locale", "geo", "device_type", "parse", "parser_type", "callback_url"], - ), - Target.WalmartProduct.value: TargetMeta( - group="Walmart", - response_format="html", - parameters=["product_id", "headless", "parse", "xhr", "markdown", "fulfillment_type", "walmart_store_id", "delivery_zip", "callback_url"], - ), - Target.WalmartSearch.value: TargetMeta( - group="Walmart", - response_format="json", - parameters=["query", "headless", "parse", "markdown", "fulfillment_type", "walmart_store_id", "delivery_zip", "callback_url"], - ), - Target.Walmart.value: TargetMeta( - group="Walmart", - response_format="html", - parameters=["url", "headless", "locale", "geo", "device_type", "store_id", "markdown", "xhr", "callback_url"], - ), - Target.TargetProduct.value: TargetMeta( - group="Target", - response_format="json", - parameters=["product_id", "headless", "parse", "device_type", "markdown", "xhr", "delivery_type", "target_store_id", "delivery_zip", "callback_url"], - ), - Target.TargetSearch.value: TargetMeta( - group="Target", - response_format="json", - parameters=["query", "headless", "parse", "device_type", "delivery_type", "target_store_id", "delivery_zip", "xhr", "markdown", "callback_url"], - ), - Target.Target.value: TargetMeta( - group="Target", - response_format="html", - parameters=["url", "headless", "device_type", "xhr", "delivery_zip", "target_store_id", "callback_url"], - ), - Target.LowesSearch.value: TargetMeta( - group="Lowe's", - response_format="json", - parameters=["query", "lowes_store_id", "headless", "delivery_zip", "user_agent_type", "free_delivery", "pickup_today", "delivery_today_tomorrow", "callback_url"], - ), - Target.Universal.value: TargetMeta( - group="Universal", - response_format="html", - parameters=["url", "payload", "proxy_pool", "http_method", "headless", "geo", "locale", "device_type", "session_id", "successful_status_codes", "headers", "cookies", "force_headers", "force_cookies", "xhr", "markdown", "callback_url"], - ), - Target.Chatgpt.value: TargetMeta( - group="AI Tools", - response_format="json", - parameters=["prompt", "search", "parse", "geo", "device_type", "markdown", "xhr", "callback_url"], - ), - Target.Perplexity.value: TargetMeta( - group="AI Tools", - response_format="json", - parameters=["prompt", "parse", "geo", "device_type", "markdown", "xhr", "callback_url"], - ), - Target.Gemini.value: TargetMeta( - group="AI Tools", - response_format="json", - parameters=["prompt", "parse", "geo", "xhr", "callback_url"], - ), - Target.Bbb.value: TargetMeta( - group="Business Reviews", - response_format="html", - parameters=["url", "headless", "geo", "device_type", "markdown", "xhr", "callback_url"], - ), - Target.Autotrader.value: TargetMeta( - group="Marketplace", - response_format="html", - parameters=["url", "headless", "geo", "device_type", "markdown", "xhr", "callback_url"], - ), - Target.Mobile.value: TargetMeta( - group="Marketplace", - response_format="html", - parameters=["url", "headless", "geo", "device_type", "markdown", "xhr", "callback_url"], - ), - Target.Airbnb.value: TargetMeta( - group="Travel", - response_format="html", - parameters=["url", "headless", "geo", "device_type", "markdown", "xhr", "callback_url"], - ), - Target.AppleAppStore.value: TargetMeta( - group="Marketplace", - response_format="html", - parameters=["url", "headless", "geo", "device_type", "markdown", "xhr", "callback_url"], - ), - Target.InstagramGraphqlProfile.value: TargetMeta( - group="Instagram", - response_format="json", - parameters=["query", "callback_url"], - ), - Target.TiktokPost.value: TargetMeta( - group="TikTok", - response_format="html", - parameters=["url", "xhr", "callback_url"], - ), - Target.TiktokShopSearch.value: TargetMeta( - group="TikTok", - response_format="html", - parameters=["query", "headless", "device_type", "markdown", "country", "callback_url"], - ), - Target.TiktokShopProduct.value: TargetMeta( - group="TikTok", - response_format="html", - parameters=["product_id", "headless", "device_type", "xhr", "markdown", "country", "callback_url"], - ), - Target.Tiktok.value: TargetMeta( - group="TikTok", - response_format="html", - parameters=["url", "headless", "user_agent_type", "callback_url"], - ), - Target.RedditPost.value: TargetMeta( - group="Reddit", - response_format="json", - parameters=["url", "locale", "geo", "callback_url"], - ), - Target.RedditSubreddit.value: TargetMeta( - group="Reddit", - response_format="json", - parameters=["url", "locale", "geo", "callback_url"], - ), - Target.RedditUser.value: TargetMeta( - group="Reddit", - response_format="json", - parameters=["url", "locale", "geo", "sort", "callback_url"], - ), - Target.YoutubeVideo.value: TargetMeta( - group="None", - response_format="json", - parameters=["query", "geo", "callback_url"], - ), - Target.YoutubeMetadata.value: TargetMeta( - group="YouTube", - response_format="json", - parameters=["query", "callback_url"], - ), - Target.YoutubeSearch.value: TargetMeta( - group="YouTube", - response_format="json", - parameters=["360", "query", "upload_date", "type", "duration", "video_sort_by", "3d", "4k", "creative_commons", "hd", "hdr", "vr180", "live", "location", "purchased", "subtitles", "callback_url"], - ), - Target.YoutubeSearchMax.value: TargetMeta( - group="YouTube", - response_format="json", - parameters=["360", "query", "upload_date", "type", "duration", "video_sort_by", "3d", "4k", "creative_commons", "hd", "hdr", "vr180", "live", "location", "purchased", "subtitles", "markdown", "callback_url"], - ), - Target.YoutubeSubtitles.value: TargetMeta( - group="YouTube", - response_format="json", - parameters=["query", "language_code", "subtitle_origin", "callback_url"], - ), - Target.YoutubeChannel.value: TargetMeta( - group="YouTube", - response_format="json", - parameters=["query", "parse", "limit", "markdown", "callback_url"], - ), -} - -ScrapeRequest = Annotated[ - UniversalEcommerceParams | GoogleSearchParams | GoogleTravelHotelsParams | GoogleTrendsExploreParams | GoogleShoppingSearchParams | GoogleShoppingProductParams | GoogleParams | GoogleSuggestParams | GoogleMapsParams | GoogleAiModeParams | GoogleAdsParams | GoogleLensParams | BingSearchParams | BingParams | YoutubeTranscriptParams | AmazonProductParams | AmazonPricingParams | AmazonSearchParams | AmazonSellersParams | AmazonBestsellersParams | AmazonParams | EcommerceParams | WalmartProductParams | WalmartSearchParams | WalmartParams | TargetProductParams | TargetSearchParams | TargetStoreParams | LowesSearchParams | UniversalParams | ChatgptParams | PerplexityParams | GeminiParams | BbbParams | AutotraderParams | MobileParams | AirbnbParams | AppleAppStoreParams | InstagramGraphqlProfileParams | TiktokPostParams | TiktokShopSearchParams | TiktokShopProductParams | TiktokParams | RedditPostParams | RedditSubredditParams | RedditUserParams | YoutubeVideoParams | YoutubeMetadataParams | YoutubeSearchParams | YoutubeSearchMaxParams | YoutubeSubtitlesParams | YoutubeChannelParams, - pydantic.Field(discriminator='target'), -] -BatchRequest = ScrapeRequest diff --git a/src/decodo/generated/__init__.py b/src/decodo/py.typed similarity index 100% rename from src/decodo/generated/__init__.py rename to src/decodo/py.typed diff --git a/src/decodo/schema/bundled_schema.py b/src/decodo/schema/bundled_schema.py index b139239..368f455 100644 --- a/src/decodo/schema/bundled_schema.py +++ b/src/decodo/schema/bundled_schema.py @@ -1,27 +1,87 @@ from __future__ import annotations +import json +import os +import warnings +from pathlib import Path from typing import Any, ClassVar, cast -from decodo.generated.request_schemas import request_json_schemas -from decodo.generated.targets import target_meta, targets +from decodo.targets import targets +from .build_target_meta import build_target_meta from .types import DecodoSchema, TargetMeta +def _load_target_meta() -> dict[str, Any] | None: + try: + from decodo.generated.targets import target_meta # noqa: PLC0415 + return cast(dict[str, Any], target_meta) + except ImportError: + return None + +_target_meta = _load_target_meta() + +_BUNDLED_IR_PATH = Path(__file__).parent.parent / "generated" / "decodo.ir.json" + + +def _find_ir_path() -> Path | None: + env = os.environ.get("DECODO_IR_PATH") + if env: + p = Path(env) + if p.is_file(): + return p + local = Path.cwd() / "decodo_generated" / "decodo.ir.json" + if local.is_file(): + return local + if _BUNDLED_IR_PATH.is_file(): + return _BUNDLED_IR_PATH + return None + + +def _load_ir_json() -> dict[str, Any] | None: + path = _find_ir_path() + if path is None: + return None + try: + with open(path, encoding="utf-8") as f: + return json.load(f) # type: ignore[no-any-return] + except (OSError, json.JSONDecodeError): + return None + + class BundledSchema: shared: ClassVar[BundledSchema] + def __init__(self) -> None: + ir = _load_ir_json() + if ir is not None: + api_targets = ir["apis"]["webScrapingApi"]["targets"] + self._request_schemas: dict[str, dict[str, Any]] = { + k: v["parameter_schema"] for k, v in api_targets.items() + } + self._target_meta = _target_meta if _target_meta is not None else build_target_meta(api_targets) + else: + warnings.warn( + "Decodo IR schema not found — payload validation is disabled. " + "Run: python -m decodo.codegen.codegen (editable install) or " + "python -m decodo.codegen.codegen --out-dir ./decodo_generated (pip install).", + RuntimeWarning, + stacklevel=2, + ) + self._request_schemas = {} + self._target_meta = _target_meta or {} + def get_request_schema(self, target: str) -> dict[str, Any] | None: - return request_json_schemas.get(target) + return self._request_schemas.get(target) def list_targets(self) -> list[str]: return list(targets) def get_target_meta(self, target: str) -> TargetMeta | None: - return cast(TargetMeta, target_meta.get(target)) + return cast(TargetMeta, self._target_meta.get(target)) def get_target_parameter_schema(self, target: str) -> dict[str, Any] | None: - return request_json_schemas.get(target) + return self._request_schemas.get(target) def get_shared_parameters(self) -> dict[str, Any]: return {} diff --git a/src/decodo/targets.py b/src/decodo/targets.py new file mode 100644 index 0000000..75412e4 --- /dev/null +++ b/src/decodo/targets.py @@ -0,0 +1,63 @@ +# Minimal target enum — committed to source. +# Run `python -m decodo.codegen.codegen` to regenerate when the IR schema changes. +from __future__ import annotations + +from enum import StrEnum + + +class Target(StrEnum): + UniversalEcommerce = "universal_ecommerce" + GoogleSearch = "google_search" + GoogleTravelHotels = "google_travel_hotels" + GoogleTrendsExplore = "google_trends_explore" + GoogleShoppingSearch = "google_shopping_search" + GoogleShoppingProduct = "google_shopping_product" + Google = "google" + GoogleSuggest = "google_suggest" + GoogleMaps = "google_maps" + GoogleAiMode = "google_ai_mode" + GoogleAds = "google_ads" + GoogleLens = "google_lens" + BingSearch = "bing_search" + Bing = "bing" + YoutubeTranscript = "youtube_transcript" + AmazonProduct = "amazon_product" + AmazonPricing = "amazon_pricing" + AmazonSearch = "amazon_search" + AmazonSellers = "amazon_sellers" + AmazonBestsellers = "amazon_bestsellers" + Amazon = "amazon" + Ecommerce = "ecommerce" + WalmartProduct = "walmart_product" + WalmartSearch = "walmart_search" + Walmart = "walmart" + TargetProduct = "target_product" + TargetSearch = "target_search" + Target = "target" + LowesSearch = "lowes_search" + Universal = "universal" + Chatgpt = "chatgpt" + Perplexity = "perplexity" + Gemini = "gemini" + Bbb = "bbb" + Autotrader = "autotrader" + Mobile = "mobile" + Airbnb = "airbnb" + AppleAppStore = "apple_app_store" + InstagramGraphqlProfile = "instagram_graphql_profile" + TiktokPost = "tiktok_post" + TiktokShopSearch = "tiktok_shop_search" + TiktokShopProduct = "tiktok_shop_product" + Tiktok = "tiktok" + RedditPost = "reddit_post" + RedditSubreddit = "reddit_subreddit" + RedditUser = "reddit_user" + YoutubeVideo = "youtube_video" + YoutubeMetadata = "youtube_metadata" + YoutubeSearch = "youtube_search" + YoutubeSearchMax = "youtube_search_max" + YoutubeSubtitles = "youtube_subtitles" + YoutubeChannel = "youtube_channel" + + +targets: list[str] = [t.value for t in Target] diff --git a/src/decodo/types/__init__.py b/src/decodo/types/__init__.py index 3d568bb..4991a2c 100644 --- a/src/decodo/types/__init__.py +++ b/src/decodo/types/__init__.py @@ -1,4 +1,7 @@ -from .requests import BatchRequest, ScrapeRequest +try: + from .requests import BatchRequest, ScrapeRequest +except ImportError: + pass from .responses import ( AsyncTaskResponse, BatchResponse, diff --git a/src/decodo/types/requests.py b/src/decodo/types/requests.py index f5fcccd..2ab1cc6 100644 --- a/src/decodo/types/requests.py +++ b/src/decodo/types/requests.py @@ -1,5 +1,7 @@ from __future__ import annotations -from decodo.generated.targets import BatchRequest, ScrapeRequest - -__all__ = ["ScrapeRequest", "BatchRequest"] +try: + from decodo.generated.targets import BatchRequest, ScrapeRequest + __all__ = ["ScrapeRequest", "BatchRequest"] +except ImportError: + __all__ = [] diff --git a/tests/conftest.py b/tests/conftest.py index d704d57..c553613 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,9 +6,17 @@ import pytest +import decodo.schema.bundled_schema as _bundled_schema_mod + MINIMAL_IR_PATH = Path(__file__).parent / "schema" / "fixtures" / "minimal_ir.json" +@pytest.fixture(autouse=True, scope="session") +def _patch_bundled_schema_ir_path() -> None: + _bundled_schema_mod._BUNDLED_IR_PATH = MINIMAL_IR_PATH + _bundled_schema_mod.BundledSchema.shared = _bundled_schema_mod.BundledSchema() + + @pytest.fixture def minimal_ir() -> dict[str, Any]: with open(MINIMAL_IR_PATH, encoding="utf-8") as f: diff --git a/tests/test_web_scraping_api.py b/tests/test_web_scraping_api.py index 6b03139..2e5832a 100644 --- a/tests/test_web_scraping_api.py +++ b/tests/test_web_scraping_api.py @@ -53,7 +53,7 @@ def test_throws_validation_error_before_http_when_schema_rejects(self) -> None: api = WebScrapingApi(http, _StrictSchema()) with pytest.raises(ValidationError): - api.scrape({"target": "google_search", "query": ""}) # type: ignore[arg-type] + api.scrape({"target": "google_search", "query": ""}) http.post.assert_not_called() @@ -61,7 +61,7 @@ def test_calls_http_when_params_pass_schema_validation(self) -> None: http = _make_http_mock() api = WebScrapingApi(http, _StrictSchema()) - api.scrape({"target": "google_search", "query": "coffee"}) # type: ignore[arg-type] + api.scrape({"target": "google_search", "query": "coffee"}) http.post.assert_called_once() @@ -69,6 +69,6 @@ def test_validates_bundled_google_search_payloads(self) -> None: http = _make_http_mock() api = WebScrapingApi(http, BundledSchema.shared) - api.scrape({"target": "google_search", "query": "coffee"}) # type: ignore[arg-type] + api.scrape({"target": "google_search", "query": "coffee"}) http.post.assert_called_once()