diff --git a/python/lib/sift_client/_internal/grpc_transport/transport.py b/python/lib/sift_client/_internal/grpc_transport/transport.py index a04dc68eb..9df13825a 100644 --- a/python/lib/sift_client/_internal/grpc_transport/transport.py +++ b/python/lib/sift_client/_internal/grpc_transport/transport.py @@ -165,6 +165,10 @@ def _compute_channel_options(opts: SiftChannelConfig) -> list[tuple[str, Any]]: # Primary cannot be overriden: # https://github.com/grpc/grpc/blob/0498194240f55d7f4b12633ad01339fb690621bf/src/core/ext/filters/http/client/http_client_filter.cc#L97 ("grpc.secondary_user_agent", _compute_user_agent()), + ( + "grpc.max_receive_message_length", + 50 * 1024 * 1024, + ), # 50 MB — headroom for 1M-row GetData pages ] enable_keepalive = opts.get("enable_keepalive", True) diff --git a/python/lib/sift_client/resources/channels.py b/python/lib/sift_client/resources/channels.py index f5cbc09d7..1d8c8d81f 100644 --- a/python/lib/sift_client/resources/channels.py +++ b/python/lib/sift_client/resources/channels.py @@ -265,6 +265,7 @@ async def get_data( start_time: datetime | None = None, end_time: datetime | None = None, limit: int | None = None, + page_size: int | None = None, ignore_cache: bool = False, show_progress: bool | None = None, ) -> dict[str, pd.DataFrame]: @@ -276,6 +277,9 @@ async def get_data( start_time: The start time to get data for. end_time: The end time to get data for. limit: The maximum number of data points to return. Will be in increments of page_size or default page size defined by the call if no page_size is provided. + page_size: Number of data points to fetch per request. Defaults to 10,000. + The server caps this at 1,000,000; values above that are coerced down. + Increase toward 1,000,000 to reduce round-trips on large datasets. ignore_cache: Whether to ignore cached data and fetch fresh data from the server. show_progress: If True, display a progress bar naming each channel as its data is fetched. Defaults to True for sync, False for async. @@ -296,6 +300,7 @@ async def get_data( start_time=start_time, end_time=end_time, max_results=limit, + page_size=page_size, ignore_cache=ignore_cache, show_progress=show_progress, ) @@ -308,6 +313,7 @@ async def get_data_as_arrow( start_time: datetime | None = None, end_time: datetime | None = None, limit: int | None = None, + page_size: int | None = None, ignore_cache: bool = False, show_progress: bool | None = None, ) -> dict[str, pa.Table]: @@ -321,6 +327,7 @@ async def get_data_as_arrow( start_time=start_time, end_time=end_time, limit=limit, + page_size=page_size, ignore_cache=ignore_cache, show_progress=show_progress, ) diff --git a/python/lib/sift_client/resources/sync_stubs/__init__.pyi b/python/lib/sift_client/resources/sync_stubs/__init__.pyi index 2419ecd35..fd6a3b6c2 100644 --- a/python/lib/sift_client/resources/sync_stubs/__init__.pyi +++ b/python/lib/sift_client/resources/sync_stubs/__init__.pyi @@ -494,6 +494,7 @@ class ChannelsAPI: start_time: datetime | None = None, end_time: datetime | None = None, limit: int | None = None, + page_size: int | None = None, ignore_cache: bool = False, show_progress: bool | None = None, ) -> dict[str, pd.DataFrame]: @@ -505,6 +506,9 @@ class ChannelsAPI: start_time: The start time to get data for. end_time: The end time to get data for. limit: The maximum number of data points to return. Will be in increments of page_size or default page size defined by the call if no page_size is provided. + page_size: Number of data points to fetch per request. Defaults to 10,000. + The server caps this at 1,000,000; values above that are coerced down. + Increase toward 1,000,000 to reduce round-trips on large datasets. ignore_cache: Whether to ignore cached data and fetch fresh data from the server. show_progress: If True, display a progress bar naming each channel as its data is fetched. Defaults to True for sync, False for async. @@ -523,6 +527,7 @@ class ChannelsAPI: start_time: datetime | None = None, end_time: datetime | None = None, limit: int | None = None, + page_size: int | None = None, ignore_cache: bool = False, show_progress: bool | None = None, ) -> dict[str, pa.Table]: