From 7131ed46401cea0c1f77aa0c86cbb8b4c0fd92e8 Mon Sep 17 00:00:00 2001 From: Jeethu Rao Date: Tue, 2 Jun 2026 21:39:06 +0100 Subject: [PATCH] Increase download chunk size to 1MB --- numerapi/utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/numerapi/utils.py b/numerapi/utils.py index 76985c0..b3534a4 100644 --- a/numerapi/utils.py +++ b/numerapi/utils.py @@ -14,6 +14,8 @@ logger = logging.getLogger(__name__) +DOWNLOAD_CHUNK_SIZE = 1024 * 1024 # 1 MiB + def load_secrets() -> tuple: """load secrets from environment variables or dotenv file""" @@ -96,9 +98,9 @@ def download_file(url: str, dest_path: str, show_progress_bars: bool = True): # Update progress bar to reflect how much of the file is already downloaded pbar.update(file_size) with open(temp_path, "ab") as dest_file: - for chunk in req.iter_content(1024): + for chunk in req.iter_content(DOWNLOAD_CHUNK_SIZE): dest_file.write(chunk) - pbar.update(1024) + pbar.update(len(chunk)) # move temp file to target destination os.replace(temp_path, dest_path) return dest_path