From d62ff994f2bede387b321bb79399b536f52ff145 Mon Sep 17 00:00:00 2001 From: Martin Varga Date: Thu, 3 Sep 2026 08:01:37 +0200 Subject: [PATCH 1/2] Allow downloading a single file by project name without a local checkout --- .gitignore | 1 + mergin/cli.py | 23 ++++++++++--- mergin/client.py | 6 ++-- mergin/client_pull.py | 69 ++++++++++++++++++++++++++++---------- mergin/test/test_client.py | 36 ++++++++++++++++++++ 5 files changed, 111 insertions(+), 24 deletions(-) diff --git a/.gitignore b/.gitignore index adb34dd..1db6fa6 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ deps venv debug.py .vscode/ +.python-version \ No newline at end of file diff --git a/mergin/cli.py b/mergin/cli.py index c20beb9..8ea33de 100755 --- a/mergin/cli.py +++ b/mergin/cli.py @@ -335,17 +335,32 @@ def share(ctx, project): @click.argument("filepath") @click.argument("output") @click.option("--version", help="Project version tag, for example 'v3'") +@click.option( + "--project", + help="Full project name ('/') to download the file directly from the server. " + "If not given, the current directory is used and must be an existing checked out project.", +) @click.pass_context -def download_file(ctx, filepath, output, version): +def download_file(ctx, filepath, output, version, project): """ - Download project file at specified version. `project` needs to be a combination of namespace/project. - If no version is given, the latest will be fetched. + Download project file at specified version. If no version is given, the latest will be fetched. """ mc = ctx.obj["client"] if mc is None: return + if project is None: + # no --project given, so we default to the current directory - make sure that's actually a checked out project + try: + MerginProject(os.getcwd()).project_full_name() + except InvalidProject: + click.secho( + "Current directory is not a Mergin Maps project. Run this command from within a " + "checked out project directory, or pass --project /.", + fg="red", + ) + return try: - job = download_file_async(mc, os.getcwd(), filepath, output, version) + job = download_file_async(mc, project or os.getcwd(), filepath, output, version) with click.progressbar(length=job.total_size) as bar: last_transferred_size = 0 while download_project_is_running(job): diff --git a/mergin/client.py b/mergin/client.py index 1555051..6c0a30d 100644 --- a/mergin/client.py +++ b/mergin/client.py @@ -1199,7 +1199,7 @@ def download_file(self, project_dir, file_path, output_filename, version=None): """ Download project file at specified version. Get the latest if no version specified. - :param project_dir: project local directory + :param project_dir: project local directory or a full project name ("/") :type project_dir: String :param file_path: relative path of file to download in the project directory :type file_path: String @@ -1401,11 +1401,11 @@ def download_files( """ Download project files at specified version. Get the latest if no version specified. - :param project_dir: project local directory + :param project_dir: project local directory or a full project name ("/") :type project_dir: String :param file_path: List of relative paths of files to download in the project directory :type file_path: List[String] - :param output_paths: List of paths for files to download to. Should be same length of as file_path. Default is `None` which means that files are downloaded into MerginProject at project_dir. + :param output_paths: List of paths for files to download to. Should be same length of as file_path. Default is `None` which means that files are downloaded into MerginProject at project_dir (only valid when project_dir is an existing local checkout). :type output_paths: List[String] :param version: optional version tag for downloaded file :type version: String diff --git a/mergin/client_pull.py b/mergin/client_pull.py index 5210089..c4d1c7b 100644 --- a/mergin/client_pull.py +++ b/mergin/client_pull.py @@ -22,11 +22,11 @@ import concurrent.futures -from .common import CHUNK_SIZE, ClientError, DeltaChangeType, PullActionType +from .common import CHUNK_SIZE, ClientError, DeltaChangeType, InvalidProject, PullActionType from .models import ProjectDelta, ProjectDeltaChange, PullAction from .merginproject import MerginProject -from .utils import cleanup_tmp_dir, save_to_file -from typing import List, Optional +from .utils import cleanup_tmp_dir, is_versioned_file, save_to_file +from typing import List, Optional, Union # status = download_project_async(...) # @@ -54,7 +54,7 @@ def __init__( update_tasks, download_queue_items, tmp_dir: tempfile.TemporaryDirectory, - mp, + mp: Union[MerginProject, "DownloadScratchContext"], project_info, ): self.project_path = project_path @@ -64,7 +64,7 @@ def __init__( self.update_tasks = update_tasks self.download_queue_items = download_queue_items self.tmp_dir = tmp_dir - self.mp = mp # MerginProject instance + self.mp = mp self.is_cancelled = False self.project_info = project_info # parsed JSON with project info returned from the server self.failure_log_file = None # log file, copied from the project directory if download fails @@ -80,6 +80,25 @@ def dump(self): print("--- END ---") +class DownloadScratchContext: + """ + Minimal stand-in for MerginProject, used by download_files_async() when downloading files + directly by project name ("/") without an existing local project checkout. + + Provides only what the shared download job code actually needs from MerginProject. + """ + + def __init__(self, mc, cache_dir: str): + self.log = mc.log + self.cache_dir = cache_dir + # only used by _cleanup_failed_download() to look for a log file + self.dir = cache_dir + + def remove_logging_handler(self): + # no-op: self.log is mc's shared logger, not owned by this throwaway context + pass + + class DownloadQueueItem: """ a piece of data from a project that should be downloaded - it can be either a chunk or it can be a diff. @@ -398,7 +417,7 @@ def __init__( self.download_queue_items = download_queue_items self.latest_version = latest_version - def apply(self, directory, mp): + def apply(self, directory, mp: Union[MerginProject, "DownloadScratchContext"]): """assemble downloaded chunks into a single file""" if self.destination_file is None: @@ -411,14 +430,14 @@ def apply(self, directory, mp): os.makedirs(file_dir, exist_ok=True) # ignore check if we download not-latest version of gpkg file (possibly reconstructed on server on demand) - check_size = self.latest_version or not mp.is_versioned_file(self.file_path) + check_size = self.latest_version or not is_versioned_file(self.file_path) # merge chunks together (and delete them afterwards) file_to_merge = DownloadFile(dest_file_path, self.download_queue_items, check_size) file_to_merge.from_chunks() # Make a copy of the file to meta dir only if there is no user-specified path for the file. - # destination_file is None for full project download and takes a meaningful value for a single file download. - if mp.is_versioned_file(self.file_path) and self.destination_file is None: + # destination_file is None for full project download and takes a meaningful value for a single file download + if self.destination_file is None and is_versioned_file(self.file_path): mp.geodiff.make_copy_sqlite(mp.fpath(self.file_path), mp.fpath_meta(self.file_path)) @@ -902,9 +921,30 @@ def download_files_async( """ Starts background download project files at specified version. Returns handle to the pending download. + + `project_dir` can either be an existing local project directory (previously fetched with + download_project()), or a full project name ("/") to download files + directly from the server without needing a local checkout. In the latter case, `output_paths` + must be provided explicitly, as there is no project directory to place files into by default. """ - mp = MerginProject(project_dir) - project_path = mp.project_full_name() + # temporary directory to stage downloaded chunks in + tmp_dir = tempfile.TemporaryDirectory(prefix="python-api-client-") + + mp: Union[MerginProject, "DownloadScratchContext"] + try: + mp = MerginProject(project_dir) + project_path = mp.project_full_name() + except InvalidProject: + # project_dir is not an existing local checkout - treat it as a full project name + # ("/") and download straight from the server instead + if output_paths is None: + cleanup_tmp_dir(mc, tmp_dir) + raise ClientError( + "output_paths must be provided when downloading files without an existing local project checkout" + ) + project_path = project_dir + mp = DownloadScratchContext(mc, tmp_dir.name) + ver_info = f"at version {version}" if version is not None else "at latest version" mp.log.info(f"Getting [{', '.join(file_paths)}] {ver_info}") latest_proj_info = mc.project_info(project_path) @@ -914,9 +954,6 @@ def download_files_async( project_info = latest_proj_info mp.log.info(f"Got project info. version {project_info['version']}") - # set temporary directory for download - tmp_dir = tempfile.TemporaryDirectory(prefix="python-api-client-") - if output_paths is None: output_paths = [] for file in file_paths: @@ -991,6 +1028,4 @@ def download_files_finalize(job: DownloadJob): for task in job.update_tasks: task.apply(job.tmp_dir, job.mp) - # Remove temporary download directory - if job.tmp_dir is not None and os.path.exists(job.tmp_dir.name): - cleanup_tmp_dir(job.mp, job.tmp_dir) + cleanup_tmp_dir(job.mp, job.tmp_dir) diff --git a/mergin/test/test_client.py b/mergin/test/test_client.py index bd987d0..d84c7b8 100644 --- a/mergin/test/test_client.py +++ b/mergin/test/test_client.py @@ -1354,6 +1354,42 @@ def test_download_file(mc): mc.download_file(project_dir, f_updated, f_downloaded, version="v5") +def test_download_file_without_checkout(mc): + """Test downloading a single file directly by project name, without an existing local checkout.""" + test_project = "test_download_file_without_checkout" + project = create_project_path(test_project, mc) + project_dir = os.path.join(TMP_DIR, test_project) + f_updated = "base.gpkg" + + create_versioned_project(mc, test_project, project_dir, f_updated) + + # download straight from the server by "workspace/project" name into a fresh directory + # that has never been used as a project checkout + download_dir = os.path.join(TMP_DIR, test_project + "_no_checkout") + remove_folders([download_dir]) + os.makedirs(download_dir, exist_ok=True) + f_downloaded = os.path.join(download_dir, f_updated) + + expected_content = "inserted_1_A.gpkg" + mc.download_file(project, f_updated, f_downloaded, version="v2") + expected = os.path.join(TEST_DATA_DIR, expected_content) + assert check_gpkg_same_content(MerginProject(project_dir), f_downloaded, expected) + assert not os.path.exists(os.path.join(download_dir, ".mergin")) + + # output_paths must be provided explicitly when there is no local checkout + with pytest.raises(ClientError, match="output_paths must be provided"): + mc.download_files(project, [f_updated]) + + # non-existent file in an existing project - same error as with a local checkout + with pytest.raises(ClientError, match=r"No \[does_not_exist\.gpkg\] exists at version v2"): + mc.download_file(project, "does_not_exist.gpkg", f_downloaded, version="v2") + + # non-existent / inaccessible project should fail clearly too + nonexistent_project = create_project_path("this_project_does_not_exist", mc) + with pytest.raises(ClientError): + mc.download_file(nonexistent_project, f_updated, f_downloaded) + + def test_download_diffs(mc): """Test download diffs for a project file between specified project versions.""" test_project = "test_download_diffs" From 4767c67770c747ebfaf2bfc8261ce4511e97bc59 Mon Sep 17 00:00:00 2001 From: Martin Varga Date: Thu, 10 Sep 2026 10:14:57 +0200 Subject: [PATCH 2/2] Separate download single file to dedicated API call --- mergin/cli.py | 26 +++++++++-------- mergin/client.py | 25 +++++++++++++++-- mergin/client_pull.py | 57 +++++++++++++++++++++++--------------- mergin/test/test_client.py | 12 ++++---- 4 files changed, 76 insertions(+), 44 deletions(-) diff --git a/mergin/cli.py b/mergin/cli.py index 8ea33de..485d16b 100755 --- a/mergin/cli.py +++ b/mergin/cli.py @@ -29,6 +29,7 @@ download_project_cancel, download_file_async, download_file_finalize, + download_project_file_async, download_project_finalize, download_project_is_running, ) @@ -348,19 +349,20 @@ def download_file(ctx, filepath, output, version, project): mc = ctx.obj["client"] if mc is None: return - if project is None: - # no --project given, so we default to the current directory - make sure that's actually a checked out project - try: - MerginProject(os.getcwd()).project_full_name() - except InvalidProject: - click.secho( - "Current directory is not a Mergin Maps project. Run this command from within a " - "checked out project directory, or pass --project /.", - fg="red", - ) - return try: - job = download_file_async(mc, project or os.getcwd(), filepath, output, version) + if project is not None: + job = download_project_file_async(mc, project, filepath, output, version) + else: + try: + MerginProject(os.getcwd()).project_full_name() + except InvalidProject: + click.secho( + "Current directory is not a Mergin Maps project. Run this command from within a " + "checked out project directory, or pass --project /.", + fg="red", + ) + return + job = download_file_async(mc, os.getcwd(), filepath, output, version) with click.progressbar(length=job.total_size) as bar: last_transferred_size = 0 while download_project_is_running(job): diff --git a/mergin/client.py b/mergin/client.py index 6c0a30d..17a616a 100644 --- a/mergin/client.py +++ b/mergin/client.py @@ -46,6 +46,7 @@ download_file_async, download_files_async, download_files_finalize, + download_project_file_async, download_diffs_async, download_project_finalize, download_project_wait, @@ -1199,7 +1200,7 @@ def download_file(self, project_dir, file_path, output_filename, version=None): """ Download project file at specified version. Get the latest if no version specified. - :param project_dir: project local directory or a full project name ("/") + :param project_dir: project local directory :type project_dir: String :param file_path: relative path of file to download in the project directory :type file_path: String @@ -1212,6 +1213,24 @@ def download_file(self, project_dir, file_path, output_filename, version=None): pull_project_wait(job) download_file_finalize(job) + def download_project_file(self, project_path, file_path, output_filename, version=None): + """ + Download a single project file at specified version directly from the server, without + needing an existing local project checkout. + + :param project_path: full project name ("/") + :type project_path: String + :param file_path: relative path of file to download in the project directory + :type file_path: String + :param output_filename: full destination path for saving the downloaded file + :type output_filename: String + :param version: optional version tag for downloaded file + :type version: String + """ + job = download_project_file_async(self, project_path, file_path, output_filename, version=version) + pull_project_wait(job) + download_file_finalize(job) + def get_file_diff(self, project_dir, file_path, output_diff, version_from, version_to): """Create concatenated diff for project file diffs between versions version_from and version_to. @@ -1401,11 +1420,11 @@ def download_files( """ Download project files at specified version. Get the latest if no version specified. - :param project_dir: project local directory or a full project name ("/") + :param project_dir: project local directory :type project_dir: String :param file_path: List of relative paths of files to download in the project directory :type file_path: List[String] - :param output_paths: List of paths for files to download to. Should be same length of as file_path. Default is `None` which means that files are downloaded into MerginProject at project_dir (only valid when project_dir is an existing local checkout). + :param output_paths: List of paths for files to download to. Should be same length of as file_path. Default is `None` which means that files are downloaded into MerginProject at project_dir. :type output_paths: List[String] :param version: optional version tag for downloaded file :type version: String diff --git a/mergin/client_pull.py b/mergin/client_pull.py index c4d1c7b..634ead0 100644 --- a/mergin/client_pull.py +++ b/mergin/client_pull.py @@ -22,7 +22,7 @@ import concurrent.futures -from .common import CHUNK_SIZE, ClientError, DeltaChangeType, InvalidProject, PullActionType +from .common import CHUNK_SIZE, ClientError, DeltaChangeType, PullActionType from .models import ProjectDelta, ProjectDeltaChange, PullAction from .merginproject import MerginProject from .utils import cleanup_tmp_dir, is_versioned_file, save_to_file @@ -82,8 +82,8 @@ def dump(self): class DownloadScratchContext: """ - Minimal stand-in for MerginProject, used by download_files_async() when downloading files - directly by project name ("/") without an existing local project checkout. + Minimal stand-in for MerginProject used when downloading a file directly by + project name ("/") without an existing local project checkout. Provides only what the shared download job code actually needs from MerginProject. """ @@ -793,6 +793,23 @@ def download_file_finalize(job): download_files_finalize(job) +def download_project_file_async(mc, project_path: str, file_path: str, output_file: str, version: str = None): + """ + Starts background download of a single project file at specified version, fetched directly + from the server without needing an existing local project checkout. + Returns handle to the pending download. + + :param project_path: full project name ("/") + :param output_file: destination path for the downloaded file + """ + if not output_file: + raise ClientError("output_file must be provided when downloading a file without a local project checkout") + + tmp_dir = tempfile.TemporaryDirectory(prefix="python-api-client-") + mp = DownloadScratchContext(mc, tmp_dir.name) + return _download_files_async(mc, mp, project_path, [file_path], [output_file], version, tmp_dir) + + def download_diffs_async(mc, project_directory, file_path, versions): """ Starts background download project file diffs for specified versions. @@ -916,35 +933,29 @@ def download_diffs_finalize(job: PullJob) -> List[str]: def download_files_async( - mc, project_dir: str, file_paths: typing.List[str], output_paths: typing.List[str], version: str + mc, project_dir: str, file_paths: typing.List[str], output_paths: typing.List[str] = None, version: str = None ): """ Starts background download project files at specified version. Returns handle to the pending download. - `project_dir` can either be an existing local project directory (previously fetched with - download_project()), or a full project name ("/") to download files - directly from the server without needing a local checkout. In the latter case, `output_paths` - must be provided explicitly, as there is no project directory to place files into by default. + `project_dir` must be an existing local project directory. """ - # temporary directory to stage downloaded chunks in + mp = MerginProject(project_dir) + project_path = mp.project_full_name() tmp_dir = tempfile.TemporaryDirectory(prefix="python-api-client-") + return _download_files_async(mc, mp, project_path, file_paths, output_paths, version, tmp_dir) - mp: Union[MerginProject, "DownloadScratchContext"] - try: - mp = MerginProject(project_dir) - project_path = mp.project_full_name() - except InvalidProject: - # project_dir is not an existing local checkout - treat it as a full project name - # ("/") and download straight from the server instead - if output_paths is None: - cleanup_tmp_dir(mc, tmp_dir) - raise ClientError( - "output_paths must be provided when downloading files without an existing local project checkout" - ) - project_path = project_dir - mp = DownloadScratchContext(mc, tmp_dir.name) +def _download_files_async( + mc, + mp: Union[MerginProject, "DownloadScratchContext"], + project_path: str, + file_paths: typing.List[str], + output_paths: typing.List[str], + version: str, + tmp_dir: tempfile.TemporaryDirectory, +): ver_info = f"at version {version}" if version is not None else "at latest version" mp.log.info(f"Getting [{', '.join(file_paths)}] {ver_info}") latest_proj_info = mc.project_info(project_path) diff --git a/mergin/test/test_client.py b/mergin/test/test_client.py index d84c7b8..cae4d2a 100644 --- a/mergin/test/test_client.py +++ b/mergin/test/test_client.py @@ -1371,23 +1371,23 @@ def test_download_file_without_checkout(mc): f_downloaded = os.path.join(download_dir, f_updated) expected_content = "inserted_1_A.gpkg" - mc.download_file(project, f_updated, f_downloaded, version="v2") + mc.download_project_file(project, f_updated, f_downloaded, version="v2") expected = os.path.join(TEST_DATA_DIR, expected_content) assert check_gpkg_same_content(MerginProject(project_dir), f_downloaded, expected) assert not os.path.exists(os.path.join(download_dir, ".mergin")) - # output_paths must be provided explicitly when there is no local checkout - with pytest.raises(ClientError, match="output_paths must be provided"): - mc.download_files(project, [f_updated]) + # output_file must be provided explicitly when there is no local checkout + with pytest.raises(ClientError, match="output_file must be provided"): + mc.download_project_file(project, f_updated, None) # non-existent file in an existing project - same error as with a local checkout with pytest.raises(ClientError, match=r"No \[does_not_exist\.gpkg\] exists at version v2"): - mc.download_file(project, "does_not_exist.gpkg", f_downloaded, version="v2") + mc.download_project_file(project, "does_not_exist.gpkg", f_downloaded, version="v2") # non-existent / inaccessible project should fail clearly too nonexistent_project = create_project_path("this_project_does_not_exist", mc) with pytest.raises(ClientError): - mc.download_file(nonexistent_project, f_updated, f_downloaded) + mc.download_project_file(nonexistent_project, f_updated, f_downloaded) def test_download_diffs(mc):