diff --git a/mergin/client.py b/mergin/client.py index 1555051..ffad3f1 100644 --- a/mergin/client.py +++ b/mergin/client.py @@ -2,7 +2,6 @@ import math import os import json -import shutil import zlib import base64 import urllib.parse @@ -67,7 +66,9 @@ int_version, is_version_acceptable, normalize_role, + long_path, ) +from . import fs from .version import __version__ try: @@ -205,7 +206,7 @@ def setup_logging(self): self.log.setLevel(logging.DEBUG) # log everything (it would otherwise log just warnings+errors) if not self.log.handlers: if client_log_file: - log_handler = logging.FileHandler(client_log_file) + log_handler = logging.FileHandler(long_path(client_log_file)) log_handler.setFormatter(logging.Formatter("%(asctime)s %(message)s")) self.log.addHandler(log_handler) else: @@ -532,7 +533,7 @@ def create_project_and_push(self, project_name, directory, is_public=False, name :param namespace: Deprecated. project_name should be full project name. Optional namespace for a new project. If empty username is used. :type namespace: String """ - if os.path.exists(os.path.join(directory, ".mergin")): + if fs.exists(os.path.join(directory, ".mergin")): raise ClientError("Directory is already assigned to a Mergin Maps project (contains .mergin sub-dir)") if namespace and "/" not in project_name: @@ -1242,11 +1243,11 @@ def get_file_diff(self, project_dir, file_path, output_diff, version_from, versi # concatenate diffs, if needed output_dir = os.path.dirname(output_diff) if len(diffs) >= 1: - os.makedirs(output_dir, exist_ok=True) + fs.makedirs(output_dir, exist_ok=True) if len(diffs) > 1: mp.geodiff.concat_changes(diffs, output_diff) elif len(diffs) == 1: - shutil.copy(diffs[0], output_diff) + fs.copy(diffs[0], output_diff) def download_file_diffs(self, project_dir, file_path, versions): """Download file diffs for specified versions if they are not present @@ -1377,7 +1378,7 @@ def reset_local_changes(self, directory: str, files_to_reset: typing.List[str] = # remove all added files for file in push_changes["added"]: if all_files or file["path"] in files_to_reset: - os.remove(mp.fpath(file["path"])) + fs.remove(mp.fpath(file["path"])) # update files get override with previous version for file in push_changes["updated"]: @@ -1612,14 +1613,14 @@ def send_logs( local_logs_file_size_to_send = int(MAX_LOG_FILE_SIZE_TO_SEND * 0.8) global_logs = b"" - if global_log_file and os.path.exists(global_log_file): - with open(global_log_file, "rb") as f: - if os.path.getsize(global_log_file) > global_logs_file_size_to_send: + if global_log_file and fs.exists(global_log_file): + with fs.open_file(global_log_file, "rb") as f: + if fs.getsize(global_log_file) > global_logs_file_size_to_send: f.seek(-global_logs_file_size_to_send, os.SEEK_END) global_logs = f.read() + b"\n--------------------------------\n\n" - with open(logfile, "rb") as f: - if os.path.getsize(logfile) > local_logs_file_size_to_send: + with fs.open_file(logfile, "rb") as f: + if fs.getsize(logfile) > local_logs_file_size_to_send: f.seek(-local_logs_file_size_to_send, os.SEEK_END) logs = f.read() diff --git a/mergin/client_pull.py b/mergin/client_pull.py index 5210089..5bc9726 100644 --- a/mergin/client_pull.py +++ b/mergin/client_pull.py @@ -26,6 +26,7 @@ from .models import ProjectDelta, ProjectDeltaChange, PullAction from .merginproject import MerginProject from .utils import cleanup_tmp_dir, save_to_file +from . import fs from typing import List, Optional # status = download_project_async(...) @@ -142,7 +143,7 @@ def download_blocking(self, mc, mp): if resp.status in [200, 206]: mp.log.debug(f"Download finished: {self.diff_id}") save_to_file(resp, self.download_file_path) - self.size = os.path.getsize(self.download_file_path) + self.size = fs.getsize(self.download_file_path) else: mp.log.error(f"Download failed: {self.diff_id}") raise ClientError(f"Failed to download of diff file {self.diff_id} to {self.download_file_path}") @@ -164,19 +165,19 @@ def __init__(self, dest_file, downloaded_items: typing.List[DownloadQueueItem], def from_chunks(self): """Merges downloaded chunks into a single file at dest_file path""" file_dir = os.path.dirname(self.dest_file) - os.makedirs(file_dir, exist_ok=True) + fs.makedirs(file_dir, exist_ok=True) - with open(self.dest_file, "wb") as final: + with fs.open_file(self.dest_file, "wb") as final: for item in self.downloaded_items: - with open(item.download_file_path, "rb") as chunk: + with fs.open_file(item.download_file_path, "rb") as chunk: shutil.copyfileobj(chunk, final) - os.remove(item.download_file_path) + fs.remove(item.download_file_path) if not self.size_check: return expected_size = sum(item.size for item in self.downloaded_items) - if os.path.getsize(self.dest_file) != expected_size: - os.remove(self.dest_file) + if fs.getsize(self.dest_file) != expected_size: + fs.remove(self.dest_file) raise ClientError("Download of file {} failed. Please try it again.".format(self.dest_file)) @@ -233,7 +234,7 @@ def _cleanup_failed_download(mergin_project: MerginProject = None): log_file = os.path.join(mergin_project.dir, ".mergin", "client-log.txt") dest_path = None - if os.path.exists(log_file): + if fs.exists(log_file): tmp_file = tempfile.NamedTemporaryFile(prefix="mergin-", suffix=".txt", delete=False) tmp_file.close() dest_path = tmp_file.name @@ -250,9 +251,9 @@ def download_project_async(mc, project_path, directory, project_version=None): if "/" not in project_path: raise ClientError("Project name needs to be fully qualified, e.g. /") - if os.path.exists(directory): + if fs.exists(directory): raise ClientError("Project directory already exists") - os.makedirs(directory) + fs.makedirs(directory) mp = MerginProject(directory) mp.log.info("--- version: " + mc.user_agent_info()) @@ -408,7 +409,7 @@ def apply(self, directory, mp): else: file_dir = os.path.dirname(os.path.normpath(self.destination_file)) dest_file_path = self.destination_file - os.makedirs(file_dir, exist_ok=True) + fs.makedirs(file_dir, exist_ok=True) # ignore check if we download not-latest version of gpkg file (possibly reconstructed on server on demand) check_size = self.latest_version or not mp.is_versioned_file(self.file_path) @@ -555,7 +556,7 @@ def pull_project_async(mc, directory) -> Optional[PullJob]: pull_action_type == PullActionType.COPY_CONFLICT and change.type == DeltaChangeType.UPDATE_DIFF ): basefile = mp.fpath_meta(change.path) - if not os.path.exists(basefile): + if not fs.exists(basefile): # The basefile does not exist for some reason. This should not happen normally (maybe user removed the file # or we removed it within previous pull because we failed to apply patch the older version for some reason). # But it's not a problem - we will download the newest version and we're sorted. @@ -722,7 +723,7 @@ def pull_project_finalize(job: PullJob): basefile = job.mp.fpath_meta(file_path) server_file = job.mp.fpath(file_path, job.tmp_dir.name) - shutil.copy(basefile, server_file) + fs.copy(basefile, server_file) diffs = [job.mp.fpath(f, job.tmp_dir.name) for f in file_diffs] patch_error = job.mp.apply_diffs(server_file, diffs) if patch_error: @@ -735,7 +736,7 @@ def pull_project_finalize(job: PullJob): job.mp.log.error("Diffs we were applying: " + str(diffs)) job.mp.log.error("Removing basefile because it would be corrupted anyway...") job.mp.log.info("--- pull aborted") - os.remove(basefile) + fs.remove(basefile) raise ClientError("Cannot patch basefile {}! Please try syncing again.".format(basefile)) conflicts = [] job.mp.log.info(f"--- applying pull actions {job.pull_actions}") @@ -830,7 +831,7 @@ def download_diffs_async(mc, project_directory, file_path, versions): diff_only=True, ) dest_file_path = mp.fpath_cache(diff["path"], version=file["version"]) - if os.path.exists(dest_file_path): + if fs.exists(dest_file_path): continue download_files.append(DownloadFile(dest_file_path, items)) download_list.extend(items) @@ -992,5 +993,5 @@ def download_files_finalize(job: DownloadJob): task.apply(job.tmp_dir, job.mp) # Remove temporary download directory - if job.tmp_dir is not None and os.path.exists(job.tmp_dir.name): + if job.tmp_dir is not None and fs.exists(job.tmp_dir.name): cleanup_tmp_dir(job.mp, job.tmp_dir) diff --git a/mergin/client_push.py b/mergin/client_push.py index 831b59b..d918844 100644 --- a/mergin/client_push.py +++ b/mergin/client_push.py @@ -18,7 +18,6 @@ import pprint import tempfile import concurrent.futures -import os import time from typing import List, Tuple, Optional, ByteString @@ -35,6 +34,7 @@ from .merginproject import MerginProject, pygeodiff from .editor import filter_changes from .utils import get_data_checksum, cleanup_tmp_dir +from . import fs POST_JSON_HEADERS = {"Content-Type": "application/json"} @@ -114,7 +114,7 @@ def upload_chunk_v2_api(self, data: ByteString, checksum: str): self.mc.upload_chunks_cache.add(checksum, self.server_chunk_id) def upload_blocking(self): - with open(self.file_path, "rb") as file_handle: + with fs.open_file(self.file_path, "rb") as file_handle: file_handle.seek(self.chunk_index * UPLOAD_CHUNK_SIZE) data = file_handle.read(UPLOAD_CHUNK_SIZE) checksum_str = get_data_checksum(data) @@ -508,8 +508,8 @@ def remove_diff_files(job: UploadJob) -> None: diff = change.get_diff() if diff: diff_file = job.mp.fpath_meta(diff.path) - if os.path.exists(diff_file): - os.remove(diff_file) + if fs.exists(diff_file): + fs.remove(diff_file) def get_push_changes_batch(mc, directory: str) -> Tuple[LocalProjectChanges, int]: diff --git a/mergin/fs.py b/mergin/fs.py new file mode 100644 index 0000000..c5a3377 --- /dev/null +++ b/mergin/fs.py @@ -0,0 +1,59 @@ +""" +Thin wrappers around the standard-library filesystem calls used by the sync code. + +Every wrapper applies utils.long_path() to its path argument(s) so that Windows paths +longer than MAX_PATH are handled transparently. + +Use these instead of calling os.* / shutil.* / open() / sqlite3.connect() directly on +project file paths. +""" + +import os +import shutil +import sqlite3 + +from .utils import long_path + + +def remove(path): + os.remove(long_path(path)) + + +def exists(path) -> bool: + return os.path.exists(long_path(path)) + + +def getsize(path) -> int: + return os.path.getsize(long_path(path)) + + +def getmtime(path) -> float: + return os.path.getmtime(long_path(path)) + + +def copy(src, dst): + return shutil.copy(long_path(src), long_path(dst)) + + +def walk(path, **kwargs): + return os.walk(long_path(path), **kwargs) + + +def makedirs(path, exist_ok=False): + os.makedirs(long_path(path), exist_ok=exist_ok) + + +def mkdir(path): + os.mkdir(long_path(path)) + + +def rmtree(path): + shutil.rmtree(long_path(path)) + + +def connect(path): + return sqlite3.connect(long_path(path)) + + +def open_file(path, *args, **kwargs): + return open(long_path(path), *args, **kwargs) diff --git a/mergin/merginproject.py b/mergin/merginproject.py index 12d798f..36d3a30 100644 --- a/mergin/merginproject.py +++ b/mergin/merginproject.py @@ -3,7 +3,6 @@ import math import os import re -import shutil from typing import List, Optional, Dict import typing import uuid @@ -19,6 +18,7 @@ from .utils import ( generate_checksum, is_versioned_file, + long_path, int_version, do_sqlite_checkpoint, unique_path_name, @@ -26,6 +26,7 @@ edit_conflict_file_name, ) from .local_changes import FileChange +from . import fs this_dir = os.path.dirname(os.path.realpath(__file__)) @@ -47,19 +48,19 @@ class MerginProject: def __init__(self, directory): self.dir = os.path.abspath(directory) - if not os.path.exists(self.dir): + if not fs.exists(self.dir): raise InvalidProject("Project directory does not exist") self.meta_dir = os.path.join(self.dir, ".mergin") - if not os.path.exists(self.meta_dir): - os.mkdir(self.meta_dir) + if not fs.exists(self.meta_dir): + fs.mkdir(self.meta_dir) # location for files from unfinished pull self.unfinished_pull_dir = os.path.join(self.meta_dir, "unfinished_pull") self.cache_dir = os.path.join(self.meta_dir, ".cache") - if not os.path.exists(self.cache_dir): - os.mkdir(self.cache_dir) + if not fs.exists(self.cache_dir): + fs.mkdir(self.cache_dir) # metadata from JSON are lazy loaded self._metadata = None @@ -95,7 +96,9 @@ def setup_logging(self, logger_name): if not self.log.handlers: # we only need to set the handler once # (otherwise we would get things logged multiple times as loggers are cached) - log_handler = logging.FileHandler(os.path.join(self.meta_dir, "client-log.txt"), encoding="utf-8") + log_handler = logging.FileHandler( + long_path(os.path.join(self.meta_dir, "client-log.txt")), encoding="utf-8" + ) log_handler.setFormatter(logging.Formatter("%(asctime)s %(message)s")) self.log.addHandler(log_handler) @@ -122,7 +125,7 @@ def fpath(self, file, other_dir=None): root = other_dir or self.dir abs_path = os.path.abspath(os.path.join(root, file)) f_dir = os.path.dirname(abs_path) - os.makedirs(f_dir, exist_ok=True) + fs.makedirs(f_dir, exist_ok=True) return abs_path def fpath_meta(self, file): @@ -230,9 +233,9 @@ def _read_metadata(self) -> None: """Loads the project's metadata from JSON""" if self._metadata is not None: return - if not os.path.exists(self.fpath_meta("mergin.json")): + if not fs.exists(self.fpath_meta("mergin.json")): raise InvalidProject("Project metadata has not been created yet") - with open(self.fpath_meta("mergin.json"), "r") as file: + with fs.open_file(self.fpath_meta("mergin.json"), "r") as file: self._metadata = json.load(file) self.is_old_metadata = "/" in self._metadata["name"] @@ -252,9 +255,9 @@ def write_metadata(project_directory: str, data: dict): (and therefore creating MerginProject would fail). """ meta_dir = os.path.join(project_directory, ".mergin") - os.makedirs(meta_dir, exist_ok=True) + fs.makedirs(meta_dir, exist_ok=True) metadata_json_file = os.path.abspath(os.path.join(meta_dir, "mergin.json")) - with open(metadata_json_file, "w") as file: + with fs.open_file(metadata_json_file, "w") as file: json.dump(data, file, indent=2) def is_versioned_file(self, file): @@ -279,7 +282,7 @@ def is_gpkg_open(self, path): f_extension = os.path.splitext(path)[1] if f_extension != ".gpkg": return False - if os.path.exists(f"{path}-wal"): + if fs.exists(f"{path}-wal"): return True return False @@ -309,21 +312,20 @@ def inspect_files(self): :rtype: list[dict] """ files_meta = [] - for root, dirs, files in os.walk(self.dir, topdown=True): + for root, dirs, files in fs.walk(self.dir, topdown=True): dirs[:] = [d for d in dirs if d not in [".mergin"]] for file in files: if self.ignore_file(file): continue - - abs_path = os.path.abspath(os.path.join(root, file)) - rel_path = os.path.relpath(abs_path, start=self.dir) + abs_path = os.path.join(root, file) + rel_path = os.path.relpath(abs_path, start=long_path(self.dir)) proj_path = "/".join(rel_path.split(os.path.sep)) # we need posix path files_meta.append( { "path": proj_path, "checksum": generate_checksum(abs_path), - "size": os.path.getsize(abs_path), - "mtime": datetime.fromtimestamp(os.path.getmtime(abs_path), tzlocal()), + "size": fs.getsize(abs_path), + "mtime": datetime.fromtimestamp(fs.getmtime(abs_path), tzlocal()), } ) return files_meta @@ -623,16 +625,16 @@ def get_local_delta(self, diff_directory: str) -> List[ProjectDeltaChange]: delta_item.size = checkpoint_size delta_item.checksum = checkpoint_checksum + diff_location = self.fpath(diff_file, diff_directory) try: - diff_location = self.fpath(diff_file, diff_directory) self.geodiff.create_changeset(origin_file, current_file, diff_location) if not self.geodiff.has_changes(diff_location): - os.remove(diff_location) + fs.remove(diff_location) continue delta_item.checksum = change.get("origin_checksum") delta_item.type = DeltaChangeType.UPDATE_DIFF - os.remove(diff_location) + fs.remove(diff_location) except (pygeodiff.GeoDiffLibError, pygeodiff.GeoDiffLibConflictError) as e: self.log.warning("failed to create changeset for " + path) # probably the database schema has been modified if geodiff cannot create changeset. @@ -680,19 +682,19 @@ def get_push_changes(self): try: self.geodiff.create_changeset(origin_file, current_file, diff_file) if self.geodiff.has_changes(diff_file): - diff_size = os.path.getsize(diff_file) + diff_size = fs.getsize(diff_file) file["checksum"] = file["origin_checksum"] # need to match basefile on server file["chunks"] = [str(uuid.uuid4()) for i in range(math.ceil(diff_size / UPLOAD_CHUNK_SIZE))] - file["mtime"] = datetime.fromtimestamp(os.path.getmtime(current_file), tzlocal()) + file["mtime"] = datetime.fromtimestamp(fs.getmtime(current_file), tzlocal()) file["diff"] = { "path": diff_name, "checksum": generate_checksum(diff_file), "size": diff_size, - "mtime": datetime.fromtimestamp(os.path.getmtime(diff_file), tzlocal()), + "mtime": datetime.fromtimestamp(fs.getmtime(diff_file), tzlocal()), } else: - if os.path.exists(diff_file): - os.remove(diff_file) + if fs.exists(diff_file): + fs.remove(diff_file) not_updated.append(file) except (pygeodiff.GeoDiffLibError, pygeodiff.GeoDiffLibConflictError) as e: self.log.warning("failed to create changeset for " + path) @@ -711,9 +713,9 @@ def copy_versioned_file_for_upload(self, f: FileChange, tmp_dir: str) -> str: path = f.path self.log.info("Making a temporary copy (full upload): " + path) tmp_file = os.path.join(tmp_dir, path) - os.makedirs(os.path.dirname(tmp_file), exist_ok=True) + fs.makedirs(os.path.dirname(tmp_file), exist_ok=True) self.geodiff.make_copy_sqlite(self.fpath(path), tmp_file) - f.size = os.path.getsize(tmp_file) + f.size = fs.getsize(tmp_file) f.checksum = generate_checksum(tmp_file) f.chunks = [str(uuid.uuid4()) for i in range(math.ceil(f.size / UPLOAD_CHUNK_SIZE))] f.upload_file = tmp_file @@ -728,10 +730,10 @@ def get_list_of_push_changes(self, push_changes): result_file = self.fpath("change_list" + str(idx), self.meta_dir) try: self.geodiff.list_changes_summary(changeset, result_file) - with open(result_file, "r") as f: + with fs.open_file(result_file, "r") as f: change = f.read() changes[file["path"]] = json.loads(change) - os.remove(result_file) + fs.remove(result_file) except (pygeodiff.GeoDiffLibError, pygeodiff.GeoDiffLibConflictError): pass return changes @@ -768,7 +770,7 @@ def apply_pull_actions(self, actions: List[PullAction], download_dir: str, serve self.geodiff.make_copy_sqlite(server_file, live_file) self.geodiff.make_copy_sqlite(server_file, basefile) else: - shutil.copy(server_file, live_file) + fs.copy(server_file, live_file) elif action_type == PullActionType.APPLY_DIFF_NO_REBASE: # simply apply the diff without rebase (no local changes or non-conflicting local changes) self.update_without_rebase(path, server_file, live_file, basefile, download_dir) @@ -794,14 +796,14 @@ def apply_pull_actions(self, actions: List[PullAction], download_dir: str, serve f_server_unfinished = self.fpath_unfinished_pull(path) self.geodiff.make_copy_sqlite(server_file, f_server_unfinished) else: - shutil.copy(server_file, live_file) + fs.copy(server_file, live_file) elif action_type == PullActionType.DELETE: # remove local file - if os.path.exists(live_file): - os.remove(live_file) - if self.is_versioned_file(path) and os.path.exists(basefile): - os.remove(basefile) + if fs.exists(live_file): + fs.remove(live_file) + if self.is_versioned_file(path) and fs.exists(basefile): + fs.remove(basefile) return conflicts @@ -930,7 +932,7 @@ def apply_push_changes(self, changes): basefile = self.fpath_meta(path) if k == "removed": - os.remove(basefile) + fs.remove(basefile) elif k == "added": self.geodiff.make_copy_sqlite(self.fpath(path), basefile) elif k == "updated": @@ -947,7 +949,7 @@ def apply_push_changes(self, changes): if patch_error: # in case of local sync issues it is safier to remove basefile, next time it will be downloaded from server self.log.warning("removing basefile (because of apply diff error) for: " + path) - os.remove(basefile) + fs.remove(basefile) else: pass @@ -961,7 +963,7 @@ def create_conflicted_copy(self, file: str, user_name: str): :rtype: str """ src = self.fpath(file) - if not os.path.exists(src): + if not fs.exists(src): return backup_path = unique_path_name( @@ -971,7 +973,7 @@ def create_conflicted_copy(self, file: str, user_name: str): if self.is_versioned_file(file): self.geodiff.make_copy_sqlite(src, backup_path) else: - shutil.copy(src, backup_path) + fs.copy(src, backup_path) return backup_path def apply_diffs(self, basefile, diffs): @@ -1012,7 +1014,7 @@ def has_unfinished_pull(self): :returns: whether there is an unfinished pull :rtype: bool """ - return os.path.exists(self.unfinished_pull_dir) + return fs.exists(self.unfinished_pull_dir) def resolve_unfinished_pull(self, user_name): """ @@ -1042,10 +1044,10 @@ def resolve_unfinished_pull(self, user_name): self.log.info("resolving unfinished pull") - for root, dirs, files in os.walk(self.unfinished_pull_dir): + for root, dirs, files in fs.walk(self.unfinished_pull_dir): for file_name in files: - src = os.path.join(root, file_name) - file_path = os.path.relpath(src, self.unfinished_pull_dir) + file_path = os.path.relpath(os.path.join(root, file_name), long_path(self.unfinished_pull_dir)) + src = self.fpath_unfinished_pull(file_path) dest = self.fpath(file_path) basefile = self.fpath_meta(file_path) @@ -1066,7 +1068,7 @@ def resolve_unfinished_pull(self, user_name): self.log.error("unable to apply changes from previous unfinished pull!") raise ClientError("Unable to resolve unfinished pull!") - shutil.rmtree(self.unfinished_pull_dir) + fs.rmtree(self.unfinished_pull_dir) self.log.info("unfinished pull resolved successfuly!") return conflicts diff --git a/mergin/report.py b/mergin/report.py index 5b9cae4..e096696 100644 --- a/mergin/report.py +++ b/mergin/report.py @@ -8,6 +8,7 @@ from . import ClientError from .merginproject import MerginProject, pygeodiff from .utils import int_version +from . import fs try: from qgis.core import ( @@ -244,14 +245,14 @@ def create_report(mc, directory, since, to, out_file): # download full gpkg in "to" version to analyze its schema to determine which col is geometry full_gpkg = mp.fpath_cache(f["path"], version=to) - if not os.path.exists(full_gpkg): + if not fs.exists(full_gpkg): mc.download_file(directory, f["path"], full_gpkg, to) # get gpkg schema schema_file = full_gpkg + "-schema.json" # geodiff writes schema into a file - if not os.path.exists(schema_file): + if not fs.exists(schema_file): mp.geodiff.schema("sqlite", "", full_gpkg, schema_file) - with open(schema_file, "r") as sf: + with fs.open_file(schema_file, "r") as sf: schema = json.load(sf).get("geodiff_schema") # add records for every version (diff) and all tables within geopackage @@ -285,8 +286,8 @@ def create_report(mc, directory, since, to, out_file): # export report to csv file out_dir = os.path.dirname(out_file) - os.makedirs(out_dir, exist_ok=True) - with open(out_file, "w", newline="") as f_csv: + fs.makedirs(out_dir, exist_ok=True) + with fs.open_file(out_file, "w", newline="") as f_csv: writer = csv.DictWriter(f_csv, fieldnames=headers) writer.writeheader() writer.writerows(records) diff --git a/mergin/utils.py b/mergin/utils.py index 91796f3..2a77433 100644 --- a/mergin/utils.py +++ b/mergin/utils.py @@ -21,7 +21,7 @@ def generate_checksum(file, chunk_size=4096): :return: sha1 checksum """ checksum = hashlib.sha1() # nosec B324 - usedforsecurity=False flag is compatible with python 3.9+ - with open(file, "rb") as f: + with open(long_path(file), "rb") as f: while True: chunk = f.read(chunk_size) if not chunk: @@ -37,9 +37,9 @@ def save_to_file(stream, path): """ directory = os.path.abspath(os.path.dirname(path)) - os.makedirs(directory, exist_ok=True) + os.makedirs(long_path(directory), exist_ok=True) - with open(path, "wb") as output: + with open(long_path(path), "wb") as output: writer = io.BufferedWriter(output, buffer_size=32768) while True: part = stream.read(4096) @@ -52,8 +52,8 @@ def save_to_file(stream, path): def move_file(src, dest): dest_dir = os.path.dirname(dest) - os.makedirs(dest_dir, exist_ok=True) - os.rename(src, dest) + os.makedirs(long_path(dest_dir), exist_ok=True) + os.rename(long_path(src), long_path(dest)) class DateTimeEncoder(json.JSONEncoder): @@ -88,10 +88,11 @@ def do_sqlite_checkpoint(path, log=None): """ new_size = None new_checksum = None - if ".gpkg" in path and os.path.exists(f"{path}-wal"): + path_lp = long_path(path) + if ".gpkg" in path and os.path.exists(f"{path_lp}-wal"): if log: log.info("checkpoint - going to add it in " + path) - conn = sqlite3.connect(path) + conn = sqlite3.connect(path_lp) cursor = conn.cursor() cursor.execute("PRAGMA wal_checkpoint=FULL") if log: @@ -99,7 +100,7 @@ def do_sqlite_checkpoint(path, log=None): cursor.execute("VACUUM") conn.commit() conn.close() - new_size = os.path.getsize(path) + new_size = os.path.getsize(path_lp) new_checksum = generate_checksum(path) if log: log.info("checkpoint - new size {} checksum {}".format(new_size, new_checksum)) @@ -166,13 +167,13 @@ def unique_path_name(path): """ unique_path = str(path) - is_dir = os.path.isdir(path) + is_dir = os.path.isdir(long_path(path)) head, tail = os.path.split(os.path.normpath(path)) ext = "".join(Path(tail).suffixes) file_name = tail.replace(ext, "") i = 0 - while os.path.exists(unique_path): + while os.path.exists(long_path(unique_path)): i += 1 if is_dir: @@ -266,6 +267,25 @@ def is_versioned_file(path: str) -> bool: return f_extension.lower() in diff_extensions +def long_path(path: str) -> str: + """ + Prefix an absolute path with the Windows "\\?\" extended-length marker, so file APIs used by + geodiff/SQLite and Python's own open() can handle long paths without raising an error. + + :param path: absolute or relative path, with either posix or windows separators + :type path: str + :returns: extended-length path on Windows, the unchanged path otherwise + :rtype: str + """ + if os.name != "nt": + return path + backslash = chr(92) + prefix = backslash + backslash + "?" + backslash + if path.startswith(prefix): + return path + return prefix + os.path.abspath(path) + + def is_qgis_file(path: str) -> bool: """ Check if file is a QGIS project file.