From 4ec2c422069db87ce30515ce39e888f2a58e348b Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 15 Sep 2026 01:28:45 +0200 Subject: [PATCH 1/2] remove the repository feature flags mechanism The manifest could carry per-operation "mandatory feature" sets that lock out borg versions not supporting them, mirrored into the cache config as ignored_features / mandatory_features to force cache rebuilds. No feature was ever defined (SUPPORTED_REPO_FEATURES was empty) and nothing ever wrote feature flags, so the whole mechanism was dead code: the compatibility checks in Manifest.load(), the cache compatibility check/wipe/update, the ManifestItem validation of config["feature_flags"] and the docs. The manifest's config dict stays (general purpose, preserved by all borg versions), but borg does not store anything in it now. MandatoryFeatureUnsupported is kept (never raised) so that its return code 25 stays reserved. The Manifest.Operation names and the compatibility= argument of the with_repository decorators are left in place as inert placeholders, they are removed in the next commit. Co-Authored-By: Claude Fable 5.1 --- docs/internals/data-structures.rst | 143 +-------------------- src/borg/cache.py | 45 +------ src/borg/item.pyx | 11 -- src/borg/manifest.py | 36 +----- src/borg/testsuite/archiver/checks_test.py | 99 -------------- 5 files changed, 8 insertions(+), 326 deletions(-) diff --git a/docs/internals/data-structures.rst b/docs/internals/data-structures.rst index 11e71f0a8e..909e47bfb8 100644 --- a/docs/internals/data-structures.rst +++ b/docs/internals/data-structures.rst @@ -253,8 +253,7 @@ manifest object if the content changed. It looks like this: Borg 2 always writes *version* 2. Reading also accepts version 1, which is what borg 1.x repositories have (they are supported read-only, e.g. for -``borg transfer``). The versions differ in the way feature flags are handled, -described below. +``borg transfer``). A *timestamp* entry, as written by borg 1.x and by older borg 2 versions, is accepted and ignored when reading. @@ -264,148 +263,12 @@ manifest, each archive has its own pointer object in the ``archives/`` namespace, see :ref:`archive`. *config* is a general-purpose location for additional metadata. All versions -of Borg preserve its contents. Borg stores these keys in there: - -*config['feature_flags']* are the feature flags of the repository, see below. +of Borg preserve its contents. Currently, borg does not store anything in there. A *config['item_keys']* list (written by older borg 2 versions) or a top-level *item_keys* list (borg 1.x) is accepted and ignored when reading: *borg check* does not validate item keys against such a list anymore, see Item_. -Feature flags -+++++++++++++ - -Feature flags are used to add features to data structures without causing -corruption if older versions are used to access or modify them. The main issues -to consider for a feature flag oriented design are flag granularity, -flag storage, and cache_ invalidation. - -Feature flags are divided in approximately three categories, detailed below. -Due to the nature of ID-based deduplication, write (i.e. creating archives) and -read access are not symmetric; it is possible to create archives referencing -chunks that are not readable with the current feature set. The third -category are operations that require accurate reference counts, for example -archive deletion and check. - -As the manifest is always read (and rewritten whenever its content changes), it is the ideal place to store -feature flags, comparable to the super-block of a file system. The only problem -is to recover from a lost manifest, i.e. how is it possible to detect which feature -flags are enabled, if there is no manifest to tell. This issue is left open at this time, -but is not expected to be a major hurdle; it doesn't have to be handled efficiently, it just -needs to be handled. - -Lastly, cache_ invalidation is handled by noting which feature -flags were and which were not understood while manipulating a cache. -This allows borg to detect whether the cache needs to be invalidated, -i.e. rebuilt from scratch. See `Cache feature flags`_ below. - -The *config* key stores the feature flags enabled on a repository: - -.. code-block:: python - - config = { - 'feature_flags': { - 'read': { - 'mandatory': ['some_feature'], - }, - 'check': { - 'mandatory': ['other_feature'], - } - 'write': ..., - 'delete': ... - }, - } - -The top-level distinction for feature flags is the operation the client intends -to perform, - -| the *read* operation includes extraction and listing of archives, -| the *write* operation includes creating new archives, -| the *delete* (archives) operation, -| the *check* operation requires full understanding of everything in the repository. -| - -These are weakly set-ordered; *check* will include everything required for *delete*, -*delete* will likely include *write* and *read*. However, *read* may require more -features than *write* (due to ID-based deduplication, *write* does not necessarily -require reading/understanding repository contents). - -Each operation can contain several sets of feature flags. Only one set, -the *mandatory* set is currently defined. - -Upon reading the manifest, the Borg client has already determined which operation -should be performed. If feature flags are found in the manifest, the set -of feature flags supported by the client is compared to the mandatory set -found in the manifest. If any unsupported flags are found (i.e. the mandatory set is -not a subset of the features supported by the Borg client used), the operation -is aborted with a *MandatoryFeatureUnsupported* error: - - Unsupported repository feature(s) {'some_feature'}. A newer version of Borg is required to access this repository. - -Older Borg releases do not have this concept and do not perform feature flags checks. -These are locked out with manifest version 2, which is what Borg 2 always writes: -the only difference between manifest versions 1 and 2 is that the latter is only -accepted by Borg releases implementing feature flags. - -.. _Cache feature flags: -.. rubric:: Cache feature flags - -:ref:`The local cache ` does not have its separate set of feature flags. -Instead, Borg stores which flags were used to create or modify a cache (as the -*mandatory_features* / *ignored_features* keys in the cache ``config`` file). - -All mandatory manifest features from all operations are gathered in one set. -Then, two sets of features are computed; - -- those features that are supported by the client and mandated by the manifest - are added to the *mandatory_features* set, -- the *ignored_features* set comprised of those features mandated by the manifest, - but not supported by the client. - -Because the client previously checked compliance with the mandatory set of features -required for the particular operation it is executing, the *mandatory_features* set -will contain all necessary features required for using the cache safely. - -Conversely, the *ignored_features* set contains only those features which were not -relevant to operating the cache. Otherwise, the client would not pass the feature -set test against the manifest. - -When opening a cache and the *mandatory_features* set is not a subset of the features -supported by the client, the cache is wiped out and rebuilt, -since a client not supporting a mandatory feature that the cache was built with -would be unable to update it correctly. -The assumption behind this behaviour is that any of the unsupported features could have -been reflected in the cache and there is no way for the client to discern whether -that is the case. -Meanwhile, it may not be practical for every feature to have clients using it track -whether the feature had an impact on the cache. -Therefore, the cache is wiped. - -When opening a cache and the intersection of *ignored_features* and the features -supported by the client contains any elements, i.e. the client possesses features -that the previous client did not have and those new features are enabled in the repository, -the cache is wiped out and rebuilt. - -While the former condition likely requires no tweaks, the latter condition is formulated -in an especially conservative way to play it safe. It seems likely that specific features -might be exempted from the latter condition. - -.. rubric:: Defined feature flags - -Currently no feature flags are defined. - -From currently planned features, some examples follow, -these may/may not be implemented and purely serve as examples. - -- A mandatory *read* feature could be using a different encryption scheme (e.g. session keys). - This may not be mandatory for the *write* operation - reading data is not strictly required for - creating an archive. -- Any additions to the way chunks are referenced (e.g. to support larger archives) would - become a mandatory *delete* and *check* feature; *delete* implies knowing correct - reference counts, so all object references need to be understood. *check* must - discover the entire object graph as well, otherwise the "orphan chunks check" - could delete data still in use. - .. _archive: Archives @@ -1400,8 +1263,6 @@ the file's name (see :ref:`the files cache ` about that name): version = 1 repository = 3c4...e59 manifest = 10e...21c - ignored_features = - mandatory_features = [integrity] manifest = 10e...21c diff --git a/src/borg/cache.py b/src/borg/cache.py index 3904806e86..a72f93b5a4 100644 --- a/src/borg/cache.py +++ b/src/borg/cache.py @@ -30,7 +30,7 @@ from .helpers import archive_hostname, archive_username from .helpers import chunkit from .helpers import CorruptPack, IntegrityError -from .helpers import hex_to_bin, bin_to_hex, parse_stringified_list +from .helpers import hex_to_bin, bin_to_hex from .helpers import format_file_size, safe_encode from .helpers import safe_ns from .helpers import ProgressIndicatorMessage, ProgressIndicatorPercent @@ -141,10 +141,6 @@ def load(self): self._check_upgrade(self.config_path) self.id = self._config.get("cache", "repository") self.manifest_id = hex_to_bin(self._config.get("cache", "manifest")) - self.ignored_features = set(parse_stringified_list(self._config.get("cache", "ignored_features", fallback=""))) - self.mandatory_features = set( - parse_stringified_list(self._config.get("cache", "mandatory_features", fallback="")) - ) try: self.integrity = dict(self._config.items("integrity")) if self._config.get("cache", "manifest") != self.integrity.pop("manifest"): @@ -163,8 +159,6 @@ def load(self): def save(self, manifest=None): if manifest: self._config.set("cache", "manifest", manifest.id_str) - self._config.set("cache", "ignored_features", ",".join(self.ignored_features)) - self._config.set("cache", "mandatory_features", ",".join(self.mandatory_features)) if not self._config.has_section("integrity"): self._config.add_section("integrity") for file, integrity_data in self.integrity.items(): @@ -1304,11 +1298,6 @@ def __init__( self.open() try: self.security_manager.assert_secure(self.key) - - if not self.check_cache_compatibility(): - self.wipe_cache() - - self.update_compatibility() except: # noqa self.close() raise @@ -1369,35 +1358,3 @@ def close(self): self.cache_config.close() pi.finish() self.cache_config = None - - def check_cache_compatibility(self): - my_features = Manifest.SUPPORTED_REPO_FEATURES - if self.cache_config.ignored_features & my_features: - # The cache might not contain references of chunks that need a feature that is mandatory for some operation - # and which this version supports. To avoid corruption while executing that operation force rebuild. - return False - if not self.cache_config.mandatory_features <= my_features: - # The cache was build with consideration to at least one feature that this version does not understand. - # This client might misinterpret the cache. Thus force a rebuild. - return False - return True - - def wipe_cache(self): - logger.warning("Discarding incompatible cache and forcing a cache rebuild") - self._chunks = ChunkIndex() - self.repository.chunks = self._chunks - self.cache_config.manifest_id = "" - self.cache_config._config.set("cache", "manifest", "") - - self.cache_config.ignored_features = set() - self.cache_config.mandatory_features = set() - - def update_compatibility(self): - operation_to_features_map = self.manifest.get_all_mandatory_features() - my_features = Manifest.SUPPORTED_REPO_FEATURES - repo_features = set() - for operation, features in operation_to_features_map.items(): - repo_features.update(features) - - self.cache_config.ignored_features.update(repo_features - my_features) - self.cache_config.mandatory_features.update(repo_features & my_features) diff --git a/src/borg/item.pyx b/src/borg/item.pyx index be53fed400..2044fc10f6 100644 --- a/src/borg/item.pyx +++ b/src/borg/item.pyx @@ -583,17 +583,6 @@ cdef class ManifestItem(PropDict): ck = fix_key(cd, ck) if ck == 'tam_required': assert isinstance(cv, bool) - if ck == 'feature_flags': - assert isinstance(cv, dict) - ops = {'read', 'check', 'write', 'delete'} - for op, specs in list(cv.items()): - op = fix_key(cv, op) - assert op in ops - for speck, specv in list(specs.items()): - speck = fix_key(specs, speck) - if speck == 'mandatory': - specs[speck] = fix_tuple_of_str(specv) - assert set(cv).issubset(ops) if k == 'item_keys': v = fix_tuple_of_str(v) self._dict[k] = v diff --git a/src/borg/manifest.py b/src/borg/manifest.py index 07329e8d5a..26a4902fbf 100644 --- a/src/borg/manifest.py +++ b/src/borg/manifest.py @@ -29,6 +29,8 @@ from .repoobj import RepoObj +# Not raised anymore: the repository feature flags mechanism was removed. The class is kept so that its +# return code stays reserved and never gets a different meaning. class MandatoryFeatureUnsupported(Error): """Unsupported repository feature(s) {}. A newer version of Borg is required to access this repository.""" @@ -508,8 +510,6 @@ class Operation(enum.StrEnum): NO_OPERATION_CHECK: Sequence[Operation] = tuple() - SUPPORTED_REPO_FEATURES: frozenset[str] = frozenset([]) - MANIFEST_ID = b"\0" * 32 def __init__(self, key, repository, ro_cls=RepoObj): @@ -552,41 +552,15 @@ def load(cls, repository, operations, key=None, *, other=False, ro_cls=RepoObj): # the list of item keys (borg 1.x: "item_keys", older borg 2 versions: config["item_keys"]). manifest.config = m.config manifest.config.pop("item_keys", None) - manifest.check_repository_compatibility(operations) return manifest - def check_repository_compatibility(self, operations): - for operation in operations: - assert isinstance(operation, self.Operation) - feature_flags = self.config.get("feature_flags", None) - if feature_flags is None: - return - if operation not in feature_flags: - continue - requirements = feature_flags[operation] - if "mandatory" in requirements: - unsupported = set(requirements["mandatory"]) - self.SUPPORTED_REPO_FEATURES - if unsupported: - raise MandatoryFeatureUnsupported(list(unsupported)) - - def get_all_mandatory_features(self): - result = {} - feature_flags = self.config.get("feature_flags", None) - if feature_flags is None: - return result - - for operation, requirements in feature_flags.items(): - if "mandatory" in requirements: - result[operation] = set(requirements["mandatory"]) - return result - def write(self): """ Store the manifest in the repository, but only if its content differs from what was loaded. - The manifest only holds the (optional) feature flags, so it usually does not change at all: - archive operations call this, but it only results in a store write when the config changed - or when the loaded manifest still had legacy entries (a timestamp, the item keys list). + The manifest content is static (borg does not store anything in its config dict currently), so + archive operations calling this usually do not result in a store write: only when the loaded + manifest still had legacy entries (a timestamp, the item keys list) is it rewritten. """ from .item import ManifestItem diff --git a/src/borg/testsuite/archiver/checks_test.py b/src/borg/testsuite/archiver/checks_test.py index 22a39a43b7..4831deb68e 100644 --- a/src/borg/testsuite/archiver/checks_test.py +++ b/src/borg/testsuite/archiver/checks_test.py @@ -7,9 +7,6 @@ from ...constants import * # NOQA from ...helpers import Location, get_security_dir, bin_to_hex from ...helpers import EXIT_ERROR -from ...manifest import Manifest, MandatoryFeatureUnsupported -from ...repository import Repository -from .. import llfuse from .. import changedir from . import cmd, _extract_repository_id, create_test_files from . import _set_repository_id, create_regular_file, assert_creates_file, generate_archiver_tests, RK_ENCRYPTION @@ -23,22 +20,6 @@ def get_security_directory(repo_path): return get_security_dir(repository_id) -def add_unknown_feature(repo_path, operation): - with Repository(repo_path, exclusive=True) as repository: - manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) - manifest.config["feature_flags"] = {operation.value: {"mandatory": ["unknown-feature"]}} - manifest.write() - - -def cmd_raises_unknown_feature(archiver, args): - if archiver.FORK_DEFAULT: - cmd(archiver, *args, exit_code=EXIT_ERROR) - else: - with pytest.raises(MandatoryFeatureUnsupported) as excinfo: - cmd(archiver, *args) - assert excinfo.value.args == (["unknown-feature"],) - - def test_repository_swap_detection(archivers, request): archiver = request.getfixturevalue(archivers) create_test_files(archiver.input_path) @@ -216,86 +197,6 @@ def test_unknown_unencrypted_empty_passphrase(archivers, request, monkeypatch): assert "stored inside the repository (repokey) and has an empty passphrase" in output -def test_unknown_feature_on_create(archivers, request): - archiver = request.getfixturevalue(archivers) - print(cmd(archiver, "repo-create", RK_ENCRYPTION)) - add_unknown_feature(archiver.repository_path, Manifest.Operation.WRITE) - cmd_raises_unknown_feature(archiver, ["create", "test", "input"]) - - -def test_unknown_feature_on_change_passphrase(archivers, request): - archiver = request.getfixturevalue(archivers) - print(cmd(archiver, "repo-create", RK_ENCRYPTION)) - add_unknown_feature(archiver.repository_path, Manifest.Operation.CHECK) - cmd_raises_unknown_feature(archiver, ["key", "change-passphrase"]) - - -def test_unknown_feature_on_read(archivers, request): - archiver = request.getfixturevalue(archivers) - print(cmd(archiver, "repo-create", RK_ENCRYPTION)) - cmd(archiver, "create", "test", "input") - add_unknown_feature(archiver.repository_path, Manifest.Operation.READ) - with changedir("output"): - cmd_raises_unknown_feature(archiver, ["extract", "test"]) - cmd_raises_unknown_feature(archiver, ["repo-list"]) - cmd_raises_unknown_feature(archiver, ["info", "-a", "test"]) - - -def test_unknown_feature_on_rename(archivers, request): - archiver = request.getfixturevalue(archivers) - print(cmd(archiver, "repo-create", RK_ENCRYPTION)) - cmd(archiver, "create", "test", "input") - add_unknown_feature(archiver.repository_path, Manifest.Operation.CHECK) - cmd_raises_unknown_feature(archiver, ["rename", "test", "other"]) - - -def test_unknown_feature_on_delete(archivers, request): - archiver = request.getfixturevalue(archivers) - print(cmd(archiver, "repo-create", RK_ENCRYPTION)) - cmd(archiver, "create", "test", "input") - add_unknown_feature(archiver.repository_path, Manifest.Operation.DELETE) - # delete of an archive raises - cmd_raises_unknown_feature(archiver, ["delete", "-a", "test"]) - cmd_raises_unknown_feature(archiver, ["prune", "--keep-daily=3"]) - # delete of the whole repository ignores features - cmd(archiver, "repo-delete") - - -@pytest.mark.skipif(not llfuse, reason="llfuse not installed") -def test_unknown_feature_on_mount(archivers, request): - archiver = request.getfixturevalue(archivers) - cmd(archiver, "repo-create", RK_ENCRYPTION) - cmd(archiver, "create", "test", "input") - add_unknown_feature(archiver.repository_path, Manifest.Operation.READ) - mountpoint = os.path.join(archiver.tmpdir, "mountpoint") - os.mkdir(mountpoint) - # XXX this might hang if it doesn't raise an error - cmd_raises_unknown_feature(archiver, ["mount", mountpoint]) - - -def test_unknown_mandatory_feature_in_cache(archivers, request): - archiver = request.getfixturevalue(archivers) - remote_repo = archiver.get_kind() == "remote" - print(cmd(archiver, "repo-create", RK_ENCRYPTION)) - - with Repository(archiver.repository_path, exclusive=True) as repository: - if remote_repo: - repository._location = Location(archiver.repository_location) - manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) - with Cache(repository, manifest) as cache: - cache.cache_config.mandatory_features = {"unknown-feature"} - - if archiver.FORK_DEFAULT: - cmd(archiver, "create", "test", "input") - - with Repository(archiver.repository_path, exclusive=True) as repository: - if remote_repo: - repository._location = Location(archiver.repository_location) - manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) - with Cache(repository, manifest) as cache: - assert cache.cache_config.mandatory_features == set() - - # Begin Remote Tests def test_remote_repo_strip_components_doesnt_leak(remote_archiver): cmd(remote_archiver, "repo-create", RK_ENCRYPTION) From f9e112f8687a2856833ad1f80202fbe5a64ceaee Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 15 Sep 2026 01:32:54 +0200 Subject: [PATCH 2/2] remove Manifest.Operation and the compatibility= decorator argument Leftovers of the feature flags mechanism removed in the previous commit: the Manifest.Operation enum, NO_OPERATION_CHECK, the operations parameter of Manifest.load(), compat_check() and the compatibility= argument of the with_repository / with_other_repository decorators, plus all their call sites in the archiver command modules and the tests. Co-Authored-By: Claude Fable 5.1 --- src/borg/archive.py | 2 +- src/borg/archiver/_common.py | 54 ++----------------- src/borg/archiver/analyze_cmd.py | 4 +- src/borg/archiver/compact_cmd.py | 3 +- src/borg/archiver/copy_cmd.py | 3 +- src/borg/archiver/create_cmd.py | 3 +- src/borg/archiver/debug_cmd.py | 13 +++-- src/borg/archiver/delete_cmd.py | 2 +- src/borg/archiver/diff_cmd.py | 3 +- src/borg/archiver/extract_cmd.py | 3 +- src/borg/archiver/find_cmd.py | 3 +- src/borg/archiver/info_cmd.py | 3 +- src/borg/archiver/key_cmds.py | 11 ++-- src/borg/archiver/list_cmd.py | 3 +- src/borg/archiver/mount_cmds.py | 3 +- src/borg/archiver/prune_cmd.py | 4 +- src/borg/archiver/recreate_cmd.py | 3 +- src/borg/archiver/rename_cmd.py | 3 +- src/borg/archiver/repo_compress_cmd.py | 3 +- src/borg/archiver/repo_create_cmd.py | 2 +- src/borg/archiver/repo_delete_cmd.py | 2 +- src/borg/archiver/repo_info_cmd.py | 3 +- src/borg/archiver/repo_list_cmd.py | 4 +- src/borg/archiver/tag_cmd.py | 3 +- src/borg/archiver/tar_cmds.py | 5 +- src/borg/archiver/transfer_cmd.py | 5 +- src/borg/archiver/undelete_cmd.py | 2 +- src/borg/archiver/webdav_cmd.py | 3 +- src/borg/manifest.py | 29 +--------- src/borg/testsuite/archiver/__init__.py | 2 +- src/borg/testsuite/archiver/check_cmd_test.py | 8 +-- .../testsuite/archiver/compact_cmd_test.py | 10 ++-- src/borg/testsuite/archiver/copy_cmd_test.py | 16 +++--- .../testsuite/archiver/create_cmd_test.py | 4 +- .../testsuite/archiver/debug_cmds_test.py | 2 +- src/borg/testsuite/archiver/diff_cmd_test.py | 2 +- .../testsuite/archiver/extract_cmd_test.py | 2 +- .../testsuite/archiver/rename_cmd_test.py | 2 +- .../archiver/repo_compress_cmd_test.py | 8 +-- .../testsuite/archiver/webdav_cmd_test.py | 8 +-- src/borg/testsuite/cache_test.py | 6 +-- 41 files changed, 81 insertions(+), 173 deletions(-) diff --git a/src/borg/archive.py b/src/borg/archive.py index e2e49fb870..572e6ef3c1 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -2322,7 +2322,7 @@ def check( rebuild_manifest = True else: try: - self.manifest = Manifest.load(repository, (Manifest.Operation.CHECK,), key=self.key) + self.manifest = Manifest.load(repository, key=self.key) except IntegrityErrorBase as exc: logger.error("Repository manifest is corrupted: %s", exc) self.error_found = True diff --git a/src/borg/archiver/_common.py b/src/borg/archiver/_common.py index 58503c7b89..6292bdc911 100644 --- a/src/borg/archiver/_common.py +++ b/src/borg/archiver/_common.py @@ -60,32 +60,7 @@ def get_repository(location, *, create, exclusive, lock_wait, lock, args, v1_leg return repository -def compat_check(*, create, manifest, key, cache, compatibility, decorator_name): - if not create and (manifest or key or cache): - if compatibility is None: - raise AssertionError(f"{decorator_name} decorator used without compatibility argument") - if type(compatibility) is not tuple: - raise AssertionError(f"{decorator_name} decorator compatibility argument must be of type tuple") - else: - if compatibility is not None: - raise AssertionError( - f"{decorator_name} called with compatibility argument, " f"but would not check {compatibility!r}" - ) - if create: - compatibility = Manifest.NO_OPERATION_CHECK - return compatibility - - -def with_repository( - create=False, - lock=True, - exclusive=False, - manifest=True, - cache=False, - secure=True, - compatibility=None, - allow_v1=False, -): +def with_repository(create=False, lock=True, exclusive=False, manifest=True, cache=False, secure=True, allow_v1=False): """ Method decorator for subcommand-handling methods: do_XYZ(self, args, repository, …) @@ -96,20 +71,8 @@ def with_repository( :param manifest: load manifest and repo_objs (key), pass them as keyword arguments :param cache: open cache, pass it as keyword argument (implies manifest) :param secure: do assert_secure after loading manifest - :param compatibility: mandatory if not create and (manifest or cache), specifies mandatory - feature categories to check :param allow_v1: (bool) allow legacy Borg 1.x repositories """ - # Note: with_repository decorator does not have a "key" argument (yet?) - compatibility = compat_check( - create=create, - manifest=manifest, - key=manifest, - cache=cache, - compatibility=compatibility, - decorator_name="with_repository", - ) - # We may need to modify `lock` inside `wrapper`. Therefore we cannot use the # `nonlocal` statement to access `lock` as modifications would also # affect the scope outside of `wrapper`. Subsequent calls would @@ -154,7 +117,7 @@ def wrapper(self, args, **kwargs): from ..legacy.repoobj import RepoObj1 ro_cls = RepoObj1 - manifest_ = Manifest.load(repository, compatibility, other=False, ro_cls=ro_cls) + manifest_ = Manifest.load(repository, other=False, ro_cls=ro_cls) kwargs["manifest"] = manifest_ if "compression" in args: manifest_.repo_objs.compressor = args.compression.compressor @@ -177,7 +140,7 @@ def wrapper(self, args, **kwargs): return decorator -def with_other_repository(manifest=False, cache=False, compatibility=None, required=False): +def with_other_repository(manifest=False, cache=False, required=False): """ this is a simplified version of "with_repository", just for the "other location". @@ -186,15 +149,6 @@ def with_other_repository(manifest=False, cache=False, compatibility=None, requi :param required: the command can not work without the other repository, refuse to run if it is not given. """ - compatibility = compat_check( - create=False, - manifest=manifest, - key=manifest, - cache=cache, - compatibility=compatibility, - decorator_name="with_other_repository", - ) - def decorator(method): @functools.wraps(method) def wrapper(self, args, **kwargs): @@ -231,7 +185,7 @@ def wrapper(self, args, **kwargs): from ..legacy.repoobj import RepoObj1 ro_cls = RepoObj1 - manifest_ = Manifest.load(repository, compatibility, other=True, ro_cls=ro_cls) + manifest_ = Manifest.load(repository, other=True, ro_cls=ro_cls) assert_secure(repository, manifest_) if manifest: kwargs["other_manifest"] = manifest_ diff --git a/src/borg/archiver/analyze_cmd.py b/src/borg/archiver/analyze_cmd.py index 51cf3cfdbd..b89aa2b8aa 100644 --- a/src/borg/archiver/analyze_cmd.py +++ b/src/borg/archiver/analyze_cmd.py @@ -9,7 +9,7 @@ from ..helpers import ProgressIndicatorPercent from ..helpers.argparsing import ArgumentParser from ..helpers import GroupBySpec -from ..manifest import AI_GROUP_BY_KEYS, Manifest, archive_group_key, format_group_key +from ..manifest import AI_GROUP_BY_KEYS, archive_group_key, format_group_key from ..repository import Repository from ..logger import create_logger @@ -445,7 +445,7 @@ def report_hotspots(self, hotspots): class AnalyzeMixIn: - @with_repository(compatibility=(Manifest.Operation.READ,)) + @with_repository() def do_analyze(self, args, repository, manifest): """Analyzes archives.""" ArchiveAnalyzer(args, repository, manifest).analyze() diff --git a/src/borg/archiver/compact_cmd.py b/src/borg/archiver/compact_cmd.py index 4ba33d1af6..b478b9cc6f 100644 --- a/src/borg/archiver/compact_cmd.py +++ b/src/borg/archiver/compact_cmd.py @@ -14,7 +14,6 @@ from ..hashindex import ChunkIndex from ..helpers import set_ec, EXIT_ERROR, Error, sig_int, format_file_size, bin_to_hex, hex_to_bin, IntegrityError from ..helpers import ProgressIndicatorPercent -from ..manifest import Manifest from ..repoobj import object_validator from ..repository import Repository @@ -456,7 +455,7 @@ def compact_packs(self): class CompactMixIn: - @with_repository(exclusive=True, compatibility=(Manifest.Operation.DELETE,)) + @with_repository(exclusive=True) def do_compact(self, args, repository, manifest): """Collects garbage in the repository.""" if not args.dry_run: diff --git a/src/borg/archiver/copy_cmd.py b/src/borg/archiver/copy_cmd.py index 92f45ae261..9cf6d8fe74 100644 --- a/src/borg/archiver/copy_cmd.py +++ b/src/borg/archiver/copy_cmd.py @@ -2,7 +2,6 @@ from ..constants import * # NOQA from ..helpers import archivename_validator, bin_to_hex from ..helpers.argparsing import ArgumentParser -from ..manifest import Manifest from ..logger import create_logger @@ -10,7 +9,7 @@ class CopyMixIn: - @with_repository(cache=True, compatibility=(Manifest.Operation.CHECK,)) + @with_repository(cache=True) @with_archive def do_copy(self, args, repository, manifest, cache, archive): """Copy an archive to a new archive name.""" diff --git a/src/borg/archiver/create_cmd.py b/src/borg/archiver/create_cmd.py index 4a42ff5f70..03e1874570 100644 --- a/src/borg/archiver/create_cmd.py +++ b/src/borg/archiver/create_cmd.py @@ -36,7 +36,6 @@ from ..helpers import MakePathSafeAction from ..helpers import Error, CommandError, BackupWarning, FileChangedWarning from ..helpers.argparsing import ArgumentParser -from ..manifest import Manifest from ..patterns import PatternMatcher from ..platform import is_win32, get_flags @@ -64,7 +63,7 @@ def stat_root(path): class CreateMixIn: - @with_repository(compatibility=(Manifest.Operation.WRITE,)) + @with_repository() def do_create(self, args, repository, manifest): """Creates a new archive.""" if args.read_special_timeout is not None and not args.read_special: diff --git a/src/borg/archiver/debug_cmd.py b/src/borg/archiver/debug_cmd.py index cb924e7974..50cc6e3e85 100644 --- a/src/borg/archiver/debug_cmd.py +++ b/src/borg/archiver/debug_cmd.py @@ -13,7 +13,6 @@ from ..helpers import archivename_validator, CompressionSpec from ..helpers import CommandError, IntegrityError, RTError from ..helpers.argparsing import ArgumentParser -from ..manifest import Manifest from ..platform import get_process_id from ..repository import Repository, LIST_SCAN_LIMIT, repo_lister from ..repoobj import RepoObj, object_validator @@ -48,7 +47,7 @@ def do_debug_info(self, args): print(sysinfo()) print("Process ID:", get_process_id()) - @with_repository(compatibility=Manifest.NO_OPERATION_CHECK) + @with_repository() def do_debug_dump_archive_items(self, args, repository, manifest): """Dumps (decrypted, decompressed) archive item metadata (not data).""" repo_objs = manifest.repo_objs @@ -62,7 +61,7 @@ def do_debug_dump_archive_items(self, args, repository, manifest): fd.write(data) print("Done.") - @with_repository(compatibility=Manifest.NO_OPERATION_CHECK) + @with_repository() def do_debug_dump_archive(self, args, repository, manifest): """Dumps decoded archive metadata (not data).""" archive_info = manifest.archives.get_one([args.name]) @@ -117,7 +116,7 @@ def output(fd): with dash_open(args.path, "w") as fd: output(fd) - @with_repository(compatibility=Manifest.NO_OPERATION_CHECK) + @with_repository() def do_debug_dump_manifest(self, args, repository, manifest): """Dumps decoded repository manifest.""" repo_objs = manifest.repo_objs @@ -222,7 +221,7 @@ def do_debug_get_obj(self, args, repository): f.write(data) print("object %s fetched." % hex_id) - @with_repository(compatibility=Manifest.NO_OPERATION_CHECK) + @with_repository() def do_debug_id_hash(self, args, repository, manifest): """Computes id-hash for file contents.""" with open(args.path, "rb") as f: @@ -231,7 +230,7 @@ def do_debug_id_hash(self, args, repository, manifest): id = key.id_hash(data) print(id.hex()) - @with_repository(compatibility=Manifest.NO_OPERATION_CHECK) + @with_repository() def do_debug_parse_obj(self, args, repository, manifest): """Parses a Borg object file into a metadata dict and data (decrypting, decompressing).""" @@ -254,7 +253,7 @@ def do_debug_parse_obj(self, args, repository, manifest): with open(args.binary_path, "wb") as f: f.write(data) - @with_repository(compatibility=Manifest.NO_OPERATION_CHECK) + @with_repository() def do_debug_format_obj(self, args, repository, manifest): """Formats file and metadata into a Borg object file.""" diff --git a/src/borg/archiver/delete_cmd.py b/src/borg/archiver/delete_cmd.py index 22c94a1db3..18ab3e0773 100644 --- a/src/borg/archiver/delete_cmd.py +++ b/src/borg/archiver/delete_cmd.py @@ -17,7 +17,7 @@ def do_delete(self, args, repository): """Deletes archives.""" self.output_list = args.output_list dry_run = args.dry_run - manifest = Manifest.load(repository, (Manifest.Operation.DELETE,)) + manifest = Manifest.load(repository) if args.name: archive_infos = [manifest.archives.get_one(archive_match_patterns(args))] else: diff --git a/src/borg/archiver/diff_cmd.py b/src/borg/archiver/diff_cmd.py index a0b83d8bba..94afbe9165 100644 --- a/src/borg/archiver/diff_cmd.py +++ b/src/borg/archiver/diff_cmd.py @@ -13,7 +13,6 @@ from ..helpers.argparsing import ArgumentParser from ..helpers.sorting import sort_spec_validator, sorted_by_spec from ..item import ItemDiff -from ..manifest import Manifest from ..logger import create_logger logger = create_logger() @@ -91,7 +90,7 @@ def __str__(self) -> str: class DiffMixIn: - @with_repository(compatibility=(Manifest.Operation.READ,)) + @with_repository() def do_diff(self, args, repository, manifest): """Finds differences between two archives.""" diff --git a/src/borg/archiver/extract_cmd.py b/src/borg/archiver/extract_cmd.py index e04562f74a..bca551e843 100644 --- a/src/borg/archiver/extract_cmd.py +++ b/src/borg/archiver/extract_cmd.py @@ -13,7 +13,6 @@ from ..helpers import ProgressIndicatorPercent from ..helpers import BackupWarning, IncludePatternNeverMatchedWarning from ..helpers.argparsing import ArgumentParser -from ..manifest import Manifest from ..logger import create_logger @@ -21,7 +20,7 @@ class ExtractMixIn: - @with_repository(compatibility=(Manifest.Operation.READ,)) + @with_repository() @with_archive def do_extract(self, args, repository, manifest, archive): """Extracts archive contents.""" diff --git a/src/borg/archiver/find_cmd.py b/src/borg/archiver/find_cmd.py index 22ec50ab0d..ea014e8e79 100644 --- a/src/borg/archiver/find_cmd.py +++ b/src/borg/archiver/find_cmd.py @@ -8,7 +8,6 @@ from ..constants import * # NOQA from ..helpers import ItemFormatter, BaseFormatter, PathSpec from ..helpers.argparsing import ArgumentParser -from ..manifest import Manifest from ..logger import create_logger @@ -16,7 +15,7 @@ class FindMixIn: - @with_repository(compatibility=(Manifest.Operation.READ,)) + @with_repository() def do_find(self, args, repository, manifest): """Find files across archives.""" matcher = build_matcher(args.patterns, args.paths) diff --git a/src/borg/archiver/info_cmd.py b/src/borg/archiver/info_cmd.py index 4a44b8841c..2b5a0e959d 100644 --- a/src/borg/archiver/info_cmd.py +++ b/src/borg/archiver/info_cmd.py @@ -6,7 +6,6 @@ from ..constants import * # NOQA from ..helpers import format_timedelta, json_print, basic_json_data, archivename_validator from ..helpers.argparsing import ArgumentParser -from ..manifest import Manifest from ..logger import create_logger @@ -14,7 +13,7 @@ class InfoMixIn: - @with_repository(cache=True, compatibility=(Manifest.Operation.READ,)) + @with_repository(cache=True) def do_info(self, args, repository, manifest, cache): """Show archive details such as disk space used""" diff --git a/src/borg/archiver/key_cmds.py b/src/borg/archiver/key_cmds.py index cd8649f854..456cc5f32a 100644 --- a/src/borg/archiver/key_cmds.py +++ b/src/borg/archiver/key_cmds.py @@ -5,7 +5,6 @@ from ..crypto.keymanager import KeyManager from ..helpers import FilesystemPathSpec, CommandError from ..helpers.argparsing import ArgumentParser -from ..manifest import Manifest from ._common import with_repository @@ -15,7 +14,7 @@ class KeysMixIn: - @with_repository(compatibility=(Manifest.Operation.CHECK,)) + @with_repository() def do_key_change_passphrase(self, args, repository, manifest): """Changes the repository key file passphrase.""" key = manifest.key @@ -27,7 +26,7 @@ def do_key_change_passphrase(self, args, repository, manifest): # print key location to make backing it up easier logger.info("Key location: %s", key.find_key()) - @with_repository(manifest=True, compatibility=(Manifest.Operation.CHECK,)) + @with_repository(manifest=True) def do_key_add(self, args, repository, manifest): """Add a new borg key (protected by an independent passphrase) to the repository.""" key = manifest.key @@ -38,7 +37,7 @@ def do_key_add(self, args, repository, manifest): if hasattr(key, "find_key"): logger.info("Key location: %s", key.find_key()) - @with_repository(manifest=True, compatibility=(Manifest.Operation.CHECK,)) + @with_repository(manifest=True) def do_key_remove(self, args, repository, manifest): """Remove a borg key from the repository.""" key = manifest.key @@ -47,7 +46,7 @@ def do_key_remove(self, args, repository, manifest): victim = key.remove_key(label=args.label, key_id=args.key, current=args.by_passphrase) logger.info("Borg key %s (label %r) removed.", victim["id"][:12], victim["label"]) - @with_repository(manifest=True, compatibility=(Manifest.Operation.CHECK,)) + @with_repository(manifest=True) def do_key_list(self, args, repository, manifest): """List the borg keys of the repository.""" key = manifest.key @@ -59,7 +58,7 @@ def do_key_list(self, args, repository, manifest): marker = "*" if bk["current"] else "" print(fmt % (marker, bk["id"][:12], bk["mode"], bk["label"] or "-", bk["algorithm"] or "-")) - @with_repository(exclusive=True, manifest=True, cache=True, compatibility=(Manifest.Operation.CHECK,)) + @with_repository(exclusive=True, manifest=True, cache=True) def do_key_change_location(self, args, repository, manifest, cache): """Changes the location of the borg key used to unlock this repository.""" key = manifest.key diff --git a/src/borg/archiver/list_cmd.py b/src/borg/archiver/list_cmd.py index c1a5d4f397..43a6d4a3de 100644 --- a/src/borg/archiver/list_cmd.py +++ b/src/borg/archiver/list_cmd.py @@ -10,7 +10,6 @@ from ..helpers import ItemFormatter, BaseFormatter, archivename_validator, PathSpec from ..helpers.argparsing import ArgumentParser from ..helpers.sorting import sort_spec_validator, sorted_by_spec -from ..manifest import Manifest from ..logger import create_logger @@ -44,7 +43,7 @@ def item_sort_key(field, item): class ListMixIn: - @with_repository(compatibility=(Manifest.Operation.READ,)) + @with_repository() def do_list(self, args, repository, manifest): """List archive contents.""" # omitting args.pattern_roots here, restricting to paths only by cli args.paths: diff --git a/src/borg/archiver/mount_cmds.py b/src/borg/archiver/mount_cmds.py index 481f693e13..d8adef372c 100644 --- a/src/borg/archiver/mount_cmds.py +++ b/src/borg/archiver/mount_cmds.py @@ -7,7 +7,6 @@ from ..helpers import location_validator from ..helpers import umount from ..helpers.argparsing import ArgumentParser -from ..manifest import Manifest from ..logger import create_logger @@ -34,7 +33,7 @@ def do_mount(self, args): self._do_mount(args) - @with_repository(compatibility=(Manifest.Operation.READ,)) + @with_repository() def _do_mount(self, args, repository, manifest): from ..fuse_impl import has_mfusepy diff --git a/src/borg/archiver/prune_cmd.py b/src/borg/archiver/prune_cmd.py index 9eaeee6c04..a7675a62cd 100644 --- a/src/borg/archiver/prune_cmd.py +++ b/src/borg/archiver/prune_cmd.py @@ -12,7 +12,7 @@ from ..helpers import GroupBySpec from ..helpers import json_print, basic_json_data from ..helpers.argparsing import ArgumentParser -from ..manifest import AI_GROUP_BY_KEYS, ArchiveInfo, Manifest, format_group_key, group_archives +from ..manifest import AI_GROUP_BY_KEYS, ArchiveInfo, format_group_key, group_archives from ..logger import create_logger @@ -195,7 +195,7 @@ def can_retain(a): class PruneMixIn: - @with_repository(compatibility=(Manifest.Operation.DELETE,)) + @with_repository() def do_prune(self, args, repository, manifest): """Prune archives according to specified rules.""" self._validate_prune_args(args) diff --git a/src/borg/archiver/recreate_cmd.py b/src/borg/archiver/recreate_cmd.py index 905fceb705..ee0d05b5de 100644 --- a/src/borg/archiver/recreate_cmd.py +++ b/src/borg/archiver/recreate_cmd.py @@ -5,7 +5,6 @@ from ..helpers import archivename_validator, comment_validator, PathSpec, ChunkerParams, bin_to_hex, CompressionSpec from ..helpers import timestamp from ..helpers.argparsing import ArgumentParser -from ..manifest import Manifest from ..logger import create_logger @@ -13,7 +12,7 @@ class RecreateMixIn: - @with_repository(cache=True, compatibility=(Manifest.Operation.CHECK,)) + @with_repository(cache=True) def do_recreate(self, args, repository, manifest, cache): """Recreate archives.""" # omitting args.pattern_roots here, restricting to paths only by cli args.paths: diff --git a/src/borg/archiver/rename_cmd.py b/src/borg/archiver/rename_cmd.py index b1d4829616..5429a91099 100644 --- a/src/borg/archiver/rename_cmd.py +++ b/src/borg/archiver/rename_cmd.py @@ -2,7 +2,6 @@ from ..constants import * # NOQA from ..helpers import archivename_validator from ..helpers.argparsing import ArgumentParser -from ..manifest import Manifest from ..logger import create_logger @@ -10,7 +9,7 @@ class RenameMixIn: - @with_repository(cache=True, compatibility=(Manifest.Operation.CHECK,)) + @with_repository(cache=True) @with_archive def do_rename(self, args, repository, manifest, cache, archive): """Rename an existing archive.""" diff --git a/src/borg/archiver/repo_compress_cmd.py b/src/borg/archiver/repo_compress_cmd.py index c97e2c81a9..9a5e773852 100644 --- a/src/borg/archiver/repo_compress_cmd.py +++ b/src/borg/archiver/repo_compress_cmd.py @@ -10,7 +10,6 @@ from ..helpers import sig_int, ProgressIndicatorPercent, Error, CompressionSpec from ..helpers import format_file_size, hex_to_bin from ..helpers.argparsing import ArgumentParser -from ..manifest import Manifest from ..repoobj import object_validator from ..repository import Repository @@ -189,7 +188,7 @@ def report(self, size_before, size_after): class RepoCompressMixIn: - @with_repository(manifest=True, exclusive=True, compatibility=(Manifest.Operation.CHECK,)) + @with_repository(manifest=True, exclusive=True) def do_repo_compress(self, args, repository, manifest): """Repository (re-)compression.""" if not isinstance(repository, Repository): diff --git a/src/borg/archiver/repo_create_cmd.py b/src/borg/archiver/repo_create_cmd.py index a7f7c1d7e9..6735c85876 100644 --- a/src/borg/archiver/repo_create_cmd.py +++ b/src/borg/archiver/repo_create_cmd.py @@ -26,7 +26,7 @@ class RepoCreateMixIn: @with_repository(create=True, exclusive=True, manifest=False) - @with_other_repository(manifest=True, compatibility=(Manifest.Operation.READ,)) + @with_other_repository(manifest=True) def do_repo_create(self, args, repository, *, other_repository=None, other_manifest=None): """Creates a new, empty repository.""" other_key = other_manifest.key if other_manifest is not None else None diff --git a/src/borg/archiver/repo_delete_cmd.py b/src/borg/archiver/repo_delete_cmd.py index 0b808fb500..6c8677d21b 100644 --- a/src/borg/archiver/repo_delete_cmd.py +++ b/src/borg/archiver/repo_delete_cmd.py @@ -28,7 +28,7 @@ def do_repo_delete(self, args, repository): location = repository._location.canonical_path() msg = [] try: - manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) + manifest = Manifest.load(repository) n_archives = manifest.archives.count() msg.append( f"You requested to DELETE the following repository completely " diff --git a/src/borg/archiver/repo_info_cmd.py b/src/borg/archiver/repo_info_cmd.py index 78fbd46a88..2c5cfb2f52 100644 --- a/src/borg/archiver/repo_info_cmd.py +++ b/src/borg/archiver/repo_info_cmd.py @@ -4,7 +4,6 @@ from ..constants import * # NOQA from ..helpers import bin_to_hex, json_print, basic_json_data from ..helpers.argparsing import ArgumentParser -from ..manifest import Manifest from ..logger import create_logger @@ -12,7 +11,7 @@ class RepoInfoMixIn: - @with_repository(cache=True, compatibility=(Manifest.Operation.READ,)) + @with_repository(cache=True) def do_repo_info(self, args, repository, manifest, cache): """Show repository information.""" key = manifest.key diff --git a/src/borg/archiver/repo_list_cmd.py b/src/borg/archiver/repo_list_cmd.py index 522abb6cf0..7133bf1b49 100644 --- a/src/borg/archiver/repo_list_cmd.py +++ b/src/borg/archiver/repo_list_cmd.py @@ -7,7 +7,7 @@ from ..helpers import BaseFormatter, ArchiveFormatter, json_print, basic_json_data from ..helpers import GroupBySpec from ..helpers.argparsing import ArgumentParser -from ..manifest import AI_GROUP_BY_KEYS, Manifest, format_group_key, group_archives +from ..manifest import AI_GROUP_BY_KEYS, format_group_key, group_archives from ..logger import create_logger @@ -18,7 +18,7 @@ class RepoListMixIn: - @with_repository(compatibility=(Manifest.Operation.READ,), allow_v1=True) + @with_repository(allow_v1=True) def do_repo_list(self, args, repository, manifest): """List the archives contained in a repository.""" if args.format is not None: diff --git a/src/borg/archiver/tag_cmd.py b/src/borg/archiver/tag_cmd.py index 27972ae231..fa65c55f7e 100644 --- a/src/borg/archiver/tag_cmd.py +++ b/src/borg/archiver/tag_cmd.py @@ -3,7 +3,6 @@ from ..constants import * # NOQA from ..helpers import bin_to_hex, archivename_validator, tag_validator from ..helpers.argparsing import ArgumentParser -from ..manifest import Manifest from ..logger import create_logger @@ -11,7 +10,7 @@ class TagMixIn: - @with_repository(cache=True, compatibility=(Manifest.Operation.WRITE,)) + @with_repository(cache=True) def do_tag(self, args, repository, manifest, cache): """Manage tags.""" diff --git a/src/borg/archiver/tar_cmds.py b/src/borg/archiver/tar_cmds.py index be2d9801eb..7297409a03 100644 --- a/src/borg/archiver/tar_cmds.py +++ b/src/borg/archiver/tar_cmds.py @@ -29,7 +29,6 @@ from ..helpers import basic_json_data, json_print from ..helpers import log_multi from ..helpers.argparsing import ArgumentParser -from ..manifest import Manifest from ._common import with_repository, with_archive, Highlander, define_exclusion_group from ._common import build_matcher, build_filter @@ -343,7 +342,7 @@ def create_zstd_filter(stream, stream_close, decompress): class TarMixIn: - @with_repository(compatibility=(Manifest.Operation.READ,)) + @with_repository() @with_archive def do_export_tar(self, args, repository, manifest, archive): """Export archive contents as a tarball""" @@ -522,7 +521,7 @@ def sparsify_tarinfo(item, tarinfo): for pattern in matcher.get_unmatched_include_patterns(): self.print_warning_instance(IncludePatternNeverMatchedWarning(pattern)) - @with_repository(cache=True, compatibility=(Manifest.Operation.WRITE,)) + @with_repository(cache=True) def do_import_tar(self, args, repository, manifest, cache): """Create a backup archive from a tarball""" self.output_filter = args.output_filter diff --git a/src/borg/archiver/transfer_cmd.py b/src/borg/archiver/transfer_cmd.py index 3f92e194bf..3ae015d56e 100644 --- a/src/borg/archiver/transfer_cmd.py +++ b/src/borg/archiver/transfer_cmd.py @@ -9,7 +9,6 @@ from ..helpers import ChunkerParams, ChunkIteratorFileWrapper, CompressionSpec from ..helpers.argparsing import ArgumentParser, ArgumentTypeError from ..item import ChunkListEntry -from ..manifest import Manifest from ..repository import Repository from ..logger import create_logger @@ -128,8 +127,8 @@ def transfer_chunks( class TransferMixIn: - @with_other_repository(manifest=True, required=True, compatibility=(Manifest.Operation.READ,)) - @with_repository(manifest=True, cache=True, compatibility=(Manifest.Operation.WRITE,)) + @with_other_repository(manifest=True, required=True) + @with_repository(manifest=True, cache=True) def do_transfer(self, args, *, repository, manifest, cache, other_repository=None, other_manifest=None): """archives transfer from other repository, optionally upgrade data format""" key = manifest.key diff --git a/src/borg/archiver/undelete_cmd.py b/src/borg/archiver/undelete_cmd.py index 38e54549ee..d936205f29 100644 --- a/src/borg/archiver/undelete_cmd.py +++ b/src/borg/archiver/undelete_cmd.py @@ -17,7 +17,7 @@ def do_undelete(self, args, repository): """Undeletes archives.""" self.output_list = args.output_list dry_run = args.dry_run - manifest = Manifest.load(repository, (Manifest.Operation.DELETE,)) + manifest = Manifest.load(repository) if args.name: archive_infos = [manifest.archives.get_one(archive_match_patterns(args), deleted=True)] else: diff --git a/src/borg/archiver/webdav_cmd.py b/src/borg/archiver/webdav_cmd.py index 50444a1316..3ce68c15dc 100644 --- a/src/borg/archiver/webdav_cmd.py +++ b/src/borg/archiver/webdav_cmd.py @@ -6,7 +6,6 @@ from ..constants import * # NOQA from ..helpers import sig_int, daemonizing, signal_handler from ..helpers.argparsing import ArgumentParser -from ..manifest import Manifest from ..logger import create_logger @@ -14,7 +13,7 @@ class WebDAVMixIn: - @with_repository(compatibility=(Manifest.Operation.READ,)) + @with_repository() def do_webdav(self, args, repository, manifest): """Serve archive contents via a read-only WebDAV / HTTP server on localhost.""" from ..webdav import make_server diff --git a/src/borg/manifest.py b/src/borg/manifest.py index 26a4902fbf..de5c6606fb 100644 --- a/src/borg/manifest.py +++ b/src/borg/manifest.py @@ -1,4 +1,3 @@ -import enum import re from collections import defaultdict, namedtuple from datetime import datetime @@ -484,32 +483,6 @@ def get_one(self, match, *, match_end=r"\Z", deleted=False): class Manifest: - @enum.unique - class Operation(enum.StrEnum): - # The comments here only roughly describe the scope of each feature. In the end, additions need to be - # based on potential problems older clients could produce when accessing newer repositories and the - # trade-offs of locking version out or still allowing access. As all older versions and their exact - # behaviours are known when introducing new features sometimes this might not match the general descriptions - # below. - - # The READ operation describes which features are needed to list and extract the archives safely in the - # repository. - READ = "read" - # The CHECK operation is for all operations that need either to understand every detail - # of the repository (for consistency checks and repairs) or are seldom used functions that just - # should use the most restrictive feature set because more fine grained compatibility tracking is - # not needed. - CHECK = "check" - # The WRITE operation is for adding archives. Features here ensure that older clients don't add archives - # in an old format, or is used to lock out clients that for other reasons can no longer safely add new - # archives. - WRITE = "write" - # The DELETE operation is for all operations (like archive deletion) that need a 100% correct reference - # count and the need to be able to find all (directly and indirectly) referenced chunks of a given archive. - DELETE = "delete" - - NO_OPERATION_CHECK: Sequence[Operation] = tuple() - MANIFEST_ID = b"\0" * 32 def __init__(self, key, repository, ro_cls=RepoObj): @@ -532,7 +505,7 @@ def id_str(self): return bin_to_hex(self.id) @classmethod - def load(cls, repository, operations, key=None, *, other=False, ro_cls=RepoObj): + def load(cls, repository, key=None, *, other=False, ro_cls=RepoObj): from .item import ManifestItem from .crypto.key import key_factory diff --git a/src/borg/testsuite/archiver/__init__.py b/src/borg/testsuite/archiver/__init__.py index d753cd7a4e..ab5d6db890 100644 --- a/src/borg/testsuite/archiver/__init__.py +++ b/src/borg/testsuite/archiver/__init__.py @@ -210,7 +210,7 @@ def create_src_archive(archiver, name, ts=None): def open_archive(repo_path, name): repository = Repository(repo_path, exclusive=True) with repository: - manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) + manifest = Manifest.load(repository) archive_info = manifest.archives.get_one([name]) archive = Archive(manifest, archive_info.id) return archive, repository diff --git a/src/borg/testsuite/archiver/check_cmd_test.py b/src/borg/testsuite/archiver/check_cmd_test.py index f5fcf712d0..e98db2a703 100644 --- a/src/borg/testsuite/archiver/check_cmd_test.py +++ b/src/borg/testsuite/archiver/check_cmd_test.py @@ -572,7 +572,7 @@ def test_spoofed_manifest(archivers, request): check_cmd_setup(archiver) archive, repository = open_archive(archiver.repository_path, "archive1") with repository: - manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) + manifest = Manifest.load(repository) cdata = manifest.repo_objs.format( Manifest.MANIFEST_ID, {}, @@ -953,7 +953,7 @@ def test_repair_finish_flushes_pack_writer(archivers, request): checker.repository = repository checker.key = checker.make_key(repository) checker.repo_objs = RepoObj(checker.key) - checker.manifest = Manifest.load(repository, (Manifest.Operation.CHECK,), key=checker.key) + checker.manifest = Manifest.load(repository, key=checker.key) # re-adding a chunk makes the chunks index no longer match the packs, so finish() rebuilds it. checker.chunks_modified = True @@ -1122,7 +1122,7 @@ def test_manifest_with_timestamp_is_accepted(archivers, request): archiver = request.getfixturevalue(archivers) check_cmd_setup(archiver) with Repository(archiver.repository_path, exclusive=True) as repository: - manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) + manifest = Manifest.load(repository) data = manifest.key.pack_metadata( { "version": 2, @@ -1157,7 +1157,7 @@ def test_items_with_unknown_keys_are_kept(archivers, request): ) ) with Repository(archiver.repository_path, exclusive=True) as repository: - manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) + manifest = Manifest.load(repository) with Cache(repository, manifest, archive_name="future") as cache: archive = Archive(manifest, "future", cache=cache, create=True) archive.items_buffer.add(item) diff --git a/src/borg/testsuite/archiver/compact_cmd_test.py b/src/borg/testsuite/archiver/compact_cmd_test.py index 0257fddcf8..531d95ce32 100644 --- a/src/borg/testsuite/archiver/compact_cmd_test.py +++ b/src/borg/testsuite/archiver/compact_cmd_test.py @@ -136,7 +136,7 @@ def test_compact_interrupted_does_not_poison_chunk_index(archivers, request, mon # unused objects), but force it to abort right before it writes the fresh, updated chunk index. repository = open_repository(archiver) with repository: - manifest = Manifest.load(repository, (Manifest.Operation.DELETE,)) + manifest = Manifest.load(repository) gc = ArchiveGarbageCollector(repository, manifest, stats=stats, threshold=40.0) def interrupt(): @@ -180,7 +180,7 @@ def test_compact_soft_interrupt_persists_valid_index(archivers, request, monkeyp pack_names_before = {info.name for info in repository.store_list("packs")} assert len(pack_names_before) >= 2 # need several packs to observe an early stop - manifest = Manifest.load(repository, (Manifest.Operation.DELETE,)) + manifest = Manifest.load(repository) gc = ArchiveGarbageCollector(repository, manifest, stats=False, threshold=10) original_store_delete = repository.store_delete @@ -422,7 +422,7 @@ def test_compact_keeps_undelete_data_when_chunks_missing(archivers, request): # sees a missing object and treats the repo as damaged. repository = open_repository(archiver) with repository: - manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) + manifest = Manifest.load(repository) kept = Archive(manifest, manifest.archives.get_one(["kept"]).id) victim = next(id for item in kept.iter_items() if "chunks" in item for id, _ in item.chunks) del repository.chunks[victim] @@ -691,7 +691,7 @@ def cached_archive_ids(): def archive_ids(): repository = open_repository(archiver) with repository: - manifest = Manifest.load(repository, (Manifest.Operation.READ,)) + manifest = Manifest.load(repository) return {bin_to_hex(info.id) for info in manifest.archives.list(sort_by=["ts"])} # no reference caches exist before the first compact @@ -819,7 +819,7 @@ def counting_build(repository, **kwargs): dry_run = scenario == "dry_run" repository = open_repository(archiver) with repository: - manifest = Manifest.load(repository, (Manifest.Operation.DELETE,)) + manifest = Manifest.load(repository) gc = ArchiveGarbageCollector(repository, manifest, stats=True, threshold=0.0, dry_run=dry_run) gc.garbage_collect() # the store must really change (or, on the dry run, really not), or the test proves nothing diff --git a/src/borg/testsuite/archiver/copy_cmd_test.py b/src/borg/testsuite/archiver/copy_cmd_test.py index 6195095dc0..d87221a577 100644 --- a/src/borg/testsuite/archiver/copy_cmd_test.py +++ b/src/borg/testsuite/archiver/copy_cmd_test.py @@ -11,7 +11,7 @@ def archive_id(archiver, name): with open_repository(archiver) as repository: - manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) + manifest = Manifest.load(repository) return manifest.archives.get_one([name]).id @@ -29,7 +29,7 @@ def test_copy(archivers, request): new_id = archive_id(archiver, "test.copy") assert new_id != old_id with open_repository(archiver) as repository: - manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) + manifest = Manifest.load(repository) assert manifest.archives.count() == 2 assert manifest.archives.exists_name_and_id("test", old_id) assert manifest.archives.exists_name_and_id("test.copy", new_id) @@ -50,7 +50,7 @@ def test_copy_by_archive_id(archivers, request): cmd(archiver, "copy", f"aid:{bin_to_hex(old_id)[:8]}", "test.copy") with open_repository(archiver) as repository: - manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) + manifest = Manifest.load(repository) assert manifest.archives.count() == 2 assert manifest.archives.exists("test.copy") @@ -65,7 +65,7 @@ def test_copy_shares_item_stream(archivers, request): cmd(archiver, "copy", "test", "test.copy") with open_repository(archiver) as repository: - manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) + manifest = Manifest.load(repository) original = Archive(manifest, manifest.archives.get_one(["test"]).id) copied = Archive(manifest, manifest.archives.get_one(["test.copy"]).id) assert original.metadata.item_ptrs == copied.metadata.item_ptrs @@ -88,7 +88,7 @@ def test_copy_delete_original(archivers, request): cmd(archiver, "compact") # actually free everything the deleted archive was the only referrer of with open_repository(archiver) as repository: - manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) + manifest = Manifest.load(repository) assert manifest.archives.count() == 1 assert manifest.archives.exists("test.copy") @@ -109,7 +109,7 @@ def test_copy_to_existing_name(archivers, request): cmd(archiver, "copy", "test", "series") with open_repository(archiver) as repository: - manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) + manifest = Manifest.load(repository) assert manifest.archives.count() == 3 assert len(list(manifest.archives.list(match=["series"]))) == 2 @@ -125,7 +125,7 @@ def test_copy_to_same_name(archivers, request): assert "can not be copied to the same name" in output with open_repository(archiver) as repository: - manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) + manifest = Manifest.load(repository) assert manifest.archives.count() == 1 @@ -141,5 +141,5 @@ def test_copy_ambiguous_name(archivers, request): assert "needed to match precisely one archive" in output with open_repository(archiver) as repository: - manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) + manifest = Manifest.load(repository) assert manifest.archives.count() == 2 diff --git a/src/borg/testsuite/archiver/create_cmd_test.py b/src/borg/testsuite/archiver/create_cmd_test.py index 7210eb9b43..a410e97505 100644 --- a/src/borg/testsuite/archiver/create_cmd_test.py +++ b/src/borg/testsuite/archiver/create_cmd_test.py @@ -694,7 +694,7 @@ def test_create_dry_run(archivers, request): cmd(archiver, "create", "--dry-run", "test", "input") # Make sure no archive has been created with Repository(archiver.repository_path) as repository: - manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) + manifest = Manifest.load(repository) assert manifest.archives.count() == 0 @@ -713,7 +713,7 @@ def test_create_dry_run_stats(archivers, request): assert "Deduplicated size:" not in output # Make sure no archive has been created with Repository(archiver.repository_path) as repository: - manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) + manifest = Manifest.load(repository) assert manifest.archives.count() == 0 diff --git a/src/borg/testsuite/archiver/debug_cmds_test.py b/src/borg/testsuite/archiver/debug_cmds_test.py index 08c31cf149..9c3a971ba1 100644 --- a/src/borg/testsuite/archiver/debug_cmds_test.py +++ b/src/borg/testsuite/archiver/debug_cmds_test.py @@ -100,7 +100,7 @@ def put_pack_with_superseded_gap(archiver): Returns ((w_id, x_id, y_id), (w_size, x_size, y_size)), the object sizes in the first pack. """ with open_repository(archiver) as repository: - repo_objs = Manifest.load(repository, Manifest.NO_OPERATION_CHECK).repo_objs + repo_objs = Manifest.load(repository).repo_objs datas = (b"W" * 100, b"X" * 100, b"Y" * 100) ids = tuple(repo_objs.id_hash(data) for data in datas) objs = [repo_objs.format(id, {}, data, ro_type=ROBJ_FILE_STREAM) for id, data in zip(ids, datas)] diff --git a/src/borg/testsuite/archiver/diff_cmd_test.py b/src/borg/testsuite/archiver/diff_cmd_test.py index 29b29c4cb2..20904af3df 100644 --- a/src/borg/testsuite/archiver/diff_cmd_test.py +++ b/src/borg/testsuite/archiver/diff_cmd_test.py @@ -284,7 +284,7 @@ def _create_archive_with_items(archiver, name, items): # metadata (uid/gid/user/group) can be controlled exactly (without needing root). repository = Repository(archiver.repository_path, exclusive=True) with repository: - manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) + manifest = Manifest.load(repository) with Cache(repository, manifest, archive_name=name) as cache: archive = Archive(manifest, name, cache=cache, create=True) for item in items: diff --git a/src/borg/testsuite/archiver/extract_cmd_test.py b/src/borg/testsuite/archiver/extract_cmd_test.py index 38bcc6625f..72c1882a71 100644 --- a/src/borg/testsuite/archiver/extract_cmd_test.py +++ b/src/borg/testsuite/archiver/extract_cmd_test.py @@ -62,7 +62,7 @@ def _create_malicious_archive(archiver, name, items): # produce a child below a symlink or a path with embedded ".."). repository = Repository(archiver.repository_path, exclusive=True) with repository: - manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) + manifest = Manifest.load(repository) with Cache(repository, manifest, archive_name=name) as cache: archive = Archive(manifest, name, cache=cache, create=True) for item in items: diff --git a/src/borg/testsuite/archiver/rename_cmd_test.py b/src/borg/testsuite/archiver/rename_cmd_test.py index d3842fedf7..6eb5185136 100644 --- a/src/borg/testsuite/archiver/rename_cmd_test.py +++ b/src/borg/testsuite/archiver/rename_cmd_test.py @@ -22,7 +22,7 @@ def test_rename(archivers, request): cmd(archiver, "extract", "test.4", "--dry-run") # Make sure both archives have been renamed with Repository(archiver.repository_path) as repository: - manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) + manifest = Manifest.load(repository) assert manifest.archives.count() == 2 assert manifest.archives.exists("test.3") assert manifest.archives.exists("test.4") diff --git a/src/borg/testsuite/archiver/repo_compress_cmd_test.py b/src/borg/testsuite/archiver/repo_compress_cmd_test.py index bac2fab466..b19935d185 100644 --- a/src/borg/testsuite/archiver/repo_compress_cmd_test.py +++ b/src/borg/testsuite/archiver/repo_compress_cmd_test.py @@ -20,7 +20,7 @@ def check_compression(ctype, clevel, olevel): """Check that all chunks in the repo are compressed/obfuscated as expected.""" repository = Repository(archiver.repository_path, exclusive=True) with repository: - manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) + manifest = Manifest.load(repository) for id, _ in repo_lister(repository, limit=LIST_SCAN_LIMIT): chunk = repository.get(id, read_data=True) meta, data = manifest.repo_objs.parse( @@ -97,7 +97,7 @@ def test_repo_compress_zstd_negative_level(archiver): cmd(archiver, "repo-compress", "-C", f"zstd,{level}") repository = Repository(archiver.repository_path, exclusive=True) with repository: - manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) + manifest = Manifest.load(repository) for id, _ in repo_lister(repository, limit=LIST_SCAN_LIMIT): chunk = repository.get(id, read_data=True) meta, data = manifest.repo_objs.parse(id, chunk, ro_type=ROBJ_DONTCARE) @@ -156,7 +156,7 @@ def test_repo_compress_multiple_packs(archiver, monkeypatch): packs_after = {info.name for info in repository.store_list("packs")} assert len(packs_before - packs_after) >= 2 # no object is left with the old zlib compression - manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) + manifest = Manifest.load(repository) for id, _ in repo_lister(repository, limit=LIST_SCAN_LIMIT): meta = manifest.repo_objs.parse_meta(id, repository.get(id, read_data=False), ro_type=ROBJ_DONTCARE) assert meta["ctype"] != ZLIB.ID @@ -204,7 +204,7 @@ def test_repo_compress_soft_interrupt_persists_valid_index(archiver, monkeypatch pack_names_before = {info.name for info in repository.store_list("packs")} assert len(pack_names_before) >= 2 # need several packs to observe an early stop - manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) + manifest = Manifest.load(repository) manifest.repo_objs.compressor = CompressionSpec("zstd,3").compressor recompressor = PackRecompressor(repository, manifest, print_stats=False) diff --git a/src/borg/testsuite/archiver/webdav_cmd_test.py b/src/borg/testsuite/archiver/webdav_cmd_test.py index e2e53b1ae0..9e8e04fef2 100644 --- a/src/borg/testsuite/archiver/webdav_cmd_test.py +++ b/src/borg/testsuite/archiver/webdav_cmd_test.py @@ -72,7 +72,7 @@ def webdav_server(archiver): ) repository = Repository(archiver.repository_path, exclusive=True) with repository: - manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) + manifest = Manifest.load(repository) server = make_server(manifest, args, port=0) thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() @@ -515,7 +515,7 @@ def test_webdav_data_cache(archivers, request, monkeypatch): ) repository = Repository(archiver.repository_path, exclusive=True) with repository: - manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) + manifest = Manifest.load(repository) server = make_server(manifest, args, port=0) data_cache = server.RequestHandlerClass.vfs.reader.data_cache assert data_cache._capacity == 8 # the env var is honored @@ -587,7 +587,7 @@ def test_webdav_file_without_chunks(archivers, request): ) repository = Repository(archiver.repository_path, exclusive=True) with repository: - manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) + manifest = Manifest.load(repository) server = make_server(manifest, args, port=0) # corrupt the in-memory tree: pretend file1 is non-empty but has no chunks vfs = server.RequestHandlerClass.vfs @@ -623,7 +623,7 @@ def test_webdav_damaged_file(archivers, request): ) repository = Repository(archiver.repository_path, exclusive=True) with repository: - manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) + manifest = Manifest.load(repository) archive = Archive(manifest, manifest.archives.get("test").id) for item in archive.iter_items(): if item.path.endswith("big"): diff --git a/src/borg/testsuite/cache_test.py b/src/borg/testsuite/cache_test.py index 2e074cd30a..f90886f9cf 100644 --- a/src/borg/testsuite/cache_test.py +++ b/src/borg/testsuite/cache_test.py @@ -51,7 +51,7 @@ def key(self, repository, monkeypatch): @pytest.fixture def manifest(self, repository, key): Manifest(key, repository).write() - return Manifest.load(repository, key=key, operations=Manifest.NO_OPERATION_CHECK) + return Manifest.load(repository, key=key) @pytest.fixture def cache(self, repository, key, manifest): @@ -482,7 +482,7 @@ def test_close_consolidates_fragments_across_sessions(tmp_path, monkeypatch): all_ids = [] for s in range(5): # each session adds 100 new chunks (< MIN), so fragments must be consolidated with Repository(loc, exclusive=True) as repository: - manifest = Manifest.load(repository, key=key, operations=Manifest.NO_OPERATION_CHECK) + manifest = Manifest.load(repository, key=key) cache = AdHocWithFilesCache(manifest) try: for i in range(s * 100, s * 100 + 100): @@ -745,7 +745,7 @@ def test_files_cache_save_tolerates_missing_chunk(tmp_path, monkeypatch): Manifest(key, repository).write() with Repository(loc, exclusive=True) as repository: - manifest = Manifest.load(repository, key=key, operations=Manifest.NO_OPERATION_CHECK) + manifest = Manifest.load(repository, key=key) # size+inode+ctime cache mode -> files cache active; archive_name -> names the cache file cache = AdHocWithFilesCache(manifest, cache_mode="cis", archive_name="test") try: