From 77db875957753683aac713e03d8bea66910ac996 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 17 Sep 2026 01:51:10 +0200 Subject: [PATCH] repository: single config/config text object replaces readme, version, id and manifest (repo version 5) Opening a repository loaded four store objects: config/readme (sanity check), config/version, config/id and config/manifest (only read for its key type byte since the manifest lost all other content). Now there is one plain text INI object, config/config, e.g.: # This is a Borg Backup repository. # See https://borgbackup.readthedocs.io/ [repository] version = 5 id = <64 hex digits> encryption = aes256-ocb id_hash = sha256 The crypto suite of the key is recorded by the --encryption / --id-hash names, so key_factory() selects the key class from the config without reading any repository object. Both names are recorded or none; a config with one but not the other is invalid, an unsupported suite raises InvalidRepositoryConfig. The repository version is 5; only version 5 is accepted, there is no code to read the old layout and no migration (betas are for new repositories only). The manifest object is gone for borg 2 repositories: Manifest is now only the in-memory container for key, repo_objs, repository and archives; its write() and all callers, get_manifest()/put_manifest(), the manifest checks and rebuild in "borg check", "borg debug dump-manifest", the key type detection from stored objects (key_from_repository) and the ROBJ_MANIFEST type are removed. borg 1.x repositories are still read via their manifest (legacy_key_factory), as it holds their archives list. NoManifestError is kept (never raised) so that rc 26 stays reserved. Repository.create() creates all namespace directories in advance (store.create_levels()) and writes the config (version, id, no key info) right away, so a repository created via the Python API is an openable key/value store. "borg repo-create" alone defers the config write (create_config=False) until after the key was created, so nothing that looks like a repository exists until repo-create succeeds: any failure or interruption before that destroys the store (and the keyfile, in keyfile mode), and a failure inside create() destroys the store as well. A store without repository config is reported as not a valid repository (InvalidRepository; DoesNotExist is only for a missing store backend, the rest:// case from #10365). repo-create on it raises IncompleteRepository (rc 11, "has no repository config"), as the store backend refuses any non-empty directory. "borg repo-delete --force" destroys such a store, so that ssh/rest/s3 users can remove it without other access to the storage, but only if it has the packs, archives, index and config namespaces every store borg created has: a data directory, a home directory or a borg 1.x repository is never destroyed by borg. key_factory() on a config without key info raises RepositoryKeyInfoMissing (rc 54). The cache config's [cache]/[integrity] cross-check (manifest id) could never detect anything and is dropped. Docs: new "Repository config" section replaces the manifest section (including how to recreate a lost config by hand, as check --repair can not), the layout list, packs and security internals (where the protection against a swapped crypto suite lives), repo-info example, error list and the remaining manifest mentions are updated. Co-Authored-By: Claude Fable 5.1 --- docs/faq.rst | 2 +- docs/internals.rst | 4 +- docs/internals/data-structures.rst | 133 ++++++----- docs/internals/frontends.rst | 4 + docs/internals/packs.rst | 25 +-- docs/internals/security.rst | 28 ++- docs/usage/debug.rst | 1 - docs/usage/repo-info.rst | 2 +- src/borg/archive.py | 48 +--- src/borg/archiver/_common.py | 49 +++- src/borg/archiver/check_cmd.py | 17 +- src/borg/archiver/copy_cmd.py | 1 - src/borg/archiver/debug_cmd.py | 42 +--- src/borg/archiver/delete_cmd.py | 1 - src/borg/archiver/prune_cmd.py | 1 - src/borg/archiver/recreate_cmd.py | 2 - src/borg/archiver/rename_cmd.py | 1 - src/borg/archiver/repo_create_cmd.py | 14 +- src/borg/archiver/repo_delete_cmd.py | 34 ++- src/borg/archiver/undelete_cmd.py | 1 - src/borg/cache.py | 22 +- src/borg/constants.py | 1 - src/borg/crypto/key.py | 117 +++++----- src/borg/crypto/keymanager.py | 7 +- src/borg/legacy/archives.py | 4 - src/borg/manifest.py | 80 +++---- src/borg/repository.py | 210 ++++++++++++++---- src/borg/testsuite/archiver/check_cmd_test.py | 128 +---------- .../testsuite/archiver/corruption_test.py | 21 -- .../testsuite/archiver/create_cmd_test.py | 13 -- .../testsuite/archiver/debug_cmds_test.py | 17 -- .../archiver/repo_create_cmd_test.py | 26 +++ .../archiver/repo_delete_cmd_test.py | 43 +++- .../testsuite/archiver/return_codes_test.py | 7 +- src/borg/testsuite/archives_test.py | 5 - src/borg/testsuite/cache_test.py | 9 +- src/borg/testsuite/crypto/key_test.py | 104 ++------- src/borg/testsuite/legacy_archives_test.py | 6 - src/borg/testsuite/repoobj_test.py | 14 +- src/borg/testsuite/repository_test.py | 141 ++++++++++++ 40 files changed, 715 insertions(+), 670 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index cb0f903457..10db082da2 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -30,7 +30,7 @@ Can I back up from multiple servers into a single repository? Yes, you can! Even simultaneously. The clocks of machines sharing a repository should be roughly synchronized -(e.g. via NTP): repository locks and archive/manifest timestamps are based on +(e.g. via NTP): repository locks and archive timestamps are based on the clients' clocks, so big clock differences between clients can cause trouble. Where the storage backend provides object timestamps (file, sftp, s3 and current rest servers - but not rclone), borg cross-checks lock staleness diff --git a/docs/internals.rst b/docs/internals.rst index 9d3c6b675b..0d460741f6 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -9,8 +9,8 @@ of Borg. Borg uses a low-level, key-value store, the :ref:`repository`, and implements a more complex data structure on top of it, which is made -up of the :ref:`manifest `, :ref:`archives `, -:ref:`items ` and data :ref:`chunks`. +up of the :ref:`archives `, :ref:`items ` and data +:ref:`chunks`. Each repository can hold multiple :ref:`archives `, which represent individual backups that contain a full archive of the files diff --git a/docs/internals/data-structures.rst b/docs/internals/data-structures.rst index a2187bfa33..9a8f67d6f4 100644 --- a/docs/internals/data-structures.rst +++ b/docs/internals/data-structures.rst @@ -38,14 +38,8 @@ It is the same for every repository and independent of the key/encryption mode (unlike the chunk id hash, which the key mode selects). config/ - readme - simple text object telling that this is a Borg repository - id - the unique repository ID encoded as hexadecimal number text - version - the repository version encoded as decimal number text - manifest - the manifest (see :ref:`manifest`), a repository object, binary + config + the repository config (see :ref:`repo_config`), a text object space-reserve.N purely random binary data to reserve space, e.g. for disk-full emergencies. These objects are created and removed by ``borg repo-space``. @@ -177,10 +171,9 @@ Repo object metadata Metadata is a MessagePack-encoded (and encrypted/authenticated) dict with: -- type (the repo object type, a one-character string: ``M`` manifest, - ``A`` archive metadata, ``C`` archive metadata stream chunk ids, - ``S`` archive metadata stream chunk, ``F`` file content stream chunk - - see the ``ROBJ_*`` constants) +- type (the repo object type, a one-character string: ``A`` archive metadata, + ``C`` archive metadata stream chunk ids, ``S`` archive metadata stream chunk, + ``F`` file content stream chunk - see the ``ROBJ_*`` constants) - ctype (compression type 0..255) - clevel (compression level, one byte, interpreted depending on ctype - see :ref:`data-compression`) @@ -235,47 +228,59 @@ More on how this helps security in :ref:`security_structural_auth`. :figwidth: 100% :width: 100% -.. _manifest: +.. _repo_config: -The manifest -~~~~~~~~~~~~ - -The manifest is a repository object stored as the ``config/manifest`` store -object (see Repository_), so it is not inside a pack file and not in the chunks -index. Different from all other repository objects, the chunk id in its object -header is not the hash of its content, but all-zero -(``Manifest.MANIFEST_ID``). - -The manifest is written when the repository is created and by ``borg check ---repair`` when it rebuilds a lost or corrupted manifest. Commands that modify -the repository also call ``Manifest.write()``, but that only stores a new -manifest object if the content changed. It looks like this: - -.. code-block:: python - - { - 'version': 2, - 'archives': {}, - 'config': {}, - } - -Borg 2 always writes *version* 2. Reading also accepts version 1, which is what -borg 1.x repositories have (they are supported read-only, e.g. for -``borg transfer``). - -A *timestamp* entry, as written by borg 1.x and by older borg 2 versions, is -accepted and ignored when reading. - -The *archives* dict is always empty: the list of archives is not part of the -manifest, each archive has its own pointer object in the ``archives/`` -namespace, see :ref:`archive`. - -*config* is a general-purpose location for additional metadata. All versions -of Borg preserve its contents. Currently, borg does not store anything in there. +Repository config +~~~~~~~~~~~~~~~~~ -A *config['item_keys']* list (written by older borg 2 versions) or a top-level -*item_keys* list (borg 1.x) is accepted and ignored when reading: *borg check* -does not validate item keys against such a list anymore, see Item_. +The repository config is the ``config/config`` store object (see Repository_), a +plain text ``INI``-style file. It is the only object borg needs to read to open a +repository. It looks like this:: + + # This is a Borg Backup repository. + # See https://borgbackup.readthedocs.io/ + + [repository] + version = 5 + id = 0a2744f216526be75ae14a5fa5b123127bb218558219f6203e09e4f220e45903 + encryption = aes256-ocb + id_hash = sha256 + +*version* is the repository version. borg refuses to open a repository whose +version it does not support (currently, only version 5 is supported). + +*id* is the unique repository ID (32 bytes, hex encoded). It does not change if +the repository is moved to another location. The keys of the repository are +bound to it (see :ref:`key_files`), and the client's cache and security +directories are named after it. + +*encryption* and *id_hash* record the crypto suite of the repository's key, by +the same names ``borg repo-create --encryption`` and ``--id-hash`` accept. This +is how borg selects the key class when opening a repository, without reading any +repository object. The config is plaintext and not authenticated; see +:ref:`remote_access_security` for what protects against a swapped crypto suite. +Where the key is stored (keyfile or repokey) is not recorded +here: that is a property of each individual key, see :ref:`key_files`. Both +entries are present, or none: a repository created via the Python API +(``Repository.create()``) without a key has none, it can be used as a key/value +store, but borg refuses to load a key for it. + +``borg repo-create`` writes the config once, after the key was created: writing +it is what makes the store a repository. A store without it (e.g. the leftover of +an interrupted ``borg repo-create``) is not a repository: borg reports it as not +a valid repository, and ``borg repo-create`` refuses to create a repository in a +non-empty location, saying whether it found a repository config there. +``borg repo-delete --force`` destroys such a store, provided it looks like a borg +store (it has the packs, archives, index and config namespaces, which +``Repository.create()`` makes in advance), so that it can be removed without other +access to the storage. + +The config is the one object ``borg check --repair`` can not restore. If a +repository lost or damaged its config, it can be recreated by hand: the version +is 5, the id is in the key (the ``BORG_KEY `` header line of a keyfile or of +a ``keys/`` object, see :ref:`key_files`), and the encryption mode and id hash +are what ``borg repo-create`` was given (the key type byte of any repository +object encodes them as well, see ``KeyType`` in ``constants.py``). .. _archive: @@ -1020,8 +1025,7 @@ version currently always an integer, 2 repository_id - the repository ID, as stored in the repository's ``config/id`` object, - see Repository_. + the repository ID, as stored in the repository config (see :ref:`repo_config`). crypt_key the initial key material used for the AEAD crypto (512 bits) @@ -1270,32 +1274,19 @@ the file's name (see :ref:`the files cache ` about that name): [cache] version = 1 repository = 3c4...e59 - manifest = 10e...21c [integrity] - manifest = 10e...21c files.9f8...a08 = {"algorithm": "SHA256", "digests": {"final": "e2a...b24"}} The chunks index is not in this list: it is not a local file, but lives in the repository below ``index/`` and has its own integrity mechanism, see :ref:`pack-index-namespace`. -The manifest ID is duplicated in the integrity section due to the way all Borg -versions handle the config file. Instead of creating a "new" config file from -an internal representation containing only the data understood by Borg, -the config file is read in entirety (using the Python ConfigParser) and modified. -This preserves all sections and values not understood by the Borg version -modifying it. - -Thus, if an older versions uses a cache with integrity data, it would preserve -the integrity section and its contents. If a integrity-aware Borg version -would read this cache, it would incorrectly report checksum errors, since -the older version did not update the checksums. - -However, by duplicating the manifest ID in the integrity section, it is -easy to tell whether the checksums concern the current state of the cache. -If they do not match, borg logs a warning and just does not use the integrity -data. +The cache config file is read in its entirety (using the Python ConfigParser), +modified and written back, so sections and values a Borg version does not +understand are preserved. There is no guard against an older Borg version +updating the files cache without updating its integrity data: every Borg +version that can open a version 5 repository knows the ``[integrity]`` section. A files cache that fails its integrity check (or can not be read at all) is discarded, not used: borg then rebuilds the files cache from the most recent diff --git a/docs/internals/frontends.rst b/docs/internals/frontends.rst index 0c598dfa82..07e4c8ec28 100644 --- a/docs/internals/frontends.rst +++ b/docs/internals/frontends.rst @@ -831,6 +831,8 @@ Errors Repository.AlreadyExists rc: 10 traceback: no A repository already exists at {}. + Repository.IncompleteRepository rc: 11 traceback: no + {} has no repository config. Repository.CheckNeeded rc: 12 traceback: yes Inconsistency detected. Please run "borg check {}". Repository.DoesNotExist rc: 13 traceback: no @@ -900,6 +902,8 @@ Errors Passphrase supplied in BORG_PASSPHRASE, by BORG_PASSCOMMAND, or via BORG_PASSPHRASE_FD is incorrect. PasswordRetriesExceeded rc: 53 traceback: no Exceeded the maximum password retries. + RepositoryKeyInfoMissing rc: 54 traceback: no + Repository {} has no key information in its config. CacheInitAbortedError rc: 60 traceback: no Cache initialization aborted diff --git a/docs/internals/packs.rst b/docs/internals/packs.rst index 4774d7ff42..a04877d5ee 100644 --- a/docs/internals/packs.rst +++ b/docs/internals/packs.rst @@ -125,7 +125,7 @@ rebuilt the index from it. Rewriting such a pack is repository-level repair, see ``OBJ_MAGIC`` occurs inside the payloads as well, so the scan accepts a candidate only when it validates like any walked header. Validating needs the key, so a -repair that cannot read the manifest walks without it. +repair that cannot load the key walks without it. In the ``none-*`` modes the tag is an unkeyed checksum, and in the ``authenticated-*`` modes it binds a blob to its chunk id and nothing else (see @@ -354,22 +354,11 @@ without decrypting any blob and without the repository key. Repository Version ------------------ -Repositories using pack files require repository version **4**, and the version is the -only gate for the pack format. +Repositories using pack files require repository version **5** or later, and the version +is the only gate for the pack format. -``Repository.create()`` stores ``4`` as the ``config/version`` store object. +``Repository.save_config()`` stores the version in the repository config (see +:ref:`repo_config`; currently ``5``, which also introduced the config object itself). ``Repository.open()`` reads it back and, if it is not in -``Repository.acceptable_repo_versions`` (currently ``(4,)``), closes the store again -and raises ``InvalidRepositoryConfig`` -- before any repository data is read. A borg -version that only accepts version 3 rejects a version 4 repository the same way, so -the version bump alone locks out every client that does not know about packs. - -Borg does have a feature flag mechanism for locking out clients more selectively -(``Manifest.check_repository_compatibility()``, fed from a ``feature_flags`` entry in -the manifest ``config`` -- see :ref:`manifest`), but it currently defines no flags at -all: ``Manifest.SUPPORTED_REPO_FEATURES`` is the empty set, and no borg code writes a -``feature_flags`` entry. On a repository borg creates, the compatibility check is -therefore a no-op; there is in particular no ``pack_files`` feature flag. - -There is no migration path from version 3 repositories to version 4. Users of the -version 3 beta format must create a new repository with ``borg repo-create``. +``Repository.acceptable_repo_versions`` (currently ``(5,)``), closes the store again +and raises ``InvalidRepositoryConfig`` -- before any repository data is read. diff --git a/docs/internals/security.rst b/docs/internals/security.rst index 9265a02ac9..cb776335ea 100644 --- a/docs/internals/security.rst +++ b/docs/internals/security.rst @@ -71,7 +71,7 @@ Above used to be all for borg 1.x and was the reason why it needed the tertiary authentication mechanism (TAM) for manifest and archives. borg 2 now stores the ro_type ("meaning") of a repo object's data into that -object's metadata (like e.g.: manifest vs. archive vs. user file content data). +object's metadata (like e.g.: archive metadata vs. user file content data). When loading data from the repo, borg verifies that the type of object it got matches the type it wanted. borg 2 does not use TAMs any more. @@ -87,7 +87,7 @@ carry an unkeyed checksum rather than a MAC, and an attacker who modifies an object can simply recompute it. What still constrains an attacker there is the object ID being the (unkeyed) hash of the plaintext: the content of an existing object can not be replaced without the ID no longer matching. But the object's -metadata, the archives list and the manifest are not anchored to anything secret, +metadata and the archives list are not anchored to anything secret, so a ``none-*`` repository provides no tamper protection - only detection of accidental corruption. @@ -384,8 +384,26 @@ used: repository objects, so the pointer object itself only reveals the archive id (a MAC over the archive metadata) plus whatever the store records about it, e.g. its modification time. -- ``config/manifest`` (an encrypted repository object), plus the plaintext - ``config/version``, ``config/id`` and ``config/readme``. +- ``config/config`` -- the plaintext repository config: version, id and the names of + the crypto suite (encryption mode, id hash), see :ref:`repo_config`. It is neither + encrypted nor authenticated, so an attacker with repository access can rewrite the + crypto suite. What protects against a swapped crypto suite is not this object, but: + + - a swap to a suite that does not encrypt (``none-*``, but also + ``authenticated-*``: its key blob carries the same key material and no suite + name, so it loads fine) would make the client write plaintext. That is caught + by the client's security directory: it records the key type of every + repository the client accessed, and borg refuses to continue with + ``EncryptionMethodMismatch`` if the suite changed. A repository that does not + encrypt and is unknown to the client is only accessed after an explicit + confirmation (``BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK``). Note that this + requires the client environment to be persistent, see `Attack model`_. + - a swap between encrypting suites can not expose plaintext: the client + would write new objects with the same secret key material under the other + cipher, and merely fail to read the existing objects. On a client that knows + the repository, the security directory catches this swap as well. The key blobs + themselves are bound to the repository id and unlocked by the passphrase, so a + client never ends up using key material of the attacker's choice. - ``keys/`` -- in ``repokey`` mode, the borg key(s), encrypted with the passphrase-derived KEK (see :ref:`key_encryption`). - ``locks/*`` and ``cache/*``. Note that the per-archive reference caches @@ -492,7 +510,7 @@ Note that the msgpack unpackers of the RPC data channel (``get_limited_unpacker( kinds ``client`` and ``server``) are deliberately configured with the maximum buffer size, because whole repository objects are transferred through them. They therefore do not bound the memory a peer can make the other side allocate; the stricter limits -of that helper apply to manifest, archive and key data. +of that helper apply to archive and key data. The msgpack implementation used (msgpack-python) has a good security track record, a large test suite and no issues found by fuzzing. It is based on the msgpack-c implementation, diff --git a/docs/usage/debug.rst b/docs/usage/debug.rst index 6f96746a8d..940029f038 100644 --- a/docs/usage/debug.rst +++ b/docs/usage/debug.rst @@ -12,7 +12,6 @@ what their name suggests: put objects into the repository / delete objects from Please note: - they will not update the chunks index about the object -- they will not update the manifest (so no automatic chunks index resync is triggered) - they will not check whether the object is in use (e.g. before delete-obj) - they will not update any metadata which may point to the object diff --git a/docs/usage/repo-info.rst b/docs/usage/repo-info.rst index 2b7fd3bd05..c07e2413bc 100644 --- a/docs/usage/repo-info.rst +++ b/docs/usage/repo-info.rst @@ -7,7 +7,7 @@ Examples $ borg repo-info Repository ID: 0a2744f216526be75ae14a5fa5b123127bb218558219f6203e09e4f220e45903 Location: /path/to/repo - Repository version: 4 + Repository version: 5 Encrypted: Yes (repokey, aes256-ocb, sha256) Security directory: /home/user/.local/share/borg/security/0a2744f216526be75ae14a5fa5b123127bb218558219f6203e09e4f220e45903 Cache: /home/user/.cache/borg/0a2744f216526be75ae14a5fa5b123127bb218558219f6203e09e4f220e45903 diff --git a/src/borg/archive.py b/src/borg/archive.py index da266d6258..432b40998c 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -23,7 +23,7 @@ from . import xattr from .chunkers import get_chunker, Chunk, release_chunk_data from .cache import ChunkListEntry, build_chunkindex_from_repo, write_chunkindex_to_repo -from .crypto.key import key_from_repository +from .crypto.key import key_factory from .constants import * # NOQA from .digests import ContentDigester from .crypto.low_level import IntegrityError as IntegrityErrorBase @@ -52,7 +52,7 @@ from .patterns import PathPrefixPattern, FnmatchPattern, IECommand from .item import Item, ArchiveItem, ItemDiff from .platform import acl_get, acl_set, set_flags, get_flags, set_times, swidth -from .repository import Repository, NoManifestError +from .repository import Repository from .repoobj import RepoObj, object_validator # macOS: SF_DATALESS marks dataless placeholder files (e.g. cloud files not materialized locally). @@ -840,7 +840,6 @@ def save(self, name=None, comment=None, timestamp=None, stats=None, additional_m # fragment referencing uncommitted objects, which compact/rebuild prunes (#10239). self.cache.write_chunks_index() self.manifest.archives.create(name, self.id, metadata.time) - self.manifest.write() return metadata def calc_stats(self, cache, want_unique=True): @@ -2265,9 +2264,8 @@ def check( # The rebuild validates every object header it walks, because a corrupt data_size parses fine # and points the walk into the middle of the pack. That costs one metadata slot read and one # decryption per object and it needs the key, so read the key here if we do not have it yet. - # manifest_only=True: the other key source make_key uses is self.chunks, built just below. if repair and self.key is None: - self.key = self.make_key(repository, manifest_only=True) + self.key = self.make_key(repository) if self.key is not None: # the validator decrypts metadata slots, so it needs a RepoObj built from the key. self.repo_objs = RepoObj(self.key) @@ -2306,22 +2304,7 @@ def check( self.repo_objs.set_assert_id_place("repair") if verify_data: self.verify_data() - rebuild_manifest = False - try: - repository.get_manifest() - except NoManifestError: - logger.error("Repository manifest is missing.") - self.error_found = True - rebuild_manifest = True - else: - try: - self.manifest = Manifest.load(repository, key=self.key) - except IntegrityErrorBase as exc: - logger.error("Repository manifest is corrupted: %s", exc) - self.error_found = True - rebuild_manifest = True - if rebuild_manifest: - self.manifest = self.rebuild_manifest() + self.manifest = Manifest.load(repository, key=self.key) # On Ctrl-C, skip any scan not yet started; a scan already running stops at its own boundary. if find_lost_archives and not sig_int: self.rebuild_archives_directory() @@ -2336,7 +2319,7 @@ def check( newer=newer, newest=newest, ) - # finish() writes the manifest and a consistent chunk index; run it on Ctrl-C too (#9850). + # finish() writes a consistent chunk index; run it on Ctrl-C too (#9850). self.finish() if sig_int: if self.error_found: @@ -2350,17 +2333,9 @@ def check( logger.info("Archive consistency check complete, no problems found.") return self.repair or not self.error_found - def make_key(self, repository, manifest_only=False): - """Return the key loaded by key_from_repository. - - manifest_only: read only the manifest, else also the objects of the chunk ids in self.chunks. - """ - - def chunk_ids(): # reads self.chunks only if the manifest does not identify the key type - for id, _ in self.chunks.iteritems(): - yield id - - return key_from_repository(repository, () if manifest_only else chunk_ids()) + def make_key(self, repository): + """Return the key of repository, see key_factory.""" + return key_factory(repository) def verify_data(self): logger.info("Starting cryptographic data integrity verification...") @@ -2448,12 +2423,6 @@ def verify_data(self): errors, ) - def rebuild_manifest(self): - """Rebuild the manifest object.""" - - logger.info("Rebuilding missing/corrupted manifest.") - return Manifest(self.key, self.repository) - def rebuild_archives_directory(self): """Rebuild the archives directory, undeleting archives. @@ -2816,7 +2785,6 @@ def finish(self): ) # drop the in-memory index so close() does not persist it over the index just written. self.repository.invalidate_chunk_index() - self.manifest.write() class ArchiveRecreater: diff --git a/src/borg/archiver/_common.py b/src/borg/archiver/_common.py index 6292bdc911..bc0ba15e16 100644 --- a/src/borg/archiver/_common.py +++ b/src/borg/archiver/_common.py @@ -29,7 +29,9 @@ logger = create_logger(__name__) -def get_repository(location, *, create, exclusive, lock_wait, lock, args, v1_legacy): +def get_repository(location, *, create, exclusive, lock_wait, lock, args, v1_legacy, allow_incomplete=False): + # create_config=False: when creating, the command (repo-create) writes the repository config itself, + # once the key exists, see Repository.create(). For an existing repository, the flag is irrelevant. if location.proto == "ssh": if v1_legacy: from ..legacy.remote import LegacyRemoteRepository @@ -47,20 +49,46 @@ def get_repository(location, *, create, exclusive, lock_wait, lock, args, v1_leg elif ( location.proto in ("rest", "sftp", "file", "http", "https", "rclone", "s3", "b2") and not v1_legacy ): # stuff directly supported by borgstore - repository = Repository(location, create=create, exclusive=exclusive, lock_wait=lock_wait, lock=lock) + repository = Repository( + location, + create=create, + create_config=False, + allow_incomplete=allow_incomplete, + exclusive=exclusive, + lock_wait=lock_wait, + lock=lock, + ) else: if v1_legacy: from ..legacy.repository import LegacyRepository - RepoCls = LegacyRepository + repository = LegacyRepository( + location.path, create=create, exclusive=exclusive, lock_wait=lock_wait, lock=lock + ) else: - RepoCls = Repository - repository = RepoCls(location.path, create=create, exclusive=exclusive, lock_wait=lock_wait, lock=lock) + repository = Repository( + location.path, + create=create, + create_config=False, + allow_incomplete=allow_incomplete, + exclusive=exclusive, + lock_wait=lock_wait, + lock=lock, + ) return repository -def with_repository(create=False, lock=True, exclusive=False, manifest=True, cache=False, secure=True, allow_v1=False): +def with_repository( + create=False, + lock=True, + exclusive=False, + manifest=True, + cache=False, + secure=True, + allow_v1=False, + allow_incomplete=False, +): """ Method decorator for subcommand-handling methods: do_XYZ(self, args, repository, …) @@ -72,6 +100,8 @@ def with_repository(create=False, lock=True, exclusive=False, manifest=True, cac :param cache: open cache, pass it as keyword argument (implies manifest) :param secure: do assert_secure after loading manifest :param allow_v1: (bool) allow legacy Borg 1.x repositories + :param allow_incomplete: (bool) also open a store without repository config (repository.incomplete is + True then, nothing else is usable), see Repository.create() - for "borg repo-delete --force". """ # We may need to modify `lock` inside `wrapper`. Therefore we cannot use the # `nonlocal` statement to access `lock` as modifications would also @@ -100,11 +130,12 @@ def wrapper(self, args, **kwargs): lock=lock, args=args, v1_legacy=v1_legacy, + allow_incomplete=allow_incomplete, ) with repository: - acceptable_versions = (1,) if v1_legacy else (4,) - if repository.version not in acceptable_versions: + acceptable_versions = (1,) if v1_legacy else (5,) + if not getattr(repository, "incomplete", False) and repository.version not in acceptable_versions: raise Error( f"This borg version only accepts version {' or '.join(str(v) for v in acceptable_versions)} " f"repos for -r/--repo, but not version {repository.version}. " @@ -171,7 +202,7 @@ def wrapper(self, args, **kwargs): ) with repository: - acceptable_versions = (1,) if v1_legacy else (4,) + acceptable_versions = (1,) if v1_legacy else (5,) if repository.version not in acceptable_versions: raise Error( f"This borg version only accepts version {' or '.join(str(v) for v in acceptable_versions)} " diff --git a/src/borg/archiver/check_cmd.py b/src/borg/archiver/check_cmd.py index 25940602f1..aa821d672b 100644 --- a/src/borg/archiver/check_cmd.py +++ b/src/borg/archiver/check_cmd.py @@ -3,8 +3,8 @@ from ._common import with_repository, Highlander from ..archive import ArchiveChecker from ..constants import * # NOQA -from ..crypto.key import key_from_repository -from ..helpers import set_ec, EXIT_WARNING, CancelledByUser, CommandError, Error, IntegrityError +from ..crypto.key import key_factory, RepositoryKeyInfoMissing +from ..helpers import set_ec, EXIT_WARNING, CancelledByUser, CommandError, Error from ..helpers import relative_time_marker_validator, yes, ArchiveFormatter, sig_int from ..helpers.argparsing import ArgumentParser from ..helpers.time import archive_ts_now, calculate_relative_offset @@ -69,12 +69,10 @@ def do_check(self, args, repository): # if we need the key later for the archives check, ask NOW for the passphrase! #1931 archive_checker = ArchiveChecker() try: - archive_checker.key = archive_checker.make_key(repository, manifest_only=True) - except IntegrityError: + archive_checker.key = archive_checker.make_key(repository) + except RepositoryKeyInfoMissing: if args.repair: - # repair needs the key to validate the index rebuild. The manifest did not give it, - # so read it from the objects the chunk index lists. - archive_checker.key = key_from_repository(repository) + raise # repair needs the key to validate the index rebuild if args.format is not None: format = args.format else: @@ -85,9 +83,8 @@ def do_check(self, args, repository): if not args.archives_only: validate = None # the object validator for the index rebuild, which only a repair does if args.repair: - # ids=(): read the key from the manifest only. Chunk objects are found through the index, - # which this check may find corrupt. - key = archive_checker.key if not args.repo_only else key_from_repository(repository, ()) + # the key class comes from the repository config, so loading the key reads no object. + key = archive_checker.key if not args.repo_only else key_factory(repository) validate = object_validator(RepoObj(key)) if not repository.check( repair=args.repair, diff --git a/src/borg/archiver/copy_cmd.py b/src/borg/archiver/copy_cmd.py index 9cf6d8fe74..8f30d5005a 100644 --- a/src/borg/archiver/copy_cmd.py +++ b/src/borg/archiver/copy_cmd.py @@ -15,7 +15,6 @@ def do_copy(self, args, repository, manifest, cache, archive): """Copy an archive to a new archive name.""" old_id = archive.id archive.copy(args.newname) - manifest.write() logger.info(f"id: {bin_to_hex(old_id):.8} -> {bin_to_hex(archive.id):.8}, name: {archive.name}.") def build_parser_copy(self, subparsers, common_parser, mid_common_parser): diff --git a/src/borg/archiver/debug_cmd.py b/src/borg/archiver/debug_cmd.py index 50cc6e3e85..66f69123bd 100644 --- a/src/borg/archiver/debug_cmd.py +++ b/src/borg/archiver/debug_cmd.py @@ -3,7 +3,8 @@ from ..archive import Archive from ..constants import * # NOQA -from ..crypto.key import key_from_repository, KeyfileInvalidError, RepoKeyNotFoundError, UnsupportedKeyFormatError +from ..crypto.key import key_factory, KeyfileInvalidError, RepoKeyNotFoundError, UnsupportedKeyFormatError +from ..crypto.key import RepositoryKeyInfoMissing from ..helpers import msgpack from ..helpers import FilesystemPathSpec from ..helpers import sysinfo @@ -11,7 +12,7 @@ from ..helpers import dash_open from ..helpers import StableDict from ..helpers import archivename_validator, CompressionSpec -from ..helpers import CommandError, IntegrityError, RTError +from ..helpers import CommandError, RTError from ..helpers.argparsing import ArgumentParser from ..platform import get_process_id from ..repository import Repository, LIST_SCAN_LIMIT, repo_lister @@ -28,14 +29,14 @@ def gap_validator(repository): """Return repoobj.object_validator for the key of repository, or None if there is no key to use. - The key is loaded with key_from_repository. There is no key to use if no stored object identifies - the key type (IntegrityError), no key is found (RepoKeyNotFoundError), or the key is invalid + The key is loaded with key_factory. There is no key to use if the repository config has no key info + (RepositoryKeyInfoMissing), no key is found (RepoKeyNotFoundError), or the key is invalid (KeyfileInvalidError, UnsupportedKeyFormatError); a warning is logged then. Other errors, e.g. a wrong passphrase, propagate. """ try: - key = key_from_repository(repository) - except (IntegrityError, RepoKeyNotFoundError, KeyfileInvalidError, UnsupportedKeyFormatError) as err: + key = key_factory(repository) + except (RepositoryKeyInfoMissing, RepoKeyNotFoundError, KeyfileInvalidError, UnsupportedKeyFormatError) as err: logger.warning(f"Could not set up the key, so rewritten packs keep their superseded gap bytes: {err}") return None return object_validator(RepoObj(key)) @@ -116,18 +117,6 @@ def output(fd): with dash_open(args.path, "w") as fd: output(fd) - @with_repository() - def do_debug_dump_manifest(self, args, repository, manifest): - """Dumps decoded repository manifest.""" - repo_objs = manifest.repo_objs - cdata = repository.get_manifest() - _, data = repo_objs.parse(manifest.MANIFEST_ID, cdata, ro_type=ROBJ_MANIFEST) - - meta = prepare_dump_dict(msgpack.unpackb(data, object_hook=StableDict)) - - with dash_open(args.path, "w") as fd: - json.dump(meta, fd, indent=4) - @with_repository(manifest=False) def do_debug_dump_repo_objs(self, args, repository): """Dumps (decrypted, decompressed) repository objects.""" @@ -142,7 +131,7 @@ def decrypt_dump(id, cdata): with open(filename, "wb") as fd: fd.write(data) - repo_objs = RepoObj(key_from_repository(repository)) + repo_objs = RepoObj(key_factory(repository)) for id, stored_size in repo_lister(repository, limit=LIST_SCAN_LIMIT): cdata = repository.get(id) decrypt_dump(id, cdata) @@ -175,7 +164,7 @@ def print_finding(info, wanted, data, offset): if not wanted: raise CommandError("search term needs to be hex:123abc or str:foobar style") - repo_objs = RepoObj(key_from_repository(repository)) + repo_objs = RepoObj(key_factory(repository)) last_data = b"" last_id = None @@ -380,19 +369,6 @@ def build_parser_debug(self, subparsers, common_parser, mid_common_parser): subparser.add_argument("name", metavar="NAME", type=archivename_validator, help="specify the archive name") subparser.add_argument("path", metavar="PATH", type=FilesystemPathSpec, help="file to dump data into") - debug_dump_manifest_epilog = process_epilog( - """ - This command dumps manifest metadata of a repository in a decoded form to a file. - """ - ) - subparser = ArgumentParser( - parents=[mid_common_parser], - description=self.do_debug_dump_manifest.__doc__, - epilog=debug_dump_manifest_epilog, - ) - debug_parsers.add_subcommand("dump-manifest", subparser, help="dump decoded repository metadata (debug)") - subparser.add_argument("path", metavar="PATH", type=FilesystemPathSpec, help="file to dump data into") - debug_dump_repo_objs_epilog = process_epilog( """ This command dumps raw (but decrypted and decompressed) repo objects to files. diff --git a/src/borg/archiver/delete_cmd.py b/src/borg/archiver/delete_cmd.py index 18ab3e0773..f524ee5fad 100644 --- a/src/borg/archiver/delete_cmd.py +++ b/src/borg/archiver/delete_cmd.py @@ -58,7 +58,6 @@ def do_delete(self, args, repository): if dry_run: logger.info("Finished dry-run.") elif deleted: - manifest.write() self.print_warning('Done. Run "borg compact" to free space.', wc=None) else: self.print_warning("Aborted.", wc=None) diff --git a/src/borg/archiver/prune_cmd.py b/src/borg/archiver/prune_cmd.py index a7675a62cd..0e5efcfe93 100644 --- a/src/borg/archiver/prune_cmd.py +++ b/src/borg/archiver/prune_cmd.py @@ -305,7 +305,6 @@ def do_prune(self, args, repository, manifest): if args.json: json_print(basic_json_data(manifest, extra={"archives": output_data})) if num_archives_deleted > 0 and not args.dry_run: - manifest.write() self.print_warning('Done. Run "borg compact" to free space.', wc=None) if sig_int: raise Error("Got Ctrl-C / SIGINT.") diff --git a/src/borg/archiver/recreate_cmd.py b/src/borg/archiver/recreate_cmd.py index ee0d05b5de..db8e2d7b64 100644 --- a/src/borg/archiver/recreate_cmd.py +++ b/src/borg/archiver/recreate_cmd.py @@ -51,8 +51,6 @@ def do_recreate(self, args, repository, manifest, cache): delete_original = True if not recreater.recreate(archive_info.id, target, delete_original, args.comment): logger.info(f"Skipped archive {name} {hex_id}: Nothing to do.") - if not args.dry_run: - manifest.write() def build_parser_recreate(self, subparsers, common_parser, mid_common_parser): from ._common import process_epilog diff --git a/src/borg/archiver/rename_cmd.py b/src/borg/archiver/rename_cmd.py index 5429a91099..3c6cdce7bd 100644 --- a/src/borg/archiver/rename_cmd.py +++ b/src/borg/archiver/rename_cmd.py @@ -14,7 +14,6 @@ class RenameMixIn: def do_rename(self, args, repository, manifest, cache, archive): """Rename an existing archive.""" archive.rename(args.newname) - manifest.write() def build_parser_rename(self, subparsers, common_parser, mid_common_parser): from ._common import process_epilog diff --git a/src/borg/archiver/repo_create_cmd.py b/src/borg/archiver/repo_create_cmd.py index 6735c85876..6f9d332d11 100644 --- a/src/borg/archiver/repo_create_cmd.py +++ b/src/borg/archiver/repo_create_cmd.py @@ -34,14 +34,20 @@ def do_repo_create(self, args, repository, *, other_repository=None, other_manif logger.info('Initializing repository at "%s"' % path) if other_key is not None: other_key.copy_crypt_key = args.copy_crypt_key + key = None try: key = key_creator(repository, args, other_key=other_key) - except (EOFError, KeyboardInterrupt): + # writing the config is what makes the store a repository, see Repository.create(). + repository.save_config(key) + except BaseException as exc: + # an interrupted or failed repo-create must not leave a (partial) repository behind, nor a keyfile. + if key is not None and key.storage == KeyBlobStorage.KEYFILE: + key.remove(key.target) repository.destroy() - raise CancelledByUser() + if isinstance(exc, (EOFError, KeyboardInterrupt)): + raise CancelledByUser() + raise manifest = Manifest(key, repository) - manifest.key = key - manifest.write() with Cache(repository, manifest, warn_if_unencrypted=False): pass if key.has_secret_key: # any key-bearing suite (everything except the "none-*" modes) diff --git a/src/borg/archiver/repo_delete_cmd.py b/src/borg/archiver/repo_delete_cmd.py index 6c8677d21b..8552e0230a 100644 --- a/src/borg/archiver/repo_delete_cmd.py +++ b/src/borg/archiver/repo_delete_cmd.py @@ -2,12 +2,13 @@ from ..cache import Cache from ..security import SecurityManager from ..constants import * # NOQA -from ..helpers import CancelledByUser +from ..helpers import CancelledByUser, Error from ..helpers import format_archive from ..helpers import bin_to_hex from ..helpers import yes from ..helpers.argparsing import ArgumentParser -from ..manifest import Manifest, NoManifestError +from ..crypto.key import RepositoryKeyInfoMissing +from ..manifest import Manifest from ..logger import create_logger @@ -15,13 +16,33 @@ class RepoDeleteMixIn: - @with_repository(exclusive=True, manifest=False) + @with_repository(exclusive=True, manifest=False, allow_incomplete=True) def do_repo_delete(self, args, repository): """Deletes a repository.""" self.output_list = args.output_list dry_run = args.dry_run keep_security_info = args.keep_security_info + if repository.incomplete: + # a store without repository config (see Repository.create()), e.g. the leftover of an + # interrupted repo-create: --force destroys it, if it looks like a store borg created. + location = repository._location.canonical_path() + if args.cache_only: + raise Error(f"{location} has no repository config, so its cache can not be identified.") + if not args.forced: + raise Error(f"{location} has no repository config. Deleting this requires the --force option.") + if not repository.looks_like_borg_store(): + raise Error( + f"{location} has no repository config and does not look like a borg store " + "(no packs, archives, index and config namespaces), so borg does not destroy it." + ) + if dry_run: + logger.info("Would destroy the store (it has no repository config).") + else: + repository.destroy() + logger.info("Store destroyed (it had no repository config).") + return + if not args.cache_only: if not args.forced: # without --force, we let the user see the archives list and confirm. id = bin_to_hex(repository.id) @@ -34,7 +55,7 @@ def do_repo_delete(self, args, repository): f"You requested to DELETE the following repository completely " f"*including* {n_archives} archives it contains:" ) - except NoManifestError: + except RepositoryKeyInfoMissing: n_archives = None msg.append( "You requested to DELETE the following repository completely " @@ -57,7 +78,7 @@ def do_repo_delete(self, args, repository): msg.append("This repository does not appear to have any archives.") else: msg.append( - "This repository seems to have no manifest, so we cannot " + "This repository has no key information in its config, so we cannot " "tell anything about its contents." ) @@ -99,6 +120,9 @@ def build_parser_repo_delete(self, subparsers, common_parser, mid_common_parser) with the ``--cache-only`` option, or keep the security info with the ``--keep-security-info`` option. + ``--force`` also destroys a store that has no repository config, provided it looks + like a borg store (it has the packs, archives, index and config namespaces). + Always first use ``--dry-run --list`` to see what would be deleted. """ ) diff --git a/src/borg/archiver/undelete_cmd.py b/src/borg/archiver/undelete_cmd.py index d936205f29..b7d1bd4ac5 100644 --- a/src/borg/archiver/undelete_cmd.py +++ b/src/borg/archiver/undelete_cmd.py @@ -51,7 +51,6 @@ def do_undelete(self, args, repository): if dry_run: logger.info("Finished dry-run.") elif undeleted: - manifest.write() self.print_warning("Done.", wc=None) else: self.print_warning("Aborted.", wc=None) diff --git a/src/borg/cache.py b/src/borg/cache.py index 50807d075a..e81e4ec2b3 100644 --- a/src/borg/cache.py +++ b/src/borg/cache.py @@ -125,9 +125,7 @@ def create(self): config.add_section("cache") config.set("cache", "version", "1") config.set("cache", "repository", self.repository.id_str) - config.set("cache", "manifest", "") config.add_section("integrity") - config.set("integrity", "manifest", "") with SaveFile(self.config_path) as fd: config.write(fd) @@ -140,30 +138,18 @@ def load(self): self._config.read_file(fd) self._check_upgrade(self.config_path) self.id = self._config.get("cache", "repository") - self.manifest_id = hex_to_bin(self._config.get("cache", "manifest")) try: self.integrity = dict(self._config.items("integrity")) - if self._config.get("cache", "manifest") != self.integrity.pop("manifest"): - # The cache config file is updated (parsed with ConfigParser, the state of the ConfigParser - # is modified and then written out.), not re-created. - # Thus, older versions will leave our [integrity] section alone, making the section's data invalid. - # Therefore, we also add the manifest ID to this section and - # can discern whether an older version interfered by comparing the manifest IDs of this section - # and the main [cache] section. - self.integrity = {} - logger.warning("Cache integrity data not available: old Borg version modified the cache.") except configparser.NoSectionError: - logger.debug("Cache integrity: No integrity data found (files, chunks). Cache is from old version.") + logger.debug("Cache integrity: no [integrity] section in the cache config, no integrity data.") self.integrity = {} - def save(self, manifest=None): - if manifest: - self._config.set("cache", "manifest", manifest.id_str) + def save(self, with_integrity=False): + if with_integrity: if not self._config.has_section("integrity"): self._config.add_section("integrity") for file, integrity_data in self.integrity.items(): self._config.set("integrity", file, integrity_data) - self._config.set("integrity", "manifest", manifest.id_str) with SaveFile(self.config_path) as fd: self._config.write(fd) @@ -1354,7 +1340,7 @@ def close(self): # of seeing a valid-looking but empty index (and so is_chunk_index_loaded reports False). self.repository.invalidate_chunk_index() pi.output("Saving cache config") - self.cache_config.save(self.manifest) + self.cache_config.save(with_integrity=True) self.cache_config.close() pi.finish() self.cache_config = None diff --git a/src/borg/constants.py b/src/borg/constants.py index 9baac146ad..7f8b9de769 100644 --- a/src/borg/constants.py +++ b/src/borg/constants.py @@ -56,7 +56,6 @@ READ_SPECIAL_TIMEOUT_DEFAULT = 1800.0 # RepoObj types -ROBJ_MANIFEST = "M" # Manifest (directory of archives, other metadata) object ROBJ_ARCHIVE_META = "A" # main archive metadata object ROBJ_ARCHIVE_CHUNKIDS = "C" # objects with a list of archive metadata stream chunkids ROBJ_ARCHIVE_STREAM = "S" # archive metadata stream chunk (containing items) diff --git a/src/borg/crypto/key.py b/src/borg/crypto/key.py index 7eeac7d7bd..6ca6800eae 100644 --- a/src/borg/crypto/key.py +++ b/src/borg/crypto/key.py @@ -3,7 +3,6 @@ import os import textwrap from hashlib import sha256 -from itertools import islice from math import ceil from pathlib import Path from typing import Any, Literal, ClassVar, Optional @@ -25,9 +24,9 @@ from ..helpers import msgpack from ..helpers import workarounds from ..item import Key, EncryptedKey -from ..manifest import Manifest, NoManifestError +from ..manifest import Manifest from ..platform import SaveFile -from ..repoobj import RepoObj, RepoObj1 +from ..repoobj import RepoObj1 from .low_level import bytes_to_int, num_cipher_blocks, hmac_sha256 @@ -240,78 +239,67 @@ def id_hash_argument_names(): return names -def identify_key(manifest_data): - # the key-type byte only identifies the crypto suite (id hash, MAC, cipher), NOT where the key is - # stored: keyfile and repokey share one class now and accept both historic type bytes. The legacy - # PASSPHRASE byte (0x01) is part of AESCTRKey.TYPES_ACCEPTABLE. All TYPES_ACCEPTABLE sets are disjoint. - key_type = manifest_data[0] - for key in LEGACY_KEY_TYPES + AVAILABLE_KEY_TYPES: - if key_type in key.TYPES_ACCEPTABLE: - return key - raise UnsupportedPayloadError(key_type) +class RepositoryKeyInfoMissing(Error): + """Repository {} has no key information in its config.""" + exit_mcode = 54 -def identify_stored_key(manifest_chunk, *, ro_cls=RepoObj): - """Return (key class, data slot) of the stored object manifest_chunk. - A stored object is an object header, a metadata slot and a data slot (see RepoObj). The first - byte of the data slot is the key type byte, which selects the key class. +def key_class_for(encryption, id_hash): + """Return the key class of the crypto suite named by the repository config values encryption and id_hash. - manifest_chunk: the stored object, e.g. the manifest. - ro_cls: the RepoObj class that parses manifest_chunk. - Raises IntegrityError if manifest_chunk is damaged (see ro_cls.extract_crypted_data), and - UnsupportedPayloadError if the key type byte selects no key class usable with ro_cls. + None if no key class implements that suite (e.g. a newer borg version wrote the config). """ - manifest_data = ro_cls.extract_crypted_data(manifest_chunk) - assert manifest_data, "manifest data must not be zero bytes long" - key_cls = identify_key(manifest_data) - if key_cls in LEGACY_KEY_TYPES and ro_cls is not RepoObj1: - # A borg 2 repository using a borg 1.x key type: that can only be a repository created by - # a borg 2 beta in the old "none" or "authenticated" mode, which have been replaced by the - # tagged envelope modes (see MACKeyBase). The legacy key classes only exist to read borg - # 1.x repositories (ro_cls is RepoObj1 then), e.g. for "borg transfer --from-borg1". - raise UnsupportedPayloadError(manifest_data[0]) - return key_cls, manifest_data + for key in AVAILABLE_KEY_TYPES: + if key.ENC_NAME == encryption and key.IDHASH_NAME == id_hash: + return key + return None -def key_factory(repository, manifest_chunk, *, other=False, ro_cls=RepoObj): - key_cls, manifest_data = identify_stored_key(manifest_chunk, ro_cls=ro_cls) - key = key_cls.detect(repository, manifest_data, other=other) - key.stored_type = manifest_data[0] - return key +def key_class_of(repository): + """Return the key class of repository, as recorded in its config (see Repository.save_config).""" + if repository.encryption is None or repository.id_hash is None: + raise RepositoryKeyInfoMissing(repository._location.canonical_path()) + key_cls = key_class_for(repository.encryption, repository.id_hash) + if key_cls is None: + from ..repository import Repository # repository imports this module, hence the local import + + raise Repository.InvalidRepositoryConfig( + repository._location.canonical_path(), + f'unsupported crypto suite: encryption "{repository.encryption}", id hash "{repository.id_hash}" ' + "(a newer version of Borg may be required)", + ) + return key_cls + +def key_factory(repository, *, other=False): + """Return the (loaded) key of repository, its class selected by the repository config.""" + return key_class_of(repository).detect(repository, None, other=other) -def key_from_repository(repository, ids=None): - """Return the key of repository, loaded from the first stored object that identifies the key type. - Stored objects are read in this order: the manifest, then the objects of the chunk ids in ids, at - most 999 of them. An object identifies the key type if identify_stored_key accepts it. Errors - loading the key, e.g. a wrong passphrase, propagate. +def legacy_key_factory(repository, manifest_chunk, *, other=False): + """Return the (loaded) key of a borg 1.x repository, its class selected by the key type byte of its manifest. - repository: the Repository whose key is loaded. - ids: iterable of chunk ids. None: the chunk ids in the chunks index of repository. - Raises IntegrityError if no object read identifies the key type. + manifest_chunk: the stored manifest object of the borg 1.x repository. The first byte of its + crypted data is the key type byte (see identify_key). """ - max_objects = 999 + manifest_data = RepoObj1.extract_crypted_data(manifest_chunk) + assert manifest_data, "manifest data must not be zero bytes long" + key_cls = identify_key(manifest_data) + key = key_cls.detect(repository, manifest_data, other=other) + key.stored_type = manifest_data[0] + return key - def stored_objects(): - try: - yield repository.get_manifest() - except NoManifestError: - pass - chunk_ids = (id for id, _ in repository.list(limit=max_objects)) if ids is None else ids - for id in islice(chunk_ids, max_objects): - yield repository.get(id) - - count = 0 - for cdata in stored_objects(): - count += 1 - try: - identify_stored_key(cdata) - except (IntegrityError, UnsupportedPayloadError): - continue - return key_factory(repository, cdata) - raise IntegrityError(f"no stored object identifies the key type ({count} objects read)") + +def identify_key(manifest_data): + # the key-type byte only identifies the crypto suite (id hash, MAC, cipher), NOT where the key is + # stored: keyfile and repokey share one class now and accept both historic type bytes. The legacy + # PASSPHRASE byte (0x01) is part of AESCTRKey.TYPES_ACCEPTABLE. All TYPES_ACCEPTABLE sets are disjoint. + key_type = manifest_data[0] + for key in LEGACY_KEY_TYPES + AVAILABLE_KEY_TYPES: + if key_type in key.TYPES_ACCEPTABLE: + return key + raise UnsupportedPayloadError(key_type) def uses_same_chunker_secret(other_key, key): @@ -1561,8 +1549,9 @@ def new_session(self): # Each of these is one unified key class per crypto suite. A key of this class may be stored either as # a keyfile or inside the repository (repokey) - that is a per-key storage property (self.storage), not -# a class distinction. The class is selected from the manifest's key-type byte (see identify_key), which -# only encodes the crypto suite (there is exactly one type byte per suite now). +# a class distinction. The class is selected from the crypto suite recorded in the repository config (see +# key_class_for); the key-type byte in every stored object only encodes the crypto suite as well (there is +# exactly one type byte per suite now). # AES-OCB has a birthday-type bound: an attacker's advantage in distinguishing the ciphertexts from # random is about 6 * sigma^2 / 2^128, sigma being the number of 128bit cipher blocks encrypted using diff --git a/src/borg/crypto/keymanager.py b/src/borg/crypto/keymanager.py index 58e7d50b06..a6b84a4d7a 100644 --- a/src/borg/crypto/keymanager.py +++ b/src/borg/crypto/keymanager.py @@ -5,11 +5,10 @@ from hashlib import sha256 from ..helpers import Error, CommandError, yes, bin_to_hex, hex_to_bin, dash_open, get_keys_dir -from ..repoobj import RepoObj from .key import keyfile_format, keyfile_parse, is_keyfile -from .key import RepoKeyNotFoundError, KeyBlobStorage, KEY_LOCATIONS, identify_key, keyfile_name_for +from .key import RepoKeyNotFoundError, KeyBlobStorage, KEY_LOCATIONS, key_class_of, keyfile_name_for class NotABorgKeyFile(Error): @@ -51,9 +50,7 @@ def __init__(self, repository): self.loaded_key_id = None self.loaded_label = None - manifest_chunk = repository.get_manifest() - manifest_data = RepoObj.extract_crypted_data(manifest_chunk) - self.key_cls = identify_key(manifest_data) + self.key_cls = key_class_of(repository) self.keyblob_storage = self.key_cls.STORAGE if self.keyblob_storage == KeyBlobStorage.NO_STORAGE: raise UnencryptedRepo() diff --git a/src/borg/legacy/archives.py b/src/borg/legacy/archives.py index 5f216b333e..ce2fc38478 100644 --- a/src/borg/legacy/archives.py +++ b/src/borg/legacy/archives.py @@ -12,7 +12,6 @@ from operator import attrgetter from ..constants import * # NOQA -from ..helpers.datastruct import StableDict from ..helpers.errors import CommandError, Error from ..helpers.parseformat import bin_to_hex from ..helpers.time import parse_timestamp, compile_date_pattern, DatePatternError @@ -46,9 +45,6 @@ def __init__(self, repository, manifest): def prepare(self, manifest, m): self._set_raw_dict(m.archives) - def finish(self, manifest): - return StableDict(self._get_raw_dict()) - def ids(self, *, deleted=False): for archive_info in self._archives.values(): yield archive_info["id"] diff --git a/src/borg/manifest.py b/src/borg/manifest.py index de5c6606fb..ff38467ebd 100644 --- a/src/borg/manifest.py +++ b/src/borg/manifest.py @@ -12,7 +12,6 @@ logger = create_logger() from .constants import * # NOQA -from .helpers.datastruct import StableDict from .helpers.parseformat import bin_to_hex, hex_to_bin from .helpers.time import ( parse_timestamp, @@ -36,6 +35,8 @@ class MandatoryFeatureUnsupported(Error): exit_mcode = 25 +# Not raised anymore: borg 2 repositories have no manifest object (their config is the config/config store +# object). The class is kept so that its return code stays reserved and never gets a different meaning. class NoManifestError(Error): """Repository has no manifest.""" @@ -129,7 +130,6 @@ class ArchivesInterface(Protocol): # pragma: no cover """ def prepare(self, manifest, m) -> None: ... - def finish(self, manifest) -> dict: ... def ids(self, *, deleted: bool = False) -> Iterator: ... def count(self) -> int: ... def names(self) -> Iterator: ... @@ -174,10 +174,7 @@ def __init__(self, repository, manifest): self.manifest = manifest def prepare(self, manifest, m): - pass # borgstore manages the archive directory; nothing to load from the manifest blob - - def finish(self, manifest): - return {} # manifest["archives"] is always empty in Borg 2 + pass # only the legacy borg 1.x manifest has an archives list to load, see LegacyArchives def ids(self, *, deleted=False): # yield the binary IDs of all archives @@ -483,7 +480,18 @@ def get_one(self, match, *, match_end=r"\Z", deleted=False): class Manifest: - MANIFEST_ID = b"\0" * 32 + """ + The repository's key, RepoObj and archives directory, bundled for the code that works with archives. + + Historically (borg 1.x), the manifest was a repository object holding the archives list and other + metadata. borg 2 repositories have no manifest object: the archives are in the archives/ namespace + and the repository config is the config/config store object (see Repository.save_config). This class + only lives on as the container the archive-level code takes its key, repo_objs, repository and + archives from. For borg 1.x repositories (read-only, e.g. "borg transfer --from-borg1"), load() + still reads the manifest object, as it holds their archives list. + """ + + MANIFEST_ID = b"\0" * 32 # legacy: the id of a borg 1.x repository's manifest object def __init__(self, key, repository, ro_cls=RepoObj): from .legacy.repository import LegacyRepository @@ -494,57 +502,37 @@ def __init__(self, key, repository, ro_cls=RepoObj): self.archives: ArchivesInterface = LegacyArchives(repository, self) else: self.archives: ArchivesInterface = Archives(repository, self) - self.config = {} self.key = key self.repo_objs = ro_cls(key) self.repository = repository - self._loaded_data = None # the packed manifest as loaded from the repository, see write() - - @property - def id_str(self): - return bin_to_hex(self.id) @classmethod def load(cls, repository, key=None, *, other=False, ro_cls=RepoObj): + """Return the Manifest of repository, loading its key (see key_factory) if key is not given.""" + from .crypto.key import key_factory # crypto.key imports this module, hence the local import + from .legacy.repository import LegacyRepository + from .legacy.remote import LegacyRemoteRepository + + if isinstance(repository, (LegacyRepository, LegacyRemoteRepository)): + return cls._load_legacy(repository, key, other=other, ro_cls=ro_cls) + if not key: + key = key_factory(repository, other=other) + return cls(key, repository, ro_cls=ro_cls) + + @classmethod + def _load_legacy(cls, repository, key, *, other, ro_cls): + # a borg 1.x repository: its manifest object identifies the key type and holds the archives list. from .item import ManifestItem - from .crypto.key import key_factory + from .crypto.key import legacy_key_factory cdata = repository.get_manifest() if not key: - key = key_factory(repository, cdata, other=other, ro_cls=ro_cls) + key = legacy_key_factory(repository, cdata, other=other) manifest = cls(key, repository, ro_cls=ro_cls) - _, data = manifest.repo_objs.parse(cls.MANIFEST_ID, cdata, ro_type=ROBJ_MANIFEST) - manifest._loaded_data = data - manifest_dict = key.unpack_manifest(data) - m = ManifestItem(internal_dict=manifest_dict) - manifest.id = manifest.repo_objs.id_hash(data) + # borg 1.x objects carry no type in their (non-existent) metadata; RepoObj1.parse ignores ro_type. + _, data = manifest.repo_objs.parse(cls.MANIFEST_ID, cdata, ro_type=ROBJ_DONTCARE) + m = ManifestItem(internal_dict=key.unpack_manifest(data)) if m.get("version") not in (1, 2): raise ValueError("Invalid manifest version") manifest.archives.prepare(manifest, m) - # a "timestamp" entry (written by borg 1.x and by older borg 2 versions) is ignored, as is - # the list of item keys (borg 1.x: "item_keys", older borg 2 versions: config["item_keys"]). - manifest.config = m.config - manifest.config.pop("item_keys", None) return manifest - - def write(self): - """ - Store the manifest in the repository, but only if its content differs from what was loaded. - - The manifest content is static (borg does not store anything in its config dict currently), so - archive operations calling this usually do not result in a store write: only when the loaded - manifest still had legacy entries (a timestamp, the item keys list) is it rewritten. - """ - from .item import ManifestItem - - manifest_archives = self.archives.finish(self) - manifest = ManifestItem(version=2, archives=manifest_archives, config=StableDict(self.config)) - data = self.key.pack_metadata(manifest.as_dict()) - self.id = self.repo_objs.id_hash(data) - if data == self._loaded_data: - logger.debug("manifest unchanged, not writing it.") - return - logger.debug("writing the manifest.") - robj = self.repo_objs.format(self.MANIFEST_ID, {}, data, ro_type=ROBJ_MANIFEST) - self.repository.put_manifest(robj) - self._loaded_data = data diff --git a/src/borg/repository.py b/src/borg/repository.py index c5b7df3548..fb19012b19 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -1,4 +1,5 @@ import io +import configparser import os import re import sys @@ -28,7 +29,6 @@ from .helpers.lrucache import LRUCache from .storelocking import Lock from .logger import create_logger -from .manifest import NoManifestError from .repoobj import RepoObj, OBJ_MAGIC, SUPPORTED_OBJ_VERSIONS from .crypto.key import is_keyfile, store_hash, STORE_HASH_NAME, STORE_HASH_SIZE @@ -767,6 +767,10 @@ def clear(self): pass +class _ConfigMissing(Exception): + """internal: the store exists, but has no repository config object (see Repository._load_config).""" + + class Repository: """borgstore-based key/value store.""" @@ -775,6 +779,11 @@ class AlreadyExists(Error): exit_mcode = 10 + class IncompleteRepository(Error): + """{} has no repository config.""" + + exit_mcode = 11 + class CheckNeeded(ErrorWithTraceback): """Inconsistency detected. Please run "borg check {}".""" @@ -866,6 +875,8 @@ def __init__( self, path_or_location, create=False, + create_config=True, + allow_incomplete=False, exclusive=False, lock_wait=1.0, lock=True, @@ -938,6 +949,10 @@ def __init__( self.permissions = None if location.proto == "rest" else permissions self.store_opened = False self.version = None + self.id = None + # the crypto suite of the repository's key, as recorded in the repository config (see save_config): + self.encryption = None # the "--encryption" name, e.g. "aes256-ocb" + self.id_hash = None # the "--id-hash" name, e.g. "sha256" # long-running repository methods which emit log or progress output are responsible for calling # the ._send_log method periodically to get log and progress output transferred to the borg client # in a timely manner, in case we have a RemoteRepository. @@ -945,7 +960,15 @@ def __init__( self._send_log = send_log_cb or (lambda: None) self.do_create = create self.created = False - self.acceptable_repo_versions = (4,) + # create_config=False: create() does not write the repository config, the caller does that via + # save_config() (used by "borg repo-create", which writes it once the key exists, see create()). + self._create_config = create_config + self._config_written = False + # allow_incomplete=True: open() also accepts a store without repository config (see create()), + # so that "borg repo-delete --force" can destroy it. self.incomplete tells whether that happened. + self._allow_incomplete = allow_incomplete + self.incomplete = False + self.acceptable_repo_versions = (5,) self.opened = False self.lock = None self.do_lock = lock @@ -974,6 +997,9 @@ def __enter__(self): self.open(exclusive=bool(self.exclusive), lock_wait=self.lock_wait, lock=self.do_lock) except Exception: self.close(aborting=True) + if self.created: + # we just created the store, but could not open it: do not leave it behind (see create()). + self.store.destroy() raise return self @@ -994,34 +1020,145 @@ def id_str(self): return bin_to_hex(self.id) def create(self): - """Create a new empty repository""" + """Create the store for a new repository, give it a fresh id and write the repository config. + + The config has no key information yet (see save_config), so the repository can be opened and + used as a key/value store, but its key can not be loaded until save_config(key) was called. + + With create_config=False (see __init__), the config is not written here, but by the caller via + save_config(). Until then, the store is not a repository (open() reports it as not a valid + repository): "borg repo-create" uses this, so that an interrupted repo-create does not leave a + repository behind. + + If anything fails after the store was created, the store is destroyed again, so a failure (e.g. + disk full, permission denied) does not leave a store without config behind either. + """ try: self.store.create() except StoreBackendAlreadyExists: - raise self.AlreadyExists(self.url) - self.store.open() + raise self._already_exists_error() from None try: - self.store.store("config/readme", REPOSITORY_README.encode()) - self.version = 4 - self.store.store("config/version", str(self.version).encode()) - self.store.store("config/id", bin_to_hex(os.urandom(32)).encode()) - # we know repo/packs/ still does not have any chunks stored in it, - # but for some stores, there might be a lot of empty directories and - # listing them all might be rather slow, so we better cache an empty - # ChunkIndex from here so that the first repo operation does not have - # to build the ChunkIndex the slow way by listing all the directories. - from borg.cache import write_chunkindex_to_repo - - write_chunkindex_to_repo(self, ChunkIndex(), clear=True, force_write=True) - finally: - self.store.close() + # create all namespace directories in advance (saves ad-hoc mkdirs later, see borgstore). This + # also is what tells a store borg created apart from any other directory, see looks_like_borg_store(). + self.store.create_levels() + self.store.open() + try: + self.version = 5 + self.id = os.urandom(32) + if self._create_config: + self.save_config() + # we know repo/packs/ still does not have any chunks stored in it, + # but for some stores, there might be a lot of empty directories and + # listing them all might be rather slow, so we better cache an empty + # ChunkIndex from here so that the first repo operation does not have + # to build the ChunkIndex the slow way by listing all the directories. + from borg.cache import write_chunkindex_to_repo + + write_chunkindex_to_repo(self, ChunkIndex(), clear=True, force_write=True) + finally: + self.store.close() + except BaseException: + # do not leave the just created store behind (see above); the original error is what matters. + try: + self.store.destroy() + except Exception as exc: + logger.warning("could not remove the incompletely created store: %s", exc) + raise + + def _already_exists_error(self): + # the store backend refused to create the store, e.g. because the directory is not empty. if there + # is a repository config, it is a repository. else we only know that there is no borg 2 repository: + # the directory may hold anything, e.g. a borg 1.x repository, or be the leftover of an interrupted + # repo-create (see create()). + try: + self.store.open() + try: + self.store.load("config/config") + except StoreObjectNotFound: + return self.IncompleteRepository(self.url) + finally: + self.store.close() + except StoreBackendError: + pass # can not look inside, so we do not know more + return self.AlreadyExists(self.url) + + def save_config(self, key=None): + """Store the repository config (the config/config store object). + + It holds the repository version and id and the crypto suite of the repository's key (encryption + mode and id hash), so that the key class is known without reading any repository object. + Writing it is what makes the store a repository: open() requires it. + + key: the repository's key, its crypto suite gets recorded. None: record the crypto suite known + from a previous config (if any) - without it, the repository can be opened, but its key can not + be loaded (see key_factory). + """ + if key is not None: + self.encryption, self.id_hash = key.ENC_NAME, key.IDHASH_NAME + config = configparser.ConfigParser(interpolation=None) + config.add_section("repository") + config.set("repository", "version", str(self.version)) + config.set("repository", "id", bin_to_hex(self.id)) + if self.encryption is not None and self.id_hash is not None: + config.set("repository", "encryption", self.encryption) + config.set("repository", "id_hash", self.id_hash) + with io.StringIO() as f: + for line in REPOSITORY_README.splitlines(): + f.write(f"# {line}\n") + f.write("\n") + config.write(f) + self.store.store("config/config", f.getvalue().encode()) + self._config_written = True + + def _load_config(self): + try: + text = self.store.load("config/config").decode() + except StoreBackendDoesNotExist: + # A rest:// store's open() does not contact the server, so for rest:// a missing repository + # only shows up here, when the first request fails with BackendDoesNotExist (#10365). + raise self.DoesNotExist(str(self._location)) from None + except StoreObjectNotFound: + # the store exists, but has no repository config: a repository that lost its config, something + # that never was a borg 2 repository, or the leftover of an interrupted repo-create (see create()). + raise _ConfigMissing() from None + except UnicodeDecodeError: + raise self.InvalidRepository(str(self._location)) from None + config = configparser.ConfigParser(interpolation=None) + try: + config.read_string(text) + self.version = config.getint("repository", "version") + self.id = hex_to_bin(config.get("repository", "id"), length=32) + self.encryption = config.get("repository", "encryption", fallback=None) + self.id_hash = config.get("repository", "id_hash", fallback=None) + except (configparser.Error, ValueError): + raise self.InvalidRepository(str(self._location)) from None + if (self.encryption is None) != (self.id_hash is None): + # the crypto suite is recorded by both entries or by none, see save_config(). + raise self.InvalidRepository(str(self._location)) + + def looks_like_borg_store(self): + """Does the (opened, config-less) store have the packs, archives, index and config namespaces? + + create() makes them, so every store borg created has them, even the leftover of an interrupted + repo-create. Any other directory (data, a home directory, a borg 1.x repository) does not, and + "borg repo-delete --force" must never destroy it. + """ + + def namespace_exists(name): + try: + next(iter(self.store.list(name)), None) # an existing namespace lists (maybe nothing) + except StoreObjectNotFound: + return False + return True + + return all(namespace_exists(name) for name in ("packs", "archives", "index", "config")) def _set_id(self, id): # for testing: change the id of an existing repository assert self.opened assert isinstance(id, bytes) and len(id) == 32 self.id = id - self.store.store("config/id", bin_to_hex(id).encode()) + self.save_config() def _lock_refresh(self): if self.lock is not None: @@ -1083,21 +1220,23 @@ def open(self, *, exclusive, lock_wait=None, lock=True): raise self.DoesNotExist(str(self._location)) from None else: self.store_opened = True - try: - readme = self.store.load("config/readme").decode() - except (StoreObjectNotFound, StoreBackendDoesNotExist): - # A rest:// store's open() does not contact the server, so for rest:// a missing repository - # only shows up here, when the first request fails with BackendDoesNotExist (#10365). - raise self.DoesNotExist(str(self._location)) from None - if readme != REPOSITORY_README: - raise self.InvalidRepository(str(self._location)) - self.version = int(self.store.load("config/version").decode()) + if self.created and not self._config_written: + pass # create() just ran and set version and id; the config is written later by save_config(). + else: + try: + self._load_config() + except _ConfigMissing: + if not self._allow_incomplete: + raise self.InvalidRepository(str(self._location)) from None + # a store without repository config: version and id are unknown, nothing gets locked, the + # only thing the caller may do with it is looks_like_borg_store() / destroy(). + self.incomplete = True + return if self.version not in self.acceptable_repo_versions: self.close() raise self.InvalidRepositoryConfig( str(self._location), "repository version %d is not supported by this borg version" % self.version ) - self.id = hex_to_bin(self.store.load("config/id").decode(), length=32) # important: lock *after* making sure that there actually is an existing, supported repository. if lock: self.lock = Lock( @@ -2025,17 +2164,6 @@ def migrate_lock(self, old_id, new_id): if self.lock is not None: self.lock.migrate_lock(old_id, new_id) - def get_manifest(self): - self._lock_refresh() - try: - return self.store.load("config/manifest") - except StoreObjectNotFound: - raise NoManifestError - - def put_manifest(self, data): - self._lock_refresh() - return self.store.store("config/manifest", data) - def store_list(self, name, *, deleted=False): self._lock_refresh() try: diff --git a/src/borg/testsuite/archiver/check_cmd_test.py b/src/borg/testsuite/archiver/check_cmd_test.py index 8706c12059..773bbaa566 100644 --- a/src/borg/testsuite/archiver/check_cmd_test.py +++ b/src/borg/testsuite/archiver/check_cmd_test.py @@ -10,8 +10,9 @@ from ... import archive as archive_module from ...archive import Archive, ArchiveChecker, ChunkBuffer from ...cache import Cache, delete_chunkindex_from_repo +from ...crypto.key import RepositoryKeyInfoMissing from ...constants import * # NOQA -from ...helpers import bin_to_hex, msgpack, CommandError, CorruptPack, Error, IntegrityError, sig_int +from ...helpers import bin_to_hex, CommandError, CorruptPack, Error, sig_int from ...helpers import BackupDamagedChunksError from ...helpers.passphrase import PassphraseWrong from ...item import Item @@ -538,63 +539,6 @@ def test_check_format_missing_archive_metadata(archivers, request): assert "Analyzing archive archive2" in output # the intact archive still uses the given format -def test_missing_manifest(archivers, request): - archiver = request.getfixturevalue(archivers) - check_cmd_setup(archiver) - archive, repository = open_archive(archiver.repository_path, "archive1") - with repository: - if isinstance(repository, Repository): - repository.store_delete("config/manifest") - else: - repository.delete(Manifest.MANIFEST_ID, validate=None) - cmd(archiver, "check", exit_code=1) - output = cmd(archiver, "check", "-v", "--repair", exit_code=0) - assert "archive1" in output - assert "archive2" in output - cmd(archiver, "check", exit_code=0) - - -def test_corrupted_manifest(archivers, request): - archiver = request.getfixturevalue(archivers) - check_cmd_setup(archiver) - archive, repository = open_archive(archiver.repository_path, "archive1") - with repository: - manifest = repository.get_manifest() - corrupted_manifest = corrupt(manifest, len(manifest) - 1) # the manifest object is small, hit the ciphertext - repository.put_manifest(corrupted_manifest) - cmd(archiver, "check", exit_code=1) - output = cmd(archiver, "check", "-v", "--repair", exit_code=0) - assert "archive1" in output - assert "archive2" in output - cmd(archiver, "check", exit_code=0) - - -def test_spoofed_manifest(archivers, request): - archiver = request.getfixturevalue(archivers) - check_cmd_setup(archiver) - archive, repository = open_archive(archiver.repository_path, "archive1") - with repository: - manifest = Manifest.load(repository) - cdata = manifest.repo_objs.format( - Manifest.MANIFEST_ID, - {}, - msgpack.packb({"version": 1, "archives": {}, "config": {}}), - # we assume that an attacker can put a file into backup src files that contains a fake manifest. - # but, the attacker can not influence the ro_type borg will use to store user file data: - ro_type=ROBJ_FILE_STREAM, # a real manifest is stored with ROBJ_MANIFEST - ) - # maybe a repo-side attacker could manage to move the fake manifest file chunk over to the manifest ID. - # we simulate this here by directly writing the fake manifest data to the manifest ID. - repository.put_manifest(cdata) - # borg should notice that the manifest has the wrong ro_type. - cmd(archiver, "check", exit_code=1) - # borg check --repair should remove the corrupted manifest and rebuild a new one. - output = cmd(archiver, "check", "-v", "--repair", exit_code=0) - assert "archive1" in output - assert "archive2" in output - cmd(archiver, "check", exit_code=0) - - def test_check_repair_rebuilds_corrupt_index(archivers, request): # A corrupt index with all packs intact: the default (full) --repair rebuilds the index from the # packs and persists it (via the archives check, see ArchiveChecker.finish), leaving the repository @@ -675,25 +619,6 @@ def test_check_repository_only_repair_aborts_on_wrong_passphrase(archivers, requ cmd(archiver, "check", "-v", "--repository-only", "--repair") -@pytest.mark.skip(reason="TODO: repair does not yet rewrite store-corrupted packs, refs #8572") -def test_manifest_rebuild_corrupted_chunk(archivers, request): - archiver = request.getfixturevalue(archivers) - check_cmd_setup(archiver) - archive, repository = open_archive(archiver.repository_path, "archive1") - with repository: - manifest = repository.get_manifest() - # flip a byte inside the encrypted manifest data so its integrity check fails and - # check --repair rebuilds the manifest. - corrupted_manifest = corrupt(manifest, len(manifest) // 3) - repository.put_manifest(corrupted_manifest) - corrupt_chunk_on_disk(repository, archive.id) - cmd(archiver, "check", exit_code=1) - output = cmd(archiver, "check", "-v", "--repair", exit_code=0) - assert "archive1" not in output - assert "archive2" in output - cmd(archiver, "check", exit_code=0) - - def test_check_undelete_archives(archivers, request): archiver = request.getfixturevalue(archivers) check_cmd_setup(archiver) # creates archive1 and archive2 @@ -720,10 +645,6 @@ def test_spoofed_archive(archivers, request): archive, repository = open_archive(archiver.repository_path, "archive1") repo_objs = archive.repo_objs with repository: - # attacker would corrupt or delete the manifest to trigger a rebuild of it: - manifest = repository.get_manifest() - corrupted_manifest = corrupt(manifest, len(manifest) - 1) # the manifest object is small, hit the ciphertext - repository.put_manifest(corrupted_manifest) archive_dict = { "command_line": "", "item_ptrs": [], @@ -747,8 +668,9 @@ def test_spoofed_archive(archivers, request): ), ) repository.flush() # make the put durable before close()/the check below - cmd(archiver, "check", exit_code=1) - cmd(archiver, "check", "--repair", "--debug", exit_code=0) + # the attacker would hope that the search for lost archives picks the fake archive up, but + # borg notices that the object has the wrong ro_type. + cmd(archiver, "check", "--repair", "--find-lost-archives", "--debug", exit_code=0) output = cmd(archiver, "repo-list") assert "archive1" in output assert "archive2" in output @@ -876,12 +798,12 @@ def test_check_without_repair_does_not_drop_a_pack_tail(archivers, request, monk real_make_key = ArchiveChecker.make_key - def make_key(self, repository, manifest_only=False): - # fail the manifest_only read, the one the rebuild's validator needs, as an unreadable key - # would. check_cmd already read the key before the check (#1931), hence the full read below. - if manifest_only: - raise IntegrityError("no key") - return real_make_key(self, repository, manifest_only=manifest_only) + def make_key(self, repository): + # fail the reads before the index rebuild (the one the rebuild's validator needs), as a + # repository config without key info would. the full read after the rebuild succeeds. + if getattr(self, "chunks", None) is None: + raise RepositoryKeyInfoMissing("no key") + return real_make_key(self, repository) real_build = archive_module.build_chunkindex_from_repo rebuilds = [] @@ -1128,34 +1050,6 @@ def test_empty_repository(archivers, request): cmd(archiver, "check", exit_code=1) -def test_manifest_with_timestamp_is_accepted(archivers, request): - # borg 1.x and older borg 2 versions wrote a "timestamp" entry into the manifest. it is not written - # anymore, but such manifests must still load (and get rewritten without it). - archiver = request.getfixturevalue(archivers) - check_cmd_setup(archiver) - with Repository(archiver.repository_path, exclusive=True) as repository: - manifest = Manifest.load(repository) - data = manifest.key.pack_metadata( - { - "version": 2, - "archives": {}, - "config": {"item_keys": tuple(sorted(ITEM_KEYS))}, - "timestamp": "2026-01-01T00:00:00.000000", - } - ) - repository.put_manifest(manifest.repo_objs.format(Manifest.MANIFEST_ID, {}, data, ro_type=ROBJ_MANIFEST)) - output = cmd(archiver, "repo-list") - assert "archive1" in output - create_src_archive(archiver, "archive3") # a writing command rewrites the manifest ... - dump_file = archiver.output_path + "/dump" - cmd(archiver, "debug", "dump-manifest", dump_file) - with open(dump_file) as f: - dump = f.read() - assert "timestamp" not in dump # ... without the timestamp - assert "item_keys" not in dump # ... and without the legacy item keys list - cmd(archiver, "check", exit_code=0) - - def test_items_with_unknown_keys_are_kept(archivers, request): # items with keys this borg version does not know (e.g. written by a newer borg) are not an error: # check warns about them once per archive (rc stays 0) and --repair writes them back unchanged. diff --git a/src/borg/testsuite/archiver/corruption_test.py b/src/borg/testsuite/archiver/corruption_test.py index bb093a42b3..bffc1d98cf 100644 --- a/src/borg/testsuite/archiver/corruption_test.py +++ b/src/borg/testsuite/archiver/corruption_test.py @@ -1,11 +1,7 @@ import json -import os -from configparser import ConfigParser -import pytest from ...constants import * # NOQA -from ...helpers import bin_to_hex from . import cmd, create_test_files, RK_ENCRYPTION @@ -13,20 +9,3 @@ def corrupt_archiver(archiver): create_test_files(archiver.input_path) cmd(archiver, "repo-create", RK_ENCRYPTION) archiver.cache_path = json.loads(cmd(archiver, "repo-info", "--json"))["cache"].get("path") - - -def test_old_version_interfered(archiver): - corrupt_archiver(archiver) - if archiver.cache_path is None: - pytest.skip("No cache path for this kind of cache implementation.") - - # Modify the main manifest ID without touching the manifest ID in the integrity section. - # This happens if a version without integrity checking modifies the cache. - config_path = os.path.join(archiver.cache_path, "config") - config = ConfigParser(interpolation=None) - config.read(config_path) - config.set("cache", "manifest", bin_to_hex(bytes(32))) - with open(config_path, "w") as fd: - config.write(fd) - out = cmd(archiver, "repo-info") - assert "Cache integrity data not available: old Borg version modified the cache." in out diff --git a/src/borg/testsuite/archiver/create_cmd_test.py b/src/borg/testsuite/archiver/create_cmd_test.py index a410e97505..a40b89df77 100644 --- a/src/borg/testsuite/archiver/create_cmd_test.py +++ b/src/borg/testsuite/archiver/create_cmd_test.py @@ -2120,16 +2120,3 @@ def test_files_cache_rebuild_group_by_invalid(archivers, request): cmd(archiver, "repo-create", RK_ENCRYPTION) output = cmd(archiver, "create", "--group-by", "", "home", "input", exit_code=2) assert "At least one group-by key is required" in output - - -def test_create_does_not_rewrite_unchanged_manifest(archivers, request): - archiver = request.getfixturevalue(archivers) - if archiver.get_kind() != "local": - pytest.skip("opens the repository directly") - create_regular_file(archiver.input_path, "file1", size=1024) - cmd(archiver, "repo-create", RK_ENCRYPTION) - with Repository(archiver.repository_path, exclusive=True) as repository: - manifest_before = repository.get_manifest() - cmd(archiver, "create", "test", "input") - with Repository(archiver.repository_path, exclusive=True) as repository: - assert repository.get_manifest() == manifest_before diff --git a/src/borg/testsuite/archiver/debug_cmds_test.py b/src/borg/testsuite/archiver/debug_cmds_test.py index 9c3a971ba1..5d3fc41aea 100644 --- a/src/borg/testsuite/archiver/debug_cmds_test.py +++ b/src/borg/testsuite/archiver/debug_cmds_test.py @@ -247,23 +247,6 @@ def test_debug_format_obj_respects_type(archivers, request): assert meta_read["type"] == ROBJ_ARCHIVE_STREAM -def test_debug_dump_manifest(archivers, request): - archiver = request.getfixturevalue(archivers) - create_regular_file(archiver.input_path, "file1", size=1024 * 80) - cmd(archiver, "repo-create", RK_ENCRYPTION) - cmd(archiver, "create", "test", "input") - dump_file = archiver.output_path + "/dump" - output = cmd(archiver, "debug", "dump-manifest", dump_file) - assert output == "" - with open(dump_file) as f: - result = json.load(f) - assert "archives" in result - assert "config" in result - assert "timestamp" not in result - assert "version" in result - assert "item_keys" not in result["config"] - - def test_debug_dump_archive(archivers, request): archiver = request.getfixturevalue(archivers) create_regular_file(archiver.input_path, "file1", size=1024 * 80) diff --git a/src/borg/testsuite/archiver/repo_create_cmd_test.py b/src/borg/testsuite/archiver/repo_create_cmd_test.py index 4cbb0171a4..9d3e6fc21f 100644 --- a/src/borg/testsuite/archiver/repo_create_cmd_test.py +++ b/src/borg/testsuite/archiver/repo_create_cmd_test.py @@ -135,3 +135,29 @@ def test_repo_create_refuse_to_overwrite_keyfile(archivers, request, monkeypatch with open(keyfile) as file: after = file.read() assert before == after + + +def test_repo_create_failure_leaves_nothing_behind(archivers, request, monkeypatch): + # not only a cancelled, also a failed repo-create must not leave a (partial) repository or a keyfile. + archiver = request.getfixturevalue(archivers) + if archiver.EXE: + pytest.skip("patches object") + keys_dir = os.path.join(archiver.tmpdir, "keys") + monkeypatch.setenv("BORG_KEYS_DIR", keys_dir) + + def failing_save_config(self, key=None): + raise OSError("simulated store failure while writing the config") + + from ...repository import Repository + + with patch.object(Repository, "save_config", failing_save_config): + if archiver.FORK_DEFAULT: + cmd(archiver, "repo-create", KF_ENCRYPTION, KF_LOCATION, exit_code=2) + else: + with pytest.raises(OSError, match="simulated store failure"): + cmd(archiver, "repo-create", KF_ENCRYPTION, KF_LOCATION) + assert not os.path.exists(archiver.repository_location) + assert not os.path.exists(keys_dir) or not os.listdir(keys_dir) # the keyfile written before the failure is gone + # and nothing stands in the way of creating the repository there now. + cmd(archiver, "repo-create", KF_ENCRYPTION, KF_LOCATION) + assert os.listdir(keys_dir) diff --git a/src/borg/testsuite/archiver/repo_delete_cmd_test.py b/src/borg/testsuite/archiver/repo_delete_cmd_test.py index b04ad37f49..12740e89f8 100644 --- a/src/borg/testsuite/archiver/repo_delete_cmd_test.py +++ b/src/borg/testsuite/archiver/repo_delete_cmd_test.py @@ -3,7 +3,7 @@ import pytest from ...constants import * # NOQA -from ...helpers import CancelledByUser +from ...helpers import CancelledByUser, Error from . import create_regular_file, cmd, generate_archiver_tests, RK_ENCRYPTION pytest_generate_tests = lambda metafunc: generate_archiver_tests(metafunc, kinds="local,binary") # NOQA @@ -40,3 +40,44 @@ def test_delete_repo_force(archivers, request): cmd(archiver, "repo-delete", "--force") # Make sure the repository is gone assert not os.path.exists(archiver.repository_path) + + +def test_delete_store_without_config(archivers, request): + # a store without repository config (e.g. the leftover of an interrupted repo-create, or a repository + # that lost its config) can only be deleted with --force. + from ...repository import Repository + + archiver = request.getfixturevalue(archivers) + if archiver.EXE: + pytest.skip("creates the store via the Python API") + with Repository(archiver.repository_path, exclusive=True, create=True, create_config=False): + pass + if archiver.FORK_DEFAULT: + output = cmd(archiver, "repo-delete", exit_code=2) + assert "requires the --force option" in output + else: + with pytest.raises(Error, match="requires the --force option"): + cmd(archiver, "repo-delete") + cmd(archiver, "repo-delete", "--force", "--dry-run") + assert os.path.exists(archiver.repository_path) + cmd(archiver, "repo-delete", "--force") + assert not os.path.exists(archiver.repository_path) + # a directory with data in it does not look like a borg store: refused even with --force. + os.mkdir(archiver.repository_path) + create_regular_file(archiver.repository_path, "file", contents=b"some data") + if archiver.FORK_DEFAULT: + output = cmd(archiver, "repo-delete", "--force", exit_code=2) + assert "does not look like a borg store" in output + else: + with pytest.raises(Error, match="does not look like a borg store"): + cmd(archiver, "repo-delete", "--force") + assert os.listdir(archiver.repository_path) == ["file"] + os.unlink(os.path.join(archiver.repository_path, "file")) + os.rmdir(archiver.repository_path) + # a repository with an archive that lost its config: --force destroys it. + cmd(archiver, "repo-create", RK_ENCRYPTION) + create_regular_file(archiver.input_path, "file1", size=1024) + cmd(archiver, "create", "test", "input") + os.unlink(os.path.join(archiver.repository_path, "config", "config")) + cmd(archiver, "repo-delete", "--force") + assert not os.path.exists(archiver.repository_path) diff --git a/src/borg/testsuite/archiver/return_codes_test.py b/src/borg/testsuite/archiver/return_codes_test.py index 21ca05b341..1a42e805c7 100644 --- a/src/borg/testsuite/archiver/return_codes_test.py +++ b/src/borg/testsuite/archiver/return_codes_test.py @@ -32,13 +32,14 @@ def test_return_codes(archivers, request): def test_exit_codes(archivers, request, monkeypatch): archiver = request.getfixturevalue(archivers) - # we create the repo path, but do NOT initialize the borg repo, - # so the borg create commands are expected to fail with DoesNotExist (was: InvalidRepository in borg 1.4). + # we create the repo path, but do NOT initialize the borg repo: the store exists, but has no repository + # config, so the borg create commands are expected to fail with InvalidRepository (DoesNotExist is only + # for a store that does not exist at all). os.makedirs(archiver.repository_path) monkeypatch.setenv("BORG_EXIT_CODES", "classic") cmd(archiver, "create", "archive", "input", fork=True, exit_code=EXIT_ERROR) monkeypatch.setenv("BORG_EXIT_CODES", "modern") - cmd(archiver, "create", "archive", "input", fork=True, exit_code=Repository.DoesNotExist.exit_mcode) + cmd(archiver, "create", "archive", "input", fork=True, exit_code=Repository.InvalidRepository.exit_mcode) def test_print_warning_instance_does_not_retain_exception(): diff --git a/src/borg/testsuite/archives_test.py b/src/borg/testsuite/archives_test.py index 59c4d33896..6a3e74d244 100644 --- a/src/borg/testsuite/archives_test.py +++ b/src/borg/testsuite/archives_test.py @@ -77,11 +77,6 @@ def test_prepare_is_noop(): m.assert_not_called() -def test_finish_returns_empty_dict(): - ar, _, manifest = _archives() - assert ar.finish(manifest) == {} - - def test_ids_empty(): ar, _, _ = _archives() assert list(ar.ids()) == [] diff --git a/src/borg/testsuite/cache_test.py b/src/borg/testsuite/cache_test.py index 05cc5c8a76..a92cd6559f 100644 --- a/src/borg/testsuite/cache_test.py +++ b/src/borg/testsuite/cache_test.py @@ -51,16 +51,13 @@ def key(self, repository, monkeypatch): @pytest.fixture def manifest(self, repository, key): - Manifest(key, repository).write() + repository.save_config(key) return Manifest.load(repository, key=key) @pytest.fixture def cache(self, repository, key, manifest): return AdHocWithFilesCache(manifest) - def test_does_not_contain_manifest(self, cache): - assert not cache.seen_chunk(Manifest.MANIFEST_ID) - def test_seen_chunk_add_chunk_size(self, cache): assert cache.add_chunk(H(1), {}, b"5678", stats=Statistics()) == (H(1), 4) @@ -478,7 +475,7 @@ def test_close_consolidates_fragments_across_sessions(tmp_path, monkeypatch): loc = os.fspath(tmp_path / "repository") with Repository(loc, exclusive=True, create=True) as repository: key = AESOCBKey.create(repository, TestKey.MockArgs()) - Manifest(key, repository).write() + repository.save_config(key) all_ids = [] for s in range(5): # each session adds 100 new chunks (< MIN), so fragments must be consolidated @@ -743,7 +740,7 @@ def test_files_cache_save_tolerates_missing_chunk(tmp_path, monkeypatch): loc = os.fspath(tmp_path / "repository") with Repository(loc, exclusive=True, create=True) as repository: key = AESOCBKey.create(repository, TestKey.MockArgs()) - Manifest(key, repository).write() + repository.save_config(key) with Repository(loc, exclusive=True) as repository: manifest = Manifest.load(repository, key=key) diff --git a/src/borg/testsuite/crypto/key_test.py b/src/borg/testsuite/crypto/key_test.py index 831b9a3b6b..930e6f1ea4 100644 --- a/src/borg/testsuite/crypto/key_test.py +++ b/src/borg/testsuite/crypto/key_test.py @@ -9,22 +9,21 @@ from ...crypto.key import ChecksumKey, Blake3ChecksumKey, keyfile_parse from ...crypto.key import AuthenticatedKey, Blake3AuthenticatedKey from ...crypto.key import AESCTRKey, Blake2AESCTRKey, Blake2AuthenticatedKey -from ...crypto.key import LegacyPlaintextKey, LegacyAuthenticatedKey +from ...crypto.key import LegacyAuthenticatedKey from ...crypto.key import AEADKeyBase from ...crypto.key import AESOCBKey, CHPOKey, Blake3AESOCBKey, Blake3CHPOKey from ...crypto.key import AES_OCB_MAX_SESSION_BLOCKS from ...crypto.key import ID_HMAC_SHA_256, ID_BLAKE2b_256, ID_BLAKE3_256 from ...crypto.key import UnsupportedManifestError, UnsupportedKeyFormatError, UnsupportedPayloadError from ...crypto.key import RepoKeyNotFoundError -from ...crypto.key import identify_key, key_from_repository +from ...crypto.key import identify_key, key_class_for, key_class_of, key_factory, RepositoryKeyInfoMissing +from ...crypto.key import AVAILABLE_KEY_TYPES from ...crypto.low_level import IntegrityError as IntegrityErrorBase from ...helpers import Error from ...helpers import IntegrityError from ...helpers import Location from ...helpers import msgpack -from ...manifest import NoManifestError -from ...repoobj import RepoObj -from ...constants import KEY_ALGORITHMS, KeyBlobStorage, KeyType, ROBJ_FILE_STREAM, ROBJ_MANIFEST +from ...constants import KEY_ALGORITHMS, KeyBlobStorage, KeyType from ...helpers import hex_to_bin, bin_to_hex @@ -602,21 +601,6 @@ def test_legacy_authenticated_no_key_key_gone(cls, monkeypatch, tmp_path): assert bytes(key.decrypt(None, envelope)) == payload -def test_dropped_borg2_beta_key_types(tmpdir): - # the borg2 beta "none"/"authenticated" formats were dropped, see #9104. A borg2 repository - # using them must be refused instead of being read with the legacy classes. - from ...repoobj import RepoObj - from ...crypto.key import key_factory - - for legacy_cls in (LegacyPlaintextKey, LegacyAuthenticatedKey): - key = legacy_cls(MagicMock(id=bytes(32))) - if legacy_cls is LegacyAuthenticatedKey: - key.id_key = bytes(32) - manifest_chunk = RepoObj(key).format(bytes(32), {}, b"manifest", ro_type=ROBJ_MANIFEST) - with pytest.raises(UnsupportedPayloadError): - key_factory(MagicMock(id=bytes(32)), manifest_chunk, ro_cls=RepoObj) - - def test_dropped_blake3_authenticated_type_byte(): with pytest.raises(UnsupportedPayloadError): identify_key(bytes([KeyType.DROPPED_BLAKE3AUTHENTICATED]) + b"payload") @@ -669,70 +653,26 @@ def test_argon2_wrong_passphrase_returns_none(monkeypatch): assert key.decrypt_key_file(a2b_base64(saved_b64), "wrong passphrase") is None -class StoredObjectsRepository: - """A repository with a stored manifest (None: no manifest) and objects by chunk id.""" - - def __init__(self, manifest, objects): - self.manifest = manifest - self.objects = objects - - def get_manifest(self): - if self.manifest is None: - raise NoManifestError - return self.manifest - - def get(self, id): - return self.objects[id] - - def list(self, limit=None): - return [(id, len(obj)) for id, obj in self.objects.items()][:limit] - - -def stored_object(key_cls, id): - return RepoObj(key_cls(MagicMock(id=bytes(32)))).format(id, {}, b"data", ro_type=ROBJ_FILE_STREAM) - - -def with_key_type(cdata, key_type): - """Return cdata with its key type byte, the first byte of the data slot, set to key_type.""" - offset = len(cdata) - len(RepoObj.extract_crypted_data(cdata)) - return cdata[:offset] + bytes([key_type]) + cdata[offset + 1 :] - - -def test_key_from_repository_reads_the_key_type_from_the_manifest(): - objects = {b"o" * 32: stored_object(ChecksumKey, b"o" * 32)} - repository = StoredObjectsRepository(stored_object(Blake3ChecksumKey, bytes(32)), objects) - assert isinstance(key_from_repository(repository), Blake3ChecksumKey) - - -def test_key_from_repository_skips_objects_that_do_not_identify_a_key_type(): - good = stored_object(ChecksumKey, b"g" * 32) - objects = { - b"d" * 32: b"damaged", - b"u" * 32: with_key_type(good, KeyType.DROPPED_BLAKE3AUTHENTICATED), - b"g" * 32: good, - } - repository = StoredObjectsRepository(b"damaged manifest", objects) - assert isinstance(key_from_repository(repository), ChecksumKey) - +@pytest.mark.parametrize("cls", AVAILABLE_KEY_TYPES) +def test_key_class_for_names_every_creatable_suite(cls): + # the repository config records the crypto suite by the --encryption / --id-hash names + assert key_class_for(cls.ENC_NAME, cls.IDHASH_NAME) is cls -def test_key_from_repository_raises_if_no_object_identifies_the_key_type(): - objects = {b"g" * 32: stored_object(ChecksumKey, b"g" * 32)} - with pytest.raises(IntegrityError): - key_from_repository(StoredObjectsRepository(None, objects), ids=()) # the manifest only - with pytest.raises(IntegrityError): - key_from_repository(StoredObjectsRepository(None, {b"d" * 32: b"damaged"})) +def test_key_class_for_unknown_suite(): + from ...repository import Repository -def test_key_from_repository_loads_the_key_once(monkeypatch): - ids = (b"a" * 32, b"b" * 32) - repository = StoredObjectsRepository(None, {id: stored_object(ChecksumKey, id) for id in ids}) - detected = [] + assert key_class_for("rot13", "sha256") is None + repository = MagicMock(encryption="rot13", id_hash="sha256") + with pytest.raises(Repository.InvalidRepositoryConfig, match="unsupported crypto suite"): + key_class_of(repository) - def detect(repository, manifest_data, *, other=False): - detected.append(manifest_data) - raise IntegrityError("the key can not be loaded") - monkeypatch.setattr(ChecksumKey, "detect", detect) - with pytest.raises(IntegrityError, match="the key can not be loaded"): - key_from_repository(repository) - assert len(detected) == 1 +def test_key_class_of_needs_key_info(): + repository = MagicMock(encryption=None, id_hash=None) + with pytest.raises(RepositoryKeyInfoMissing): + key_class_of(repository) + with pytest.raises(RepositoryKeyInfoMissing): + key_factory(repository) + repository = MagicMock(encryption="none-sha256", id_hash="sha256") + assert key_class_of(repository) is ChecksumKey diff --git a/src/borg/testsuite/legacy_archives_test.py b/src/borg/testsuite/legacy_archives_test.py index 29d8aa6617..56a2382b1b 100644 --- a/src/borg/testsuite/legacy_archives_test.py +++ b/src/borg/testsuite/legacy_archives_test.py @@ -87,12 +87,6 @@ def test_prepare(): assert la._archives == {"x": {"id": _id(5), "time": TS}} -def test_finish(): - la, _, manifest = _archives([("a", _id(1), TS)]) - result = la.finish(manifest) - assert result == {"a": {"id": _id(1), "time": TS}} - - def test_ids(): la, _, _ = _archives([("a", _id(1), TS), ("b", _id(2), TS)]) assert list(la.ids()) == [_id(1), _id(2)] diff --git a/src/borg/testsuite/repoobj_test.py b/src/borg/testsuite/repoobj_test.py index ee5fe41b34..bd582a3746 100644 --- a/src/borg/testsuite/repoobj_test.py +++ b/src/borg/testsuite/repoobj_test.py @@ -1,6 +1,6 @@ import pytest -from ..constants import ROBJ_FILE_STREAM, ROBJ_MANIFEST, ROBJ_ARCHIVE_META +from ..constants import ROBJ_FILE_STREAM, ROBJ_ARCHIVE_META from ..crypto.key import AESOCBKey, ChecksumKey, AuthenticatedKey, CHPOKey, LegacyPlaintextKey from ..helpers import CompressionSpec, msgpack from ..helpers.errors import Error, IntegrityError @@ -176,18 +176,6 @@ def test_malformed_object_inconsistent_sizes(key): repo_objs.parse(id, hdr, ro_type=ROBJ_FILE_STREAM) -def test_spoof_manifest(key): - repo_objs = RepoObj(key) - data = b"fake or malicious manifest data" # File content could be provided by an attacker. - id = repo_objs.id_hash(data) - # Create a repository object containing user data (file content data). - cdata = repo_objs.format(id, {}, data, ro_type=ROBJ_FILE_STREAM) - # Let's assume an attacker managed to replace the manifest with that repository object. - # As Borg always gives the ro_type it intends to read, this should fail: - with pytest.raises(IntegrityError): - repo_objs.parse(id, cdata, ro_type=ROBJ_MANIFEST) - - def test_spoof_archive(key): repo_objs = RepoObj(key) data = b"fake or malicious archive data" # File content could be provided by an attacker. diff --git a/src/borg/testsuite/repository_test.py b/src/borg/testsuite/repository_test.py index 80525924c4..e83465cb9e 100644 --- a/src/borg/testsuite/repository_test.py +++ b/src/borg/testsuite/repository_test.py @@ -2651,3 +2651,144 @@ def test_superseded_gap_ranges_ends_at_an_object_reaching_past_the_gap(tmp_path, f"pack {bin_to_hex(THIS_PACK)}: object reaching past its gap at offset 0 in a gap, " f"keeping the remaining {len(obj) - 1} bytes of the gap." in caplog.text ) + + +def test_config_roundtrip(tmp_path): + # the config/config store object holds version, id and the crypto suite of the key, see Repository.save_config + from ..crypto.key import Blake3CHPOKey + + location = os.fspath(tmp_path / "repo") + with Repository(location, exclusive=True, create=True) as repository: + # create() wrote a config without key info: + text = repository.store_load("config/config").decode() + assert text.startswith("# This is a Borg Backup repository.\n") + assert "[repository]\nversion = 5\n" in text + assert "encryption" not in text and "id_hash" not in text + assert (repository.encryption, repository.id_hash) == (None, None) + key = Blake3CHPOKey(repository) + repository.save_config(key) + id = repository.id + text = repository.store_load("config/config").decode() + assert "encryption = chacha20-poly1305\nid_hash = blake3\n" in text + with Repository(location, exclusive=True) as repository: + assert repository.version == 5 + assert repository.id == id + assert (repository.encryption, repository.id_hash) == ("chacha20-poly1305", "blake3") + repository.save_config() # without a key: the recorded crypto suite is kept + with Repository(location, exclusive=True) as repository: + assert (repository.encryption, repository.id_hash) == ("chacha20-poly1305", "blake3") + + +def test_store_without_config_is_not_a_repository(tmp_path): + # with create_config=False (as repo-create uses it), create() does not write the config, save_config() + # does: until then, the store is not a repository. + location = os.fspath(tmp_path / "repo") + with Repository(location, exclusive=True, create=True, create_config=False): + pass + assert os.path.exists(location) + with pytest.raises(Repository.InvalidRepository): + with Repository(location, exclusive=True): + pass + with pytest.raises(Repository.IncompleteRepository): + with Repository(location, exclusive=True, create=True): + pass + # allow_incomplete (as repo-delete --force uses it): opens, tells, and the store can be destroyed. + with Repository(location, exclusive=True, allow_incomplete=True) as repository: + assert repository.incomplete and repository.version is None and repository.id is None + assert repository.looks_like_borg_store() # create() made the namespaces + repository.destroy() + assert not os.path.exists(location) + + +def test_repository_that_lost_its_config(tmp_path): + # a repository with data whose config got lost is not a valid repository (not "does not exist"). + location = os.fspath(tmp_path / "repo") + with Repository(location, exclusive=True, create=True) as repository: + repository.put(H(1), fchunk(b"DATA", chunk_id=H(1))) + repository.flush() + os.unlink(os.path.join(location, "config", "config")) + with pytest.raises(Repository.InvalidRepository): + with Repository(location, exclusive=True): + pass + with Repository(location, exclusive=True, allow_incomplete=True) as repository: + assert repository.incomplete + assert repository.looks_like_borg_store() + + +def test_create_refuses_non_empty_directory(tmp_path): + # a non-empty directory that is no repository: the backend refuses to create the store there, and borg + # only knows that there is no repository config (it can not tell what the directory holds). + location = os.fspath(tmp_path / "data") + os.mkdir(location) + with open(os.path.join(location, "file"), "w") as f: + f.write("some data") + with pytest.raises(Repository.IncompleteRepository): + with Repository(location, exclusive=True, create=True): + pass + assert os.listdir(location) == ["file"] # and it leaves the directory alone + with Repository(location, exclusive=True, allow_incomplete=True) as repository: + assert repository.incomplete + assert not repository.looks_like_borg_store() # no borg namespaces: never destroyed by borg + assert os.listdir(location) == ["file"] + # a repository (with config) is reported as existing, too: + location = os.fspath(tmp_path / "repo") + with Repository(location, exclusive=True, create=True): + pass + with pytest.raises(Repository.AlreadyExists): + with Repository(location, exclusive=True, create=True): + pass + + +def test_open_refuses_bad_config(tmp_path): + location = os.fspath(tmp_path / "repo") + with Repository(location, exclusive=True, create=True): + pass + config_path = os.path.join(location, "config", "config") # the store is a directory, write the object directly + with open(config_path, "rb") as f: + good = f.read() + + def write_config(data): + with open(config_path, "wb") as f: + f.write(data) + + write_config(good.replace(b"version = 5", b"version = 4")) + with pytest.raises(Repository.InvalidRepositoryConfig): + with Repository(location, exclusive=True): + pass + bad_configs = [ + b"", # no [repository] section + b"[repository]\nversion = 5\n", # no id + b"[repository]\nversion = x\nid = 00\n", # invalid version and id + good + b"encryption = aes256-ocb\n", # key info must be complete (encryption AND id_hash) + good + b"id_hash = sha256\n", + b"version = 5\n", # not an INI file (no section header) + b"\xff\xfe", # not even text + ] + for bad in bad_configs: + write_config(bad) + with pytest.raises(Repository.InvalidRepository): + with Repository(location, exclusive=True): + pass + write_config(good) + with Repository(location, exclusive=True): + pass + + +def test_create_failure_leaves_no_store_behind(tmp_path, monkeypatch): + # a failure inside create() after the store was created (e.g. disk full while writing the empty chunk + # index) must not leave a store without config behind. + from .. import cache as cache_module + + def failing_write(*args, **kwargs): + raise OSError("simulated disk full") + + monkeypatch.setattr(cache_module, "write_chunkindex_to_repo", failing_write) + location = os.fspath(tmp_path / "repo") + with pytest.raises(OSError, match="simulated disk full"): + with Repository(location, exclusive=True, create=True): + pass + assert not os.path.exists(location) + monkeypatch.undo() + with Repository(location, exclusive=True, create=True): # and creating it afterwards works + pass + assert os.path.exists(os.path.join(location, "config", "config"))