From de1807ee2316e156b0469e73880c1a08a50a2a79 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 28 Jul 2026 01:25:00 +0200 Subject: [PATCH] store: support memoryview values additionally to bytes, fixes #200 This enables callers to avoid copying, e.g. by giving a slice of a bigger buffer they already have. A memoryview is normalized (cast to a 1-dimensional view of bytes), so len(value) always is the number of bytes - that matters for quota tracking and the volume statistics. A non-contiguous memoryview is rejected, because the value must be stored as one consecutive sequence of bytes. posixfs, rest and rclone hand the memoryview to the code doing the write / the request without copying. sftp and s3 create a bytes object first, because paramiko (only accepts str/bytes) and boto3 (parameter validation rejects a memoryview Body) can not deal with it. Co-Authored-By: Claude Opus 5 --- README.rst | 3 +- docs/backends.rst | 8 ++++ docs/changes.rst | 5 ++ docs/store.rst | 29 +++++++++++- src/borgstore/backends/_base.py | 38 +++++++++++++++- src/borgstore/backends/posixfs.py | 3 +- src/borgstore/backends/rclone.py | 6 ++- src/borgstore/backends/rest.py | 6 ++- src/borgstore/backends/s3.py | 4 +- src/borgstore/backends/sftp.py | 5 +- src/borgstore/store.py | 15 ++++-- tests/test_backends.py | 76 ++++++++++++++++++++++++++++++- tests/test_cache.py | 21 +++++++++ tests/test_posixfs_quota.py | 11 +++++ tests/test_server_rest.py | 16 +++++++ tests/test_store.py | 27 +++++++++++ 16 files changed, 258 insertions(+), 15 deletions(-) diff --git a/README.rst b/README.rst index 268ff4f..e146b1a 100644 --- a/README.rst +++ b/README.rst @@ -7,7 +7,8 @@ Overview -------- Keys are simple strings like `config/main` or `data/0123456789abcdef` `[str]` -(config and data are namespaces here). Values are binary objects `[bytes]`. +(config and data are namespaces here). Values are binary objects `[bytes]` - +when storing, a `memoryview` is accepted as well, so callers can avoid copying. The `Store` class is the high-level API, so you can comfortably work with the kv store without caring for low-level details. diff --git a/docs/backends.rst b/docs/backends.rst index 575a8a6..ef9b63f 100644 --- a/docs/backends.rst +++ b/docs/backends.rst @@ -4,6 +4,10 @@ Backends The backend API is rather simple; one only needs to provide some very basic operations. +``store(name, value)`` accepts a ``bytes`` or a ``memoryview`` value, see +:ref:`store-values`. Backends that can not give a ``memoryview`` to the library +they use internally create a ``bytes`` object (one copy) - this is noted below. + Existing backends are listed below; more might come in the future. See also :doc:`store_caching` for optional Store-level caching with a secondary backend. @@ -115,6 +119,8 @@ Use storage on an SFTP server: - Users must know the full absolute path of the space they are permitted to use. - Namespaces: directories - Values: in key-named files +- store: a ``memoryview`` value is copied into a ``bytes`` object first, because + paramiko does not accept a ``memoryview``. - hash: runs the hexdigest computation server-side (if server supports check-file). "blake3" is not part of the check-file extension, so it is always computed client-side. @@ -155,6 +161,8 @@ Use storage on an S3-compliant cloud service: - Namespaces: directories - Values: in key-named files +- store: a ``memoryview`` value is copied into a ``bytes`` object first, because + boto3 does not accept a ``memoryview``. REST (http/https) diff --git a/docs/changes.rst b/docs/changes.rst index 09b3eb6..efce972 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -11,6 +11,11 @@ New features: For backends that hash server-side, it needs to be installed on the server. Note: sftp always computes blake3 client-side, because the SFTP "check-file" extension does not support blake3. +- store: accept a memoryview value additionally to bytes, #200. + This enables callers to avoid copying, e.g. by giving a slice of a bigger + buffer they already have. posixfs, rest and rclone store it without copying, + sftp and s3 internally create a bytes object first, because paramiko / boto3 + do not accept a memoryview. Version 0.5.5 (2026-07-10) diff --git a/docs/store.rst b/docs/store.rst index f0c2914..223008f 100644 --- a/docs/store.rst +++ b/docs/store.rst @@ -112,10 +112,37 @@ Please note: can't add another namespace later, because that would create nested namespaces. +.. _store-values: + Values ------ -Values can be any arbitrary binary data (bytes). +Values can be any arbitrary binary data. + +``store(name, value)`` accepts the value either as ``bytes`` or as a +``memoryview`` of bytes, so a caller that already has the data inside a bigger +buffer can avoid copying it: + +.. code-block:: python + + buffer = bytearray(...) # e.g. a big buffer, filled with multiple objects + store.store("data/0123456789abcdef", memoryview(buffer)[offset : offset + size]) + +Notes about ``memoryview`` values: + +- The view must be C-contiguous (a plain slice of a buffer is). Otherwise, a + ``ValueError`` is raised - the value must be storable as one consecutive + sequence of bytes. +- If the view has an itemsize > 1 (e.g. a view of an ``array.array("I", ...)``), + the bytes it consists of are stored, so the stored size is + ``len(value) * value.itemsize``. +- The value is only used while ``store`` runs. Afterwards, the caller may reuse + or release the underlying buffer. +- Not all backends can hand a ``memoryview`` to the library they use: ``posixfs``, + ``rest`` and ``rclone`` store it without copying, ``sftp`` and ``s3`` internally + create a ``bytes`` object (one copy) first. + +Loading always gives ``bytes``. Automatic Nesting ----------------- diff --git a/src/borgstore/backends/_base.py b/src/borgstore/backends/_base.py index dced871..d4ad89f 100644 --- a/src/borgstore/backends/_base.py +++ b/src/borgstore/backends/_base.py @@ -14,6 +14,35 @@ # atime is the last read access UNIX timestamp [s] or 0 if not implemented ItemInfo = namedtuple("ItemInfo", "name exists size directory atime", defaults=(0,)) +# type of a value given to store: a memoryview is accepted additionally to bytes, +# so callers can avoid copying (e.g. give a slice of a big buffer they already have). +StoreValue = bytes | memoryview + + +def validate_value(value: StoreValue) -> StoreValue: + """Validate/normalize a value given to store. + + bytes (and other bytes-like objects) are returned unchanged. + A memoryview is cast to a 1-dimensional view of bytes, so len(value) always + gives the number of bytes (even if the caller had a view with a bigger itemsize). + """ + if isinstance(value, memoryview): + try: + return value.cast("B") + except TypeError: + # cast only works for C-contiguous views, but the backends need to + # write the value out as one consecutive sequence of bytes. + raise ValueError("value must be a C-contiguous memoryview") from None + return value + + +def to_bytes(value: StoreValue) -> bytes: + """Get a bytes object for code that can not deal with a memoryview. + + Note: this copies the data, except if it already is a bytes object. + """ + return value if isinstance(value, bytes) else bytes(value) + def validate_name(name): """Validate a backend key/name.""" @@ -108,8 +137,13 @@ def load(self, name: str, *, size=None, offset=0) -> bytes: """ @abstractmethod - def store(self, name: str, value: bytes) -> None: - """store into """ + def store(self, name: str, value: StoreValue) -> None: + """store into + + is either bytes or a memoryview of bytes (see StoreValue). + Backends must not keep a reference to a memoryview value after returning, + because the caller may reuse or release the underlying buffer. + """ @abstractmethod def delete(self, name: str) -> None: diff --git a/src/borgstore/backends/posixfs.py b/src/borgstore/backends/posixfs.py index e3622f3..7d4d3e2 100644 --- a/src/borgstore/backends/posixfs.py +++ b/src/borgstore/backends/posixfs.py @@ -21,7 +21,7 @@ except ImportError: fcntl = None # not available on Windows -from ._base import BackendBase, ItemInfo, validate_name +from ._base import BackendBase, ItemInfo, validate_name, validate_value from .errors import BackendError, BackendAlreadyExists, BackendDoesNotExist, BackendMustNotBeOpen, BackendMustBeOpen from .errors import ObjectNotFound, PermissionDenied, QuotaExceeded from ..constants import TMP_SUFFIX, QUOTA_STORE_NAME, QUOTA_PERSIST_DELTA, QUOTA_PERSIST_INTERVAL @@ -232,6 +232,7 @@ def _write_to_tempfile(self, path, value, suffix=TMP_SUFFIX, do_fsync=False): def store(self, name, value): if not self.opened: raise BackendMustBeOpen() + value = validate_value(value) # writing a memoryview is supported, no copy needed path = self._validate_join(name) overwrite = path.exists() self._check_permission(name, "W" if overwrite else "wW") diff --git a/src/borgstore/backends/rclone.py b/src/borgstore/backends/rclone.py index 3d17362..66060e2 100644 --- a/src/borgstore/backends/rclone.py +++ b/src/borgstore/backends/rclone.py @@ -16,7 +16,7 @@ except ImportError: requests = None -from ._base import BackendBase, ItemInfo, validate_name +from ._base import BackendBase, ItemInfo, validate_name, validate_value, StoreValue from ._utils import make_range_header, ignore_sigint from .errors import ( BackendError, @@ -288,9 +288,11 @@ def load(self, name: str, *, size=None, offset=0) -> bytes: content = content[:size] return content - def store(self, name: str, value: bytes) -> None: + def store(self, name: str, value: StoreValue) -> None: """Store into .""" validate_name(name) + # requests puts a memoryview into the multipart body as it is, no copy needed. + value = validate_value(value) files = {"file": (os.path.basename(name), value, "application/octet-stream")} params = {"fs": self.fs, "remote": os.path.dirname(name)} self._rpc("operations/uploadfile", None, tries=self.TRIES, params=params, files=files) diff --git a/src/borgstore/backends/rest.py b/src/borgstore/backends/rest.py index b177e60..c5970f0 100644 --- a/src/borgstore/backends/rest.py +++ b/src/borgstore/backends/rest.py @@ -29,7 +29,7 @@ except ImportError: pass -from ._base import BackendBase, ItemInfo, validate_name +from ._base import BackendBase, ItemInfo, validate_name, validate_value, StoreValue from ._utils import make_range_header, ignore_sigint from .errors import ( ObjectNotFound, @@ -557,9 +557,11 @@ def load(self, name: str, *, size=None, offset=0) -> bytes: return content @with_reconnect - def store(self, name: str, value: bytes) -> None: + def store(self, name: str, value: StoreValue) -> None: self._assert_open() validate_name(name) + # requests sends a memoryview body as it is (with a correct Content-Length), no copy needed. + value = validate_value(value) algorithm = "sha256" headers = {f"X-Content-hash-{algorithm}": hashlib.new(algorithm, value).hexdigest()} response = self._request("post", self._url(name), data=value, headers=headers) diff --git a/src/borgstore/backends/s3.py b/src/borgstore/backends/s3.py index 7a75101..438e12b 100644 --- a/src/borgstore/backends/s3.py +++ b/src/borgstore/backends/s3.py @@ -12,7 +12,7 @@ from typing import Optional import urllib.parse -from ._base import BackendBase, ItemInfo, validate_name +from ._base import BackendBase, ItemInfo, validate_name, validate_value, to_bytes from ._utils import make_range_header from .errors import BackendError, BackendMustBeOpen, BackendMustNotBeOpen, BackendDoesNotExist, BackendAlreadyExists from .errors import ObjectNotFound @@ -179,6 +179,8 @@ def store(self, name, value): if not self.opened: raise BackendMustBeOpen() validate_name(name) + # botocore rejects a memoryview Body (parameter validation), so we need a copy. + value = to_bytes(validate_value(value)) key = self.base_path + name self.s3.put_object(Bucket=self.bucket, Key=key, Body=value) diff --git a/src/borgstore/backends/sftp.py b/src/borgstore/backends/sftp.py index 3b483a0..a14aee2 100644 --- a/src/borgstore/backends/sftp.py +++ b/src/borgstore/backends/sftp.py @@ -19,7 +19,7 @@ except ImportError: paramiko = None -from ._base import BackendBase, ItemInfo, validate_name +from ._base import BackendBase, ItemInfo, validate_name, validate_value, to_bytes from .errors import BackendError, BackendMustBeOpen, BackendMustNotBeOpen, BackendDoesNotExist, BackendAlreadyExists from .errors import ObjectNotFound from ..constants import TMP_SUFFIX @@ -437,6 +437,9 @@ def _write_to_tmpfile(): if not self.opened: raise BackendMustBeOpen() validate_name(name) + # paramiko's file.write only deals with str/bytes, a memoryview would blow up + # deeply inside paramiko ("Unknown type for encoding"), so we need a copy here. + value = to_bytes(validate_value(value)) tmp_dir = Path(name).parent # write to a differently named temp file in same directory first, # so the store never sees partially written data. diff --git a/src/borgstore/store.py b/src/borgstore/store.py index cd229a3..b0f227f 100644 --- a/src/borgstore/store.py +++ b/src/borgstore/store.py @@ -19,7 +19,7 @@ from typing import Iterator, NamedTuple, Optional from .utils.nesting import nest, unnest -from .backends._base import ItemInfo, BackendBase +from .backends._base import ItemInfo, BackendBase, StoreValue, validate_value from .backends.errors import ObjectNotFound, NoBackendGiven, BackendURLInvalid, ReadRangeError # noqa from .backends.posixfs import get_file_backend from .backends.rclone import get_rclone_backend @@ -433,7 +433,7 @@ def load(self, name: str, *, size=None, offset=0, deleted=False) -> bytes: self._stats_update_volume("load", len(result)) return result - def _cache_store(self, nested_name: str, value: bytes) -> None: + def _cache_store(self, nested_name: str, value: StoreValue) -> None: if self.cache_backend is None or self._cache_disabled: return self._stats["cache_store_calls"] += 1 @@ -444,10 +444,19 @@ def _cache_store(self, nested_name: str, value: bytes) -> None: logger.warning(f"borgstore: cache store failed for {nested_name!r}: {err!r}") self._stats["cache_errors"] += 1 - def store(self, name: str, value: bytes) -> None: + def store(self, name: str, value: StoreValue) -> None: + """ + store into item . + + is either bytes or a memoryview of bytes - a memoryview enables callers + to avoid copying (e.g. by giving a slice of a bigger buffer they already have). + The value is only used while this method runs, so the caller may reuse or + release the underlying buffer afterwards. + """ # note: using .find here will: # - overwrite an existing item (level stays same) # - write to the last level if no existing item is found. + value = validate_value(value) # normalize, so len(value) is the number of bytes with self._stats_updater("store", f"store({name!r})"): nested_name = self.find(name) self._backend_call(lambda: self.backend.store(nested_name, value), key="store", volume=len(value)) diff --git a/tests/test_backends.py b/tests/test_backends.py index eceb76b..9faefe6 100644 --- a/tests/test_backends.py +++ b/tests/test_backends.py @@ -2,6 +2,7 @@ Generic tests for the backend implementations. """ +import array import hashlib import os import sys @@ -37,7 +38,7 @@ from borgstore.backends.posixfs import PosixFS, get_file_backend from borgstore.backends.sftp import Sftp, get_sftp_backend from borgstore.backends.rclone import get_rclone_backend -from borgstore.backends.s3 import S3, get_s3_backend +from borgstore.backends.s3 import S3, boto3, get_s3_backend from borgstore.backends.rest import REST, get_rest_backend from borgstore.constants import ROOTNS @@ -564,6 +565,79 @@ def test_scalability_size(tested_backends, exp, request): assert backend.load(key) == value +def test_store_memoryview(tested_backends, request): + with get_backend_from_fixture(tested_backends, request) as backend: + # a memoryview of a slice of a bigger buffer (that's what callers use to avoid copies) + buffer = bytearray(b"0123456789" * 10) + value = memoryview(buffer)[10:30] + backend.store("key", value) + assert backend.load("key") == bytes(value) + + # a memoryview with itemsize > 1: we store the bytes it consists of + array_value = array.array("I", range(16)) # itemsize 4 + backend.store("key_array", memoryview(array_value)) + assert backend.load("key_array") == array_value.tobytes() + + # empty memoryview + backend.store("key_empty", memoryview(b"")) + assert backend.load("key_empty") == b"" + + # a non-contiguous memoryview can not be stored (we would have to copy it first) + with pytest.raises(ValueError, match="C-contiguous"): + backend.store("key_invalid", memoryview(buffer)[::2]) + + +def test_sftp_store_memoryview_gives_bytes_to_paramiko(): + # paramiko can not deal with a memoryview, so the sftp backend must convert it. + written = [] + + class FakeFile: + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def set_pipelined(self, pipelined): + pass + + def write(self, data): + written.append(data) + + class FakeClient: + def open(self, name, mode): + return FakeFile() + + def posix_rename(self, curr_name, new_name): + pass + + backend = Sftp(hostname="localhost", path="/some/path") + backend.opened = True + backend.client = FakeClient() + buffer = bytearray(b"0123456789" * 10) + backend.store("key", memoryview(buffer)[10:30]) + assert written == [b"0123456789" * 2] + assert type(written[0]) is bytes + + +@pytest.mark.skipif(boto3 is None, reason="boto3 is not installed") +def test_s3_store_memoryview_gives_bytes_to_boto3(): + # boto3 rejects a memoryview Body (parameter validation), so the s3 backend must convert it. + put_objects = [] + + class FakeS3: + def put_object(self, **kwargs): + put_objects.append(kwargs) + + backend = S3(bucket="bucket", path="/path", is_b2=False) + backend.opened = True + backend.s3 = FakeS3() + buffer = bytearray(b"0123456789" * 10) + backend.store("key", memoryview(buffer)[10:30]) + assert put_objects[0]["Body"] == b"0123456789" * 2 + assert type(put_objects[0]["Body"]) is bytes + + def test_load_partial(tested_backends, request): with get_backend_from_fixture(tested_backends, request) as backend: backend.store("key", b"0123456789") diff --git a/tests/test_cache.py b/tests/test_cache.py index 96b91b8..82fdfad 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -36,6 +36,27 @@ def make_store(tmp_path, *, config=None, with_cache_backend=True): return Store(**kwargs), cache_root +def test_cache_store_memoryview(tmp_path): + """A memoryview value is written to the primary backend as well as to the cache backend.""" + store, _ = make_store(tmp_path, config=make_config({"data/": {"cache": "writethrough"}})) + store.create() + try: + with store: + buffer = bytearray(b"0123456789" * 10) + name, value = "data/00000000", memoryview(buffer)[10:30] + store.store(name, value) + assert store.stats["cache_store_calls"] == 1 + assert store.stats["cache_store_volume"] == 20 + # served from the cache: + assert store.load(name) == bytes(value) + assert store.stats["cache_hits"] == 1 + # ... and the primary backend has it, too: + store.cache_invalidate(name) + assert store.load(name) == bytes(value) + finally: + store.destroy() + + def test_cache_disabled_by_default(tmp_path): store, cache_root = make_store(tmp_path, config=None, with_cache_backend=False) store.create() diff --git a/tests/test_posixfs_quota.py b/tests/test_posixfs_quota.py index 49a789c..bad58be 100644 --- a/tests/test_posixfs_quota.py +++ b/tests/test_posixfs_quota.py @@ -1,5 +1,6 @@ """Tests for PosixFS quota support.""" +import array import time import pytest @@ -42,6 +43,16 @@ def test_store_and_delete_track_size(self, backend): assert backend._quota_use == 0 backend.close() + def test_memoryview_tracks_size_in_bytes(self, backend): + """A memoryview value is accounted with its size in bytes.""" + backend.open() + backend.store("testobj", memoryview(bytearray(b"x" * 200))[50:150]) + assert backend._quota_use == 100 + # itemsize 4, so this is 64 bytes (not 16) + backend.store("testobj", memoryview(array.array("I", range(16)))) + assert backend._quota_use == 64 + backend.close() + def test_overwrite_tracks_delta(self, backend): """Overwriting an object tracks the size delta correctly.""" backend.open() diff --git a/tests/test_server_rest.py b/tests/test_server_rest.py index 253f5aa..0bc90b1 100644 --- a/tests/test_server_rest.py +++ b/tests/test_server_rest.py @@ -59,6 +59,22 @@ def rest_server_with_auth(tmp_path): server.server_close() +def test_rest_server_store_memoryview(rest_server_with_auth): + # the rest backend sends a memoryview as request body and also computes the + # X-Content-hash-sha256 header from it, so the server must see the same data. + be = rest_server_with_auth + be.create() + be.open() + try: + buffer = bytearray(b"0123456789" * 1000) + value = memoryview(buffer)[10:9000] + be.store("test/item", value) + assert be.load("test/item") == bytes(value) + assert be.info("test/item").size == 8990 + finally: + be.close() + + def test_rest_server_basic_ops(rest_server_with_auth): be = rest_server_with_auth be.create() diff --git a/tests/test_store.py b/tests/test_store.py index 735ea14..b82980a 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -2,6 +2,7 @@ Tests for the high-level Store API. """ +import array import hashlib import pytest @@ -101,6 +102,32 @@ def test_basics(posixfs_store_created): assert store.stats["backend_delete_calls"] == 1 +def test_store_memoryview(posixfs_store_created): + ns = "two" + with posixfs_store_created as store: + # a memoryview of a slice of a bigger buffer (that's what callers use to avoid copies) + buffer = bytearray(b"0123456789" * 10) + value = memoryview(buffer)[10:30] + store.store(ns + "/" + key(0), value) + assert store.load(ns + "/" + key(0)) == bytes(value) + assert store.info(ns + "/" + key(0)).size == 20 + + # the caller may reuse/modify its buffer afterwards, the stored value must not change. + buffer[10:30] = b"x" * 20 + assert store.load(ns + "/" + key(0)) == b"0123456789" * 2 + + # a memoryview with itemsize > 1: we store the bytes it consists of and + # the stats volume is the number of bytes (not the number of items). + array_value = array.array("I", range(16)) # itemsize 4 + store.store(ns + "/" + key(1), memoryview(array_value)) + assert store.load(ns + "/" + key(1)) == array_value.tobytes() + assert store.stats["store_volume"] == 20 + 64 + + # a non-contiguous memoryview can not be stored (we would have to copy it first) + with pytest.raises(ValueError, match="C-contiguous"): + store.store(ns + "/" + key(2), memoryview(buffer)[::2]) + + def test_defrag_nested(posixfs_store_created): ns = "two" # nested! CONFIG has {"two/": {"levels": [2]}} v1 = b"0123456789"