diff --git a/.pylintrc b/.pylintrc index 8b58011..e9c1c31 100644 --- a/.pylintrc +++ b/.pylintrc @@ -341,7 +341,7 @@ indent-string=' ' max-line-length=100 # Maximum number of lines in a module. -max-module-lines=4300 +max-module-lines=4400 # Allow the body of a class to be on the same line as the declaration if body # contains single statement. diff --git a/CHANGELOG.md b/CHANGELOG.md index d7931fa..3fcc114 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ diskcache-community forked from DiskCache at upstream commit ebfa37c (3 March 2024), two commits after the 5.6.3 release. Each entry names the pull request in this repository. Imported upstream pull requests are referenced by number, and their commits keep their original authors in the git history. +## Unreleased + +### Signed entries + +This is a new, security-sensitive API in a patch release. Nothing existing changes unless you call it: no other disk behaves differently, and the `cache.db` schema and the file layout are unchanged. + +- `SafeDisk.with_signing()` returns a `SafeDisk` subclass that stores every value it pickles as a full pickle with an HMAC-SHA256 tag over the pickle and the cache key, and checks the tag before anything is unpickled. A library can then cache its own objects that no allowlist can hold, such as parsed databases, torch models, application objects and responses. A pickled value without a matching tag, including every unsigned pickle, raises `UnpicklingError`, or reads as missing with `on_error='miss'`. Values stored without pickling are stored as before, and keys stay on the allowlist (PR #46). +- The key is read from a per-user file outside the cache directory, `$DISKCACHE_SIGNING_KEY_FILE` or `signing.key` in the user's local data directory, which is created with a new random key and mode `0600` on first use. Opening a cache refuses a key file inside the cache directory, or, on POSIX, one that other users can write or, unless it belongs to root, read. `key=` takes a key kept only in memory, for caches that live in one process; a cache that uses it cannot be pickled (PR #46). +- Rows written by a signing disk raise `UnpicklingError` in every other disk and in diskcache 5.6.3 without being parsed; a `SafeDisk` without signing says the row is a signed entry. A signing disk does not combine with `with_numpy()`, and `SafeJSONDisk.with_signing()` raises (PR #46). + ## 5.8.3 (2026-11-03) Fixes a regression from 5.8.2 and several `SafeDisk` issues found by testing downstream libraries, and updates the documentation. The `cache.db` schema and the file layout are unchanged. diff --git a/SECURITY.md b/SECURITY.md index bdfa195..922ac8e 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -20,6 +20,7 @@ Where this fork stands: - 5.8.0 is imported as `diskcache_community`, next to the upstream `diskcache` package. `SafeDisk.with_types` allows your own classes and the classes defined in named modules while everything else stays refused; numpy classes cannot be allowed, because numpy unpickling is not memory-safe. `configure()` and `DISKCACHE_DISK=safe` select `SafeDisk` for caches created without a disk, including those that libraries create through `diskcache_community`. `Disk.with_options(on_error='miss')` deletes entries that cannot be read and treats them as missing. The steps are in [Fix CVE-2025-69872 in diskcache](https://diskcache-community.readthedocs.io/en/latest/fix-cve-2025-69872.html). - `JSONDisk` writes no pickles but still reads one from a crafted row, as it does in 5.6.3. `SafeJSONDisk` reads and writes the same files and never reads a pickle; `SafeDisk` does not decode what `JSONDisk` wrote. - `SafeDisk.with_numpy()` returns a `SafeDisk` that also stores numpy arrays, in the `.npy` format rather than as pickles. numpy pickles stay refused, because unpickling an array is not memory-safe. +- `SafeDisk.with_signing()` returns a `SafeDisk` that stores every value it pickles as a full pickle with an HMAC-SHA256 tag over the pickle and the cache key, and checks the tag before anything is unpickled; a pickled value without a matching tag raises `UnpicklingError`. It is for libraries that cache their own objects, which no allowlist can hold. The key is the trust boundary: it is kept in a per-user file outside the cache directory, which is refused when it lies inside the cache directory or when other users can write or read it, and anyone who can read it can sign a row that runs code when read. - The default stays as it is. Existing caches hold objects outside any allowlist, such as instances of your own classes and Django responses, so `SafeDisk` remains an explicit choice. ## Dependencies diff --git a/diskcache_community/core.py b/diskcache_community/core.py index b2dcb86..b9b954e 100644 --- a/diskcache_community/core.py +++ b/diskcache_community/core.py @@ -862,7 +862,8 @@ class SafeDisk(Disk): Objects outside the allowlist, such as instances of your own classes and Django model instances and responses, need either the default - :class:`Disk` or a disk from :meth:`with_types` that allows them. The + :class:`Disk`, a disk from :meth:`with_types` that allows them, or a disk + from :meth:`with_signing`, which signs what it writes instead. The allowed classes themselves can be stored too, so ``memoize(typed=True)`` works. @@ -871,10 +872,22 @@ class SafeDisk(Disk): safe_types: frozenset = frozenset() safe_modules: tuple = () safe_numpy: bool = False + signing = None + + def __init_subclass__(cls, **kwargs): + super().__init_subclass__(**kwargs) + # A subclass of a signing disk is passed key= too, so an override of + # fetch that cannot take it fails here rather than on the first read. + if cls.signing is not None: + signing.check_fetch(cls) def __init__(self, directory, min_file_size=0, pickle_protocol=0): super().__init__(directory, min_file_size, pickle_protocol) self._real_directory = op.realpath(directory) + if self.signing is not None: + self._signing_key, self._signing_path = self.signing.load( + directory + ) @classmethod def with_types(cls, *types, modules=()): @@ -974,10 +987,59 @@ def with_numpy(cls): True :return: SafeDisk subclass to pass as ``disk=`` + :raises TypeError: if the class signs entries (:meth:`with_signing`) """ + if cls.signing is not None: + raise TypeError(signing.NUMPY_REFUSED) return _derive_disk(cls, {'safe_numpy': True}) + @classmethod + def with_signing(cls, key_file=None, *, key=None): + """Return a SafeDisk subclass that signs the values it pickles. + + Every value the returned class pickles is stored as a full pickle + with an HMAC-SHA256 tag over the pickle and the cache key, and the + tag is checked before anything is unpickled. A pickled value without + a tag, or whose tag does not match, raises + :exc:`pickle.UnpicklingError`, which ``on_error='miss'`` treats as a + miss. Values that are stored without pickling, such as ``str`` and + ``int``, are stored as on every disk, and keys stay on the allowlist. + + The key is read from `key_file`, which is created with a new random + key if it is missing, or from the default key file when neither + argument is given: ``$DISKCACHE_SIGNING_KEY_FILE``, or + ``signing.key`` in a folder ``diskcache-community`` of the user's + local data directory. Opening a cache raises :exc:`ValueError` when + the key file is inside the cache directory or, on POSIX, when other + users can write it, or read it unless it belongs to root. `key` takes + the key itself, at least 32 random bytes, kept only in memory; a + cache that uses it cannot be pickled. Every other disk, and + diskcache 5.6.3, raises :exc:`pickle.UnpicklingError` on the pickled + rows a signing disk writes. It combines with :meth:`with_types`, + which then widens only what keys may hold, and with + :meth:`with_options`. + + >>> import secrets + >>> KeyDisk = SafeDisk.with_signing(key=secrets.token_bytes(32)) + >>> KeyDisk.signing # doctest: +ELLIPSIS + + + :param key_file: path of the key file (default None, the default key + file) + :param bytes key: the key, instead of a key file + :return: SafeDisk subclass to pass as ``disk=`` + :raises TypeError: if both `key_file` and `key` are given, if `key` + is not bytes, if the class stores numpy arrays + (:meth:`with_numpy`), or if its ``fetch`` does not accept the + keyword argument ``key`` + :raises ValueError: if `key` is shorter than 32 bytes + + """ + return _derive_disk( + cls, {'signing': signing.source(cls, key_file, key)} + ) + def _plain_unpickler(self, file): return SafeUnpickler( file, @@ -1142,9 +1204,14 @@ def _confine(self, filename): ) def put(self, key): - # pylint: disable=unidiomatic-typecheck + # pylint: disable=unidiomatic-typecheck,comparison-with-itself if type(key) not in (bytes, str, int, float): self._check_object(key, 'key') + elif self.signing is not None and key != key: + raise TypeError( + 'SafeDisk.with_signing() cannot store a NaN key, which SQLite' + ' stores as NULL' + ) return super().put(key) def get(self, key, raw): @@ -1155,6 +1222,10 @@ def get(self, key, raw): def store(self, value, read, key=UNKNOWN): # pylint: disable=unidiomatic-typecheck + if self.signing is not None and not ( + read or signing.stored_raw(value) + ): + return signing.store(self, value, key) if read or type(value) in (bytes, str, int, float): return super().store(value, read, key=key) numpy = sys.modules.get('numpy') if self.safe_numpy else None @@ -1163,15 +1234,25 @@ def store(self, value, read, key=UNKNOWN): self._check_object(value, 'value') return super().store(value, read, key=key) - def fetch(self, mode, filename, value, read): + def fetch(self, mode, filename, value, read, key=None): + """Convert fields `mode`, `filename`, and `value` from Cache table to + value. A signing disk (:meth:`with_signing`) is also given the row's + `key`, a ``(db_key, raw)`` pair, and refuses a signed row without + it.""" if filename is not None: self._confine(filename) if mode == MODE_PICKLE: + if self.signing is not None: + return signing.fetch(self, filename, value, key) if value is None: with super().fetch(MODE_BINARY, filename, None, True) as file: return self._unpickler(file).load() - return self._unpickler(io.BytesIO(value)).load() + try: + return self._unpickler(io.BytesIO(value)).load() + except pickle.UnpicklingError: + signing.refuse_signed(value) + raise return super().fetch(mode, filename, value, read) @@ -1204,6 +1285,13 @@ class SafeJSONDisk(JSONDisk, SafeDisk): def with_numpy(cls): raise TypeError('SafeJSONDisk stores only JSON, not numpy arrays') + @classmethod + def with_signing(cls, key_file=None, *, key=None): + # pylint: disable=unused-argument + raise TypeError( + 'SafeJSONDisk never pickles a value, so it has none to sign' + ) + def get(self, key, raw): if not raw: raise pickle.UnpicklingError('SafeJSONDisk refuses a pickled key') @@ -1702,7 +1790,7 @@ def _after_fork_in_child(): ) -class Cache: +class Cache: # pylint: disable=too-many-instance-attributes """Disk and file backed cache.""" def __init__( @@ -1818,6 +1906,11 @@ def __init__( if key.startswith('disk_') } self._disk = disk(directory, **kwargs) + # A signing disk checks the cache key of each row it reads. Only a + # SafeDisk signs, whatever other disks name their attributes. + self._fetch_with_key = ( + isinstance(self._disk, SafeDisk) and self._disk.signing is not None + ) self._init_settings(sql, sets, current_settings) @@ -2761,6 +2854,10 @@ def get( mode, filename, db_value = row try: + if self._fetch_with_key: + return self._disk.fetch( + mode, filename, db_value, read, key=(db_key, raw) + ) return self._disk.fetch(mode, filename, db_value, read) except IOError: # Key was deleted before we could retrieve result. @@ -2819,7 +2916,12 @@ def get( db_store_time = None try: - value = self._disk.fetch(mode, filename, db_value, read) + if self._fetch_with_key: + value = self._disk.fetch( + mode, filename, db_value, read, key=(db_key, raw) + ) + else: + value = self._disk.fetch(mode, filename, db_value, read) except IOError: # Key was deleted before we could retrieve result. return default @@ -2871,7 +2973,14 @@ def get( db_store_time = None try: - value = self._disk.fetch(mode, filename, db_value, read) + if self._fetch_with_key: + value = self._disk.fetch( + mode, filename, db_value, read, key=(db_key, raw) + ) + else: + value = self._disk.fetch( + mode, filename, db_value, read + ) except IOError: # Key was deleted before we could retrieve result. if statistics: @@ -2997,7 +3106,12 @@ def pop( sql('DELETE FROM Cache WHERE rowid = ?', (rowid,)) try: - value = self._disk.fetch(mode, filename, db_value, False) + if self._fetch_with_key: + value = self._disk.fetch( + mode, filename, db_value, False, key=(db_key, raw) + ) + else: + value = self._disk.fetch(mode, filename, db_value, False) except IOError: # Key was deleted before we could retrieve result. return default @@ -3168,6 +3282,12 @@ def push( else: db_key = num + if self._fetch_with_key: + # The disk hashed the value before the transaction, so only + # the key chosen here is added to its tag. + db_value = signing.finish(db_value, (db_key, raw)) + columns = columns[:-1] + (db_value,) + self._row_insert(db_key, raw, now, columns) self._cull(now, sql, cleanup) @@ -3275,7 +3395,12 @@ def pull( break try: - value = self._disk.fetch(mode, name, db_value, False) + if self._fetch_with_key: + value = self._disk.fetch( + mode, name, db_value, False, key=(key, True) + ) + else: + value = self._disk.fetch(mode, name, db_value, False) except IOError: # Key was deleted before we could retrieve result. continue @@ -3398,7 +3523,12 @@ def peek( break try: - value = self._disk.fetch(mode, name, db_value, False) + if self._fetch_with_key: + value = self._disk.fetch( + mode, name, db_value, False, key=(key, True) + ) + else: + value = self._disk.fetch(mode, name, db_value, False) except IOError: # Key was deleted before we could retrieve result. continue @@ -3483,7 +3613,12 @@ def peekitem(self, last=True, expire_time=False, tag=False, retry=False): try: key = self._disk.get(db_key, raw) - value = self._disk.fetch(mode, name, db_value, False) + if self._fetch_with_key: + value = self._disk.fetch( + mode, name, db_value, False, key=(db_key, raw) + ) + else: + value = self._disk.fetch(mode, name, db_value, False) except IOError: # Key was deleted before we could retrieve result. continue @@ -4228,6 +4363,7 @@ def reset(self, key, value=ENOVAL, update=True): return value -# arrays imports the classes above, so it is imported once they exist. +# arrays and signing import the classes above, so they are imported once they +# exist. # pylint: disable=wrong-import-position,cyclic-import -from . import arrays # noqa: E402 # isort: skip +from . import arrays, signing # noqa: E402 # isort: skip diff --git a/diskcache_community/core.pyi b/diskcache_community/core.pyi index a607ba3..35d14e7 100644 --- a/diskcache_community/core.pyi +++ b/diskcache_community/core.pyi @@ -14,7 +14,7 @@ from typing import ( overload, ) -from _typeshed import StrOrBytesPath +from _typeshed import StrOrBytesPath, StrPath from typing_extensions import Self, TypeVar, Unpack from ._types import ( @@ -40,6 +40,7 @@ from ._types import ( Tag, UnknownType, ) +from .signing import InMemoryKey, KeyFile _T = TypeVar('_T') _BoundStrT = TypeVar('_BoundStrT', bound=str) @@ -191,16 +192,36 @@ class SafeDisk(Disk): safe_types: frozenset[type] safe_modules: tuple[str, ...] safe_numpy: bool + signing: KeyFile | InMemoryKey | None @classmethod def with_types( cls, *types: type, modules: str | Iterable[str] = ... ) -> type[Self]: ... @classmethod def with_numpy(cls) -> type[Self]: ... + @classmethod + def with_signing( + cls, key_file: StrPath | None = ..., *, key: bytes | None = ... + ) -> type[Self]: ... + def fetch( # type: ignore[override] + self, + mode: ModeLiteral, + filename: str | None, + value: Any, + read: bool, + key: tuple[Any, bool] | None = ..., + ) -> Any: ... class SafeJSONDisk(JSONDisk, SafeDisk): @classmethod def with_numpy(cls) -> NoReturn: ... + @classmethod + def with_signing( + cls, key_file: StrPath | None = ..., *, key: bytes | None = ... + ) -> NoReturn: ... + def fetch( # type: ignore[override] + self, mode: ModeLiteral, filename: str | None, value: Any, read: bool + ) -> Any: ... def configure(disk: type[Disk] | None) -> None: ... def default_disk() -> type[Disk]: ... diff --git a/diskcache_community/fanout.py b/diskcache_community/fanout.py index 7bbc33c..26e8770 100644 --- a/diskcache_community/fanout.py +++ b/diskcache_community/fanout.py @@ -14,10 +14,12 @@ ENOVAL, Cache, Timeout, + _makedirs_private, _meta_result, default_disk, ) from .persistent import Deque, Index +from .signing import check_key_location def _default_with_meta(default, expire_time, tag, store_time=False): @@ -58,6 +60,14 @@ def __init__( default_size_limit = DEFAULT_SETTINGS['size_limit'] size_limit = settings.pop('size_limit', default_size_limit) / shards + # Each shard's disk checks the key file against the shard's + # directory only, so check it against this one before any shard can + # create it here. + if not op.isdir(directory): + with cl.suppress(OSError): + _makedirs_private(op.abspath(directory)) + check_key_location(disk, directory) + self._count = shards self._directory = directory self._disk = disk diff --git a/diskcache_community/signing.py b/diskcache_community/signing.py new file mode 100644 index 0000000..edbf8bf --- /dev/null +++ b/diskcache_community/signing.py @@ -0,0 +1,598 @@ +"""Signed entries for SafeDisk.with_signing(). + +A signing disk stores every value it pickles as a full pickle with an +HMAC-SHA256 tag and checks the tag before it unpickles anything. The tag +covers the cache key the row is stored under, so a value moved to another key +is refused, and a pickled row without a tag is refused too. + +The row's mode stays MODE_PICKLE. Its value column starts with a 45-byte +prefix, the magic, a version, the key id and the tag, followed by the pickle +for an inline row; a file row keeps the prefix alone in the value column and +the pickle alone in its value file. The magic starts with ``0xff``, which is +not a pickle opcode, so every reader that unpickles the value column, from +diskcache 5.6.3 on, raises before reading anything else. +""" + +import errno +import hashlib +import hmac +import inspect +import os +import os.path as op +import pickle +import secrets +import sqlite3 +import stat +import struct +import sys +import time + +from .core import ( + MODE_PICKLE, + UNKNOWN, + SafeDisk, + _makedirs_private, + native_filename, +) + +ENV_KEY_FILE = 'DISKCACHE_SIGNING_KEY_FILE' + +MAGIC = b'\xffDCS' +VERSION = 1 +HEADER_SIZE = 13 # magic, version and key id +PREFIX_SIZE = HEADER_SIZE + 32 # header and tag +MIN_KEY_SIZE = 32 + +_MAX_KEY_FILE_SIZE = 4096 +_CHUNK_SIZE = 2**20 +_KEY_ID_LABEL = b'diskcache-community key id' +_MAC_LABEL = b'diskcache-community signed entry v1' +_INT64 = 2**63 + +NUMPY_REFUSED = ( + 'with_signing() and with_numpy() do not combine: a signing disk pickles' + ' every value in full, numpy arrays included' +) + + +def _key_too_short(size): + """Raise ValueError for a key of `size` bytes. + + Only the size is passed, so a traceback that records the locals of the + raising frame holds no key bytes. + + """ + raise ValueError( + 'a signing key must be at least %d bytes, not %d' + % (MIN_KEY_SIZE, size) + ) + + +class _Key: + """A signing key, reduced to its key id and a keyed HMAC object. + + The key's bytes are not kept. The derived MAC key lives in the HMAC + object's C state, which ``vars()``, ``repr()`` and pickle do not show. That + is hygiene against accidental exposure, not protection: anything that can + read the process's memory can reach it. + + """ + + __slots__ = ('key_id', '_hmac') + + def __init__(self, key): + self.key_id = hmac.new(key, _KEY_ID_LABEL, hashlib.sha256).digest()[:8] + self._hmac = hmac.new( + hmac.new(key, _MAC_LABEL, hashlib.sha256).digest(), + digestmod=hashlib.sha256, + ) + + def __repr__(self): + return '' % self.key_id.hex() + + def __reduce__(self): + raise TypeError('a signing key cannot be pickled') + + def mac(self): + """Return a fresh HMAC object keyed with the derived MAC key.""" + return self._hmac.copy() + + +class KeyFile: + """Key source of a signing disk: a key file, or the default one.""" + + __slots__ = ('path',) + + def __init__(self, path): + self.path = path + + def __repr__(self): + if self.path is None: + return '' + return '' % (self.path,) + + def __reduce__(self): + return KeyFile, (self.path,) + + def load(self, directory): + """Return the key and the key file's real path for a cache in + `directory`, creating the file if it is missing.""" + path = default_key_file() if self.path is None else self.path + return load_key_file(path, directory) + + +class InMemoryKey: + """Key source of a signing disk: a key held only in this process.""" + + __slots__ = ('key',) + + def __init__(self, key): + self.key = _Key(key) + + def __repr__(self): + return '' % self.key.key_id.hex() + + def __reduce__(self): + raise TypeError( + 'a disk with an in-memory signing key cannot be pickled; use' + ' key_file= to share the key with other processes' + ) + + def load(self, directory): + """Return the key; it has no file.""" + # pylint: disable=unused-argument + return self.key, None + + +def source(cls, key_file, key): + """Return the key source of `cls`.with_signing(), or raise.""" + if cls.safe_numpy: + raise TypeError(NUMPY_REFUSED) + check_fetch(cls) + + if key is not None: + if key_file is not None: + raise TypeError('with_signing takes key_file or key, not both') + if not isinstance(key, bytes): + raise TypeError('key must be bytes, not %s' % type(key).__name__) + if len(key) < MIN_KEY_SIZE: + _key_too_short(len(key)) + return InMemoryKey(key) + + if key_file is None: + return KeyFile(None) + return KeyFile(op.abspath(op.expanduser(os.fsdecode(key_file)))) + + +def check_fetch(cls): + """Raise TypeError unless the `fetch` of `cls` accepts ``key=``.""" + method = cls.fetch + if method is SafeDisk.fetch: + return + + try: + parameters = inspect.signature(method).parameters + except (TypeError, ValueError): + parameters = {} + + if 'key' in parameters or any( + parameter.kind is parameter.VAR_KEYWORD + for parameter in parameters.values() + ): + return + + raise TypeError( + '%s.fetch does not accept the keyword argument key, which a signing' + ' disk is passed' % cls.__qualname__ + ) + + +def default_key_file(): + """Return the path of the default key file. + + It is ``$DISKCACHE_SIGNING_KEY_FILE`` when that is set, and otherwise a + file named ``signing.key`` in a folder ``diskcache-community`` of the + user's local data directory. + + """ + path = os.environ.get(ENV_KEY_FILE) + if path: + return op.abspath(op.expanduser(path)) + + base = data_directory(os.environ, sys.platform, op.expanduser('~')) + if base is None: + raise OSError( + errno.ENOENT, + 'there is no home directory for the default signing key file;' + ' set %s, or pass key_file= or key= to with_signing()' + % ENV_KEY_FILE, + ) + return op.join(base, 'diskcache-community', 'signing.key') + + +def data_directory(environ, platform, home): + """Return the user's local data directory, or None if there is none.""" + if platform == 'win32': + base = environ.get('LOCALAPPDATA') + elif platform == 'darwin': + base = op.join(home, 'Library', 'Application Support') + else: + base = environ.get('XDG_DATA_HOME') + if not base or not op.isabs(base): + base = op.join(home, '.local', 'share') + if not base or not op.isabs(base): + return None + return base + + +def load_key_file(path, directory): + """Return the key in the file `path` and the file's real path. + + The file is created with a new random key if it is missing. It is refused + when it lies inside the cache `directory`, and on POSIX when its + permissions let other users replace or read it. + + """ + _refuse_inside(path, directory) + + try: + reader = open(path, 'rb') # pylint: disable=consider-using-with + except FileNotFoundError: + _create_key_file(path) + reader = open(path, 'rb') # pylint: disable=consider-using-with + + with reader: + real_path = op.realpath(path) + check_permissions(os.fstat(reader.fileno()), real_path) + return _Key(_read_key(reader)), real_path + + +def _create_key_file(path): + """Create the key file `path` with a new key, unless it now exists. + + The key is written to a temporary file in the same folder and hard-linked + into place, so a process that starts at the same time ends up with the + same key and never reads a half-written file. + + """ + folder = op.dirname(path) + try: + _makedirs_private(folder) + temporary = op.join(folder, '.signing-%s.tmp' % secrets.token_hex(8)) + _write_new(temporary, secrets.token_hex(32).encode('ascii') + b'\n') + except OSError as error: + raise OSError( + error.errno, + 'cannot create the signing key file %r (%s); set %s, or pass' + ' key_file= or key= to with_signing()' + % (path, error.strerror, ENV_KEY_FILE), + ) from error + + try: + os.link(temporary, path) + except FileExistsError: + pass + except OSError: + # A file system without hard links: create the file in place, which + # a concurrent reader may see before the key is written. + try: + with open(temporary, 'rb') as reader: + _write_new(path, reader.read()) + except FileExistsError: + pass + finally: + os.remove(temporary) + + +def _write_new(path, data): + """Write `data` to the new file `path` with mode ``0o600``.""" + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0) + descriptor = os.open(path, flags, 0o600) + with open(descriptor, 'wb') as writer: + writer.write(data) + writer.flush() + os.fsync(writer.fileno()) + + +def _read_key(reader): + """Return the key in the open key file `reader`.""" + for _ in range(10): + reader.seek(0) + data = reader.read(_MAX_KEY_FILE_SIZE + 1) + if data.endswith(b'\r\n'): + data = data[:-2] + elif data.endswith(b'\n'): + data = data[:-1] + if len(data) >= MIN_KEY_SIZE: + break + # The file may be in the middle of being created in place. + time.sleep(0.01) + + size = len(data) + if size > _MAX_KEY_FILE_SIZE: + del data + raise ValueError( + 'a signing key file must be at most %d bytes' % _MAX_KEY_FILE_SIZE + ) + if size < MIN_KEY_SIZE: + del data + _key_too_short(size) + return data + + +def check_permissions(status, path): + """Raise ValueError unless the key file `path`, whose ``os.stat`` result + is `status`, is safe from other users. Windows is not checked.""" + if os.name == 'nt': + return + + mode = status.st_mode + uid = os.geteuid() + + if not stat.S_ISREG(mode): + raise ValueError('the signing key file %r is not a file' % path) + if mode & (stat.S_IWGRP | stat.S_IWOTH): + raise ValueError( + 'the signing key file %r can be written by other users' % path + ) + if status.st_uid not in (uid, 0): + raise ValueError( + 'the signing key file %r belongs to uid %d, not to this user' + ' (uid %d) or root' % (path, status.st_uid, uid) + ) + # A file owned by root that others can read is accepted, since container + # platforms mount secrets that way. + if status.st_uid == uid != 0 and mode & (stat.S_IRGRP | stat.S_IROTH): + raise ValueError( + 'the signing key file %r can be read by other users; run' + ' chmod 600 on it' % path + ) + + folder = os.stat(op.dirname(path)).st_mode + if folder & (stat.S_IWGRP | stat.S_IWOTH) and not folder & stat.S_ISVTX: + raise ValueError( + 'the folder of the signing key file %r can be written by other' + ' users' % path + ) + + +def _inside(path, directory): + """Return True when the real path `path` lies inside `directory`. + + Folders are compared by device and inode, which also matches spellings + that ``realpath`` leaves alone, such as another letter case on a + case-insensitive volume. A file system that reports inode 0 is compared + by normalized real path instead. Folders that do not exist yet are + skipped. + + """ + target = os.stat(directory) + real_directory = op.normcase(op.realpath(directory)) + folder = op.dirname(path) + + while True: + try: + status = os.stat(folder) + except FileNotFoundError: + status = None + + if status is None: + pass + elif target.st_ino and status.st_ino: + if (status.st_dev, status.st_ino) == ( + target.st_dev, + target.st_ino, + ): + return True + elif op.normcase(folder) == real_directory: + return True + + parent = op.dirname(folder) + if parent == folder: + return False + folder = parent + + +def check_key_location(disk, directory): + """Raise ValueError if the key file `disk` reads is inside `directory`. + + FanoutCache calls this for its own directory before it creates any + shard: each shard's disk sees only the shard's directory, and would + create a missing key file in the fanout directory before a later check + could refuse it. A missing `directory` is skipped. + + """ + source = getattr(disk, 'signing', None) + if not isinstance(source, KeyFile) or not op.isdir(directory): + return + path = default_key_file() if source.path is None else source.path + _refuse_inside(path, directory) + + +def _refuse_inside(path, directory): + if _inside(op.realpath(path), directory): + raise ValueError( + 'the signing key file %r is inside the cache directory %r' + % (path, directory) + ) + + +def stored_raw(value): + """Return True when Disk.store stores `value` without pickling it.""" + # pylint: disable=unidiomatic-typecheck,comparison-with-itself + kind = type(value) + if kind is bytes or kind is str: + return True + if kind is int: + return -_INT64 <= value < _INT64 + return kind is float and value == value + + +def _key_bytes(db_key, raw): + """Return the canonical encoding of the stored key `db_key`. + + SQLite matches an integer and a real of equal value, so an integral float + in the 64-bit range is encoded as the integer, and ``1.0`` and ``1``, + ``0.0`` and ``-0.0``, encode alike. Values of different storage classes + never match, so their type bytes keep them apart. + + """ + # pylint: disable=unidiomatic-typecheck + if not raw: + return b'p' + bytes(db_key) + if db_key is None: + raise pickle.UnpicklingError( + 'SafeDisk.with_signing() refuses a row whose key is NULL' + ) + + kind = type(db_key) + if kind is float: + if db_key.is_integer() and -_INT64 <= db_key < _INT64: + return b'i%d' % int(db_key) + return b'f' + struct.pack('!d', db_key) + if kind is int: + return b'i%d' % db_key + if kind is str: + return b's' + db_key.encode('utf-8', 'surrogatepass') + return b'b' + bytes(db_key) + + +def _finish(mac, key): + """Add the cache `key`, a ``(db_key, raw)`` pair, and return the tag.""" + encoded = _key_bytes(*key) + mac.update(encoded) + mac.update(struct.pack('>Q', len(encoded))) + return mac.digest() + + +class _Pending: # pylint: disable=too-few-public-methods + """The value column of a signed row, waiting for the key it is stored + under. The header and pickle are already hashed.""" + + __slots__ = ('_header', '_mac', '_data') + + def __init__(self, header, mac, data): + self._header = header + self._mac = mac + self._data = data + + def sign(self, key): + """Return the value column for the cache `key`.""" + prefix = self._header + _finish(self._mac.copy(), key) + if self._data is None: + return sqlite3.Binary(prefix) + return sqlite3.Binary(prefix + self._data) + + +def store(disk, value, key): + """Return the Cache table fields of `value`, pickled and signed. + + Without a `key`, as from ``Cache.push``, the value column is left pending + and :func:`finish` signs it once the key is chosen. The header and the + pickle are hashed here in either case, so only the key is added later. + + """ + try: + data = pickle.dumps(value, protocol=disk.pickle_protocol) + except (pickle.PicklingError, AttributeError) as error: + raise TypeError( + 'SafeDisk cannot store value: %s' % (error,) + ) from error + + signing_key = disk._signing_key # pylint: disable=protected-access + header = MAGIC + bytes((VERSION,)) + signing_key.key_id + mac = signing_key.mac() + mac.update(header) + mac.update(data) + + if PREFIX_SIZE + len(data) < disk.min_file_size: + size, filename = 0, None + else: + filename, full_path = disk.filename(key, value) + disk._write(full_path, data, 'xb') # pylint: disable=protected-access + size, data = len(data), None + + pending = _Pending(header, mac, data) + if key is UNKNOWN: + return size, MODE_PICKLE, filename, pending + return size, MODE_PICKLE, filename, pending.sign(disk.put(key)) + + +def finish(db_value, key): + """Return the value column `db_value` signed for `key` if it is pending.""" + if isinstance(db_value, _Pending): + return db_value.sign(key) + return db_value + + +def _refuse(message): + raise pickle.UnpicklingError('SafeDisk.with_signing() refuses ' + message) + + +def fetch(disk, filename, value, key): + """Return the value of the signed row with columns `filename` and + `value`, stored under `key`, after checking its tag.""" + view = None if value is None else memoryview(value) + + if view is None or view[:4] != MAGIC: + _refuse('an unsigned pickled value') + if (filename is None) == (len(view) == PREFIX_SIZE) or len(view) < ( + PREFIX_SIZE + ): + _refuse('a signed row of the wrong shape') + if view[4] != VERSION: + _refuse('a signed row of unknown version %d' % view[4]) + + signing_key = disk._signing_key # pylint: disable=protected-access + key_id = bytes(view[5:HEADER_SIZE]) + if key_id != signing_key.key_id: + _refuse('a row signed with another key (key id %s)' % key_id.hex()) + if key is None: + _refuse('to read a signed row without its cache key') + + header = bytes(view[:HEADER_SIZE]) + tag = bytes(view[HEADER_SIZE:PREFIX_SIZE]) + + if filename is None: + data = view[PREFIX_SIZE:] + else: + full_path = op.join( + disk._directory, # pylint: disable=protected-access + native_filename(filename), + ) + with open(full_path, 'rb') as reader: + # The streaming pass needs no memory for a planted large file, + # and the second read is capped at what the first pass hashed. + mac = signing_key.mac() + mac.update(header) + size = 0 + for chunk in iter(lambda: reader.read(_CHUNK_SIZE), b''): + mac.update(chunk) + size += len(chunk) + _check_tag(_finish(mac, key), tag) + reader.seek(0) + data = reader.read(size) + + mac = signing_key.mac() + mac.update(header) + mac.update(data) + _check_tag(_finish(mac, key), tag) + return pickle.loads(data) + + +def _check_tag(expected, tag): + if not hmac.compare_digest(expected, tag): + _refuse( + 'a signed row whose tag does not match: the row was changed, or' + ' moved from another key' + ) + + +def refuse_signed(value): + """Raise a clear UnpicklingError if `value` is a signed row's value + column; a disk without signing calls this when its unpickler fails.""" + if value is not None and bytes(value[:4]) == MAGIC: + raise pickle.UnpicklingError( + 'this row is a signed entry; read it with SafeDisk.with_signing()' + ' and the key that wrote it' + ) diff --git a/diskcache_community/signing.pyi b/diskcache_community/signing.pyi new file mode 100644 index 0000000..bdac3c8 --- /dev/null +++ b/diskcache_community/signing.pyi @@ -0,0 +1,52 @@ +import os +import sqlite3 +from typing import Any, Final + +from diskcache_community.core import SafeDisk + +ENV_KEY_FILE: Final[str] +MAGIC: Final[bytes] +VERSION: Final[int] +HEADER_SIZE: Final[int] +PREFIX_SIZE: Final[int] +MIN_KEY_SIZE: Final[int] +NUMPY_REFUSED: Final[str] + +class _Key: + key_id: bytes + def __init__(self, key: bytes) -> None: ... + def mac(self) -> Any: ... + +class KeyFile: + path: str | None + def __init__(self, path: str | None) -> None: ... + def load(self, directory: str) -> tuple[_Key, str]: ... + +class InMemoryKey: + key: _Key + def __init__(self, key: bytes) -> None: ... + def load(self, directory: str) -> tuple[_Key, None]: ... + +def source( + cls: type[SafeDisk], key_file: Any, key: bytes | None +) -> KeyFile | InMemoryKey: ... +def check_fetch(cls: type[SafeDisk]) -> None: ... +def default_key_file() -> str: ... +def data_directory( + environ: os._Environ[str] | dict[str, str], platform: str, home: str +) -> str | None: ... +def load_key_file(path: str, directory: str) -> tuple[_Key, str]: ... +def check_permissions(status: os.stat_result, path: str) -> None: ... +def check_key_location(disk: Any, directory: str) -> None: ... +def stored_raw(value: Any) -> bool: ... +def store( + disk: SafeDisk, value: Any, key: Any +) -> tuple[int, int, str | None, Any]: ... +def finish(db_value: Any, key: tuple[Any, bool]) -> Any: ... +def fetch( + disk: SafeDisk, + filename: str | None, + value: bytes | sqlite3.Binary | None, + key: tuple[Any, bool] | None, +) -> Any: ... +def refuse_signed(value: Any) -> None: ... diff --git a/docs/api.rst b/docs/api.rst index 5fe4afc..a20e995 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -158,6 +158,13 @@ Read the :ref:`SafeDisk tutorial ` for details. crafted pickle of one can corrupt memory or run code: numpy, scipy, torch, bitstruct and pyahocorasick. +.. data:: diskcache_community.signing.ENV_KEY_FILE + + ``'DISKCACHE_SIGNING_KEY_FILE'``, the environment variable that names the + default key file of :meth:`SafeDisk.with_signing + `. Read the :ref:`signing + tutorial ` for details. + SafeJSONDisk ------------ diff --git a/docs/fix-cve-2025-69872.rst b/docs/fix-cve-2025-69872.rst index dc2c7f0..06162d4 100644 --- a/docs/fix-cve-2025-69872.rst +++ b/docs/fix-cve-2025-69872.rst @@ -137,13 +137,36 @@ Steps To select it from the environment, name a class like ``ArrayDisk`` in ``DISKCACHE_DISK``, such as ``myapp.cache:ArrayDisk``. The same lists apply. -5. **Deal with entries already in the cache.** A cache written with the +5. **Sign objects no allowlist can hold.** A library that caches its own + objects, whose pickles call functions or name classes it cannot list in + advance, such as parsed databases, models, framework objects or responses, + can sign them with :meth:`SafeDisk.with_signing + ` instead of falling back to + :class:`Disk `: + + .. code-block:: python + + from diskcache_community import Cache, SafeDisk + + SignedDisk = SafeDisk.with_signing().with_options(on_error='miss') + cache = Cache('/var/tmp/mycache', disk=SignedDisk) + + Every value the disk pickles is stored with an HMAC-SHA256 tag over the + pickle and the cache key, made with a per-user key kept outside the cache + directory. The tag is checked before anything is unpickled, and a pickled + value without a matching tag raises :exc:`pickle.UnpicklingError`, or reads + as missing with ``on_error='miss'``. Anyone who can read the key can sign + a row, so the key file is the trust boundary. :ref:`tutorial-signing` in + the tutorial covers the key file, containers and CI, and what signing does + not cover. + +6. **Deal with entries already in the cache.** A cache written with the default :class:`Disk ` may hold objects that :class:`SafeDisk ` refuses; those entries can be cleared or treated as missing (:ref:`tutorial-unreadable-entries`). -6. **Check the result.** ``type(cache.disk)`` names the disk a cache uses, and +7. **Check the result.** ``type(cache.disk)`` names the disk a cache uses, and :func:`default_disk() ` returns the process default. Run ``pip show diskcache`` to see whether the upstream package is still installed, and, with pipdeptree installed, diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 0d3516c..9b36a25 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -1125,6 +1125,135 @@ and keeps it, even with ``on_error='miss'``. Arrays that the default :class:`Disk ` pickled earlier are refused like any other pickle outside the allowlist, so compute them again. +.. _tutorial-signing: + +Signing entries +............... + +Some caches hold objects that no allowlist can hold: a parsed database whose +pickle calls functions, a torch model, instances of an application's own +classes, a ``requests.Response``. :meth:`SafeDisk.with_signing +` returns a :class:`SafeDisk +` subclass that stores such values as full +pickles with an HMAC-SHA256 tag, and checks the tag before it unpickles +anything: + +.. code-block:: python + + from diskcache_community import Cache, SafeDisk + + SignedDisk = SafeDisk.with_signing().with_options(on_error='miss') + cache = Cache('/var/tmp/mycache', disk=SignedDisk) + cache['database'] = parse('vehicle.dbc') + +* Every value the disk pickles is signed. Values stored without pickling, + ``str``, ``bytes`` and ``int`` and ``float`` values, are stored as on every + disk and are not signed. +* The tag covers the pickle and the cache key, so a signed value moved under + another key, or a value file pointed at by another row, is refused. + Numbers that SQLite treats as one key, such as ``1`` and ``1.0``, are the + same key here too. A NaN key raises :exc:`TypeError`, since SQLite stores it + as NULL. +* A pickled value without a tag, including every pickle written by another + disk or by diskcache 5.6.3, is refused. Rows written before you switched to + signing are therefore refused once, and written again signed as they are + computed again. +* Keys stay on the allowlist, as in :class:`SafeDisk + `; :meth:`with_types + ` widens what keys may hold. + Store the name of a function in a key rather than the function. + +A refused value raises :exc:`pickle.UnpicklingError` with the reason. With +``with_options(on_error='miss')`` it reads as missing and is deleted, which +suits a cache whose values can be computed again. Keep the default, +``'raise'``, when they cannot, so that a refused value stops the program with +a reason rather than vanishing. + +The key file +"""""""""""" + +Without arguments, the key is read from a file outside every cache directory: + +* ``$DISKCACHE_SIGNING_KEY_FILE``, if it is set; +* on Linux and other POSIX systems, + ``$XDG_DATA_HOME/diskcache-community/signing.key``, by default + ``~/.local/share/diskcache-community/signing.key``; +* on macOS, ``~/Library/Application Support/diskcache-community/signing.key``; +* on Windows, ``%LOCALAPPDATA%\diskcache-community\signing.key``. + +The file is created with a new random key the first time a cache opens, with +mode ``0600`` in a folder with mode ``0700``. Pass ``key_file=`` to use a file +of your own; it holds the key as text, and only a trailing newline is +removed. Opening a cache raises :exc:`ValueError` when the key file is inside +the cache directory and, on POSIX, when other users can write it or the +folder it is in, when it belongs to another user, or when other users can +read it, unless it belongs to root, as container platforms mount secrets. A +root-owned key that other users can read hands the key to all of them, so +mount it that way only in a container. + +When there is no home directory, or it cannot be written, opening the cache +raises :exc:`OSError`. Set ``DISKCACHE_SIGNING_KEY_FILE`` in that case, or +catch the error where your library opens its cache and fall back to no +caching. + +In a container, the default works whenever ``HOME`` can be written, and every +process in the container shares the key. Mount the key as a secret and set +``DISKCACHE_SIGNING_KEY_FILE`` when entries must outlive the container, when +several containers or CI runners share a cache, or when the root file system +is read-only. Never put the key in an image, and never on the cache's own +volume. + +For a cache that lives only as long as one process, pass the key instead: + +.. code-block:: python + + import secrets + + RunDisk = SafeDisk.with_signing(key=secrets.token_bytes(32)) + +The key is then kept only in memory, and pickling a cache that uses it raises +:exc:`TypeError`. A key must be at least 32 random bytes: a guessable key can +be recovered from any signed row. + +Changing the key and sharing a directory +"""""""""""""""""""""""""""""""""""""""" + +To change the key, point the disk at a new key file. Rows signed with the old +key are then refused once, and are computed again with ``on_error='miss'``; +clear the directory if they cannot be computed again. Restart every process +that shares the directory, since a process that still holds the old key keeps +writing rows the others refuse. + +Every process that shares a directory must use a signing disk with the same +key. Any other disk, and diskcache 5.6.3, raises +:exc:`pickle.UnpicklingError` on a signed row without reading it, or deletes +it with ``on_error='miss'``; an older version of a library that shares the +directory with a newer one therefore raises on the newer one's entries. Users +with different keys who share a directory refuse each other's rows. + +What signing does not cover +""""""""""""""""""""""""""" + +* Anyone who can read the key can sign a row that runs code when it is read: + the same account, administrators, backups of the home directory, a copy of + the key on a shared volume. +* Entries a publisher built on another machine and shipped with a package + cannot be verified with your key. +* Someone who can write the cache can put back an older signed row for the + same key, change expiry times and tags, delete rows, or plant ``str``, + ``bytes`` and numbers, which are not signed. +* A row whose tag matches is unpickled in full, so whatever the cached object + does when it is unpickled runs, as with :class:`Disk + `. + +:meth:`with_signing ` combines with +:meth:`with_types ` and +:meth:`with_options ` in any order. It +does not combine with :meth:`with_numpy +`, since a signing disk pickles +arrays in full, and :class:`SafeJSONDisk ` +has nothing to sign. + .. _tutorial-default-disk: Default disk diff --git a/tests/test_golden.py b/tests/test_golden.py index a5313c0..17b25a5 100644 --- a/tests/test_golden.py +++ b/tests/test_golden.py @@ -125,3 +125,59 @@ def test_current_code_writes_what_563_reads(tmp_path): ) assert result.returncode == 0, result.stdout + result.stderr assert 'diskcache %s' % recipe.WHEEL_VERSION in result.stdout + + +def _execute(path): + """Marker that a pickle ran code: it creates `path`.""" + with open(path, 'w', encoding='utf-8') as writer: + writer.write('executed') + + +class _Poison: + def __init__(self, path): + self.path = path + + def __reduce__(self): + return (_execute, (self.path,)) + + +READ_SIGNED_ROWS = """ +import os, pickle, sys +import diskcache +directory, marker = sys.argv[1:] +with diskcache.Cache(directory) as cache: + for key in ('inline', 'file'): + try: + cache[key] + except pickle.UnpicklingError as error: + print(key, 'raised', error) +print('marker', os.path.exists(marker)) +""" + + +def test_563_raises_on_rows_a_signing_disk_wrote(tmp_path): + python = os.environ.get(REVERSE_PYTHON) + if not python: + pytest.skip('%s is not set' % REVERSE_PYTHON) + + import diskcache_community as dc + + directory = str(tmp_path / 'signed') + marker = str(tmp_path / 'marker') + disk = dc.SafeDisk.with_signing(tmp_path / 'signing.key') + for key, size in (('inline', 2**20), ('file', 0)): + with dc.Cache(directory, disk=disk, disk_min_file_size=size) as cache: + cache[key] = _Poison(marker) + + result = sp.run( + [python, '-I', '-c', READ_SIGNED_ROWS, directory, marker], + cwd=str(tmp_path), + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "inline raised invalid load key, '\\xff'." in result.stdout + assert "file raised invalid load key, '\\xff'." in result.stdout + assert 'marker False' in result.stdout + assert not op.exists(marker) diff --git a/tests/test_safedisk_signing.py b/tests/test_safedisk_signing.py new file mode 100644 index 0000000..a4d8a78 --- /dev/null +++ b/tests/test_safedisk_signing.py @@ -0,0 +1,1340 @@ +"""Test SafeDisk.with_signing().""" + +import collections as co +import contextlib +import errno +import functools +import hashlib +import hmac +import math +import multiprocessing +import os +import os.path as op +import pickle +import sqlite3 +import stat +import threading +import tracemalloc +import types +import typing +import warnings + +import pytest + +import diskcache_community as dc +from diskcache_community import signing +from diskcache_community.core import MODE_PICKLE, MODE_RAW + +pytestmark = pytest.mark.filterwarnings('ignore', category=dc.EmptyDirWarning) + +KEY = bytes(range(32)) +OTHER_KEY = bytes(range(1, 33)) +POSIX = os.name != 'nt' +ROOT = POSIX and os.geteuid() == 0 +INLINE, FILE = 2**20, 0 + +Signed = dc.SafeDisk.with_signing() +Miss = Signed.with_options(on_error='miss') + + +def _execute(path): + """Marker that a pickle ran code: it creates `path`.""" + with open(path, 'w', encoding='utf-8') as writer: + writer.write('executed') + return 'executed' + + +class Poison: + """Object whose pickle calls `_execute` when loaded.""" + + def __init__(self, path): + self.path = path + + def __reduce__(self): + return (_execute, (self.path,)) + + +class Point: + """A class outside every allowlist.""" + + def __init__(self, x, y): + self.x = x + self.y = y + + def __eq__(self, other): + return type(other) is Point and (self.x, self.y) == (other.x, other.y) + + def __hash__(self): + return hash((self.x, self.y)) + + def norm(self): + return abs(self.x) + abs(self.y) + + +def square(number): + return number * number + + +T = typing.TypeVar('T') + + +class Box(typing.Generic[T]): + def __init__(self, item): + self.item = item + + +Pair = co.namedtuple('Pair', 'left right') + + +@pytest.fixture(autouse=True) +def key_file(tmp_path, monkeypatch): + """Point the default key file of every test at its own folder.""" + path = tmp_path / 'keys' / 'signing.key' + monkeypatch.setenv(signing.ENV_KEY_FILE, str(path)) + return path + + +@pytest.fixture +def directory(tmp_path): + return str(tmp_path / 'cache') + + +def connect(directory): + return contextlib.closing(sqlite3.connect(op.join(directory, 'cache.db'))) + + +def row(directory, key): + """Return the mode, filename and value columns of the row of `key`.""" + with connect(directory) as con: + return con.execute( + 'SELECT mode, filename, value FROM Cache WHERE key = ?', (key,) + ).fetchone() + + +def update(directory, which, **columns): + """Set `columns` of the row whose key is `which`.""" + names = ', '.join('%s = ?' % name for name in columns) + with connect(directory) as con: + con.execute( + 'UPDATE Cache SET %s WHERE key = ?' % names, + tuple(columns.values()) + (which,), + ) + con.commit() + + +def flip(data, index): + data = bytearray(data) + data[index] ^= 1 + return bytes(data) + + +def mac_key(key): + return hmac.new(key, signing._MAC_LABEL, hashlib.sha256).digest() + + +def write_key_file(path, data, mode=0o600): + path.parent.mkdir(parents=True, exist_ok=True) + os.chmod(path.parent, 0o700) + path.write_bytes(data) + os.chmod(path, mode) + return path + + +# Round trips + + +@pytest.mark.parametrize('size', [INLINE, FILE], ids=['inline', 'file']) +def test_objects_no_allowlist_holds_round_trip(directory, size): + point = Point(1, -2) + value = { + 'point': point, + 'function': square, + 'bound method': point.norm, + 'alias': Box[int], + 'instance of an alias': Box[int](3), + 'partial': functools.partial(max, 1), + 'ordered': co.OrderedDict(a=1), + } + + with dc.Cache(directory, disk=Signed, disk_min_file_size=size) as cache: + cache['key'] = value + result = cache['key'] + assert cache.get('key', read=True)['point'] == point + + assert result['point'] == point + assert result['function'] is square + assert result['bound method']() == 3 + assert result['alias'] == Box[int] + assert result['instance of an alias'].item == 3 + assert result['partial'](0) == 1 + assert result['ordered'] == co.OrderedDict(a=1) + + mode, filename, db_value = row(directory, 'key') + assert mode == MODE_PICKLE + assert db_value[:4] == signing.MAGIC + if size == FILE: + assert len(db_value) == signing.PREFIX_SIZE + path = op.join(directory, filename) + assert open(path, 'rb').read()[:1] == b'\x80' + else: + assert filename is None + assert len(db_value) > signing.PREFIX_SIZE + + +def test_values_stored_without_pickling_are_stored_as_on_every_disk( + directory, +): + values = {'str': 'text', 'bytes': b'\x00\xff', 'int': -7, 'float': 1.5} + + with dc.Cache(directory, disk=Signed) as cache: + for key, value in values.items(): + cache[key] = value + + for key in values: + assert row(directory, key)[0] == MODE_RAW + + with dc.Cache(directory, disk=dc.SafeDisk) as cache: + assert {key: cache[key] for key in values} == values + + +def test_nan_and_large_int_values_are_signed(directory): + with dc.Cache(directory, disk=Signed) as cache: + cache['nan'] = float('nan') + cache['large'] = 2**70 + assert math.isnan(cache['nan']) + assert cache['large'] == 2**70 + + assert row(directory, 'nan')[2][:4] == signing.MAGIC + assert row(directory, 'large')[2][:4] == signing.MAGIC + + +def test_values_that_cannot_be_pickled_raise_type_error(directory): + with dc.Cache(directory, disk=Signed) as cache: + with pytest.raises(TypeError, match='cannot store value'): + cache['key'] = lambda: None + with pytest.raises(TypeError): + cache['key'] = threading.Lock() + assert 'key' not in cache + + +@pytest.mark.parametrize( + 'settings', + [{}, {'statistics': True}, {'eviction_policy': 'least-recently-used'}], + ids=['metadata', 'statistics', 'eviction policy'], +) +def test_every_path_of_get_checks_the_cache_key(directory, settings): + with dc.Cache(directory, disk=Signed, **settings) as cache: + cache.set('a', Point(1, 1), expire=60, tag='first') + cache.set('b', Point(2, 2)) + value, _, tag = cache.get('a', expire_time=True, tag=True) + assert (value, tag) == (Point(1, 1), 'first') + assert cache['a'] == Point(1, 1) + + _, _, value = row(directory, 'a') + update(directory, 'b', value=value) + + with dc.Cache(directory, disk=Signed, **settings) as cache: + with pytest.raises(pickle.UnpicklingError, match='does not match'): + cache.get('b', tag=True) + with pytest.raises(pickle.UnpicklingError, match='does not match'): + cache['b'] + + +def test_read_only_cache_reads_signed_rows(directory): + with dc.Cache(directory, disk=Signed) as cache: + cache['key'] = Point(1, 2) + + with dc.Cache(directory, disk=Signed, read_only=True) as cache: + assert cache['key'] == Point(1, 2) + + +# Refusals + + +@pytest.mark.parametrize('size', [INLINE, FILE], ids=['inline', 'file']) +@pytest.mark.parametrize('planted', ['poison', 'plain dict']) +def test_unsigned_pickles_are_refused(directory, tmp_path, size, planted): + marker = tmp_path / 'marker' + value = Poison(str(marker)) if planted == 'poison' else {'plain': 1} + + with dc.Cache(directory, disk=dc.Disk, disk_min_file_size=size) as cache: + cache['key'] = value + + with dc.Cache(directory, disk=Signed) as cache: + with pytest.raises(pickle.UnpicklingError, match='unsigned'): + cache['key'] + assert 'key' in cache + + with dc.Cache(directory, disk=Miss) as cache: + assert cache.get('key') is None + assert 'key' not in cache + + assert not marker.exists() + + +@pytest.mark.parametrize('size', [INLINE, FILE], ids=['inline', 'file']) +@pytest.mark.parametrize('where', ['tag', 'pickle']) +def test_tampered_rows_are_refused(directory, size, where): + with dc.Cache(directory, disk=Signed, disk_min_file_size=size) as cache: + cache['key'] = Point(1, 2) + + _, filename, value = row(directory, 'key') + if where == 'tag': + update(directory, 'key', value=flip(value, 20)) + elif filename is None: + update(directory, 'key', value=flip(value, len(value) - 2)) + else: + path = op.join(directory, filename) + with open(path, 'rb') as reader: + data = reader.read() + with open(path, 'wb') as writer: + writer.write(flip(data, len(data) - 2)) + + with dc.Cache(directory, disk=Signed) as cache: + with pytest.raises(pickle.UnpicklingError, match='does not match'): + cache['key'] + + +def test_rows_signed_with_another_key_are_refused(directory, tmp_path): + with dc.Cache(directory, disk=Signed) as cache: + cache['key'] = Point(1, 2) + + others = [ + dc.SafeDisk.with_signing(tmp_path / 'other.key'), + dc.SafeDisk.with_signing(key=KEY), + ] + for disk in others: + with dc.Cache(directory, disk=disk) as cache: + with pytest.raises(pickle.UnpicklingError, match='another key'): + cache['key'] + + +def test_users_with_different_keys_delete_each_others_rows_under_miss( + directory, tmp_path +): + with dc.Cache(directory, disk=Signed) as cache: + cache['key'] = Point(1, 2) + + other = dc.SafeDisk.with_signing(tmp_path / 'other.key') + with dc.Cache( + directory, disk=other.with_options(on_error='miss') + ) as cache: + assert cache.get('key') is None + assert len(cache) == 0 + + +@pytest.mark.parametrize('size', [INLINE, FILE], ids=['inline', 'file']) +def test_a_value_moved_under_another_key_is_refused(directory, size): + with dc.Cache(directory, disk=Signed, disk_min_file_size=size) as cache: + cache['a'] = Point(1, 1) + cache['b'] = Point(2, 2) + + _, filename_a, value_a = row(directory, 'a') + _, filename_b, value_b = row(directory, 'b') + update(directory, 'a', filename=filename_b, value=value_b) + update(directory, 'b', filename=filename_a, value=value_a) + + with dc.Cache(directory, disk=Signed) as cache: + for key in 'ab': + with pytest.raises(pickle.UnpicklingError, match='does not match'): + cache[key] + + +def test_rows_of_the_wrong_shape_or_version_are_refused(directory): + with dc.Cache(directory, disk=Signed, disk_min_file_size=FILE) as cache: + cache['file'] = Point(1, 1) + with dc.Cache(directory, disk=Signed, disk_min_file_size=INLINE) as cache: + cache['inline'] = Point(1, 1) + + _, _, prefix = row(directory, 'file') + _, _, inline = row(directory, 'inline') + cases = [ + ('inline', dict(value=prefix), 'wrong shape'), + ('inline', dict(value=signing.MAGIC + b'\x01'), 'wrong shape'), + ('file', dict(value=prefix + b'\x00'), 'wrong shape'), + ('inline', dict(value=inline[:4] + b'\x02' + inline[5:]), 'version 2'), + ] + + for key, columns, message in cases: + original = row(directory, key) + update(directory, key, **columns) + with dc.Cache(directory, disk=Signed) as cache: + with pytest.raises(pickle.UnpicklingError, match=message): + cache[key] + update(directory, key, filename=original[1], value=original[2]) + + +def test_a_signed_row_read_without_its_cache_key_is_refused(directory): + with dc.Cache(directory, disk=Signed) as cache: + cache['key'] = Point(1, 2) + mode, filename, value = row(directory, 'key') + with pytest.raises(pickle.UnpicklingError, match='without its cache'): + cache.disk.fetch(mode, filename, value, False) + + +def _planted_large_file(directory): + with dc.Cache(directory, disk=Signed, disk_min_file_size=FILE) as cache: + cache['key'] = Point(1, 2) + _, filename, _ = row(directory, 'key') + return op.join(directory, filename) + + +def _peak_while(function): + tracemalloc.start() + try: + function() + return tracemalloc.get_traced_memory()[1] + finally: + tracemalloc.stop() + + +def test_a_large_planted_file_is_refused_in_constant_memory(directory): + path = _planted_large_file(directory) + with open(path, 'r+b') as writer: + writer.truncate(2**25) + + def read(): + with dc.Cache(directory, disk=Signed) as cache: + with pytest.raises(pickle.UnpicklingError, match='does not match'): + cache['key'] + + assert _peak_while(read) < 2**23 + + with dc.Cache(directory, disk=Miss) as cache: + assert cache.get('key') is None + assert not op.exists(path) + + +def test_a_file_rewritten_between_the_two_passes_is_refused( + directory, monkeypatch +): + path = _planted_large_file(directory) + check_tag = signing._check_tag + calls = [] + + def check_then_rewrite(expected, tag): + check_tag(expected, tag) + if not calls: + calls.append(1) + with open(path, 'r+b') as writer: + writer.write(b'\x80' * 64) + writer.truncate(2**25) + + monkeypatch.setattr(signing, '_check_tag', check_then_rewrite) + + def read(): + with dc.Cache(directory, disk=Signed) as cache: + with pytest.raises(pickle.UnpicklingError, match='does not match'): + cache['key'] + + assert _peak_while(read) < 2**23 + assert calls == [1] + + +# Cache keys + + +def test_numeric_keys_that_sqlite_matches_verify_on_every_path(directory): + with dc.Cache(directory, disk=Signed) as cache: + cache[1.0] = Point(1, 1) + cache[0.0] = Point(0, 0) + cache[-(2.0**63)] = Point(-1, -1) + cache[2.0**63] = Point(9, 9) + + assert cache.get(1) == Point(1, 1) + assert cache[-0.0] == Point(0, 0) + assert cache[-(2**63)] == Point(-1, -1) + assert cache[2.0**63] == Point(9, 9) + assert cache.peekitem(last=False) == (1.0, Point(1, 1)) + assert cache.peekitem() == (2.0**63, Point(9, 9)) + assert cache.peek() == (1.0, Point(1, 1)) + assert cache.pull() == (1.0, Point(1, 1)) + assert cache.pop(0) == Point(0, 0) + + +def test_nan_keys_are_refused(directory): + with dc.Cache(directory, disk=Signed) as cache: + with pytest.raises(TypeError, match='NaN'): + cache[float('nan')] = Point(1, 2) + with pytest.raises(TypeError, match='NaN'): + cache.get(float('nan')) + + +def test_a_row_whose_key_is_null_is_refused(directory): + with dc.Cache(directory, disk=Signed) as cache: + cache['key'] = Point(1, 2) + update(directory, 'key', key=None) + + with dc.Cache(directory, disk=Signed) as cache: + with pytest.raises(pickle.UnpicklingError, match='NULL'): + cache.peekitem() + + with dc.Cache(directory, disk=Miss) as cache: + with pytest.raises(KeyError): + cache.peekitem() + assert len(cache) == 0 + + +@pytest.mark.parametrize('size', [INLINE, FILE], ids=['inline', 'file']) +def test_push_and_pull_round_trip(directory, size): + with dc.Cache(directory, disk=Signed, disk_min_file_size=size) as cache: + for number in range(3): + cache.push(Point(number, number)) + cache.push(Point(9, 9), prefix='queue', side='front') + cache.push('plain') + + assert cache.peek()[1] == Point(0, 0) + assert [cache.pull()[1] for _ in range(4)] == [ + Point(number, number) for number in range(3) + ] + ['plain'] + assert cache.pull(prefix='queue')[1] == Point(9, 9) + + +def test_str_bytes_and_pickled_keys_are_kept_apart(directory): + with dc.Cache(directory, disk=Signed) as cache: + cache['k'] = Point(1, 1) + cache[b'k'] = Point(2, 2) + cache[('k',)] = Point(3, 3) + assert [cache['k'], cache[b'k'], cache[('k',)]] == [ + Point(1, 1), + Point(2, 2), + Point(3, 3), + ] + + _, _, value = row(directory, 'k') + update(directory, b'k', value=value) + with dc.Cache(directory, disk=Signed) as cache: + with pytest.raises(pickle.UnpicklingError, match='does not match'): + cache[b'k'] + + +def test_a_pushed_value_moved_to_another_key_is_refused(directory): + with dc.Cache(directory, disk=Signed) as cache: + first = cache.push(Point(1, 1)) + second = cache.push(Point(2, 2)) + cache['other'] = 'plain' + + _, _, value = row(directory, first) + update(directory, 'other', mode=MODE_PICKLE, value=value) + update(directory, second, value=value) + + with dc.Cache(directory, disk=Signed) as cache: + assert cache.pull() == (first, Point(1, 1)) + for read in (lambda: cache['other'], cache.pull): + with pytest.raises(pickle.UnpicklingError, match='does not match'): + read() + + +def test_push_hashes_the_value_before_its_transaction(directory, monkeypatch): + events = [] + store, finish = signing.store, signing.finish + + def recording_store(*args): + result = store(*args) + events.append('hashed') + return result + + def recording_finish(*args): + events.append('signed') + return finish(*args) + + monkeypatch.setattr(signing, 'store', recording_store) + monkeypatch.setattr(signing, 'finish', recording_finish) + + with dc.Cache(directory, disk=Signed) as cache: + transact = cache._transact + + @contextlib.contextmanager + def recording_transact(*args, **kwargs): + with transact(*args, **kwargs) as result: + events.append('began') + yield result + + monkeypatch.setattr(cache, '_transact', recording_transact) + cache.push(Point(1, 1)) + assert cache.pull()[1] == Point(1, 1) + + assert events[:3] == ['hashed', 'began', 'signed'] + + +def _push_many(directory, count): + with dc.Cache(directory, disk=Signed) as cache: + for number in range(count): + cache.push(Point(os.getpid(), number)) + + +def test_two_processes_pushing_at_once(directory): + with dc.Cache(directory, disk=Signed) as cache: + cache.push(Point(0, 0)) + + context = multiprocessing.get_context('spawn') + processes = [ + context.Process(target=_push_many, args=(directory, 50)) + for _ in range(2) + ] + for process in processes: + process.start() + for process in processes: + process.join(60) + assert [process.exitcode for process in processes] == [0, 0] + + with dc.Cache(directory, disk=Signed) as cache: + values = [cache.pull()[1] for _ in range(101)] + assert cache.pull() == (None, None) + assert all(type(value) is Point for value in values) + assert len(set(values)) == 101 + + +def test_housekeeping_on_signed_file_rows(directory): + with dc.Cache( + directory, disk=Signed, disk_min_file_size=FILE, tag_index=True + ) as cache: + cache.set('a', Point(1, 1), tag='first') + cache.set('b', Point(2, 2), tag='second') + + with warnings.catch_warnings(): + warnings.simplefilter('error') + assert cache.check(fix=True) == [] + + assert cache.evict('first') == 1 + assert cache['b'] == Point(2, 2) + assert cache.volume() > 0 + + cache.reset('size_limit', 0) + assert cache.cull() >= 1 + assert len(cache) == 0 + + files = [name for _, _, names in os.walk(directory) for name in names] + assert all(name.startswith('cache.db') for name in files) + + +# Key files + + +def test_a_key_file_inside_the_cache_directory_is_refused(directory): + os.makedirs(directory) + for path in ('signing.key', op.join('sub', 'signing.key')): + disk = dc.SafeDisk.with_signing(op.join(directory, path)) + with pytest.raises(ValueError, match='inside the cache directory'): + dc.Cache(directory, disk=disk) + + +@pytest.mark.skipif(not POSIX, reason='symlinks need privileges on Windows') +def test_a_symlink_to_a_key_file_inside_the_cache_is_refused( + directory, tmp_path +): + os.makedirs(directory) + target = write_key_file( + tmp_path / 'cache' / 'hidden.key', KEY.hex().encode() + ) + link = tmp_path / 'link.key' + link.symlink_to(target) + + with pytest.raises(ValueError, match='inside the cache directory'): + dc.Cache(directory, disk=dc.SafeDisk.with_signing(link)) + + +def test_a_key_file_spelled_with_another_case_is_refused(directory): + os.makedirs(directory) + swapped = op.join(op.dirname(directory), op.basename(directory).upper()) + if not op.exists(swapped) or not op.samefile(swapped, directory): + pytest.skip('the file system is case-sensitive') + + disk = dc.SafeDisk.with_signing(op.join(swapped, 'signing.key')) + with pytest.raises(ValueError, match='inside the cache directory'): + dc.Cache(directory, disk=disk) + + +@pytest.mark.parametrize('exists', [False, True], ids=['new', 'existing']) +def test_a_key_file_in_a_fanout_cache_directory_is_refused(tmp_path, exists): + root = tmp_path / 'fanout' + if exists: + os.makedirs(root) + disk = dc.SafeDisk.with_signing(root / 'signing.key') + with pytest.raises(ValueError, match='inside the cache directory'): + dc.FanoutCache(str(root), disk=disk, shards=2) + # Refused before any shard could create the key file there. + assert os.listdir(root) == [] + + +def test_a_default_key_file_in_a_fanout_cache_directory_is_refused( + tmp_path, monkeypatch +): + root = tmp_path / 'fanout' + monkeypatch.setenv( + signing.ENV_KEY_FILE, str(root / 'keys' / 'signing.key') + ) + with pytest.raises(ValueError, match='inside the cache directory'): + dc.FanoutCache(str(root), disk=dc.SafeDisk.with_signing(), shards=2) + assert os.listdir(root) == [] + + +def test_key_location_skips_a_missing_directory(tmp_path): + disk = dc.SafeDisk.with_signing(tmp_path / 'fanout' / 'signing.key') + signing.check_key_location(disk, str(tmp_path / 'fanout')) + assert not op.exists(tmp_path / 'fanout') + + +def test_a_zero_inode_falls_back_to_comparing_paths( + directory, tmp_path, monkeypatch +): + os.makedirs(directory) + os.makedirs(tmp_path / 'keys') + real_stat = os.stat + + def stat_without_inode(path, *args, **kwargs): + result = real_stat(path, *args, **kwargs) + return os.stat_result(result[:1] + (0,) + result[2:10]) + + monkeypatch.setattr(signing.os, 'stat', stat_without_inode) + inside = op.join(op.realpath(directory), 'signing.key') + outside = op.join(op.realpath(tmp_path), 'keys', 'signing.key') + assert signing._inside(inside, directory) + assert not signing._inside(outside, directory) + + +def test_the_environment_variable_names_the_default_key_file( + tmp_path, monkeypatch, directory +): + path = tmp_path / 'elsewhere' / 'my.key' + monkeypatch.setenv(signing.ENV_KEY_FILE, str(path)) + assert signing.default_key_file() == str(path) + + with dc.Cache(directory, disk=Signed): + pass + assert path.exists() + + +# A root the running system treats as absolute: on Windows, Python 3.13 and +# later need a drive for that, so a path like '/data' would not do. +ANCHOR = op.abspath(os.sep) +HOME = op.join(ANCHOR, 'home', 'u') + + +@pytest.mark.parametrize( + 'platform, environ, home, expected', + [ + ( + 'linux', + {'XDG_DATA_HOME': op.join(ANCHOR, 'data')}, + HOME, + op.join(ANCHOR, 'data'), + ), + ('linux', {}, HOME, op.join(HOME, '.local', 'share')), + ( + 'linux', + {'XDG_DATA_HOME': 'relative'}, + HOME, + op.join(HOME, '.local', 'share'), + ), + ( + 'darwin', + {}, + HOME, + op.join(HOME, 'Library', 'Application Support'), + ), + ( + 'win32', + {'LOCALAPPDATA': op.join(ANCHOR, 'appdata')}, + HOME, + op.join(ANCHOR, 'appdata'), + ), + ('win32', {}, HOME, None), + ('linux', {}, '~', None), + ], +) +def test_data_directory(platform, environ, home, expected): + assert signing.data_directory(environ, platform, home) == expected + + +def test_the_default_key_file_is_in_the_data_directory( + tmp_path, monkeypatch, directory +): + monkeypatch.delenv(signing.ENV_KEY_FILE) + monkeypatch.setenv('HOME', str(tmp_path / 'home')) + monkeypatch.setenv('XDG_DATA_HOME', str(tmp_path / 'data')) + monkeypatch.setenv('LOCALAPPDATA', str(tmp_path / 'data')) + + path = signing.default_key_file() + assert path.startswith(str(tmp_path)) + assert path.endswith(op.join('diskcache-community', 'signing.key')) + + with dc.Cache(directory, disk=Signed): + pass + assert op.exists(path) + + +def test_no_home_directory_raises_with_a_hint(monkeypatch): + monkeypatch.delenv(signing.ENV_KEY_FILE) + monkeypatch.setattr(signing, 'data_directory', lambda *args: None) + with pytest.raises(OSError, match=signing.ENV_KEY_FILE): + signing.default_key_file() + + +@pytest.mark.skipif(not POSIX, reason='POSIX modes') +def test_a_new_key_file_is_private(directory, key_file): + with dc.Cache(directory, disk=Signed): + pass + + assert stat.S_IMODE(os.stat(key_file).st_mode) == 0o600 + assert stat.S_IMODE(os.stat(key_file.parent).st_mode) == 0o700 + content = key_file.read_bytes() + assert len(content) == 65 and content.endswith(b'\n') + bytes.fromhex(content.decode('ascii')) + assert os.listdir(key_file.parent) == ['signing.key'] + + +def test_concurrent_first_use_creates_one_key(tmp_path, directory): + os.makedirs(directory) + path = str(tmp_path / 'new' / 'signing.key') + barrier = threading.Barrier(16) + key_ids = [] + + def load(): + barrier.wait() + key_ids.append(signing.KeyFile(path).load(directory)[0].key_id) + + threads = [threading.Thread(target=load) for _ in range(16)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + assert len(key_ids) == 16 + assert len(set(key_ids)) == 1 + assert os.listdir(op.dirname(path)) == ['signing.key'] + + +def test_a_file_system_without_hard_links(tmp_path, directory, monkeypatch): + os.makedirs(directory) + + def no_link(source, target): + raise OSError(errno.EPERM, 'hard links are not supported') + + monkeypatch.setattr(signing.os, 'link', no_link) + path = str(tmp_path / 'new' / 'signing.key') + barrier = threading.Barrier(8) + key_ids = [] + + def load(): + barrier.wait() + key_ids.append(signing.KeyFile(path).load(directory)[0].key_id) + + threads = [threading.Thread(target=load) for _ in range(8)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + assert len(key_ids) == 8 + assert len(set(key_ids)) == 1 + assert os.listdir(op.dirname(path)) == ['signing.key'] + + +@pytest.mark.skipif(not POSIX or ROOT, reason='POSIX modes, not as root') +def test_a_key_file_that_cannot_be_created_raises_with_a_hint( + tmp_path, directory +): + os.makedirs(directory) + folder = tmp_path / 'read-only' + folder.mkdir() + folder.chmod(0o500) + try: + disk = dc.SafeDisk.with_signing(folder / 'keys' / 'signing.key') + with pytest.raises(OSError, match=signing.ENV_KEY_FILE): + dc.Cache(directory, disk=disk) + finally: + folder.chmod(0o700) + + +@pytest.mark.skipif(not POSIX or ROOT, reason='POSIX modes, not as root') +@pytest.mark.parametrize( + 'mode, message', + [ + (0o620, 'written by other users'), + (0o602, 'written by other users'), + (0o640, 'chmod 600'), + (0o604, 'chmod 600'), + ], +) +def test_key_files_other_users_can_use_are_refused( + directory, key_file, mode, message +): + with dc.Cache(directory, disk=Signed): + pass + key_file.chmod(mode) + + with pytest.raises(ValueError, match=message): + dc.Cache(directory, disk=Signed) + + +@pytest.mark.skipif(not POSIX or ROOT, reason='POSIX modes, not as root') +def test_a_key_folder_others_can_write_is_refused_unless_sticky( + directory, key_file +): + with dc.Cache(directory, disk=Signed): + pass + + key_file.parent.chmod(0o770) + with pytest.raises(ValueError, match='folder'): + dc.Cache(directory, disk=Signed) + + key_file.parent.chmod(0o1777) + with dc.Cache(directory, disk=Signed): + pass + key_file.parent.chmod(0o700) + + +@pytest.mark.skipif(not POSIX or ROOT, reason='POSIX modes, not as root') +def test_the_environment_variable_cannot_name_a_shared_key( + tmp_path, monkeypatch, directory +): + path = write_key_file(tmp_path / 'shared.key', KEY.hex().encode(), 0o644) + monkeypatch.setenv(signing.ENV_KEY_FILE, str(path)) + with pytest.raises(ValueError, match='chmod 600'): + dc.Cache(directory, disk=Signed) + + +@pytest.mark.skipif(not POSIX, reason='POSIX modes') +def test_owners_of_key_files(key_file): + write_key_file(key_file, KEY.hex().encode()) + path = str(key_file) + uid = os.geteuid() + + def status(mode, owner): + return os.stat_result((mode, 0, 0, 1, owner, 0, 0, 0, 0, 0)) + + # A root-owned secret that others can read, as containers mount them. + signing.check_permissions(status(stat.S_IFREG | 0o444, 0), path) + signing.check_permissions(status(stat.S_IFREG | 0o600, uid), path) + + with pytest.raises(ValueError, match='belongs to uid'): + signing.check_permissions(status(stat.S_IFREG | 0o600, uid + 1), path) + with pytest.raises(ValueError, match='not a file'): + signing.check_permissions(status(stat.S_IFDIR | 0o700, uid), path) + + +def test_permissions_are_not_checked_on_windows(key_file, monkeypatch): + write_key_file(key_file, KEY.hex().encode()) + shared = os.stat_result( + (stat.S_IFREG | 0o666, 0, 0, 1, 12345, 0) + (0,) * 4 + ) + monkeypatch.setattr(os, 'name', 'nt') + signing.check_permissions(shared, str(key_file)) + + +def test_only_a_trailing_newline_is_stripped(tmp_path, directory): + os.makedirs(directory) + expected = signing.InMemoryKey(KEY).key.key_id + + def key_id(name, content): + path = write_key_file(tmp_path / name, content) + return signing.KeyFile(str(path)).load(directory)[0].key_id + + assert key_id('plain', KEY) == expected + assert key_id('newline', KEY + b'\n') == expected + assert key_id('crlf', KEY + b'\r\n') == expected + assert key_id('space', KEY + b' \n') != expected + assert key_id('leading', b' ' + KEY) != expected + + +def test_key_files_of_the_wrong_size_are_refused(tmp_path, directory): + os.makedirs(directory) + short = write_key_file(tmp_path / 'short', b'x' * 31 + b'\n') + long = write_key_file(tmp_path / 'long', b'x' * 5000) + + with pytest.raises(ValueError, match='at least 32 bytes, not 31'): + signing.KeyFile(str(short)).load(directory) + with pytest.raises(ValueError, match='at most'): + signing.KeyFile(str(long)).load(directory) + + +# Keeping the key out of sight + + +def _contains(value, secrets): + if isinstance(value, (bytes, bytearray, memoryview)): + data = bytes(value) + elif isinstance(value, str): + data = value.encode('utf-8', 'surrogatepass') + else: + data = repr(value).encode('utf-8', 'backslashreplace') + return any(secret in data for secret in secrets) + + +def _secrets(key): + derived = mac_key(key) + return [key, key.hex().encode(), derived, derived.hex().encode()] + + +def test_the_key_never_appears_in_pickles_or_reprs(directory, tmp_path): + with dc.Cache(directory, disk=Signed) as cache: + cache['key'] = Point(1, 2) + with open(signing.default_key_file(), 'rb') as reader: + secrets = _secrets(reader.read().rstrip(b'\n')) + shown = [ + pickle.dumps(cache), + pickle.dumps(type(cache.disk)), + repr(type(cache.disk)), + repr(cache.disk.signing), + repr(vars(cache.disk)), + repr(cache.disk._signing_key), + ] + + with dc.FanoutCache(str(tmp_path / 'fanout'), disk=Signed) as cache: + shown.append(pickle.dumps(cache)) + shown.append(pickle.dumps(dc.Deque(directory=str(tmp_path / 'deque')))) + shown.append(pickle.dumps(dc.Index(str(tmp_path / 'index')))) + + for item in shown: + assert not _contains(item, secrets) + + +def test_an_in_memory_key_is_never_pickled(directory, tmp_path): + disk = dc.SafeDisk.with_signing(key=KEY) + assert KEY.hex() not in repr(disk) + assert signing.InMemoryKey(KEY).key.key_id.hex() in repr(disk) + + with pytest.raises(TypeError, match='in-memory'): + pickle.dumps(disk) + with dc.Cache(directory, disk=disk) as cache: + with pytest.raises(TypeError, match='in-memory'): + pickle.dumps(cache) + with pytest.raises(TypeError, match='cannot be pickled'): + pickle.dumps(cache.disk._signing_key) + with dc.FanoutCache(str(tmp_path / 'fanout'), disk=disk) as cache: + with pytest.raises(TypeError, match='in-memory'): + pickle.dumps(cache) + + +def _library_locals(traceback): + package = op.dirname(dc.__file__) + while traceback is not None: + frame = traceback.tb_frame + if frame.f_code.co_filename.startswith(package): + yield from frame.f_locals.values() + traceback = traceback.tb_next + + +def test_tracebacks_of_refused_rows_hold_no_key_bytes(directory): + disk = dc.SafeDisk.with_signing(key=KEY) + with dc.Cache(directory, disk=disk) as cache: + cache['key'] = Point(1, 2) + update(directory, 'key', value=flip(row(directory, 'key')[2], 20)) + + with dc.Cache(directory, disk=disk) as cache: + with pytest.raises(pickle.UnpicklingError) as info: + cache['key'] + + values = list(_library_locals(info.value.__traceback__)) + assert values + assert not any(_contains(value, _secrets(KEY)) for value in values) + + +def test_tracebacks_of_short_keys_hold_no_key_bytes(tmp_path, directory): + os.makedirs(directory) + content = b'secret-but-too-short\n' + path = write_key_file(tmp_path / 'short', content) + + with pytest.raises(ValueError) as info: + signing.KeyFile(str(path)).load(directory) + for value in _library_locals(info.value.__traceback__): + assert not _contains(value, [content.strip()]) + + with pytest.raises(ValueError) as info: + dc.SafeDisk.with_signing(key=b'secret-but-too-short') + assert 'secret' not in str(info.value) + + +# Combinations + + +def test_with_types_widens_only_what_keys_may_hold(directory): + disk = dc.SafeDisk.with_types(Pair).with_signing() + with dc.Cache(directory, disk=disk) as cache: + cache[Pair(1, 2)] = Point(1, 2) + assert cache[Pair(1, 2)] == Point(1, 2) + with pytest.raises(TypeError): + cache[Point(1, 1)] = 1 + + +@pytest.mark.parametrize( + 'make', + [ + lambda: dc.SafeDisk.with_signing().with_options(on_error='miss'), + lambda: dc.SafeDisk.with_options(on_error='miss').with_signing(), + lambda: dc.SafeDisk.with_types(Pair) + .with_options(on_error='miss') + .with_signing(), + lambda: dc.SafeDisk.with_signing() + .with_types(Pair) + .with_options(on_error='miss'), + ], +) +def test_options_combine_in_any_order(directory, tmp_path, make): + disk = make() + assert disk.on_error == 'miss' + assert isinstance(disk.signing, signing.KeyFile) + + marker = tmp_path / 'marker' + with dc.Cache(directory, disk=dc.Disk) as cache: + cache['planted'] = Poison(str(marker)) + with dc.Cache(directory, disk=disk) as cache: + cache['signed'] = Point(1, 2) + assert cache['signed'] == Point(1, 2) + assert cache.get('planted') is None + assert not marker.exists() + + pickled = pickle.loads(pickle.dumps(disk)) + assert pickled.on_error == 'miss' + assert isinstance(pickled.signing, signing.KeyFile) + + +def test_with_signing_again_replaces_the_key(): + first = dc.SafeDisk.with_signing(key=KEY) + second = first.with_signing(key=OTHER_KEY) + assert ( + second.signing.key.key_id == signing.InMemoryKey(OTHER_KEY).key.key_id + ) + assert first.signing.key.key_id == signing.InMemoryKey(KEY).key.key_id + + +def test_with_numpy_does_not_combine(): + with pytest.raises(TypeError, match='with_numpy'): + dc.SafeDisk.with_numpy().with_signing() + with pytest.raises(TypeError, match='with_numpy'): + dc.SafeDisk.with_signing().with_numpy() + + +def test_safejsondisk_has_nothing_to_sign(): + with pytest.raises(TypeError, match='SafeJSONDisk'): + dc.SafeJSONDisk.with_signing() + + +def test_with_signing_argument_errors(tmp_path): + with pytest.raises(TypeError, match='not both'): + dc.SafeDisk.with_signing(tmp_path / 'key', key=KEY) + with pytest.raises(TypeError, match='bytes'): + dc.SafeDisk.with_signing(key=KEY.hex()) + with pytest.raises(ValueError, match='at least 32 bytes, not 31'): + dc.SafeDisk.with_signing(key=KEY[:31]) + + disk = dc.SafeDisk.with_signing(op.join('~', 'relative.key')) + assert op.isabs(disk.signing.path) + assert dc.SafeDisk.with_signing(b'/bytes.key').signing.path == ( + op.abspath('/bytes.key') + ) + + +class WithKey(dc.SafeDisk): + def fetch(self, mode, filename, value, read, key=None): + return super().fetch(mode, filename, value, read, key=key) + + +class WithKeywords(dc.SafeDisk): + def fetch(self, *args, **kwargs): + return super().fetch(*args, **kwargs) + + +class WithArguments(dc.SafeDisk): + def fetch(self, *args): + return super().fetch(*args) + + +class WithoutKey(dc.SafeDisk): + def fetch(self, mode, filename, value, read): + return super().fetch(mode, filename, value, read) + + +@pytest.mark.parametrize('disk', [WithKey, WithKeywords]) +def test_subclasses_whose_fetch_accepts_key(directory, disk): + with dc.Cache(directory, disk=disk.with_signing()) as cache: + cache['key'] = Point(1, 2) + assert cache['key'] == Point(1, 2) + + +@pytest.mark.parametrize('disk', [WithArguments, WithoutKey]) +def test_subclasses_whose_fetch_refuses_key(disk): + with pytest.raises(TypeError, match='keyword argument key'): + disk.with_signing() + + +def test_subclasses_of_a_signing_disk_whose_fetch_refuses_key(): + with pytest.raises(TypeError, match='Later.fetch does not accept'): + + class Later(dc.SafeDisk.with_signing()): + def fetch(self, mode, filename, value, read): + return super().fetch(mode, filename, value, read) + + +def test_subclasses_of_a_signing_disk_whose_fetch_accepts_key(directory): + class Later(dc.SafeDisk.with_signing()): + def fetch(self, mode, filename, value, read, key=None): + return super().fetch(mode, filename, value, read, key=key) + + with dc.Cache(directory, disk=Later) as cache: + cache['key'] = Point(1, 2) + assert cache['key'] == Point(1, 2) + + +def test_other_disks_with_a_signing_attribute_are_not_signing_disks( + directory, +): + class Mine(dc.Disk): + signing = 'a name of their own' + + with dc.Cache(directory, disk=Mine) as cache: + cache['key'] = Point(1, 2) + assert cache['key'] == Point(1, 2) + + +def test_subclasses_whose_fetch_cannot_be_inspected(monkeypatch): + def cannot_inspect(function): + raise ValueError('no signature') + + monkeypatch.setattr( + signing, 'inspect', types.SimpleNamespace(signature=cannot_inspect) + ) + with pytest.raises(TypeError, match='keyword argument key'): + WithKey.with_signing() + dc.SafeDisk.with_signing() + + +# Older readers + + +@pytest.mark.parametrize('size', [INLINE, FILE], ids=['inline', 'file']) +@pytest.mark.parametrize( + 'disk, message', + [ + (dc.Disk, 'invalid load key'), + (dc.SafeDisk, 'signed entry'), + (dc.SafeDisk.with_types(Pair), 'signed entry'), + ], + ids=['Disk', 'SafeDisk', 'with_types'], +) +def test_disks_without_signing_raise_on_signed_rows( + directory, tmp_path, size, disk, message +): + marker = tmp_path / 'marker' + with dc.Cache(directory, disk=Signed, disk_min_file_size=size) as cache: + cache['key'] = Poison(str(marker)) + + with dc.Cache(directory, disk=disk) as cache: + with pytest.raises(pickle.UnpicklingError, match=message): + cache['key'] + with pytest.raises(pickle.UnpicklingError): + cache.get('key', read=True) + + with dc.Cache(directory, disk=disk.with_options(on_error='miss')) as cache: + assert cache.get('key') is None + assert len(cache) == 0 + + assert not marker.exists() + + +# Golden version 1 rows: the writer must keep producing them, and later +# releases must keep reading them, byte for byte. They were computed from the +# format's description, not with this code. + +GOLDEN_VALUE = {'golden': [1, 2.5, 'three'], 'nested': ('a', b'b')} +GOLDEN_PICKLE = bytes.fromhex( + '80049538000000000000007d94288c06676f6c64656e945d94284b01474004000000' + '0000008c05746872656594658c066e6573746564948c016194430162948694752e' +) +GOLDEN_PREFIX = bytes.fromhex( + 'ff44435301d6501f4f9886a1f2529ffd0cb33d012488d809c1f055a2bbc0cac46fca' + '2125ee85dbe9050b0b8e4c' +) + + +@pytest.mark.parametrize('size', [INLINE, FILE], ids=['inline', 'file']) +def test_golden_rows_of_version_1(directory, size): + disk = dc.SafeDisk.with_signing(key=KEY) + settings = dict(disk_pickle_protocol=4, disk_min_file_size=size) + + with dc.Cache(directory, disk=disk, **settings) as cache: + cache['golden'] = GOLDEN_VALUE + + _, filename, value = row(directory, 'golden') + if pickle.dumps(GOLDEN_VALUE, protocol=4) == GOLDEN_PICKLE: + if size == INLINE: + assert value == GOLDEN_PREFIX + GOLDEN_PICKLE + else: + assert value == GOLDEN_PREFIX + with open(op.join(directory, filename), 'rb') as reader: + assert reader.read() == GOLDEN_PICKLE + + if size == INLINE: + update(directory, 'golden', value=GOLDEN_PREFIX + GOLDEN_PICKLE) + else: + with open(op.join(directory, filename), 'wb') as writer: + writer.write(GOLDEN_PICKLE) + update(directory, 'golden', value=GOLDEN_PREFIX) + + with dc.Cache(directory, disk=disk) as cache: + assert cache['golden'] == GOLDEN_VALUE + + +# The rest of the API + + +def test_memoize(directory): + calls = [] + + with dc.Cache(directory, disk=Signed) as cache: + + @cache.memoize(typed=True) + def make(number): + calls.append(number) + return Point(number, number) + + assert make(1) == make(1) == Point(1, 1) + assert make(1.0) == Point(1.0, 1.0) + + assert calls == [1, 1.0] + + +def test_deque_and_index(tmp_path): + deque = dc.Deque(directory=str(tmp_path / 'deque'), disk=Signed) + deque.append(Point(1, 1)) + deque.appendleft(Point(0, 0)) + deque[1] = Point(2, 2) + assert list(deque) == [Point(0, 0), Point(2, 2)] + assert deque.pop() == Point(2, 2) + + index = dc.Index(str(tmp_path / 'index'), disk=Signed) + index['a'] = Point(1, 1) + index.update(b=Point(2, 2)) + assert dict(index.items()) == {'a': Point(1, 1), 'b': Point(2, 2)} + assert index.peekitem() == ('b', Point(2, 2)) + + +def test_fanout_cache(tmp_path): + with dc.FanoutCache(str(tmp_path / 'fanout'), disk=Signed) as cache: + cache['a'] = Point(1, 1) + assert cache['a'] == Point(1, 1) + + sub = cache.cache('sub') + sub['b'] = Point(2, 2) + assert sub['b'] == Point(2, 2) + + deque = cache.deque('deque') + deque.append(Point(3, 3)) + assert deque.popleft() == Point(3, 3) + + index = cache.index('index') + index['c'] = Point(4, 4) + assert index['c'] == Point(4, 4) + + +def test_djangocache(tmp_path): + pytest.importorskip('django') + from diskcache_community.djangocache import DjangoCache + + cache = DjangoCache(str(tmp_path / 'django'), {'DISK': Signed}) + try: + cache.set('a', Point(1, 1)) + assert cache.get('a') == Point(1, 1) + finally: + cache.close() diff --git a/tests/typecheck_stubs.py b/tests/typecheck_stubs.py index 79d2204..448ed1e 100644 --- a/tests/typecheck_stubs.py +++ b/tests/typecheck_stubs.py @@ -28,3 +28,15 @@ def with_types_one_module() -> type[dc.SafeDisk]: def with_types_module_list() -> type[dc.SafeDisk]: return dc.SafeDisk.with_types(int, modules=['myapp.models', 'myapp.api']) + + +def with_signing_default_key() -> type[dc.SafeDisk]: + return dc.SafeDisk.with_signing().with_options(on_error='miss') + + +def with_signing_key_file(path: str) -> dc.Cache: + return dc.Cache(disk=dc.SafeDisk.with_signing(path)) + + +def with_signing_in_memory_key(key: bytes) -> type[dc.SafeDisk]: + return dc.SafeDisk.with_types(int).with_signing(key=key)