Add SafeDisk.with_signing() to cache objects no allowlist can hold as signed pickles

.pylintrc

@@ -341,7 +341,7 @@ indent-string='    '
 max-line-length=100
 
 # Maximum number of lines in a module.
-max-module-lines=4300
+max-module-lines=4400
 
 # Allow the body of a class to be on the same line as the declaration if body
 # contains single statement.

CHANGELOG.md

@@ -2,6 +2,16 @@
 
 diskcache-community forked from DiskCache at upstream commit ebfa37c (3 March 2024), two commits after the 5.6.3 release. Each entry names the pull request in this repository. Imported upstream pull requests are referenced by number, and their commits keep their original authors in the git history.
 
+## Unreleased
+
+### Signed entries
+
+This is a new, security-sensitive API in a patch release. Nothing existing changes unless you call it: no other disk behaves differently, and the `cache.db` schema and the file layout are unchanged.
+
+- `SafeDisk.with_signing()` returns a `SafeDisk` subclass that stores every value it pickles as a full pickle with an HMAC-SHA256 tag over the pickle and the cache key, and checks the tag before anything is unpickled. A library can then cache its own objects that no allowlist can hold, such as parsed databases, torch models, application objects and responses. A pickled value without a matching tag, including every unsigned pickle, raises `UnpicklingError`, or reads as missing with `on_error='miss'`. Values stored without pickling are stored as before, and keys stay on the allowlist (PR #46).
+- The key is read from a per-user file outside the cache directory, `$DISKCACHE_SIGNING_KEY_FILE` or `signing.key` in the user's local data directory, which is created with a new random key and mode `0600` on first use. Opening a cache refuses a key file inside the cache directory, or, on POSIX, one that other users can write or, unless it belongs to root, read. `key=` takes a key kept only in memory, for caches that live in one process; a cache that uses it cannot be pickled (PR #46).
+- Rows written by a signing disk raise `UnpicklingError` in every other disk and in diskcache 5.6.3 without being parsed; a `SafeDisk` without signing says the row is a signed entry. A signing disk does not combine with `with_numpy()`, and `SafeJSONDisk.with_signing()` raises (PR #46).
+
 ## 5.8.3 (2026-11-03)
 
 Fixes a regression from 5.8.2 and several `SafeDisk` issues found by testing downstream libraries, and updates the documentation. The `cache.db` schema and the file layout are unchanged.

SECURITY.md

@@ -20,6 +20,7 @@ Where this fork stands:
 - 5.8.0 is imported as `diskcache_community`, next to the upstream `diskcache` package. `SafeDisk.with_types` allows your own classes and the classes defined in named modules while everything else stays refused; numpy classes cannot be allowed, because numpy unpickling is not memory-safe. `configure()` and `DISKCACHE_DISK=safe` select `SafeDisk` for caches created without a disk, including those that libraries create through `diskcache_community`. `Disk.with_options(on_error='miss')` deletes entries that cannot be read and treats them as missing. The steps are in [Fix CVE-2025-69872 in diskcache](https://diskcache-community.readthedocs.io/en/latest/fix-cve-2025-69872.html).
 - `JSONDisk` writes no pickles but still reads one from a crafted row, as it does in 5.6.3. `SafeJSONDisk` reads and writes the same files and never reads a pickle; `SafeDisk` does not decode what `JSONDisk` wrote.
 - `SafeDisk.with_numpy()` returns a `SafeDisk` that also stores numpy arrays, in the `.npy` format rather than as pickles. numpy pickles stay refused, because unpickling an array is not memory-safe.
+- `SafeDisk.with_signing()` returns a `SafeDisk` that stores every value it pickles as a full pickle with an HMAC-SHA256 tag over the pickle and the cache key, and checks the tag before anything is unpickled; a pickled value without a matching tag raises `UnpicklingError`. It is for libraries that cache their own objects, which no allowlist can hold. The key is the trust boundary: it is kept in a per-user file outside the cache directory, which is refused when it lies inside the cache directory or when other users can write or read it, and anyone who can read it can sign a row that runs code when read.
 - The default stays as it is. Existing caches hold objects outside any allowlist, such as instances of your own classes and Django responses, so `SafeDisk` remains an explicit choice.
 
 ## Dependencies

diskcache_community/core.py

@@ -862,7 +862,8 @@ class SafeDisk(Disk):
 
     Objects outside the allowlist, such as instances of your own classes and
     Django model instances and responses, need either the default
-    :class:`Disk` or a disk from :meth:`with_types` that allows them. The
+    :class:`Disk`, a disk from :meth:`with_types` that allows them, or a disk
+    from :meth:`with_signing`, which signs what it writes instead. The
     allowed classes themselves can be stored too, so ``memoize(typed=True)``
     works.
 
@@ -871,10 +872,22 @@ class SafeDisk(Disk):
     safe_types: frozenset = frozenset()
     safe_modules: tuple = ()
     safe_numpy: bool = False
+    signing = None
+
+    def __init_subclass__(cls, **kwargs):
+        super().__init_subclass__(**kwargs)
+        # A subclass of a signing disk is passed key= too, so an override of
+        # fetch that cannot take it fails here rather than on the first read.
+        if cls.signing is not None:
+            signing.check_fetch(cls)
 
     def __init__(self, directory, min_file_size=0, pickle_protocol=0):
         super().__init__(directory, min_file_size, pickle_protocol)
         self._real_directory = op.realpath(directory)
+        if self.signing is not None:
+            self._signing_key, self._signing_path = self.signing.load(
+                directory
+            )
 
     @classmethod
     def with_types(cls, *types, modules=()):
@@ -974,10 +987,59 @@ def with_numpy(cls):
         True
 
         :return: SafeDisk subclass to pass as ``disk=``
+        :raises TypeError: if the class signs entries (:meth:`with_signing`)
 
         """
+        if cls.signing is not None:
+            raise TypeError(signing.NUMPY_REFUSED)
         return _derive_disk(cls, {'safe_numpy': True})
 
+    @classmethod
+    def with_signing(cls, key_file=None, *, key=None):
+        """Return a SafeDisk subclass that signs the values it pickles.
+
+        Every value the returned class pickles is stored as a full pickle
+        with an HMAC-SHA256 tag over the pickle and the cache key, and the
+        tag is checked before anything is unpickled. A pickled value without
+        a tag, or whose tag does not match, raises
+        :exc:`pickle.UnpicklingError`, which ``on_error='miss'`` treats as a
+        miss. Values that are stored without pickling, such as ``str`` and
+        ``int``, are stored as on every disk, and keys stay on the allowlist.
+
+        The key is read from `key_file`, which is created with a new random
+        key if it is missing, or from the default key file when neither
+        argument is given: ``$DISKCACHE_SIGNING_KEY_FILE``, or
+        ``signing.key`` in a folder ``diskcache-community`` of the user's
+        local data directory. Opening a cache raises :exc:`ValueError` when
+        the key file is inside the cache directory or, on POSIX, when other
+        users can write it, or read it unless it belongs to root. `key` takes
+        the key itself, at least 32 random bytes, kept only in memory; a
+        cache that uses it cannot be pickled. Every other disk, and
+        diskcache 5.6.3, raises :exc:`pickle.UnpicklingError` on the pickled
+        rows a signing disk writes. It combines with :meth:`with_types`,
+        which then widens only what keys may hold, and with
+        :meth:`with_options`.
+
+        >>> import secrets
+        >>> KeyDisk = SafeDisk.with_signing(key=secrets.token_bytes(32))
+        >>> KeyDisk.signing  # doctest: +ELLIPSIS
+        <in-memory key ...>
+
+        :param key_file: path of the key file (default None, the default key
+            file)
+        :param bytes key: the key, instead of a key file
+        :return: SafeDisk subclass to pass as ``disk=``
+        :raises TypeError: if both `key_file` and `key` are given, if `key`
+            is not bytes, if the class stores numpy arrays
+            (:meth:`with_numpy`), or if its ``fetch`` does not accept the
+            keyword argument ``key``
+        :raises ValueError: if `key` is shorter than 32 bytes
+
+        """
+        return _derive_disk(
+            cls, {'signing': signing.source(cls, key_file, key)}
+        )
+
     def _plain_unpickler(self, file):
         return SafeUnpickler(
             file,
@@ -1142,9 +1204,14 @@ def _confine(self, filename):
             )
 
     def put(self, key):
-        # pylint: disable=unidiomatic-typecheck
+        # pylint: disable=unidiomatic-typecheck,comparison-with-itself
         if type(key) not in (bytes, str, int, float):
             self._check_object(key, 'key')
+        elif self.signing is not None and key != key:
+            raise TypeError(
+                'SafeDisk.with_signing() cannot store a NaN key, which SQLite'
+                ' stores as NULL'
+            )
         return super().put(key)
 
     def get(self, key, raw):
@@ -1155,6 +1222,10 @@ def get(self, key, raw):
 
     def store(self, value, read, key=UNKNOWN):
         # pylint: disable=unidiomatic-typecheck
+        if self.signing is not None and not (
+            read or signing.stored_raw(value)
+        ):
+            return signing.store(self, value, key)
         if read or type(value) in (bytes, str, int, float):
             return super().store(value, read, key=key)
         numpy = sys.modules.get('numpy') if self.safe_numpy else None
@@ -1163,15 +1234,25 @@ def store(self, value, read, key=UNKNOWN):
         self._check_object(value, 'value')
         return super().store(value, read, key=key)
 
-    def fetch(self, mode, filename, value, read):
+    def fetch(self, mode, filename, value, read, key=None):
+        """Convert fields `mode`, `filename`, and `value` from Cache table to
+        value. A signing disk (:meth:`with_signing`) is also given the row's
+        `key`, a ``(db_key, raw)`` pair, and refuses a signed row without
+        it."""
         if filename is not None:
             self._confine(filename)
 
         if mode == MODE_PICKLE:
+            if self.signing is not None:
+                return signing.fetch(self, filename, value, key)
             if value is None:
                 with super().fetch(MODE_BINARY, filename, None, True) as file:
                     return self._unpickler(file).load()
-            return self._unpickler(io.BytesIO(value)).load()
+            try:
+                return self._unpickler(io.BytesIO(value)).load()
+            except pickle.UnpicklingError:
+                signing.refuse_signed(value)
+                raise
 
         return super().fetch(mode, filename, value, read)
 
@@ -1204,6 +1285,13 @@ class SafeJSONDisk(JSONDisk, SafeDisk):
     def with_numpy(cls):
         raise TypeError('SafeJSONDisk stores only JSON, not numpy arrays')
 
+    @classmethod
+    def with_signing(cls, key_file=None, *, key=None):
+        # pylint: disable=unused-argument
+        raise TypeError(
+            'SafeJSONDisk never pickles a value, so it has none to sign'
+        )
+
     def get(self, key, raw):
         if not raw:
             raise pickle.UnpicklingError('SafeJSONDisk refuses a pickled key')
@@ -1702,7 +1790,7 @@ def _after_fork_in_child():
     )
 
 
-class Cache:
+class Cache:  # pylint: disable=too-many-instance-attributes
     """Disk and file backed cache."""
 
     def __init__(
@@ -1818,6 +1906,11 @@ def __init__(
             if key.startswith('disk_')
         }
         self._disk = disk(directory, **kwargs)
+        # A signing disk checks the cache key of each row it reads. Only a
+        # SafeDisk signs, whatever other disks name their attributes.
+        self._fetch_with_key = (
+            isinstance(self._disk, SafeDisk) and self._disk.signing is not None
+        )
 
         self._init_settings(sql, sets, current_settings)
 
@@ -2761,6 +2854,10 @@ def get(
             mode, filename, db_value = row
 
             try:
+                if self._fetch_with_key:
+                    return self._disk.fetch(
+                        mode, filename, db_value, read, key=(db_key, raw)
+                    )
                 return self._disk.fetch(mode, filename, db_value, read)
             except IOError:
                 # Key was deleted before we could retrieve result.
@@ -2819,7 +2916,12 @@ def get(
                 db_store_time = None
 
             try:
-                value = self._disk.fetch(mode, filename, db_value, read)
+                if self._fetch_with_key:
+                    value = self._disk.fetch(
+                        mode, filename, db_value, read, key=(db_key, raw)
+                    )
+                else:
+                    value = self._disk.fetch(mode, filename, db_value, read)
             except IOError:
                 # Key was deleted before we could retrieve result.
                 return default
@@ -2871,7 +2973,14 @@ def get(
                     db_store_time = None
 
                 try:
-                    value = self._disk.fetch(mode, filename, db_value, read)
+                    if self._fetch_with_key:
+                        value = self._disk.fetch(
+                            mode, filename, db_value, read, key=(db_key, raw)
+                        )
+                    else:
+                        value = self._disk.fetch(
+                            mode, filename, db_value, read
+                        )
                 except IOError:
                     # Key was deleted before we could retrieve result.
                     if statistics:
@@ -2997,7 +3106,12 @@ def pop(
             sql('DELETE FROM Cache WHERE rowid = ?', (rowid,))
 
         try:
-            value = self._disk.fetch(mode, filename, db_value, False)
+            if self._fetch_with_key:
+                value = self._disk.fetch(
+                    mode, filename, db_value, False, key=(db_key, raw)
+                )
+            else:
+                value = self._disk.fetch(mode, filename, db_value, False)
         except IOError:
             # Key was deleted before we could retrieve result.
             return default
@@ -3168,6 +3282,12 @@ def push(
             else:
                 db_key = num
 
+            if self._fetch_with_key:
+                # The disk hashed the value before the transaction, so only
+                # the key chosen here is added to its tag.
+                db_value = signing.finish(db_value, (db_key, raw))
+                columns = columns[:-1] + (db_value,)
+
             self._row_insert(db_key, raw, now, columns)
             self._cull(now, sql, cleanup)
 
@@ -3275,7 +3395,12 @@ def pull(
                         break
 
             try:
-                value = self._disk.fetch(mode, name, db_value, False)
+                if self._fetch_with_key:
+                    value = self._disk.fetch(
+                        mode, name, db_value, False, key=(key, True)
+                    )
+                else:
+                    value = self._disk.fetch(mode, name, db_value, False)
             except IOError:
                 # Key was deleted before we could retrieve result.
                 continue
@@ -3398,7 +3523,12 @@ def peek(
                         break
 
             try:
-                value = self._disk.fetch(mode, name, db_value, False)
+                if self._fetch_with_key:
+                    value = self._disk.fetch(
+                        mode, name, db_value, False, key=(key, True)
+                    )
+                else:
+                    value = self._disk.fetch(mode, name, db_value, False)
             except IOError:
                 # Key was deleted before we could retrieve result.
                 continue
@@ -3483,7 +3613,12 @@ def peekitem(self, last=True, expire_time=False, tag=False, retry=False):
 
             try:
                 key = self._disk.get(db_key, raw)
-                value = self._disk.fetch(mode, name, db_value, False)
+                if self._fetch_with_key:
+                    value = self._disk.fetch(
+                        mode, name, db_value, False, key=(db_key, raw)
+                    )
+                else:
+                    value = self._disk.fetch(mode, name, db_value, False)
             except IOError:
                 # Key was deleted before we could retrieve result.
                 continue
@@ -4228,6 +4363,7 @@ def reset(self, key, value=ENOVAL, update=True):
         return value
 
 
-# arrays imports the classes above, so it is imported once they exist.
+# arrays and signing import the classes above, so they are imported once they
+# exist.
 # pylint: disable=wrong-import-position,cyclic-import
-from . import arrays  # noqa: E402  # isort: skip
+from . import arrays, signing  # noqa: E402  # isort: skip

diskcache_community/core.pyi

@@ -14,7 +14,7 @@ from typing import (
     overload,
 )
 
-from _typeshed import StrOrBytesPath
+from _typeshed import StrOrBytesPath, StrPath
 from typing_extensions import Self, TypeVar, Unpack
 
 from ._types import (
@@ -40,6 +40,7 @@ from ._types import (
     Tag,
     UnknownType,
 )
+from .signing import InMemoryKey, KeyFile
 
 _T = TypeVar('_T')
 _BoundStrT = TypeVar('_BoundStrT', bound=str)
@@ -191,16 +192,36 @@ class SafeDisk(Disk):
     safe_types: frozenset[type]
     safe_modules: tuple[str, ...]
     safe_numpy: bool
+    signing: KeyFile | InMemoryKey | None
     @classmethod
     def with_types(
         cls, *types: type, modules: str | Iterable[str] = ...
     ) -> type[Self]: ...
     @classmethod
     def with_numpy(cls) -> type[Self]: ...
+    @classmethod
+    def with_signing(
+        cls, key_file: StrPath | None = ..., *, key: bytes | None = ...
+    ) -> type[Self]: ...
+    def fetch(  # type: ignore[override]
+        self,
+        mode: ModeLiteral,
+        filename: str | None,
+        value: Any,
+        read: bool,
+        key: tuple[Any, bool] | None = ...,
+    ) -> Any: ...
 
 class SafeJSONDisk(JSONDisk, SafeDisk):
     @classmethod
     def with_numpy(cls) -> NoReturn: ...
+    @classmethod
+    def with_signing(
+        cls, key_file: StrPath | None = ..., *, key: bytes | None = ...
+    ) -> NoReturn: ...
+    def fetch(  # type: ignore[override]
+        self, mode: ModeLiteral, filename: str | None, value: Any, read: bool
+    ) -> Any: ...
 
 def configure(disk: type[Disk] | None) -> None: ...
 def default_disk() -> type[Disk]: ...