diff --git a/CHANGELOG.md b/CHANGELOG.md index 61ba1f8..93faf0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,10 @@ diskcache-community forked from DiskCache at upstream commit ebfa37c (3 March 20 - A float NaN stored as a value is kept, on every disk. It was written to SQLite, which stores NaN as NULL, so it read back as `None`. It is now stored as a pickle, which 5.6.3 reads too (PR #34). - `memoize_stampede` works with `SafeDisk`: the key of its early recomputation marker held the internal `ENOVAL` constant, which `SafeDisk` refuses, so a call raised `TypeError` once a cached value neared expiry (PR #32). +### Packaging + +- The package installs on Python 3.8 and later (PR #36). + ## 5.8.1 (2026-10-18) Ships type stubs and installs on Python 3.9. The code and the files on disk are unchanged from 5.8.0. diff --git a/diskcache_community/arrays.py b/diskcache_community/arrays.py index f3beab4..8cf13ed 100644 --- a/diskcache_community/arrays.py +++ b/diskcache_community/arrays.py @@ -235,12 +235,9 @@ def persistent_id(self, obj): class _ArrayUnpickler(SafeUnpickler): """Unpickler that reads back what _ArrayPickler wrote. - A forged row can follow an array with BUILD, which calls its - ``__setstate__``. numpy restores an array's internal flags from the dtype - in that state without checking them, which is why numpy pickles are - refused. The call is harmless here because the dtype slot accepts only a - ``numpy.dtype``, and neither the allowlist nor a persistent ID can make - one; a test pins that. + Array state is rebuilt from the checked ``.npy`` header alone: the dtype + slot of a trailing ``BUILD`` accepts only a ``numpy.dtype``, which + neither the allowlist nor a persistent ID can produce. """ diff --git a/diskcache_community/core.py b/diskcache_community/core.py index e1b7944..76b3bfa 100644 --- a/diskcache_community/core.py +++ b/diskcache_community/core.py @@ -601,7 +601,7 @@ def fetch(self, mode, filename, value, read): # the internal flags of a dtype from its pickle without checking them, so a # crafted pickle can make an array read raw bytes as object pointers. -UNSAFE_MODULES = ('numpy',) +_UNSAFE_MODULES = ('numpy',) _PICKLE_STRING_OPS = frozenset( {'BINUNICODE', 'BINUNICODE8', 'SHORT_BINUNICODE', 'UNICODE'} @@ -707,17 +707,16 @@ class SafeUnpickler(pickle.Unpickler): Every pickle opcode that can call code (GLOBAL, STACK_GLOBAL, INST, OBJ, NEWOBJ, NEWOBJ_EX and REDUCE) obtains its callable through `find_class`, - so restricting `find_class` to a fixed set of constructors is enough to - keep a crafted pickle from running code. + so restricting `find_class` to a fixed set of constructors keeps every + name a pickle resolves inside the allowlist. By default the allowlist is SAFE_PICKLE_NAMES. `names` replaces that mapping of module to names, `classes` adds classes by identity, and `modules` adds every class defined in the given modules or below them. A class is looked up attribute by attribute and every step must be a - class, so a name such as ``os.system`` that reaches a function through a - module imported by a trusted module is refused. Classes are looked up - only in modules that are already imported; reading a pickle never - imports a module, so it cannot run a module's top-level code. + class, so dotted names that pass through a module or a dunder attribute + are refused. Classes are looked up only in modules that are already + imported; reading a pickle never imports a module. """ @@ -869,8 +868,7 @@ def with_types(cls, *types, modules=()): do nothing beyond storing their fields, such as dataclasses, named tuples, enums and pydantic models. Reading looks classes up only in modules that are already imported, so import them before reading. - numpy classes are refused, see ``UNSAFE_MODULES``; use - :meth:`with_numpy` to store arrays. + numpy classes are refused; use :meth:`with_numpy` to store arrays. >>> import collections >>> Point = collections.namedtuple('Point', 'x y') @@ -884,13 +882,13 @@ def with_types(cls, *types, modules=()): :return: SafeDisk subclass to pass as ``disk=`` :raises TypeError: if an entry of `types` is not a class :raises ValueError: if an entry of `modules` is not a module name, or - a type or module is in ``UNSAFE_MODULES`` + a type or module is numpy (arrays belong to :meth:`with_numpy`) """ for entry in types: if not isinstance(entry, type): raise TypeError('with_types takes classes, not %r' % (entry,)) - if _in_modules(entry.__module__, UNSAFE_MODULES): + if _in_modules(entry.__module__, _UNSAFE_MODULES): raise ValueError( 'with_types refuses the class %r; use with_numpy() to' ' store arrays' % (entry,) @@ -908,7 +906,7 @@ def with_types(cls, *types, modules=()): if any( _in_modules(prefix, (unsafe,)) or _in_modules(unsafe, (prefix,)) - for unsafe in UNSAFE_MODULES + for unsafe in _UNSAFE_MODULES ): raise ValueError( 'with_types refuses the module %r; use with_numpy() to' @@ -930,21 +928,21 @@ def with_types(cls, *types, modules=()): def with_numpy(cls): """Return a SafeDisk subclass that also stores numpy arrays. - :class:`SafeDisk` refuses numpy, because unpickling an array is not - memory-safe (see ``UNSAFE_MODULES``). The returned class writes - arrays through pickle's persistent IDs in numpy's ``.npy`` format - with ``allow_pickle=False``, anywhere in a value, and checks each - header before it reads an array back. numpy scalars keep their type. + The returned class writes arrays through pickle's persistent IDs in + numpy's ``.npy`` format with ``allow_pickle=False``, anywhere in a + value except in keys, and checks each header before it reads an + array back. (Plain :class:`SafeDisk` refuses numpy pickles, because + unpickling an array is not memory-safe.) numpy scalars keep their + type. Object arrays, ``ndarray`` subclasses and other numpy scalars raise :exc:`TypeError` when written. It combines with :meth:`with_types` and :meth:`with_options` in any order. A value with no numpy in it is an ordinary pickle. An entry holding - numpy data cannot be read by any other disk, or by diskcache 5.6.3: - they raise :exc:`pickle.UnpicklingError` for it, and a disk with - ``on_error='miss'`` deletes it, so select this disk in every process - that shares the directory. A process without numpy raises - :exc:`ImportError` for such an entry and keeps it. + numpy data cannot be read by any other disk, or by diskcache 5.6.3, + so select this disk in every process that shares the directory. A + process without numpy raises :exc:`ImportError` for such an entry + and keeps it. >>> ArrayDisk = SafeDisk.with_numpy() >>> ArrayDisk.safe_numpy @@ -1097,7 +1095,7 @@ def put(self, key): def get(self, key, raw): if raw: return super().get(key, raw) - # Keys never hold numpy, so a key row gets no persistent IDs. + # Keys exclude numpy, so a key row gets no persistent IDs. return self._plain_unpickler(io.BytesIO(key)).load() def store(self, value, read, key=UNKNOWN): @@ -1131,9 +1129,9 @@ class SafeJSONDisk(JSONDisk, SafeDisk): """Disk that stores JSON like :class:`JSONDisk` and never reads a pickle. :class:`JSONDisk` writes only compressed JSON, but it inherits the - :func:`pickle.load` calls of :class:`Disk`, so a row crafted to say that - it holds a pickle still runs code in the process that reads it - (CVE-2025-69872). :class:`SafeJSONDisk` closes that path: + :func:`pickle.load` calls of :class:`Disk`, so it still reads pickles + from crafted rows (CVE-2025-69872). :class:`SafeJSONDisk` closes that + path: * A pickled key or value raises :exc:`pickle.UnpicklingError` without being parsed, whatever it contains, because this disk never writes one. diff --git a/diskcache_community/core.pyi b/diskcache_community/core.pyi index 2dc0971..9fa0a99 100644 --- a/diskcache_community/core.pyi +++ b/diskcache_community/core.pyi @@ -65,7 +65,7 @@ EVICTION_POLICY: Final[dict[EvictionPolicyKey, EvictionPolicyItem]] SETTINGS_TRIGGERS: Final[tuple[tuple[str, str], ...]] SAFE_TYPES: Final[frozenset[type]] SAFE_PICKLE_NAMES: Final[dict[str, frozenset[str]]] -UNSAFE_MODULES: Final[tuple[str, ...]] +_UNSAFE_MODULES: Final[tuple[str, ...]] ENV_DISK: Final[str] ENV_SAFE_TYPES: Final[str] ENV_SAFE_MODULES: Final[str] diff --git a/docs/api.rst b/docs/api.rst index 85c419b..7ca394a 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -1,7 +1,7 @@ DiskCache API Reference ======================= -.. py:module:: diskcache +.. py:module:: diskcache_community The :doc:`tutorial` provides a helpful walkthrough of most methods. @@ -151,7 +151,7 @@ Read the :ref:`SafeDisk tutorial ` for details. The Python types that :class:`SafeDisk ` accepts for writing; exactly the types the names above read back. -.. data:: diskcache_community.core.UNSAFE_MODULES +.. data:: diskcache_community.core._UNSAFE_MODULES Packages whose classes :meth:`SafeDisk.with_types ` refuses to allow, because diff --git a/docs/benchmarks.rst b/docs/benchmarks.rst index 7f267c6..731d01f 100644 --- a/docs/benchmarks.rst +++ b/docs/benchmarks.rst @@ -20,8 +20,8 @@ Environment - memcached: memcached 1.6.45 - Redis: 8.10.2 -- diskcache-community Cache: diskcache-community 5.7.0 -- diskcache-community FanoutCache(8): diskcache-community 5.7.0 +- diskcache-community Cache: diskcache-community 5.7.0 (measured 2026-10-12) +- diskcache-community FanoutCache(8): diskcache-community 5.7.0 (measured 2026-10-12) - diskcache 5.6.3 Cache: diskcache 5.6.3 - memcached, pymemcache: pymemcache 4.0.0 - memcached, pylibmc: pylibmc 1.6.3, libmemcached 1.0.18 diff --git a/docs/fix-cve-2025-69872.rst b/docs/fix-cve-2025-69872.rst index 78bca96..6d1cfce 100644 --- a/docs/fix-cve-2025-69872.rst +++ b/docs/fix-cve-2025-69872.rst @@ -129,19 +129,9 @@ Steps 5. **Deal with entries already in the cache.** A cache written with the default :class:`Disk ` may hold objects that - :class:`SafeDisk ` refuses, and reading them - raises :exc:`pickle.UnpicklingError`. If the entries can be computed again, - clear the cache once with :meth:`clear `, - or let reads drop them with :meth:`with_options - `: - - .. code-block:: python - - AppDisk = SafeDisk.with_options(on_error='miss') - - An unreadable entry is then deleted and treated as missing, and the first - one is logged as a warning on the ``diskcache_community`` logger. See - :ref:`tutorial-unreadable-entries`. + :class:`SafeDisk ` refuses; those entries + can be cleared or treated as missing + (:ref:`tutorial-unreadable-entries`). 6. **Check the result.** ``type(cache.disk)`` names the disk a cache uses, and :func:`default_disk() ` returns the @@ -166,12 +156,6 @@ reads a pickle. :class:`SafeDisk ` does not decode what :class:`JSONDisk ` wrote. See :ref:`tutorial-safejsondisk`. -Vulnerability scanners match on the distribution name. ``pip-audit``, -Dependabot and similar tools report CVE-2025-69872 for ``diskcache`` and -nothing for ``diskcache-community``, whichever disk you use. A clean scan -means the upstream package is gone, not that your caches use :class:`SafeDisk -`. - What a restricted disk has to cover ----------------------------------- @@ -181,49 +165,33 @@ values are unpickled is not enough. A disk that closes this vulnerability covers all of the following, and :class:`SafeDisk ` does each one: -* **Keys as well as values.** Keys that are not plain strings, bytes or - numbers are pickled too, and :meth:`Disk.get ` - unpickles the ``key`` column whenever the cache returns a key, for example - while iterating or in :meth:`peekitem `. - A disk that restricts only :meth:`fetch ` - still runs a crafted key. :class:`SafeDisk ` - reads keys with the same unpickler as values. +* **Keys as well as values.** Non-trivial keys are pickled too, and unpickled + whenever the cache returns a key. :class:`SafeDisk + ` reads keys with the same unpickler as values. * **The filename column.** Large values live in files whose path is stored in - ``cache.db``. Reads, deletes, expiry and culling all follow that path, and - :meth:`Disk.remove ` also removes the - parent directories it leaves empty. A crafted row can therefore point outside - the cache and have a file read or deleted, including by a delete that runs - after a failed read. :class:`SafeDisk ` refuses - any path that leaves the cache directory, on reads and on removal. - -* **Exact classes, not module prefixes.** With protocol 4 and later, which the - writer of the pickle chooses, a pickle names a global as a dotted path that - the unpickler follows attribute by attribute. Trusting every name under a - package such as ``myapp.types`` therefore also trusts whatever those modules - import: a name like ``os.system`` or ``builtins.eval`` resolves through any - trusted module that imports ``os`` or ``builtins``. - :meth:`SafeDisk.with_types ` - accepts a name only when every step is a class and the final class is - defined in a trusted module, and it refuses dunder attributes. + ``cache.db``, and reads, deletes, expiry and culling all follow that path. + :class:`SafeDisk ` refuses any path that + leaves the cache directory, on reads and on removal. + +* **Exact classes, not module prefixes.** A pickle can name a global through a + module that merely imports it. :meth:`SafeDisk.with_types + ` accepts a name only when every + step is a class defined in a trusted module, and it refuses dunder + attributes. -* **No numpy.** numpy's array and dtype reconstructors restore internal dtype - state from the pickle without checking it, so a crafted array can make - numpy read arbitrary memory as object pointers. Allowing those names turns - the cache into a crash or memory-corruption vector. +* **No numpy.** Array pickles can corrupt memory. :meth:`SafeDisk.with_types ` - refuses numpy classes even when asked for them. + refuses numpy classes even when asked for them; use :meth:`with_numpy + ` to store arrays instead. -* **No imports while reading.** Resolving a name by importing its module runs - that module's top-level code, chosen by the pickle. :class:`SafeDisk +* **No imports while reading.** :class:`SafeDisk ` looks classes up only in modules that are already imported. -* **Refuse on write.** A value the reader would refuse should fail when it is - written, not when a later process reads it. :class:`SafeDisk - ` checks the names in a value's pickle against - the same allowlist before anything is written and raises :exc:`TypeError` - naming the class. +* **Refuse on write.** A value the reader would refuse fails when it is + written, with :exc:`TypeError` naming the class, not when a later process + reads it. To report a problem with :class:`SafeDisk `, follow ``SECURITY.md`` in the repository. diff --git a/docs/migrating.rst b/docs/migrating.rst index 7653bcb..81e0e2f 100644 --- a/docs/migrating.rst +++ b/docs/migrating.rst @@ -93,8 +93,10 @@ requires it. What behaves differently ------------------------ -These changes from 5.6.3 can be noticed by an application. The -``CHANGELOG.md`` in the repository lists every change. +Most applications need no changes for the following: existing cache +directories open as they are and the defaults behave as before. The cases +that can be noticed are listed here, and the ``CHANGELOG.md`` in the +repository lists every change. * **New directories are created with mode 0700.** That includes missing parent directories and the value subdirectories. Existing directories keep diff --git a/docs/tutorial.rst b/docs/tutorial.rst index a461648..6d8bfc6 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -18,7 +18,7 @@ Pip & PyPI Installing :doc:`DiskCache ` is simple with `pip `_:: - $ pip install --upgrade diskcache + $ pip install --upgrade diskcache-community The versioning scheme uses `major.minor.micro` with `micro` intended for bug fixes, `minor` intended for small features or improvements, and `major` @@ -1186,10 +1186,12 @@ unreadable entry of each cache is logged as a warning on the since they cannot move past it. A value file that has disappeared is a miss in either mode, as before. -The entry is deleted for every process that shares the directory, so give -every reader and writer of a shared cache the same disk: a reader whose -allowlist is narrower than a writer's would delete entries the writer can -still read. +.. note:: + + The entry is deleted for every process that shares the directory, so give + every reader and writer of a shared cache the same disk: a reader whose + allowlist is narrower than a writer's would delete entries the writer can + still read. .. _tutorial-read-only: