[3.15] gh-159739: Fully clear the unpickler memo (GH-160173) (#160328)

Clearing an Unpickler memo left its sequential index and sparse entries intact, so reusing the unpickler for an independent record could raise UnpicklingError or retrieve an object from a previous record. Replace the memo and reset all of its state before releasing the old entries. Backport to 3.15; manually backport only the counter reset to 3.14, which has no sparse memo.
(cherry picked from commit cc36604)

Co-authored-by: Jelle Zijlstra <jelle@openai.com>

Author: miss-islington

Lib/test/pickletester.py

@@ -5186,6 +5186,22 @@ def test_clear_pickler_memo(self):
             self.assertNotEqual(first_pickled, second_pickled)
             self.assertEqual(first_pickled, third_pickled)
 
+    def test_clear_unpickler_memo(self):
+        value = [b'repeated'] * 2
+        data = pickle.dumps(value, protocol=4)
+        unpickler = self.unpickler_class(io.BytesIO(data * 2))
+        self.assertEqual(unpickler.load(), value)
+        unpickler.memo.clear()
+        self.assertEqual(unpickler.load(), value)
+
+    def test_clear_unpickler_memo_sparse(self):
+        # Store None at index 1000, then retrieve it in a second record.
+        unpickler = self.unpickler_class(io.BytesIO(b'Np1000\n.g1000\n.'))
+        self.assertIsNone(unpickler.load())
+        unpickler.memo.clear()
+        with self.assertRaises(pickle.UnpicklingError):
+            unpickler.load()
+
     def test_priming_pickler_memo(self):
         # Verify that we can set the Pickler's memo attribute.
         data = ["abcdefg", "abcdefg", 44]

Misc/NEWS.d/next/Library/2026-10-22-03-55-33.gh-issue-159739.a8bf30.rst

@@ -0,0 +1,2 @@
+Fix clearing a :class:`pickle.Unpickler` memo so that the unpickler can be
+reused for independent records.

Modules/_pickle.c

@@ -1747,16 +1747,16 @@ _Unpickler_NewMemo(Py_ssize_t new_size)
     return memo;
 }
 
-/* Free the unpickler's memo, taking care to decref any items left in it. */
+/* Replace the unpickler's memo and release the old entries. */
 static void
-_Unpickler_MemoCleanup(UnpicklerObject *self)
+_Unpickler_MemoCleanup(UnpicklerObject *self, PyObject **replacement)
 {
     Py_ssize_t i;
     PyObject **memo = self->memo;
 
-    if (self->memo == NULL)
+    self->memo = replacement;
+    if (memo == NULL)
         return;
-    self->memo = NULL;
     i = self->memo_size;
     while (--i >= 0) {
         Py_XDECREF(memo[i]);
@@ -7577,7 +7577,7 @@ Unpickler_clear(PyObject *op)
     }
 
     Py_CLEAR(self->memo_dict);
-    _Unpickler_MemoCleanup(self);
+    _Unpickler_MemoCleanup(self, NULL);
     PyMem_Free(self->marks);
     self->marks = NULL;
     PyMem_Free(self->input_line);
@@ -7724,10 +7724,16 @@ static PyObject *
 _pickle_UnpicklerMemoProxy_clear_impl(UnpicklerMemoProxyObject *self)
 /*[clinic end generated code: output=d20cd43f4ba1fb1f input=b1df7c52e7afd9bd]*/
 {
-    _Unpickler_MemoCleanup(self->unpickler);
-    self->unpickler->memo = _Unpickler_NewMemo(self->unpickler->memo_size);
-    if (self->unpickler->memo == NULL)
+    UnpicklerObject *unpickler = self->unpickler;
+    PyObject **memo = _Unpickler_NewMemo(unpickler->memo_size);
+    if (memo == NULL) {
         return NULL;
+    }
+    PyObject *memo_dict = unpickler->memo_dict;
+    unpickler->memo_dict = NULL;
+    unpickler->memo_len = 0;
+    _Unpickler_MemoCleanup(unpickler, memo);
+    Py_XDECREF(memo_dict);
     Py_RETURN_NONE;
 }
 
@@ -7932,7 +7938,7 @@ Unpickler_set_memo(PyObject *op, PyObject *obj, void *Py_UNUSED(closure))
         return -1;
     }
 
-    _Unpickler_MemoCleanup(self);
+    _Unpickler_MemoCleanup(self, NULL);
     self->memo_size = new_memo_size;
     self->memo = new_memo;