gh-159811: Raise OpenSSL errors in _hashlib.HASH after re-attaching the thread state (#159817)

When `EVP_DigestUpdate()` failed, `_hashlib_HASH_hash()` raised a Python
exception, however, `_hashlib.HASH.update()` and the `_hashlib.HASH`
constructor release the GIL before calling it when the data is larger
than `HASHLIB_GIL_MINSIZE` bytes.

`_hashlib_HASH_hash()` now only returns an error status and its callers
raise the exception once they have re-attached the thread state.

Author: picnixz

Misc/NEWS.d/next/Library/2026-10-19-23-25-29.gh-issue-159811.cMB4DO.rst

@@ -0,0 +1,3 @@
+Fix a crash in :mod:`hashlib` when the OpenSSL routine used by
+:meth:`hash.update() <hashlib.hash.update>` or by hash object constructors
+fails on data larger than 2048 bytes. Patch by Bénédikt Tran.

Modules/_hashopenssl.c

@@ -847,6 +847,12 @@ new_hash_object(PyTypeObject *type)
     return retval;
 }
 
+/*
+ * Update the digest context of 'self' with 'len' bytes from 'vp'.
+ *
+ * Return 0 on success and -1 if EVP_DigestUpdate() failed. No exception
+ * is set, so that this can be called without an attached thread state.
+ */
 static int
 _hashlib_HASH_hash(HASHobject *self, const void *vp, Py_ssize_t len)
 {
@@ -858,7 +864,6 @@ _hashlib_HASH_hash(HASHobject *self, const void *vp, Py_ssize_t len)
         else
             process = Py_SAFE_DOWNCAST(len, Py_ssize_t, unsigned int);
         if (!EVP_DigestUpdate(self->ctx, (const void*)cp, process)) {
-            notify_ssl_error_occurred_in(Py_STRINGIFY(EVP_DigestUpdate));
             return -1;
         }
         len -= process;
@@ -991,7 +996,11 @@ _hashlib_HASH_update_impl(HASHobject *self, PyObject *obj)
         result = _hashlib_HASH_hash(self, view.buf, view.len)
     );
     PyBuffer_Release(&view);
-    return result < 0 ? NULL : Py_None;
+    if (result < 0) {
+        notify_ssl_error_occurred_in(Py_STRINGIFY(EVP_DigestUpdate));
+        return NULL;
+    }
+    Py_RETURN_NONE;
 }
 
 static PyMethodDef HASH_methods[] = {
@@ -1305,7 +1314,7 @@ _hashlib_HASH(_hashlibstate *state, const char *digestname, PyObject *data_obj,
             result = _hashlib_HASH_hash(self, view.buf, view.len)
         );
         if (result == -1) {
-            assert(PyErr_Occurred());
+            notify_ssl_error_occurred_in(Py_STRINGIFY(EVP_DigestUpdate));
             Py_CLEAR(self);
             goto exit;
         }