Add sys._set_exception() to expose the PyErr_SetHandledExecption function

Exposing this function allows Python code to clear/set the current
exception context as returned by `sys.exception()`. It is prefixed by an
underscore since this functionality is not intended to be accessed by
user-written code.

In order to allow `sys._set_exception(None)` to clear the current
exception `_PyErr_GetTopmostException` was changed to consider `Py_None`
a valid exception instead of an indication to keep traversing the stack
like `NULL` is. Additionally, `PyErr_SetHandledException` was updated to
add `Py_None` to the exception stack instead of `NULL`.

Put together, these changes allow `PyErr_SetHandledException(NULL)` to
mask the entire exception chain and "clear" the current exception state
as documented in `Doc/c-api/exceptions.rst`. Furthermore, since
`sys._set_exception()` uses `PyErr_SetHandledException`, this allows
`sys._set_exception(None)` to clear the current exception context
instead of just exposing the next exception in the stack.

Author: pR0Ps

Lib/test/test_sys.py

@@ -161,6 +161,123 @@ def f():
         self.assertIsInstance(e, ValueError)
         self.assertIs(exc, e)
 
+class SetExceptionTests(unittest.TestCase):
+
+    def tearDown(self):
+        # make sure we don't leave the global exception set
+        sys._set_exception(None);
+
+    def test_set_exc_invalid_values(self):
+        for x in (0, "1", b"2"):
+            with self.assertRaises(TypeError):
+                sys._set_exception(x);
+
+    def test_clear_exc(self):
+        try:
+            raise ValueError()
+        except ValueError:
+            self.assertIsInstance(sys.exception(), ValueError)
+            sys._set_exception(None)
+            self.assertIsNone(sys.exception())
+
+    def test_set_exc(self):
+        exc = ValueError()
+        self.assertIsNone(sys.exception())
+        sys._set_exception(exc)
+        self.assertIs(sys.exception(), exc)
+
+    def test_set_exc_replaced_by_new_exception_and_restored(self):
+        exc = ValueError()
+        sys._set_exception(exc)
+        self.assertIs(sys.exception(), exc)
+        try:
+            raise TypeError()
+        except TypeError:
+            self.assertIsInstance(sys.exception(), TypeError)
+            self.assertIs(sys.exception().__context__, exc)
+
+        self.assertIs(sys.exception(), exc)
+
+    def test_set_exc_popped_on_exit_except(self):
+        exc = ValueError()
+        try:
+            raise TypeError()
+        except TypeError:
+            self.assertIsInstance(sys.exception(), TypeError)
+            sys._set_exception(exc)
+            self.assertIs(sys.exception(), exc)
+        self.assertIsNone(sys.exception())
+
+    def test_cleared_exc_overridden_and_restored(self):
+        try:
+            raise ValueError()
+        except ValueError:
+            try:
+                raise TypeError()
+            except TypeError:
+                self.assertIsInstance(sys.exception(), TypeError)
+                sys._set_exception(None)
+                self.assertIsNone(sys.exception())
+                try:
+                    raise IndexError()
+                except IndexError:
+                    self.assertIsInstance(sys.exception(), IndexError)
+                    self.assertIsNone(sys.exception().__context__)
+                self.assertIsNone(sys.exception())
+            self.assertIsInstance(sys.exception(), ValueError)
+        self.assertIsNone(sys.exception())
+
+    def test_clear_exc_in_generator(self):
+        def inner():
+            self.assertIsNone(sys.exception())
+            yield
+            self.assertIsInstance(sys.exception(), ValueError)
+            sys._set_exception(None)
+            self.assertIsNone(sys.exception())
+            yield
+            self.assertIsNone(sys.exception())
+
+        # with a single exception in exc_info stack
+        g = inner()
+        next(g)
+        try:
+            raise ValueError()
+        except:
+            self.assertIsInstance(sys.exception(), ValueError)
+            next(g)
+            self.assertIsInstance(sys.exception(), ValueError)
+        self.assertIsNone(sys.exception())
+        with self.assertRaises(StopIteration):
+            next(g)
+        self.assertIsNone(sys.exception())
+
+        # with multiple exceptions in exc_info stack by chaining generators
+        def outer():
+            g = inner()
+            self.assertIsNone(sys.exception())
+            yield next(g)
+            self.assertIsInstance(sys.exception(), TypeError)
+            try:
+                raise ValueError()
+            except:
+                self.assertIsInstance(sys.exception(), ValueError)
+                self.assertIsInstance(sys.exception().__context__, TypeError)
+                yield next(g)
+                self.assertIsInstance(sys.exception(), ValueError)
+            self.assertIsInstance(sys.exception(), TypeError)
+
+        g = outer()
+        next(g)
+        try:
+            raise TypeError()
+        except:
+            self.assertIsInstance(sys.exception(), TypeError)
+            next(g)
+            self.assertIsInstance(sys.exception(), TypeError)
+        self.assertIsNone(sys.exception())
+        with self.assertRaises(StopIteration):
+            next(g)
+        self.assertIsNone(sys.exception())
 
 class ExceptHookTest(unittest.TestCase):
 

Misc/NEWS.d/next/Core_and_Builtins/2023-12-19-17-27-08.gh-issue-111375.XeHOl_.rst

@@ -0,0 +1,2 @@
+Add sys._set_exception() function that can set/clear the current exception
+context. Patch by Carey Metcalfe.

Python/clinic/sysmodule.c.h

@@ -122,7 +122,6 @@ _PyErr_GetTopmostException(PyThreadState *tstate)
     {
         exc_info = exc_info->previous_item;
     }
-    assert(!Py_IsNone(exc_info->exc_value));
     return exc_info;
 }
 
@@ -598,7 +597,7 @@ PyErr_GetHandledException(void)
 void
 _PyErr_SetHandledException(PyThreadState *tstate, PyObject *exc)
 {
-    Py_XSETREF(tstate->exc_info->exc_value, Py_XNewRef(exc == Py_None ? NULL : exc));
+    Py_XSETREF(tstate->exc_info->exc_value, Py_XNewRef(exc ? exc : Py_None));
 }
 
 void

Python/errors.c

@@ -853,6 +853,32 @@ sys_exception_impl(PyObject *module)
     Py_RETURN_NONE;
 }
 
+/*[clinic input]
+sys._set_exception
+    exception: object
+
+Set the current exception.
+
+Subsequent calls to sys.exception()/sys.exc_info() will return
+the provided exception until another exception is raised in the
+current thread or the execution stack returns to a frame where
+another exception is being handled.
+[clinic start generated code]*/
+
+static PyObject *
+sys__set_exception_impl(PyObject *module, PyObject *exception)
+/*[clinic end generated code: output=39e119ee6b747085 input=46da3b45313a1cfa]*/
+{
+    if (!Py_IsNone(exception) && !PyExceptionInstance_Check(exception)){
+        PyErr_SetString(
+            PyExc_TypeError,
+            "must be an exception/None"
+        );
+        return NULL;
+    }
+    PyErr_SetHandledException(exception);
+    Py_RETURN_NONE;
+}
 
 /*[clinic input]
 sys.exc_info
@@ -2772,6 +2798,7 @@ static PyMethodDef sys_methods[] = {
     SYS__CURRENT_EXCEPTIONS_METHODDEF
     SYS_DISPLAYHOOK_METHODDEF
     SYS_EXCEPTION_METHODDEF
+    SYS__SET_EXCEPTION_METHODDEF
     SYS_EXC_INFO_METHODDEF
     SYS_EXCEPTHOOK_METHODDEF
     SYS_EXIT_METHODDEF