From 86c93a5b843eeec288b1557369f6f08745ae92ad Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 18 Sep 2026 14:09:09 +0200 Subject: [PATCH 1/6] gh-155742: Use PyBytesWriter in _Py_strhex_impl() (#157734) Replace soft deprecated PyBytes_FromStringAndSize() with PyBytesWriter. Replace also PyBytes_FromStringAndSize(NULL, 0) with Py_GetConstant(Py_CONSTANT_EMPTY_BYTES) in other functions. Replace also PyBytes_FromStringAndSize(NULL, 0) with PyBytes_FromStringAndSize("", 0) in _Py_GetConstant_Init(). --- Modules/_io/bytesio.c | 4 ++-- Modules/_io/winconsoleio.c | 2 +- Modules/_sqlite/blob.c | 2 +- Objects/object.c | 2 +- Python/pystrhex.c | 19 +++++++++++-------- 5 files changed, 16 insertions(+), 13 deletions(-) diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index b487d6d7beca93..b7c1555c2637dd 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -486,7 +486,7 @@ peek_bytes_lock_held(bytesio *self, Py_ssize_t size) is beyond the size of self->buf. Assert above validates size is always in bounds. When self->pos is out of bounds calling code sets size to 0. */ if (size == 0) { - return PyBytes_FromStringAndSize(NULL, 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); } output = PyBytes_AS_STRING(self->buf) + self->pos; @@ -1109,7 +1109,7 @@ bytesio_new(PyTypeObject *type, PyObject *args, PyObject *kwds) /* tp_alloc initializes all the fields to zero. So we don't have to initialize them here. */ - self->buf = PyBytes_FromStringAndSize(NULL, 0); + self->buf = Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); if (self->buf == NULL) { Py_DECREF(self); return PyErr_NoMemory(); diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index bc375e3dfe7de8..11e29227d9276e 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -942,7 +942,7 @@ _io__WindowsConsoleIO_readall_impl(winconsoleio *self) if (len == 0 && _buflen(self) == 0) { /* when the result starts with ^Z we return an empty buffer */ PyMem_Free(buf); - return PyBytes_FromStringAndSize(NULL, 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); } if (len) { diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 43cee9e0f308df..ae318ca19fa0b8 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -447,7 +447,7 @@ subscript_slice(pysqlite_Blob *self, PyObject *item) } if (len == 0) { - return PyBytes_FromStringAndSize(NULL, 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); } if (step == 1) { diff --git a/Objects/object.c b/Objects/object.c index e3f29b71301695..971ac1b7a68669 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -3464,7 +3464,7 @@ _Py_GetConstant_Init(void) constants[Py_CONSTANT_ZERO] = _PyLong_GetZero(); constants[Py_CONSTANT_ONE] = _PyLong_GetOne(); constants[Py_CONSTANT_EMPTY_STR] = PyUnicode_New(0, 0); - constants[Py_CONSTANT_EMPTY_BYTES] = PyBytes_FromStringAndSize(NULL, 0); + constants[Py_CONSTANT_EMPTY_BYTES] = PyBytes_FromStringAndSize("", 0); constants[Py_CONSTANT_EMPTY_TUPLE] = PyTuple_New(0); #ifndef NDEBUG for (size_t i=0; i < Py_ARRAY_LENGTH(constants); i++) { diff --git a/Python/pystrhex.c b/Python/pystrhex.c index 8fb1fa36f85e73..ff6ee830fea7ab 100644 --- a/Python/pystrhex.c +++ b/Python/pystrhex.c @@ -168,15 +168,16 @@ _Py_strhex_impl(const char* argbuf, Py_ssize_t arglen, abs_bytes_per_sep = 0; } - PyObject *retval; + PyObject *retval = NULL; + PyBytesWriter *bytes_writer = NULL; Py_UCS1 *retbuf; if (return_bytes) { /* If _PyBytes_FromSize() were public we could avoid malloc+copy. */ - retval = PyBytes_FromStringAndSize(NULL, resultlen); - if (!retval) { + bytes_writer = PyBytesWriter_Create(resultlen); + if (!bytes_writer) { return NULL; } - retbuf = (Py_UCS1 *)PyBytes_AS_STRING(retval); + retbuf = PyBytesWriter_GetData(bytes_writer); } else { retval = PyUnicode_New(resultlen, 127); @@ -244,13 +245,15 @@ _Py_strhex_impl(const char* argbuf, Py_ssize_t arglen, } } + if (return_bytes) { + return PyBytesWriter_Finish(bytes_writer); + } + else { #ifdef Py_DEBUG - if (!return_bytes) { assert(_PyUnicode_CheckConsistency(retval, 1)); - } #endif - - return retval; + return retval; + } } PyObject * _Py_strhex(const char* argbuf, Py_ssize_t arglen) From c7dbdb5f6be5c6eba382a0aabe66a2e36c884fa1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 18 Sep 2026 15:10:42 +0200 Subject: [PATCH 2/6] gh-155742: Use PyBytesWriter in PySSL_RAND() (#157731) Replace soft deprecated PyBytes_FromStringAndSize() with PyBytesWriter. Remove 'pseudo' parameter of PySSL_RAND(): it's no longer needed since ssl.RAND_pseudo_bytes() has been removed (in 2022, commit d435a18c537a62a89a70005885e6e09f58997d8a). Add a test on ssl.RAND_bytes(0). --- Lib/test/test_ssl.py | 2 ++ Modules/_ssl.c | 23 +++++++++-------------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 37323b7ebc6b1a..abd7710a1d570a 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -469,6 +469,8 @@ def test_random(self): if v: data = ssl.RAND_bytes(16) self.assertEqual(len(data), 16) + + self.assertEqual(ssl.RAND_bytes(0), b'') else: self.assertRaises(ssl.SSLError, ssl.RAND_bytes, 16) diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 9bddb9ce62d5b9..7a780dce967c79 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -6243,10 +6243,9 @@ _ssl_RAND_add_impl(PyObject *module, Py_buffer *view, double entropy) } static PyObject * -PySSL_RAND(PyObject *module, int len, int pseudo) +PySSL_RAND(PyObject *module, int len) { int ok; - PyObject *bytes; unsigned long err; const char *errstr; PyObject *v; @@ -6256,20 +6255,16 @@ PySSL_RAND(PyObject *module, int len, int pseudo) return NULL; } - bytes = PyBytes_FromStringAndSize(NULL, len); - if (bytes == NULL) + PyBytesWriter *writer = PyBytesWriter_Create(len); + if (writer == NULL) { return NULL; - if (pseudo) { - ok = RAND_bytes((unsigned char*)PyBytes_AS_STRING(bytes), len); - if (ok == 0 || ok == 1) - return Py_BuildValue("NO", bytes, ok == 1 ? Py_True : Py_False); } - else { - ok = RAND_bytes((unsigned char*)PyBytes_AS_STRING(bytes), len); - if (ok == 1) - return bytes; + + ok = RAND_bytes(PyBytesWriter_GetData(writer), len); + if (ok == 1) { + return PyBytesWriter_Finish(writer); } - Py_DECREF(bytes); + PyBytesWriter_Discard(writer); err = ERR_get_error(); errstr = ERR_reason_error_string(err); @@ -6294,7 +6289,7 @@ static PyObject * _ssl_RAND_bytes_impl(PyObject *module, int n) /*[clinic end generated code: output=977da635e4838bc7 input=2e78ce1e86336776]*/ { - return PySSL_RAND(module, n, 0); + return PySSL_RAND(module, n); } From c0bc6fad254479629bbe5cf523a69263cbd28d49 Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Fri, 18 Sep 2026 18:16:57 +0300 Subject: [PATCH 3/6] gh-156865: Correctly detect overflows for array's "e", "f" and "Zf" type codes (#156869) --- Lib/test/test_array.py | 13 +++++++++ ...-09-03-04-51-46.gh-issue-156864.Pbe7Tl.rst | 2 ++ Modules/arraymodule.c | 29 ++++++++++++------- 3 files changed, 34 insertions(+), 10 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-09-03-04-51-46.gh-issue-156864.Pbe7Tl.rst diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index c931be6df5fdaa..ba9c25c835bc39 100755 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -1607,10 +1607,19 @@ class HalfFloatTest(FPTest, unittest.TestCase): typecode = 'e' minitemsize = 2 + def test_overflows(self): + # Overflows half-float type: + self.assertRaises(OverflowError, array.array, self.typecode, [123456]) + # Overflows also float type: + self.assertRaises(OverflowError, array.array, self.typecode, [1e300]) + class FloatTest(FPTest, unittest.TestCase): typecode = 'f' minitemsize = 4 + def test_overflows(self): + self.assertRaises(OverflowError, array.array, self.typecode, [1e300]) + class DoubleTest(FPTest, unittest.TestCase): typecode = 'd' minitemsize = 8 @@ -1637,6 +1646,10 @@ class ComplexFloatTest(CFPTest, unittest.TestCase): typecode = 'Zf' minitemsize = 8 + def test_overflows(self): + self.assertRaises(OverflowError, array.array, self.typecode, [1e300]) + self.assertRaises(OverflowError, array.array, self.typecode, [1e300j]) + class ComplexDoubleTest(CFPTest, unittest.TestCase): typecode = 'Zd' minitemsize = 16 diff --git a/Misc/NEWS.d/next/Library/2026-09-03-04-51-46.gh-issue-156864.Pbe7Tl.rst b/Misc/NEWS.d/next/Library/2026-09-03-04-51-46.gh-issue-156864.Pbe7Tl.rst new file mode 100644 index 00000000000000..ee89b830d3d486 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-09-03-04-51-46.gh-issue-156864.Pbe7Tl.rst @@ -0,0 +1,2 @@ +:func:`array.array` setter now correctly detects overflows for the ``'e'``, +``'f'`` and ``'Zf'`` type codes. Patch by Sergey B Kirpichev. diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index a0181c083a6036..ef492d143d5644 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -584,8 +584,8 @@ e_getitem(arrayobject *ap, Py_ssize_t i) static int e_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) { - float x; - if (!PyArg_Parse(v, "f;array item must be float", &x)) { + double x; + if (!PyArg_Parse(v, "d;array item must be float", &x)) { return -1; } @@ -607,14 +607,16 @@ f_getitem(arrayobject *ap, Py_ssize_t i) static int f_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) { - float x; - if (!PyArg_Parse(v, "f;array item must be float", &x)) + double x; + if (!PyArg_Parse(v, "d;array item must be float", &x)) return -1; CHECK_ARRAY_BOUNDS(ap, i); - if (i >= 0) - ((float *)ap->ob_item)[i] = x; + if (i >= 0) { + return PyFloat_Pack4(x, ap->ob_item + sizeof(float)*i, + PY_LITTLE_ENDIAN); + } return 0; } @@ -651,7 +653,6 @@ static int cf_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) { Py_complex x; - float f[2]; if (!PyArg_Parse(v, "D;array item must be complex", &x)) { return -1; @@ -659,10 +660,18 @@ cf_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) CHECK_ARRAY_BOUNDS(ap, i); - f[0] = (float)x.real; - f[1] = (float)x.imag; if (i >= 0) { - memcpy(ap->ob_item + i*sizeof(f), &f, sizeof(f)); + char f[8]; + int ret = PyFloat_Pack4(x.real, f, PY_LITTLE_ENDIAN); + + if (ret) { + return ret; + } + ret = PyFloat_Pack4(x.imag, f + sizeof(float), PY_LITTLE_ENDIAN); + if (!ret) { + memcpy(ap->ob_item + i*sizeof(f), &f, sizeof(f)); + } + return ret; } return 0; } From b62e0286858fcd9345b8ec40f490754a306a2186 Mon Sep 17 00:00:00 2001 From: Brittany Reynoso Date: Fri, 18 Sep 2026 12:06:44 -0400 Subject: [PATCH 4/6] gh-149640: Add new GitHub action to test lazy_imports=all against test suite (#151105) * Add new github action to test lazy imports all against stdlib. * Adjust GH Action naming to better match existing checks * Address comments for reusability * Fix double typo + add flaky module * Update test name * Remove concurrency configeration for reusable-test-lazy-imports-all.yml * bikeshed renames * Take a swing at adding exclusion checks * remove allegedly passing modules? * clean up * Fix bug with env var not flowing through and bring back exclusions * Accidentally added random files * Add more modules * Address incorrect exclusions * Address feedback: update ubuntu version and CODEOWNERS file * Prune test_idle and test_zoneinfo from the exclusion list * Address comments. Variety of small nits and cleanups. * Remove accidental file and remove myself from codeowners :( * Deleting other random empty files * Minor change to trigger tests again * Minor change to trigger tests again 2 --- .github/CODEOWNERS | 10 ++- .github/workflows/build.yml | 8 ++ .../reusable-test-lazy-imports-all.yml | 79 +++++++++++++++++++ Lib/test/lazy_imports_all_exclude.txt | 40 ++++++++++ 4 files changed, 134 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/reusable-test-lazy-imports-all.yml create mode 100644 Lib/test/lazy_imports_all_exclude.txt diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d12eaee2adb396..0a0100fcb2ebb5 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -290,9 +290,10 @@ Tools/jit/ @brandtbucher @savannahostrowski @diegorusso InternalDocs/jit.md @brandtbucher @savannahostrowski @diegorusso @AA-Turner # Lazy imports (PEP 810) -Objects/lazyimportobject.c @yhg1s @DinoV @pablogsal -Include/internal/pycore_lazyimportobject.h @yhg1s @DinoV @pablogsal -Lib/test/test_lazy_import @yhg1s @DinoV @pablogsal +.github/workflows/reusable-test-lazy-imports-all.yml @yhg1s @DinoV @pablogsal +Objects/lazyimportobject.c @yhg1s @DinoV @pablogsal +Include/internal/pycore_lazyimportobject.h @yhg1s @DinoV @pablogsal +Lib/test/test_lazy_import @yhg1s @DinoV @pablogsal # Micro-op / μop / Tier 2 Optimiser Python/optimizer.c @markshannon @Fidget-Spinner @@ -655,5 +656,8 @@ Objects/**/clinic/ PC/**/clinic/ Python/**/clinic/ +# Exclude Lazy Imports=all CI carve out file +Lib/test/lazy_imports_all_exclude.txt + # Exclude HTML IDs list Doc/tools/removed-ids.txt diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7fdc81ae0ade1d..e11e6aa6b6d3dc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -476,6 +476,12 @@ jobs: name: hypothesis-example-db path: ${{ env.CPYTHON_BUILDDIR }}/.hypothesis/examples/ + test-lazy-imports-all: + name: 'Lazy imports enabled' + needs: build-context + if: fromJSON(needs.build-context.outputs.run-tests) + uses: ./.github/workflows/reusable-test-lazy-imports-all.yml + build-asan: name: 'Address sanitizer' runs-on: ${{ matrix.os }} @@ -648,6 +654,7 @@ jobs: - build-emscripten - build-wasi - test-hypothesis + - test-lazy-imports-all - build-asan - build-san - cross-build-linux @@ -705,4 +712,5 @@ jobs: ${{ !fromJSON(needs.build-context.outputs.run-ios) && 'build-ios,' || '' }} ${{ !fromJSON(needs.build-context.outputs.run-emscripten) && 'build-emscripten,' || '' }} ${{ !fromJSON(needs.build-context.outputs.run-wasi) && 'build-wasi,' || '' }} + ${{ !fromJSON(needs.build-context.outputs.run-tests) && 'test-lazy-imports-all,' || '' }} jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/reusable-test-lazy-imports-all.yml b/.github/workflows/reusable-test-lazy-imports-all.yml new file mode 100644 index 00000000000000..3754308e89cad4 --- /dev/null +++ b/.github/workflows/reusable-test-lazy-imports-all.yml @@ -0,0 +1,79 @@ +name: Reusable Lazy Imports Tests + +# Run the CPython test suite with global lazy imports forced on +# (``-X lazy_imports=all``). +# +# Modules that are known to fail under lazy imports are listed in +# Lib/test/lazy_imports_all_exclude.txt and skipped here. Remove entries from +# that file as the modules are fixed so this workflow starts guarding them +# against regressions. Excluded modules are also checked separately so the +# workflow fails when one starts passing and its exclusion should be removed. + +on: + workflow_call: + +permissions: + contents: read + +env: + FORCE_COLOR: 1 + +jobs: + test-lazy-imports-all: + name: 'Run Tests with lazy_imports=all' + runs-on: ubuntu-26.04 + timeout-minutes: 60 + env: + EXCLUDE_FILE: Lib/test/lazy_imports_all_exclude.txt + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - name: Register gcc problem matcher + run: echo "::add-matcher::.github/problem-matchers/gcc.json" + - name: Install dependencies + run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Configure CPython + run: ./configure --config-cache --with-pydebug + - name: Build CPython + run: make -j4 + - name: Display build info + run: make pythoninfo + - name: Verify lazy imports are fully enabled + run: ./python -X lazy_imports=all -c "import sys; assert sys.flags.lazy_imports == 1, sys.flags.lazy_imports; print('lazy imports all enabled')" + - name: Build test list (all tests minus the known-failing exclusions) + run: | + set -euo pipefail + ./python -m test --list-tests > all_tests.txt + # Strip comments/blank lines from the exclusion file, then drop those + # exact test names (whole-line, fixed-string match) from the run list. + grep -vE '^\s*(#.*)?$' "$EXCLUDE_FILE" > exclude_tests.txt || true + grep -vxF -f exclude_tests.txt all_tests.txt > run_tests.txt + # Fail loudly if any exclusion entry matched nothing: a stale or + # mistyped name (or a change in `--list-tests` output) would otherwise + # silently stop excluding a module and let it fail the run. + stale=$(comm -23 <(sort -u exclude_tests.txt) <(sort -u all_tests.txt)) + if [ -n "$stale" ]; then + echo "::error::Stale entries in $EXCLUDE_FILE (no longer match 'python -m test --list-tests'); remove or fix them:" + echo "$stale" + exit 1 + fi + echo "Excluding $(wc -l < exclude_tests.txt) module(s); running $(wc -l < run_tests.txt) of $(wc -l < all_tests.txt)." + - name: Run tests with lazy imports + run: xvfb-run xargs -a run_tests.txt ./python -X lazy_imports=all -m test --fast-ci --timeout=900 < /dev/null + - name: Verify excluded tests still need exclusion + run: | + set -euo pipefail + unexpected_passes=() + while IFS= read -r test_name; do + [ -n "$test_name" ] || continue + echo "Checking excluded test: $test_name" + if xvfb-run ./python -X lazy_imports=all -m test --fast-ci --timeout=900 "$test_name"; then + unexpected_passes+=("$test_name") + fi + done < exclude_tests.txt + if [ "${#unexpected_passes[@]}" -ne 0 ]; then + echo "::error::These tests still appear in $EXCLUDE_FILE but now pass with -X lazy_imports=all. Remove them from the exclude file:" + printf '%s\n' "${unexpected_passes[@]}" + exit 1 + fi diff --git a/Lib/test/lazy_imports_all_exclude.txt b/Lib/test/lazy_imports_all_exclude.txt new file mode 100644 index 00000000000000..2680d3b1e4357b --- /dev/null +++ b/Lib/test/lazy_imports_all_exclude.txt @@ -0,0 +1,40 @@ +# Test modules that currently FAIL under global lazy imports +# (``-X lazy_imports=all`` / ``PYTHON_LAZY_IMPORTS=all``). +# +# The "Lazy Imports All" CI workflow +# (.github/workflows/reusable-test-lazy-imports-all.yml) runs the whole test +# suite with lazy_imports=all, skipping every module listed here. Exclusion is +# whole-module: a listed module is skipped entirely, so any passing tests it +# contains are not covered until its line is removed. As each module is fixed, +# delete its line so the workflow starts guarding it against regressions. The +# workflow also checks listed modules separately and fails if one now passes, +# so accidental fixes prompt cleanup of this file. +# +# Format: one test name per line, exactly as printed by +# ``python -m test --list-tests``. Lines starting with ``#`` and blank lines +# are ignored. Note that split test packages use a dotted path +# (e.g. test.test_future_stmt.test_future) while ordinary modules use the bare +# name (e.g. test_builtin). + +test.test_inspect.test_inspect +test___all__ +test__interpreters +test_builtin +test_clinic +test_crossinterp +test_datetime +test_generated_cases +test_heapq +test_import +test_importlib +test_json +test_pkg +test_profile +test_profiling +test_pyrepl +test_subprocess +test_symtable +test_tools +test_trace +test_type_annotations +test_unittest From 92db1519aa089de78da70535ae6e4680050c0b5f Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 18 Sep 2026 20:03:25 +0300 Subject: [PATCH 5/6] gh-86427: Determine the stdio encoding for every standard stream on Windows (GH-155416) It was the ANSI code page instead of the encoding of the device the stream is connected to, as in 3.7. The stdio encoding is now left undefined in this mode and determined for every standard stream. --- Lib/test/test_cmd_line.py | 9 +++-- ...09-05-30-00.gh-issue-86427.legacystdio.rst | 3 ++ ...8-09-07-00-00.gh-issue-86427.consolecp.rst | 3 -- Objects/unicodeobject.c | 4 ++ Python/initconfig.c | 39 +++++++------------ Python/pylifecycle.c | 13 ++++++- 6 files changed, 37 insertions(+), 34 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2026-08-09-05-30-00.gh-issue-86427.legacystdio.rst delete mode 100644 Misc/NEWS.d/next/Windows/2026-08-09-07-00-00.gh-issue-86427.consolecp.rst diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 2200770b4a45f0..4c1abb15c0cb14 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -1069,8 +1069,9 @@ def test_python_legacy_windows_stdio(self): @unittest.skipUnless(support.MS_WINDOWS, 'Test only applicable on Windows') def test_python_legacy_windows_stdio_encoding(self): - # gh-86427: In the legacy mode the encoding of the standard streams - # is the encoding of the console. + # gh-86427: In the legacy mode the encoding of a standard stream is + # the encoding of the console it is connected to, which can differ + # for input and output. import ctypes kernel32 = ctypes.WinDLL('kernel32', use_last_error=True) try: @@ -1080,7 +1081,7 @@ def test_python_legacy_windows_stdio_encoding(self): # We cannot use PIPE, because the standard streams should be # connected to the console. So we use the exit code. code = ("import sys; sys.exit(sys.stdin.encoding != 'cp850' or " - "sys.stdout.encoding != 'cp850')") + "sys.stdout.encoding != 'cp437')") env = os.environ.copy() env['PYTHONLEGACYWINDOWSSTDIO'] = '1' env['PYTHONUTF8'] = '0' @@ -1091,7 +1092,7 @@ def test_python_legacy_windows_stdio_encoding(self): try: if not kernel32.SetConsoleCP(850): self.skipTest('cannot set the console input code page') - if not kernel32.SetConsoleOutputCP(850): + if not kernel32.SetConsoleOutputCP(437): self.skipTest('cannot set the console output code page') proc = subprocess.run([sys.executable, '-c', code], env=env, stdin=fin, stdout=fout, diff --git a/Misc/NEWS.d/next/Windows/2026-08-09-05-30-00.gh-issue-86427.legacystdio.rst b/Misc/NEWS.d/next/Windows/2026-08-09-05-30-00.gh-issue-86427.legacystdio.rst new file mode 100644 index 00000000000000..c3e2332defabfa --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2026-08-09-05-30-00.gh-issue-86427.legacystdio.rst @@ -0,0 +1,3 @@ +Fix the encoding of the standard streams in the legacy Windows stdio mode +(:envvar:`PYTHONLEGACYWINDOWSSTDIO`). It is now the encoding of the device +the stream is connected to, as in Python 3.7, not the ANSI code page. diff --git a/Misc/NEWS.d/next/Windows/2026-08-09-07-00-00.gh-issue-86427.consolecp.rst b/Misc/NEWS.d/next/Windows/2026-08-09-07-00-00.gh-issue-86427.consolecp.rst deleted file mode 100644 index 1cbf188c12e407..00000000000000 --- a/Misc/NEWS.d/next/Windows/2026-08-09-07-00-00.gh-issue-86427.consolecp.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix the encoding of the standard streams in the legacy Windows stdio mode -(:envvar:`PYTHONLEGACYWINDOWSSTDIO`). It is now the code page of the -console, as in Python 3.7, not the ANSI code page. diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 7ee9d17b7f77ff..d2faa28722b0e0 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -15229,6 +15229,10 @@ init_stdio_encoding(PyInterpreterState *interp) { /* Update the stdio encoding to the normalized Python codec name. */ PyConfig *config = (PyConfig*)_PyInterpreterState_GetConfig(interp); + if (config->stdio_encoding == NULL) { + /* gh-86427: The encoding is determined for every stream. */ + return _PyStatus_OK(); + } if (config_get_codec_name(&config->stdio_encoding) < 0) { return _PyStatus_ERR("failed to get the Python codec name " "of the stdio encoding"); diff --git a/Python/initconfig.c b/Python/initconfig.c index 69d47a5872f5e3..49f1beb37bb920 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -196,7 +196,7 @@ static const PyConfigSpec PYCONFIG_SPEC[] = { SPEC(show_ref_count, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), SPEC(site_import, BOOL, READ_ONLY, NO_SYS, GLOBAL(&Py_NoSiteFlag, 1)), // sys.flags.no_site SPEC(skip_source_first_line, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), - SPEC(stdio_encoding, WSTR, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(stdio_encoding, WSTR_OPT, READ_ONLY, NO_SYS, NO_GLOBAL), SPEC(stdio_errors, WSTR, READ_ONLY, NO_SYS, NO_GLOBAL), SPEC(tracemalloc, UINT, READ_ONLY, NO_SYS, NO_GLOBAL), SPEC(use_frozen_modules, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), @@ -1074,11 +1074,14 @@ config_check_consistency(const PyConfig *config) assert(config->module_search_paths_set >= 0); assert(config->filesystem_encoding != NULL); assert(config->filesystem_errors != NULL); - assert(config->stdio_encoding != NULL); - assert(config->stdio_errors != NULL); #ifdef MS_WINDOWS + /* stdio_encoding can be NULL in the legacy Windows stdio mode. */ + assert(config->stdio_encoding != NULL || config->legacy_windows_stdio); assert(config->legacy_windows_stdio >= 0); +#else + assert(config->stdio_encoding != NULL); #endif + assert(config->stdio_errors != NULL); /* -c and -m options are exclusive */ assert(!(config->run_command != NULL && config->run_module != NULL)); assert(config->check_hash_pycs_mode != NULL); @@ -2709,31 +2712,15 @@ config_init_stdio_encoding(PyConfig *config, } /* Choose the default error handler based on the current locale. */ - if (config->stdio_encoding == NULL) { + if (config->stdio_encoding == NULL #ifdef MS_WINDOWS - /* gh-86427: use the console code page. Only one encoding can be - specified, so the output code page is used: it affects two - streams of three. */ - UINT cp = config->legacy_windows_stdio ? GetConsoleOutputCP() : 0; - if (cp != 0) { - if (cp == CP_UTF8) { - status = PyConfig_SetString(config, &config->stdio_encoding, - L"utf-8"); - } - else { - wchar_t encoding[20]; - swprintf(encoding, Py_ARRAY_LENGTH(encoding), L"cp%u", - (unsigned int)cp); - status = PyConfig_SetString(config, &config->stdio_encoding, - encoding); - } - } - else + /* gh-86427: it is determined for each stream: create_stdio() uses + _Py_device_encoding(), falling back to the locale encoding. */ + && !config->legacy_windows_stdio #endif - { - status = config_get_locale_encoding(config, preconfig, - &config->stdio_encoding); - } + ) { + status = config_get_locale_encoding(config, preconfig, + &config->stdio_encoding); if (_PyStatus_EXCEPTION(status)) { return status; } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 6074d4439531f4..a9c98d73fd0f72 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -3080,7 +3080,18 @@ create_stdio(const PyConfig *config, PyObject* io, newline = "\n"; #endif - PyObject *encoding_str = PyUnicode_FromWideChar(encoding, -1); + PyObject *encoding_str; + if (encoding != NULL) { + encoding_str = PyUnicode_FromWideChar(encoding, -1); + } + else { + /* gh-86427: use the encoding of the device. */ + encoding_str = _Py_device_encoding(fd); + if (encoding_str == Py_None) { + Py_DECREF(encoding_str); + encoding_str = _Py_GetLocaleEncodingObject(); + } + } if (encoding_str == NULL) { Py_CLEAR(buf); goto error; From 76f22f9cf2f8a822f1c428ffdcb060b914e66b67 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 18 Sep 2026 20:14:25 +0300 Subject: [PATCH 6/6] gh-156942: Raise the exception where the marshalling error is detected (GH-156944) Previously the marshal writer recorded an error code and converted it into an exception at the end, replacing the exception which was already raised with ValueError("unmarshallable object"). Error messages now name the type of the unsupported object and the required version. Co-Authored-By: Claude Opus 5 (1M context) --- Lib/test/test_capi/test_marshal.py | 4 +- Lib/test/test_marshal.py | 87 +++++++++--- ...-09-04-16-12-30.gh-issue-156942.Qk7Rw2.rst | 4 + Python/marshal.c | 132 +++++++----------- 4 files changed, 125 insertions(+), 102 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-09-04-16-12-30.gh-issue-156942.Qk7Rw2.rst diff --git a/Lib/test/test_capi/test_marshal.py b/Lib/test/test_capi/test_marshal.py index 972ff4ed53d687..4680c45f32dd4c 100644 --- a/Lib/test/test_capi/test_marshal.py +++ b/Lib/test/test_capi/test_marshal.py @@ -125,7 +125,7 @@ def test_write_object_to_file(self): with self.assertRaises(SystemError): write_object_to_file(NULL, filename, version) - with self.assertRaisesRegex(ValueError, 'unmarshallable object'): + with self.assertRaisesRegex(ValueError, 'cannot marshal object objects'): write_object_to_file(UNMARSHALLABLE, filename, version) def test_read_short_from_file(self): @@ -225,7 +225,7 @@ def test_pymarshal_writeobjecttostring(self): obj2 = marshal.loads(data) self.check_object(obj2, obj) - with self.assertRaisesRegex(ValueError, 'unmarshallable object'): + with self.assertRaisesRegex(ValueError, 'cannot marshal object objects'): writeobjecttostring(UNMARSHALLABLE, version) with self.assertRaises(SystemError): diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index d7db3d480ff1e2..449704d1c347b6 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -162,14 +162,16 @@ def test_no_allow_code(self): data = {'a': [({co, 0},)]} dump = marshal.dumps(data, allow_code=True) self.assertEqual(marshal.loads(dump, allow_code=True), data) - with self.assertRaises(ValueError): + with self.assertRaisesRegex(ValueError, + 'marshalling code objects is disallowed'): marshal.dumps(data, allow_code=False) with self.assertRaises(ValueError): marshal.loads(dump, allow_code=False) marshal.dump(data, io.BytesIO(), allow_code=True) self.assertEqual(marshal.load(io.BytesIO(dump), allow_code=True), data) - with self.assertRaises(ValueError): + with self.assertRaisesRegex(ValueError, + 'marshalling code objects is disallowed'): marshal.dump(data, io.BytesIO(), allow_code=False) with self.assertRaises(ValueError): marshal.load(io.BytesIO(dump), allow_code=False) @@ -347,16 +349,29 @@ def test_reference_loop_dict(self): self.assertIsInstance(b, dict) self.assertIs(b[None], b) + def check_reference_loop(self, a, typename, minversion, + oldmsg='object too deeply nested to marshal'): + # Only versions supporting references to the type detect the loop; + # older versions fail for a different reason. + for v in range(minversion): + with self.subTest(version=v): + with self.assertRaisesRegex(ValueError, oldmsg): + marshal.dumps(a, v) + for v in range(minversion, marshal.version + 1): + with self.subTest(version=v): + with self.assertRaisesRegex( + ValueError, + f'cannot marshal recursion {typename} objects'): + marshal.dumps(a, v) + def test_reference_loop_tuple(self): a = ([],) a[0].append(a) - for v in range(marshal.version + 1): - self.assertRaises(ValueError, marshal.dumps, a, v) + self.check_reference_loop(a, 'tuple', 3) a = ({},) a[0][None] = a - for v in range(marshal.version + 1): - self.assertRaises(ValueError, marshal.dumps, a, v) + self.check_reference_loop(a, 'tuple', 3) def test_shared_reference_tuple(self): # A tuple referenced more than once still round-trips with the @@ -381,30 +396,28 @@ def f(): # so we need to break the loop manually. See gh-148722. self.addCleanup(a.clear) a.append(code) - for v in range(marshal.version + 1): - self.assertRaises(ValueError, marshal.dumps, code, v) + self.check_reference_loop(code, 'code', 3) def test_reference_loop_slice(self): + oldmsg = 'marshalling slice objects requires version 5 or higher' a = slice([], None) a.start.append(a) - for v in range(marshal.version + 1): - self.assertRaises(ValueError, marshal.dumps, a, v) + self.check_reference_loop(a, 'slice', 5, oldmsg) a = slice(None, []) a.stop.append(a) - for v in range(marshal.version + 1): - self.assertRaises(ValueError, marshal.dumps, a, v) + self.check_reference_loop(a, 'slice', 5, oldmsg) a = slice(None, None, []) a.step.append(a) - for v in range(marshal.version + 1): - self.assertRaises(ValueError, marshal.dumps, a, v) + self.check_reference_loop(a, 'slice', 5, oldmsg) def test_reference_loop_frozendict(self): a = frozendict({None: []}) a[None].append(a) - for v in range(marshal.version + 1): - self.assertRaises(ValueError, marshal.dumps, a, v) + self.check_reference_loop( + a, 'frozendict', 6, + 'marshalling frozendict objects requires version 6 or higher') def test_shared_reference_frozendict(self): # A frozendict referenced more than once must round-trip with the @@ -475,7 +488,9 @@ def test_exact_type_match(self): # Note: str subclasses are not tested because they get handled # by marshal's routines for objects supporting the buffer API. subtyp = type('subtyp', (typ,), {}) - self.assertRaises(ValueError, marshal.dumps, subtyp()) + with self.assertRaisesRegex(ValueError, + r'cannot marshal \S*subtyp objects'): + marshal.dumps(subtyp()) # Issue #1792 introduced a change in how marshal increases the size of its # internal buffer; this test ensures that the new code is exercised. @@ -578,9 +593,25 @@ def test_unmarshallable(self): ('code', code)) for name, arg in cases: with self.subTest(name, arg=arg): - with self.assertRaisesRegex(ValueError, "unmarshallable object"): + with self.assertRaisesRegex(ValueError, + "cannot marshal type objects"): marshal.dumps((arg, memoryview(b''))) + def test_error_in_set_item(self): + # Set items are sorted by their marshalled representation, and NaNs + # are only distinguished by identity, so they are compared as + # complex numbers. + nan = float('nan') + with self.assertRaisesRegex(TypeError, "'<' not supported"): + marshal.dumps({complex(nan, 0), complex(nan, 0)}) + + def test_error_in_buffer(self): + # The BufferError raised for a non-contiguous buffer is not replaced + # with a generic error. + step2 = slice(None, None, 2) + with self.assertRaises(BufferError): + marshal.dumps(memoryview(bytearray(b'abcdef'))[step2]) + LARGE_SIZE = 2**31 pointer_size = 8 if sys.maxsize > 0xFFFFFFFF else 4 @@ -591,8 +622,14 @@ def write(self, s): @unittest.skipIf(LARGE_SIZE > sys.maxsize, "test cannot run on 32-bit systems") class LargeValuesTestCase(unittest.TestCase): - def check_unmarshallable(self, data): - self.assertRaises(ValueError, marshal.dump, data, NullWriter()) + def check_unmarshallable(self, data, msg='object too large to marshal'): + with self.assertRaisesRegex(ValueError, msg): + marshal.dump(data, NullWriter()) + + @support.bigmemtest(size=LARGE_SIZE, memuse=4, dry_run=False) + def test_int(self, size): + # An int with more than SIZE32_MAX 15-bit digits. + self.check_unmarshallable(1 << (15 * size), 'int too large to marshal') @support.bigmemtest(size=LARGE_SIZE, memuse=2, dry_run=False) def test_bytes(self, size): @@ -725,7 +762,10 @@ def testFrozenDict(self): self.helper(dictobj) for version in range(6): - with self.assertRaises(ValueError): + with self.assertRaisesRegex( + ValueError, + 'marshalling frozendict objects requires ' + 'version 6 or higher'): marshal.dumps(dictobj, version) def testModule(self): @@ -794,7 +834,10 @@ def test_slice(self): self.helper(obj) for version in range(5): - with self.assertRaises(ValueError): + with self.assertRaisesRegex( + ValueError, + 'marshalling slice objects requires ' + 'version 5 or higher'): marshal.dumps(obj, version) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-04-16-12-30.gh-issue-156942.Qk7Rw2.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-04-16-12-30.gh-issue-156942.Qk7Rw2.rst new file mode 100644 index 00000000000000..24bbeef0461efb --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-04-16-12-30.gh-issue-156942.Qk7Rw2.rst @@ -0,0 +1,4 @@ +:mod:`marshal` no longer replaces the exception raised while marshalling the +value with a generic ``ValueError("unmarshallable object")``. Error messages +now name the type of the unsupported object and, for the types supported only +by newer data formats, the required version. diff --git a/Python/marshal.c b/Python/marshal.c index 420c3ee115a737..6aa99bb4693117 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -100,17 +100,9 @@ module marshal #define FLAG_REF '\x80' /* with a type, add obj to index */ -// Error codes: -#define WFERR_OK 0 -#define WFERR_UNMARSHALLABLE 1 -#define WFERR_NESTEDTOODEEP 2 -#define WFERR_NOMEMORY 3 -#define WFERR_CODE_NOT_ALLOWED 4 -#define WFERR_EXCEPTION_SET 5 /* An exception has already been raised. */ - typedef struct { FILE *fp; - int error; /* see WFERR_* values */ + bool error; /* An exception has been raised. */ int depth; PyObject *str; char *ptr; @@ -132,10 +124,10 @@ static void w_file_error(WFILE *p) { int saved_errno = errno; - if (p->error != WFERR_OK) { + if (p->error) { return; } - p->error = WFERR_EXCEPTION_SET; + p->error = true; if (PyErr_CheckSignals()) { /* The signal handler has raised an exception. */ return; @@ -174,12 +166,14 @@ w_reserve(WFILE *p, Py_ssize_t needed) delta = size + 1024; delta = Py_MAX(delta, needed); if (delta > PY_SSIZE_T_MAX - size) { - p->error = WFERR_NOMEMORY; + PyErr_NoMemory(); + p->error = true; return 0; } size += delta; if (_PyBytes_Resize(&p->str, size) != 0) { p->end = p->ptr = p->buf = NULL; + p->error = true; return 0; } else { @@ -236,13 +230,15 @@ w_long(long x, WFILE *p) #define SIZE32_MAX 0x7FFFFFFF #if SIZEOF_SIZE_T > 4 -# define W_SIZE(n, p) do { \ - if ((n) > SIZE32_MAX) { \ - (p)->depth--; \ - (p)->error = WFERR_UNMARSHALLABLE; \ - return; \ - } \ - w_long((long)(n), p); \ +# define W_SIZE(n, p) do { \ + if ((n) > SIZE32_MAX) { \ + (p)->depth--; \ + PyErr_SetString(PyExc_ValueError, \ + "object too large to marshal"); \ + (p)->error = true; \ + return; \ + } \ + w_long((long)(n), p); \ } while(0) #else # define W_SIZE w_long @@ -295,7 +291,8 @@ _r_digits##bitsize(const uint ## bitsize ## _t *digits, Py_ssize_t n, \ } while (d != 0); \ if (l > SIZE32_MAX) { \ p->depth--; \ - p->error = WFERR_UNMARSHALLABLE; \ + PyErr_SetString(PyExc_ValueError, "int too large to marshal"); \ + p->error = true; \ return; \ } \ w_long((long)(negative ? -l : l), p); \ @@ -331,7 +328,7 @@ w_PyLong(const PyLongObject *ob, char flag, WFILE *p) if (PyLong_Export((PyObject *)ob, &long_export) < 0) { p->depth--; - p->error = WFERR_UNMARSHALLABLE; + p->error = true; return; } if (!long_export.digits) { @@ -384,7 +381,7 @@ w_float_bin(double v, WFILE *p) { char buf[8]; if (PyFloat_Pack8(v, buf, 1) < 0) { - p->error = WFERR_UNMARSHALLABLE; + p->error = true; return; } w_string(buf, 8, p); @@ -395,7 +392,7 @@ w_float_str(double v, WFILE *p) { char *buf = PyOS_double_to_string(v, 'g', 17, 0, NULL); if (!buf) { - p->error = WFERR_NOMEMORY; + p->error = true; return; } w_short_pstring(buf, strlen(buf), p); @@ -449,13 +446,14 @@ w_ref(PyObject *v, char *flag, WFILE *p) if (_Py_hashtable_set(p->hashtable, Py_NewRef(v), (void *)(uintptr_t)w) < 0) { Py_DECREF(v); + PyErr_NoMemory(); goto err; } *flag |= FLAG_REF; return 0; } err: - p->error = WFERR_UNMARSHALLABLE; + p->error = true; return 1; } @@ -488,14 +486,16 @@ w_object(PyObject *v, WFILE *p) { char flag = '\0'; - if (p->error != WFERR_OK) { + if (p->error) { return; } p->depth++; if (p->depth > MAX_MARSHAL_STACK_DEPTH) { - p->error = WFERR_NESTEDTOODEEP; + PyErr_SetString(PyExc_ValueError, + "object too deeply nested to marshal"); + p->error = true; } else if (v == NULL) { w_byte(TYPE_NULL, p); @@ -598,7 +598,7 @@ w_complex_object(PyObject *v, char flag, WFILE *p) utf8 = PyUnicode_AsEncodedString(v, "utf8", "surrogatepass"); if (utf8 == NULL) { p->depth--; - p->error = WFERR_UNMARSHALLABLE; + p->error = true; return; } if (p->version >= 3 && PyUnicode_CHECK_INTERNED(v)) @@ -638,7 +638,10 @@ w_complex_object(PyObject *v, char flag, WFILE *p) if (PyFrozenDict_CheckExact(v)) { if (p->version < 6) { w_byte(TYPE_UNKNOWN, p); - p->error = WFERR_UNMARSHALLABLE; + PyErr_Format(PyExc_ValueError, + "marshalling %T objects requires version 6 " + "or higher", v); + p->error = true; return; } @@ -675,7 +678,7 @@ w_complex_object(PyObject *v, char flag, WFILE *p) // use an order equivalent to sorted(v, key=marshal.dumps): PyObject *pairs = PyList_New(n); if (pairs == NULL) { - p->error = WFERR_NOMEMORY; + p->error = true; return; } Py_ssize_t i = 0; @@ -684,25 +687,25 @@ w_complex_object(PyObject *v, char flag, WFILE *p) PyObject *dump = _PyMarshal_WriteObjectToString(value, p->version, p->allow_code); if (dump == NULL) { - p->error = WFERR_UNMARSHALLABLE; + p->error = true; Py_DECREF(value); break; } PyObject *pair = _PyTuple_FromPairSteal(dump, value); if (pair == NULL) { - p->error = WFERR_NOMEMORY; + p->error = true; break; } PyList_SET_ITEM(pairs, i++, pair); } Py_END_CRITICAL_SECTION(); - if (p->error == WFERR_UNMARSHALLABLE || p->error == WFERR_NOMEMORY) { + if (p->error) { Py_DECREF(pairs); return; } assert(i == n); if (PyList_Sort(pairs)) { - p->error = WFERR_NOMEMORY; + p->error = true; Py_DECREF(pairs); return; } @@ -715,13 +718,15 @@ w_complex_object(PyObject *v, char flag, WFILE *p) } else if (PyCode_Check(v)) { if (!p->allow_code) { - p->error = WFERR_CODE_NOT_ALLOWED; + PyErr_SetString(PyExc_ValueError, + "marshalling code objects is disallowed"); + p->error = true; return; } PyCodeObject *co = (PyCodeObject *)v; PyObject *co_code = _PyCode_GetCode(co); if (co_code == NULL) { - p->error = WFERR_NOMEMORY; + p->error = true; return; } W_TYPE(TYPE_CODE, p); @@ -750,7 +755,7 @@ w_complex_object(PyObject *v, char flag, WFILE *p) if (PyObject_GetBuffer(v, &view, PyBUF_SIMPLE) != 0) { w_byte(TYPE_UNKNOWN, p); p->depth--; - p->error = WFERR_UNMARSHALLABLE; + p->error = true; return; } W_TYPE(TYPE_STRING, p); @@ -760,7 +765,10 @@ w_complex_object(PyObject *v, char flag, WFILE *p) else if (PySlice_Check(v)) { if (p->version < 5) { w_byte(TYPE_UNKNOWN, p); - p->error = WFERR_UNMARSHALLABLE; + PyErr_Format(PyExc_ValueError, + "marshalling %T objects requires version 5 " + "or higher", v); + p->error = true; return; } PySliceObject *slice = (PySliceObject *)v; @@ -772,7 +780,8 @@ w_complex_object(PyObject *v, char flag, WFILE *p) } else { W_TYPE(TYPE_UNKNOWN, p); - p->error = WFERR_UNMARSHALLABLE; + PyErr_Format(PyExc_ValueError, "cannot marshal %T objects", v); + p->error = true; } } @@ -806,35 +815,6 @@ w_clear_refs(WFILE *wf) } } -/* Set the exception indicator according to the recorded error. */ -static void -w_set_exception(WFILE *p) -{ - assert(p->error != WFERR_OK); - switch (p->error) { - case WFERR_NOMEMORY: - PyErr_NoMemory(); - break; - case WFERR_NESTEDTOODEEP: - PyErr_SetString(PyExc_ValueError, - "object too deeply nested to marshal"); - break; - case WFERR_CODE_NOT_ALLOWED: - PyErr_SetString(PyExc_ValueError, - "marshalling code objects is disallowed"); - break; - case WFERR_EXCEPTION_SET: - /* An exception has already been raised. */ - assert(PyErr_Occurred()); - break; - default: - case WFERR_UNMARSHALLABLE: - PyErr_SetString(PyExc_ValueError, - "unmarshallable object"); - break; - } -} - /* version currently has no effect for writing ints. */ void PyMarshal_WriteLongToFile(long x, FILE *fp, int version) @@ -845,13 +825,11 @@ PyMarshal_WriteLongToFile(long x, FILE *fp, int version) wf.fp = fp; wf.ptr = wf.buf = buf; wf.end = wf.ptr + sizeof(buf); - wf.error = WFERR_OK; + wf.error = false; wf.version = version; w_long(x, &wf); w_flush(&wf); - if (wf.error != WFERR_OK) { - w_set_exception(&wf); - } + assert(!wf.error || PyErr_Occurred()); } void @@ -866,7 +844,7 @@ PyMarshal_WriteObjectToFile(PyObject *x, FILE *fp, int version) wf.fp = fp; wf.ptr = wf.buf = buf; wf.end = wf.ptr + sizeof(buf); - wf.error = WFERR_OK; + wf.error = false; wf.version = version; wf.allow_code = 1; if (w_init_refs(&wf, version)) { @@ -875,9 +853,7 @@ PyMarshal_WriteObjectToFile(PyObject *x, FILE *fp, int version) w_object(x, &wf); w_clear_refs(&wf); w_flush(&wf); - if (wf.error != WFERR_OK) { - w_set_exception(&wf); - } + assert(!wf.error || PyErr_Occurred()); } typedef struct { @@ -2005,7 +1981,7 @@ _PyMarshal_WriteObjectToString(PyObject *x, int version, int allow_code) return NULL; wf.ptr = wf.buf = PyBytes_AS_STRING(wf.str); wf.end = wf.ptr + PyBytes_GET_SIZE(wf.str); - wf.error = WFERR_OK; + wf.error = false; wf.version = version; wf.allow_code = allow_code; if (w_init_refs(&wf, version)) { @@ -2019,9 +1995,9 @@ _PyMarshal_WriteObjectToString(PyObject *x, int version, int allow_code) if (_PyBytes_Resize(&wf.str, (Py_ssize_t)(wf.ptr - base)) < 0) return NULL; } - if (wf.error != WFERR_OK) { + if (wf.error) { + assert(PyErr_Occurred()); Py_XDECREF(wf.str); - w_set_exception(&wf); return NULL; } return wf.str;