diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d12eaee2adb396..0a0100fcb2ebb5 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -290,9 +290,10 @@ Tools/jit/ @brandtbucher @savannahostrowski @diegorusso InternalDocs/jit.md @brandtbucher @savannahostrowski @diegorusso @AA-Turner # Lazy imports (PEP 810) -Objects/lazyimportobject.c @yhg1s @DinoV @pablogsal -Include/internal/pycore_lazyimportobject.h @yhg1s @DinoV @pablogsal -Lib/test/test_lazy_import @yhg1s @DinoV @pablogsal +.github/workflows/reusable-test-lazy-imports-all.yml @yhg1s @DinoV @pablogsal +Objects/lazyimportobject.c @yhg1s @DinoV @pablogsal +Include/internal/pycore_lazyimportobject.h @yhg1s @DinoV @pablogsal +Lib/test/test_lazy_import @yhg1s @DinoV @pablogsal # Micro-op / μop / Tier 2 Optimiser Python/optimizer.c @markshannon @Fidget-Spinner @@ -655,5 +656,8 @@ Objects/**/clinic/ PC/**/clinic/ Python/**/clinic/ +# Exclude Lazy Imports=all CI carve out file +Lib/test/lazy_imports_all_exclude.txt + # Exclude HTML IDs list Doc/tools/removed-ids.txt diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7fdc81ae0ade1d..e11e6aa6b6d3dc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -476,6 +476,12 @@ jobs: name: hypothesis-example-db path: ${{ env.CPYTHON_BUILDDIR }}/.hypothesis/examples/ + test-lazy-imports-all: + name: 'Lazy imports enabled' + needs: build-context + if: fromJSON(needs.build-context.outputs.run-tests) + uses: ./.github/workflows/reusable-test-lazy-imports-all.yml + build-asan: name: 'Address sanitizer' runs-on: ${{ matrix.os }} @@ -648,6 +654,7 @@ jobs: - build-emscripten - build-wasi - test-hypothesis + - test-lazy-imports-all - build-asan - build-san - cross-build-linux @@ -705,4 +712,5 @@ jobs: ${{ !fromJSON(needs.build-context.outputs.run-ios) && 'build-ios,' || '' }} ${{ !fromJSON(needs.build-context.outputs.run-emscripten) && 'build-emscripten,' || '' }} ${{ !fromJSON(needs.build-context.outputs.run-wasi) && 'build-wasi,' || '' }} + ${{ !fromJSON(needs.build-context.outputs.run-tests) && 'test-lazy-imports-all,' || '' }} jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/reusable-test-lazy-imports-all.yml b/.github/workflows/reusable-test-lazy-imports-all.yml new file mode 100644 index 00000000000000..3754308e89cad4 --- /dev/null +++ b/.github/workflows/reusable-test-lazy-imports-all.yml @@ -0,0 +1,79 @@ +name: Reusable Lazy Imports Tests + +# Run the CPython test suite with global lazy imports forced on +# (``-X lazy_imports=all``). +# +# Modules that are known to fail under lazy imports are listed in +# Lib/test/lazy_imports_all_exclude.txt and skipped here. Remove entries from +# that file as the modules are fixed so this workflow starts guarding them +# against regressions. Excluded modules are also checked separately so the +# workflow fails when one starts passing and its exclusion should be removed. + +on: + workflow_call: + +permissions: + contents: read + +env: + FORCE_COLOR: 1 + +jobs: + test-lazy-imports-all: + name: 'Run Tests with lazy_imports=all' + runs-on: ubuntu-26.04 + timeout-minutes: 60 + env: + EXCLUDE_FILE: Lib/test/lazy_imports_all_exclude.txt + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - name: Register gcc problem matcher + run: echo "::add-matcher::.github/problem-matchers/gcc.json" + - name: Install dependencies + run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Configure CPython + run: ./configure --config-cache --with-pydebug + - name: Build CPython + run: make -j4 + - name: Display build info + run: make pythoninfo + - name: Verify lazy imports are fully enabled + run: ./python -X lazy_imports=all -c "import sys; assert sys.flags.lazy_imports == 1, sys.flags.lazy_imports; print('lazy imports all enabled')" + - name: Build test list (all tests minus the known-failing exclusions) + run: | + set -euo pipefail + ./python -m test --list-tests > all_tests.txt + # Strip comments/blank lines from the exclusion file, then drop those + # exact test names (whole-line, fixed-string match) from the run list. + grep -vE '^\s*(#.*)?$' "$EXCLUDE_FILE" > exclude_tests.txt || true + grep -vxF -f exclude_tests.txt all_tests.txt > run_tests.txt + # Fail loudly if any exclusion entry matched nothing: a stale or + # mistyped name (or a change in `--list-tests` output) would otherwise + # silently stop excluding a module and let it fail the run. + stale=$(comm -23 <(sort -u exclude_tests.txt) <(sort -u all_tests.txt)) + if [ -n "$stale" ]; then + echo "::error::Stale entries in $EXCLUDE_FILE (no longer match 'python -m test --list-tests'); remove or fix them:" + echo "$stale" + exit 1 + fi + echo "Excluding $(wc -l < exclude_tests.txt) module(s); running $(wc -l < run_tests.txt) of $(wc -l < all_tests.txt)." + - name: Run tests with lazy imports + run: xvfb-run xargs -a run_tests.txt ./python -X lazy_imports=all -m test --fast-ci --timeout=900 < /dev/null + - name: Verify excluded tests still need exclusion + run: | + set -euo pipefail + unexpected_passes=() + while IFS= read -r test_name; do + [ -n "$test_name" ] || continue + echo "Checking excluded test: $test_name" + if xvfb-run ./python -X lazy_imports=all -m test --fast-ci --timeout=900 "$test_name"; then + unexpected_passes+=("$test_name") + fi + done < exclude_tests.txt + if [ "${#unexpected_passes[@]}" -ne 0 ]; then + echo "::error::These tests still appear in $EXCLUDE_FILE but now pass with -X lazy_imports=all. Remove them from the exclude file:" + printf '%s\n' "${unexpected_passes[@]}" + exit 1 + fi diff --git a/Lib/test/lazy_imports_all_exclude.txt b/Lib/test/lazy_imports_all_exclude.txt new file mode 100644 index 00000000000000..2680d3b1e4357b --- /dev/null +++ b/Lib/test/lazy_imports_all_exclude.txt @@ -0,0 +1,40 @@ +# Test modules that currently FAIL under global lazy imports +# (``-X lazy_imports=all`` / ``PYTHON_LAZY_IMPORTS=all``). +# +# The "Lazy Imports All" CI workflow +# (.github/workflows/reusable-test-lazy-imports-all.yml) runs the whole test +# suite with lazy_imports=all, skipping every module listed here. Exclusion is +# whole-module: a listed module is skipped entirely, so any passing tests it +# contains are not covered until its line is removed. As each module is fixed, +# delete its line so the workflow starts guarding it against regressions. The +# workflow also checks listed modules separately and fails if one now passes, +# so accidental fixes prompt cleanup of this file. +# +# Format: one test name per line, exactly as printed by +# ``python -m test --list-tests``. Lines starting with ``#`` and blank lines +# are ignored. Note that split test packages use a dotted path +# (e.g. test.test_future_stmt.test_future) while ordinary modules use the bare +# name (e.g. test_builtin). + +test.test_inspect.test_inspect +test___all__ +test__interpreters +test_builtin +test_clinic +test_crossinterp +test_datetime +test_generated_cases +test_heapq +test_import +test_importlib +test_json +test_pkg +test_profile +test_profiling +test_pyrepl +test_subprocess +test_symtable +test_tools +test_trace +test_type_annotations +test_unittest diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index c931be6df5fdaa..ba9c25c835bc39 100755 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -1607,10 +1607,19 @@ class HalfFloatTest(FPTest, unittest.TestCase): typecode = 'e' minitemsize = 2 + def test_overflows(self): + # Overflows half-float type: + self.assertRaises(OverflowError, array.array, self.typecode, [123456]) + # Overflows also float type: + self.assertRaises(OverflowError, array.array, self.typecode, [1e300]) + class FloatTest(FPTest, unittest.TestCase): typecode = 'f' minitemsize = 4 + def test_overflows(self): + self.assertRaises(OverflowError, array.array, self.typecode, [1e300]) + class DoubleTest(FPTest, unittest.TestCase): typecode = 'd' minitemsize = 8 @@ -1637,6 +1646,10 @@ class ComplexFloatTest(CFPTest, unittest.TestCase): typecode = 'Zf' minitemsize = 8 + def test_overflows(self): + self.assertRaises(OverflowError, array.array, self.typecode, [1e300]) + self.assertRaises(OverflowError, array.array, self.typecode, [1e300j]) + class ComplexDoubleTest(CFPTest, unittest.TestCase): typecode = 'Zd' minitemsize = 16 diff --git a/Lib/test/test_capi/test_marshal.py b/Lib/test/test_capi/test_marshal.py index 972ff4ed53d687..4680c45f32dd4c 100644 --- a/Lib/test/test_capi/test_marshal.py +++ b/Lib/test/test_capi/test_marshal.py @@ -125,7 +125,7 @@ def test_write_object_to_file(self): with self.assertRaises(SystemError): write_object_to_file(NULL, filename, version) - with self.assertRaisesRegex(ValueError, 'unmarshallable object'): + with self.assertRaisesRegex(ValueError, 'cannot marshal object objects'): write_object_to_file(UNMARSHALLABLE, filename, version) def test_read_short_from_file(self): @@ -225,7 +225,7 @@ def test_pymarshal_writeobjecttostring(self): obj2 = marshal.loads(data) self.check_object(obj2, obj) - with self.assertRaisesRegex(ValueError, 'unmarshallable object'): + with self.assertRaisesRegex(ValueError, 'cannot marshal object objects'): writeobjecttostring(UNMARSHALLABLE, version) with self.assertRaises(SystemError): diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 2200770b4a45f0..4c1abb15c0cb14 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -1069,8 +1069,9 @@ def test_python_legacy_windows_stdio(self): @unittest.skipUnless(support.MS_WINDOWS, 'Test only applicable on Windows') def test_python_legacy_windows_stdio_encoding(self): - # gh-86427: In the legacy mode the encoding of the standard streams - # is the encoding of the console. + # gh-86427: In the legacy mode the encoding of a standard stream is + # the encoding of the console it is connected to, which can differ + # for input and output. import ctypes kernel32 = ctypes.WinDLL('kernel32', use_last_error=True) try: @@ -1080,7 +1081,7 @@ def test_python_legacy_windows_stdio_encoding(self): # We cannot use PIPE, because the standard streams should be # connected to the console. So we use the exit code. code = ("import sys; sys.exit(sys.stdin.encoding != 'cp850' or " - "sys.stdout.encoding != 'cp850')") + "sys.stdout.encoding != 'cp437')") env = os.environ.copy() env['PYTHONLEGACYWINDOWSSTDIO'] = '1' env['PYTHONUTF8'] = '0' @@ -1091,7 +1092,7 @@ def test_python_legacy_windows_stdio_encoding(self): try: if not kernel32.SetConsoleCP(850): self.skipTest('cannot set the console input code page') - if not kernel32.SetConsoleOutputCP(850): + if not kernel32.SetConsoleOutputCP(437): self.skipTest('cannot set the console output code page') proc = subprocess.run([sys.executable, '-c', code], env=env, stdin=fin, stdout=fout, diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index d7db3d480ff1e2..449704d1c347b6 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -162,14 +162,16 @@ def test_no_allow_code(self): data = {'a': [({co, 0},)]} dump = marshal.dumps(data, allow_code=True) self.assertEqual(marshal.loads(dump, allow_code=True), data) - with self.assertRaises(ValueError): + with self.assertRaisesRegex(ValueError, + 'marshalling code objects is disallowed'): marshal.dumps(data, allow_code=False) with self.assertRaises(ValueError): marshal.loads(dump, allow_code=False) marshal.dump(data, io.BytesIO(), allow_code=True) self.assertEqual(marshal.load(io.BytesIO(dump), allow_code=True), data) - with self.assertRaises(ValueError): + with self.assertRaisesRegex(ValueError, + 'marshalling code objects is disallowed'): marshal.dump(data, io.BytesIO(), allow_code=False) with self.assertRaises(ValueError): marshal.load(io.BytesIO(dump), allow_code=False) @@ -347,16 +349,29 @@ def test_reference_loop_dict(self): self.assertIsInstance(b, dict) self.assertIs(b[None], b) + def check_reference_loop(self, a, typename, minversion, + oldmsg='object too deeply nested to marshal'): + # Only versions supporting references to the type detect the loop; + # older versions fail for a different reason. + for v in range(minversion): + with self.subTest(version=v): + with self.assertRaisesRegex(ValueError, oldmsg): + marshal.dumps(a, v) + for v in range(minversion, marshal.version + 1): + with self.subTest(version=v): + with self.assertRaisesRegex( + ValueError, + f'cannot marshal recursion {typename} objects'): + marshal.dumps(a, v) + def test_reference_loop_tuple(self): a = ([],) a[0].append(a) - for v in range(marshal.version + 1): - self.assertRaises(ValueError, marshal.dumps, a, v) + self.check_reference_loop(a, 'tuple', 3) a = ({},) a[0][None] = a - for v in range(marshal.version + 1): - self.assertRaises(ValueError, marshal.dumps, a, v) + self.check_reference_loop(a, 'tuple', 3) def test_shared_reference_tuple(self): # A tuple referenced more than once still round-trips with the @@ -381,30 +396,28 @@ def f(): # so we need to break the loop manually. See gh-148722. self.addCleanup(a.clear) a.append(code) - for v in range(marshal.version + 1): - self.assertRaises(ValueError, marshal.dumps, code, v) + self.check_reference_loop(code, 'code', 3) def test_reference_loop_slice(self): + oldmsg = 'marshalling slice objects requires version 5 or higher' a = slice([], None) a.start.append(a) - for v in range(marshal.version + 1): - self.assertRaises(ValueError, marshal.dumps, a, v) + self.check_reference_loop(a, 'slice', 5, oldmsg) a = slice(None, []) a.stop.append(a) - for v in range(marshal.version + 1): - self.assertRaises(ValueError, marshal.dumps, a, v) + self.check_reference_loop(a, 'slice', 5, oldmsg) a = slice(None, None, []) a.step.append(a) - for v in range(marshal.version + 1): - self.assertRaises(ValueError, marshal.dumps, a, v) + self.check_reference_loop(a, 'slice', 5, oldmsg) def test_reference_loop_frozendict(self): a = frozendict({None: []}) a[None].append(a) - for v in range(marshal.version + 1): - self.assertRaises(ValueError, marshal.dumps, a, v) + self.check_reference_loop( + a, 'frozendict', 6, + 'marshalling frozendict objects requires version 6 or higher') def test_shared_reference_frozendict(self): # A frozendict referenced more than once must round-trip with the @@ -475,7 +488,9 @@ def test_exact_type_match(self): # Note: str subclasses are not tested because they get handled # by marshal's routines for objects supporting the buffer API. subtyp = type('subtyp', (typ,), {}) - self.assertRaises(ValueError, marshal.dumps, subtyp()) + with self.assertRaisesRegex(ValueError, + r'cannot marshal \S*subtyp objects'): + marshal.dumps(subtyp()) # Issue #1792 introduced a change in how marshal increases the size of its # internal buffer; this test ensures that the new code is exercised. @@ -578,9 +593,25 @@ def test_unmarshallable(self): ('code', code)) for name, arg in cases: with self.subTest(name, arg=arg): - with self.assertRaisesRegex(ValueError, "unmarshallable object"): + with self.assertRaisesRegex(ValueError, + "cannot marshal type objects"): marshal.dumps((arg, memoryview(b''))) + def test_error_in_set_item(self): + # Set items are sorted by their marshalled representation, and NaNs + # are only distinguished by identity, so they are compared as + # complex numbers. + nan = float('nan') + with self.assertRaisesRegex(TypeError, "'<' not supported"): + marshal.dumps({complex(nan, 0), complex(nan, 0)}) + + def test_error_in_buffer(self): + # The BufferError raised for a non-contiguous buffer is not replaced + # with a generic error. + step2 = slice(None, None, 2) + with self.assertRaises(BufferError): + marshal.dumps(memoryview(bytearray(b'abcdef'))[step2]) + LARGE_SIZE = 2**31 pointer_size = 8 if sys.maxsize > 0xFFFFFFFF else 4 @@ -591,8 +622,14 @@ def write(self, s): @unittest.skipIf(LARGE_SIZE > sys.maxsize, "test cannot run on 32-bit systems") class LargeValuesTestCase(unittest.TestCase): - def check_unmarshallable(self, data): - self.assertRaises(ValueError, marshal.dump, data, NullWriter()) + def check_unmarshallable(self, data, msg='object too large to marshal'): + with self.assertRaisesRegex(ValueError, msg): + marshal.dump(data, NullWriter()) + + @support.bigmemtest(size=LARGE_SIZE, memuse=4, dry_run=False) + def test_int(self, size): + # An int with more than SIZE32_MAX 15-bit digits. + self.check_unmarshallable(1 << (15 * size), 'int too large to marshal') @support.bigmemtest(size=LARGE_SIZE, memuse=2, dry_run=False) def test_bytes(self, size): @@ -725,7 +762,10 @@ def testFrozenDict(self): self.helper(dictobj) for version in range(6): - with self.assertRaises(ValueError): + with self.assertRaisesRegex( + ValueError, + 'marshalling frozendict objects requires ' + 'version 6 or higher'): marshal.dumps(dictobj, version) def testModule(self): @@ -794,7 +834,10 @@ def test_slice(self): self.helper(obj) for version in range(5): - with self.assertRaises(ValueError): + with self.assertRaisesRegex( + ValueError, + 'marshalling slice objects requires ' + 'version 5 or higher'): marshal.dumps(obj, version) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 37323b7ebc6b1a..abd7710a1d570a 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -469,6 +469,8 @@ def test_random(self): if v: data = ssl.RAND_bytes(16) self.assertEqual(len(data), 16) + + self.assertEqual(ssl.RAND_bytes(0), b'') else: self.assertRaises(ssl.SSLError, ssl.RAND_bytes, 16) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-04-16-12-30.gh-issue-156942.Qk7Rw2.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-04-16-12-30.gh-issue-156942.Qk7Rw2.rst new file mode 100644 index 00000000000000..24bbeef0461efb --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-04-16-12-30.gh-issue-156942.Qk7Rw2.rst @@ -0,0 +1,4 @@ +:mod:`marshal` no longer replaces the exception raised while marshalling the +value with a generic ``ValueError("unmarshallable object")``. Error messages +now name the type of the unsupported object and, for the types supported only +by newer data formats, the required version. diff --git a/Misc/NEWS.d/next/Library/2026-09-03-04-51-46.gh-issue-156864.Pbe7Tl.rst b/Misc/NEWS.d/next/Library/2026-09-03-04-51-46.gh-issue-156864.Pbe7Tl.rst new file mode 100644 index 00000000000000..ee89b830d3d486 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-09-03-04-51-46.gh-issue-156864.Pbe7Tl.rst @@ -0,0 +1,2 @@ +:func:`array.array` setter now correctly detects overflows for the ``'e'``, +``'f'`` and ``'Zf'`` type codes. Patch by Sergey B Kirpichev. diff --git a/Misc/NEWS.d/next/Windows/2026-08-09-05-30-00.gh-issue-86427.legacystdio.rst b/Misc/NEWS.d/next/Windows/2026-08-09-05-30-00.gh-issue-86427.legacystdio.rst new file mode 100644 index 00000000000000..c3e2332defabfa --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2026-08-09-05-30-00.gh-issue-86427.legacystdio.rst @@ -0,0 +1,3 @@ +Fix the encoding of the standard streams in the legacy Windows stdio mode +(:envvar:`PYTHONLEGACYWINDOWSSTDIO`). It is now the encoding of the device +the stream is connected to, as in Python 3.7, not the ANSI code page. diff --git a/Misc/NEWS.d/next/Windows/2026-08-09-07-00-00.gh-issue-86427.consolecp.rst b/Misc/NEWS.d/next/Windows/2026-08-09-07-00-00.gh-issue-86427.consolecp.rst deleted file mode 100644 index 1cbf188c12e407..00000000000000 --- a/Misc/NEWS.d/next/Windows/2026-08-09-07-00-00.gh-issue-86427.consolecp.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix the encoding of the standard streams in the legacy Windows stdio mode -(:envvar:`PYTHONLEGACYWINDOWSSTDIO`). It is now the code page of the -console, as in Python 3.7, not the ANSI code page. diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index b487d6d7beca93..b7c1555c2637dd 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -486,7 +486,7 @@ peek_bytes_lock_held(bytesio *self, Py_ssize_t size) is beyond the size of self->buf. Assert above validates size is always in bounds. When self->pos is out of bounds calling code sets size to 0. */ if (size == 0) { - return PyBytes_FromStringAndSize(NULL, 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); } output = PyBytes_AS_STRING(self->buf) + self->pos; @@ -1109,7 +1109,7 @@ bytesio_new(PyTypeObject *type, PyObject *args, PyObject *kwds) /* tp_alloc initializes all the fields to zero. So we don't have to initialize them here. */ - self->buf = PyBytes_FromStringAndSize(NULL, 0); + self->buf = Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); if (self->buf == NULL) { Py_DECREF(self); return PyErr_NoMemory(); diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index bc375e3dfe7de8..11e29227d9276e 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -942,7 +942,7 @@ _io__WindowsConsoleIO_readall_impl(winconsoleio *self) if (len == 0 && _buflen(self) == 0) { /* when the result starts with ^Z we return an empty buffer */ PyMem_Free(buf); - return PyBytes_FromStringAndSize(NULL, 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); } if (len) { diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 43cee9e0f308df..ae318ca19fa0b8 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -447,7 +447,7 @@ subscript_slice(pysqlite_Blob *self, PyObject *item) } if (len == 0) { - return PyBytes_FromStringAndSize(NULL, 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); } if (step == 1) { diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 9bddb9ce62d5b9..7a780dce967c79 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -6243,10 +6243,9 @@ _ssl_RAND_add_impl(PyObject *module, Py_buffer *view, double entropy) } static PyObject * -PySSL_RAND(PyObject *module, int len, int pseudo) +PySSL_RAND(PyObject *module, int len) { int ok; - PyObject *bytes; unsigned long err; const char *errstr; PyObject *v; @@ -6256,20 +6255,16 @@ PySSL_RAND(PyObject *module, int len, int pseudo) return NULL; } - bytes = PyBytes_FromStringAndSize(NULL, len); - if (bytes == NULL) + PyBytesWriter *writer = PyBytesWriter_Create(len); + if (writer == NULL) { return NULL; - if (pseudo) { - ok = RAND_bytes((unsigned char*)PyBytes_AS_STRING(bytes), len); - if (ok == 0 || ok == 1) - return Py_BuildValue("NO", bytes, ok == 1 ? Py_True : Py_False); } - else { - ok = RAND_bytes((unsigned char*)PyBytes_AS_STRING(bytes), len); - if (ok == 1) - return bytes; + + ok = RAND_bytes(PyBytesWriter_GetData(writer), len); + if (ok == 1) { + return PyBytesWriter_Finish(writer); } - Py_DECREF(bytes); + PyBytesWriter_Discard(writer); err = ERR_get_error(); errstr = ERR_reason_error_string(err); @@ -6294,7 +6289,7 @@ static PyObject * _ssl_RAND_bytes_impl(PyObject *module, int n) /*[clinic end generated code: output=977da635e4838bc7 input=2e78ce1e86336776]*/ { - return PySSL_RAND(module, n, 0); + return PySSL_RAND(module, n); } diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index a0181c083a6036..ef492d143d5644 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -584,8 +584,8 @@ e_getitem(arrayobject *ap, Py_ssize_t i) static int e_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) { - float x; - if (!PyArg_Parse(v, "f;array item must be float", &x)) { + double x; + if (!PyArg_Parse(v, "d;array item must be float", &x)) { return -1; } @@ -607,14 +607,16 @@ f_getitem(arrayobject *ap, Py_ssize_t i) static int f_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) { - float x; - if (!PyArg_Parse(v, "f;array item must be float", &x)) + double x; + if (!PyArg_Parse(v, "d;array item must be float", &x)) return -1; CHECK_ARRAY_BOUNDS(ap, i); - if (i >= 0) - ((float *)ap->ob_item)[i] = x; + if (i >= 0) { + return PyFloat_Pack4(x, ap->ob_item + sizeof(float)*i, + PY_LITTLE_ENDIAN); + } return 0; } @@ -651,7 +653,6 @@ static int cf_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) { Py_complex x; - float f[2]; if (!PyArg_Parse(v, "D;array item must be complex", &x)) { return -1; @@ -659,10 +660,18 @@ cf_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) CHECK_ARRAY_BOUNDS(ap, i); - f[0] = (float)x.real; - f[1] = (float)x.imag; if (i >= 0) { - memcpy(ap->ob_item + i*sizeof(f), &f, sizeof(f)); + char f[8]; + int ret = PyFloat_Pack4(x.real, f, PY_LITTLE_ENDIAN); + + if (ret) { + return ret; + } + ret = PyFloat_Pack4(x.imag, f + sizeof(float), PY_LITTLE_ENDIAN); + if (!ret) { + memcpy(ap->ob_item + i*sizeof(f), &f, sizeof(f)); + } + return ret; } return 0; } diff --git a/Objects/object.c b/Objects/object.c index e3f29b71301695..971ac1b7a68669 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -3464,7 +3464,7 @@ _Py_GetConstant_Init(void) constants[Py_CONSTANT_ZERO] = _PyLong_GetZero(); constants[Py_CONSTANT_ONE] = _PyLong_GetOne(); constants[Py_CONSTANT_EMPTY_STR] = PyUnicode_New(0, 0); - constants[Py_CONSTANT_EMPTY_BYTES] = PyBytes_FromStringAndSize(NULL, 0); + constants[Py_CONSTANT_EMPTY_BYTES] = PyBytes_FromStringAndSize("", 0); constants[Py_CONSTANT_EMPTY_TUPLE] = PyTuple_New(0); #ifndef NDEBUG for (size_t i=0; i < Py_ARRAY_LENGTH(constants); i++) { diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 7ee9d17b7f77ff..d2faa28722b0e0 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -15229,6 +15229,10 @@ init_stdio_encoding(PyInterpreterState *interp) { /* Update the stdio encoding to the normalized Python codec name. */ PyConfig *config = (PyConfig*)_PyInterpreterState_GetConfig(interp); + if (config->stdio_encoding == NULL) { + /* gh-86427: The encoding is determined for every stream. */ + return _PyStatus_OK(); + } if (config_get_codec_name(&config->stdio_encoding) < 0) { return _PyStatus_ERR("failed to get the Python codec name " "of the stdio encoding"); diff --git a/Python/initconfig.c b/Python/initconfig.c index 69d47a5872f5e3..49f1beb37bb920 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -196,7 +196,7 @@ static const PyConfigSpec PYCONFIG_SPEC[] = { SPEC(show_ref_count, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), SPEC(site_import, BOOL, READ_ONLY, NO_SYS, GLOBAL(&Py_NoSiteFlag, 1)), // sys.flags.no_site SPEC(skip_source_first_line, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), - SPEC(stdio_encoding, WSTR, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(stdio_encoding, WSTR_OPT, READ_ONLY, NO_SYS, NO_GLOBAL), SPEC(stdio_errors, WSTR, READ_ONLY, NO_SYS, NO_GLOBAL), SPEC(tracemalloc, UINT, READ_ONLY, NO_SYS, NO_GLOBAL), SPEC(use_frozen_modules, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), @@ -1074,11 +1074,14 @@ config_check_consistency(const PyConfig *config) assert(config->module_search_paths_set >= 0); assert(config->filesystem_encoding != NULL); assert(config->filesystem_errors != NULL); - assert(config->stdio_encoding != NULL); - assert(config->stdio_errors != NULL); #ifdef MS_WINDOWS + /* stdio_encoding can be NULL in the legacy Windows stdio mode. */ + assert(config->stdio_encoding != NULL || config->legacy_windows_stdio); assert(config->legacy_windows_stdio >= 0); +#else + assert(config->stdio_encoding != NULL); #endif + assert(config->stdio_errors != NULL); /* -c and -m options are exclusive */ assert(!(config->run_command != NULL && config->run_module != NULL)); assert(config->check_hash_pycs_mode != NULL); @@ -2709,31 +2712,15 @@ config_init_stdio_encoding(PyConfig *config, } /* Choose the default error handler based on the current locale. */ - if (config->stdio_encoding == NULL) { + if (config->stdio_encoding == NULL #ifdef MS_WINDOWS - /* gh-86427: use the console code page. Only one encoding can be - specified, so the output code page is used: it affects two - streams of three. */ - UINT cp = config->legacy_windows_stdio ? GetConsoleOutputCP() : 0; - if (cp != 0) { - if (cp == CP_UTF8) { - status = PyConfig_SetString(config, &config->stdio_encoding, - L"utf-8"); - } - else { - wchar_t encoding[20]; - swprintf(encoding, Py_ARRAY_LENGTH(encoding), L"cp%u", - (unsigned int)cp); - status = PyConfig_SetString(config, &config->stdio_encoding, - encoding); - } - } - else + /* gh-86427: it is determined for each stream: create_stdio() uses + _Py_device_encoding(), falling back to the locale encoding. */ + && !config->legacy_windows_stdio #endif - { - status = config_get_locale_encoding(config, preconfig, - &config->stdio_encoding); - } + ) { + status = config_get_locale_encoding(config, preconfig, + &config->stdio_encoding); if (_PyStatus_EXCEPTION(status)) { return status; } diff --git a/Python/marshal.c b/Python/marshal.c index 420c3ee115a737..6aa99bb4693117 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -100,17 +100,9 @@ module marshal #define FLAG_REF '\x80' /* with a type, add obj to index */ -// Error codes: -#define WFERR_OK 0 -#define WFERR_UNMARSHALLABLE 1 -#define WFERR_NESTEDTOODEEP 2 -#define WFERR_NOMEMORY 3 -#define WFERR_CODE_NOT_ALLOWED 4 -#define WFERR_EXCEPTION_SET 5 /* An exception has already been raised. */ - typedef struct { FILE *fp; - int error; /* see WFERR_* values */ + bool error; /* An exception has been raised. */ int depth; PyObject *str; char *ptr; @@ -132,10 +124,10 @@ static void w_file_error(WFILE *p) { int saved_errno = errno; - if (p->error != WFERR_OK) { + if (p->error) { return; } - p->error = WFERR_EXCEPTION_SET; + p->error = true; if (PyErr_CheckSignals()) { /* The signal handler has raised an exception. */ return; @@ -174,12 +166,14 @@ w_reserve(WFILE *p, Py_ssize_t needed) delta = size + 1024; delta = Py_MAX(delta, needed); if (delta > PY_SSIZE_T_MAX - size) { - p->error = WFERR_NOMEMORY; + PyErr_NoMemory(); + p->error = true; return 0; } size += delta; if (_PyBytes_Resize(&p->str, size) != 0) { p->end = p->ptr = p->buf = NULL; + p->error = true; return 0; } else { @@ -236,13 +230,15 @@ w_long(long x, WFILE *p) #define SIZE32_MAX 0x7FFFFFFF #if SIZEOF_SIZE_T > 4 -# define W_SIZE(n, p) do { \ - if ((n) > SIZE32_MAX) { \ - (p)->depth--; \ - (p)->error = WFERR_UNMARSHALLABLE; \ - return; \ - } \ - w_long((long)(n), p); \ +# define W_SIZE(n, p) do { \ + if ((n) > SIZE32_MAX) { \ + (p)->depth--; \ + PyErr_SetString(PyExc_ValueError, \ + "object too large to marshal"); \ + (p)->error = true; \ + return; \ + } \ + w_long((long)(n), p); \ } while(0) #else # define W_SIZE w_long @@ -295,7 +291,8 @@ _r_digits##bitsize(const uint ## bitsize ## _t *digits, Py_ssize_t n, \ } while (d != 0); \ if (l > SIZE32_MAX) { \ p->depth--; \ - p->error = WFERR_UNMARSHALLABLE; \ + PyErr_SetString(PyExc_ValueError, "int too large to marshal"); \ + p->error = true; \ return; \ } \ w_long((long)(negative ? -l : l), p); \ @@ -331,7 +328,7 @@ w_PyLong(const PyLongObject *ob, char flag, WFILE *p) if (PyLong_Export((PyObject *)ob, &long_export) < 0) { p->depth--; - p->error = WFERR_UNMARSHALLABLE; + p->error = true; return; } if (!long_export.digits) { @@ -384,7 +381,7 @@ w_float_bin(double v, WFILE *p) { char buf[8]; if (PyFloat_Pack8(v, buf, 1) < 0) { - p->error = WFERR_UNMARSHALLABLE; + p->error = true; return; } w_string(buf, 8, p); @@ -395,7 +392,7 @@ w_float_str(double v, WFILE *p) { char *buf = PyOS_double_to_string(v, 'g', 17, 0, NULL); if (!buf) { - p->error = WFERR_NOMEMORY; + p->error = true; return; } w_short_pstring(buf, strlen(buf), p); @@ -449,13 +446,14 @@ w_ref(PyObject *v, char *flag, WFILE *p) if (_Py_hashtable_set(p->hashtable, Py_NewRef(v), (void *)(uintptr_t)w) < 0) { Py_DECREF(v); + PyErr_NoMemory(); goto err; } *flag |= FLAG_REF; return 0; } err: - p->error = WFERR_UNMARSHALLABLE; + p->error = true; return 1; } @@ -488,14 +486,16 @@ w_object(PyObject *v, WFILE *p) { char flag = '\0'; - if (p->error != WFERR_OK) { + if (p->error) { return; } p->depth++; if (p->depth > MAX_MARSHAL_STACK_DEPTH) { - p->error = WFERR_NESTEDTOODEEP; + PyErr_SetString(PyExc_ValueError, + "object too deeply nested to marshal"); + p->error = true; } else if (v == NULL) { w_byte(TYPE_NULL, p); @@ -598,7 +598,7 @@ w_complex_object(PyObject *v, char flag, WFILE *p) utf8 = PyUnicode_AsEncodedString(v, "utf8", "surrogatepass"); if (utf8 == NULL) { p->depth--; - p->error = WFERR_UNMARSHALLABLE; + p->error = true; return; } if (p->version >= 3 && PyUnicode_CHECK_INTERNED(v)) @@ -638,7 +638,10 @@ w_complex_object(PyObject *v, char flag, WFILE *p) if (PyFrozenDict_CheckExact(v)) { if (p->version < 6) { w_byte(TYPE_UNKNOWN, p); - p->error = WFERR_UNMARSHALLABLE; + PyErr_Format(PyExc_ValueError, + "marshalling %T objects requires version 6 " + "or higher", v); + p->error = true; return; } @@ -675,7 +678,7 @@ w_complex_object(PyObject *v, char flag, WFILE *p) // use an order equivalent to sorted(v, key=marshal.dumps): PyObject *pairs = PyList_New(n); if (pairs == NULL) { - p->error = WFERR_NOMEMORY; + p->error = true; return; } Py_ssize_t i = 0; @@ -684,25 +687,25 @@ w_complex_object(PyObject *v, char flag, WFILE *p) PyObject *dump = _PyMarshal_WriteObjectToString(value, p->version, p->allow_code); if (dump == NULL) { - p->error = WFERR_UNMARSHALLABLE; + p->error = true; Py_DECREF(value); break; } PyObject *pair = _PyTuple_FromPairSteal(dump, value); if (pair == NULL) { - p->error = WFERR_NOMEMORY; + p->error = true; break; } PyList_SET_ITEM(pairs, i++, pair); } Py_END_CRITICAL_SECTION(); - if (p->error == WFERR_UNMARSHALLABLE || p->error == WFERR_NOMEMORY) { + if (p->error) { Py_DECREF(pairs); return; } assert(i == n); if (PyList_Sort(pairs)) { - p->error = WFERR_NOMEMORY; + p->error = true; Py_DECREF(pairs); return; } @@ -715,13 +718,15 @@ w_complex_object(PyObject *v, char flag, WFILE *p) } else if (PyCode_Check(v)) { if (!p->allow_code) { - p->error = WFERR_CODE_NOT_ALLOWED; + PyErr_SetString(PyExc_ValueError, + "marshalling code objects is disallowed"); + p->error = true; return; } PyCodeObject *co = (PyCodeObject *)v; PyObject *co_code = _PyCode_GetCode(co); if (co_code == NULL) { - p->error = WFERR_NOMEMORY; + p->error = true; return; } W_TYPE(TYPE_CODE, p); @@ -750,7 +755,7 @@ w_complex_object(PyObject *v, char flag, WFILE *p) if (PyObject_GetBuffer(v, &view, PyBUF_SIMPLE) != 0) { w_byte(TYPE_UNKNOWN, p); p->depth--; - p->error = WFERR_UNMARSHALLABLE; + p->error = true; return; } W_TYPE(TYPE_STRING, p); @@ -760,7 +765,10 @@ w_complex_object(PyObject *v, char flag, WFILE *p) else if (PySlice_Check(v)) { if (p->version < 5) { w_byte(TYPE_UNKNOWN, p); - p->error = WFERR_UNMARSHALLABLE; + PyErr_Format(PyExc_ValueError, + "marshalling %T objects requires version 5 " + "or higher", v); + p->error = true; return; } PySliceObject *slice = (PySliceObject *)v; @@ -772,7 +780,8 @@ w_complex_object(PyObject *v, char flag, WFILE *p) } else { W_TYPE(TYPE_UNKNOWN, p); - p->error = WFERR_UNMARSHALLABLE; + PyErr_Format(PyExc_ValueError, "cannot marshal %T objects", v); + p->error = true; } } @@ -806,35 +815,6 @@ w_clear_refs(WFILE *wf) } } -/* Set the exception indicator according to the recorded error. */ -static void -w_set_exception(WFILE *p) -{ - assert(p->error != WFERR_OK); - switch (p->error) { - case WFERR_NOMEMORY: - PyErr_NoMemory(); - break; - case WFERR_NESTEDTOODEEP: - PyErr_SetString(PyExc_ValueError, - "object too deeply nested to marshal"); - break; - case WFERR_CODE_NOT_ALLOWED: - PyErr_SetString(PyExc_ValueError, - "marshalling code objects is disallowed"); - break; - case WFERR_EXCEPTION_SET: - /* An exception has already been raised. */ - assert(PyErr_Occurred()); - break; - default: - case WFERR_UNMARSHALLABLE: - PyErr_SetString(PyExc_ValueError, - "unmarshallable object"); - break; - } -} - /* version currently has no effect for writing ints. */ void PyMarshal_WriteLongToFile(long x, FILE *fp, int version) @@ -845,13 +825,11 @@ PyMarshal_WriteLongToFile(long x, FILE *fp, int version) wf.fp = fp; wf.ptr = wf.buf = buf; wf.end = wf.ptr + sizeof(buf); - wf.error = WFERR_OK; + wf.error = false; wf.version = version; w_long(x, &wf); w_flush(&wf); - if (wf.error != WFERR_OK) { - w_set_exception(&wf); - } + assert(!wf.error || PyErr_Occurred()); } void @@ -866,7 +844,7 @@ PyMarshal_WriteObjectToFile(PyObject *x, FILE *fp, int version) wf.fp = fp; wf.ptr = wf.buf = buf; wf.end = wf.ptr + sizeof(buf); - wf.error = WFERR_OK; + wf.error = false; wf.version = version; wf.allow_code = 1; if (w_init_refs(&wf, version)) { @@ -875,9 +853,7 @@ PyMarshal_WriteObjectToFile(PyObject *x, FILE *fp, int version) w_object(x, &wf); w_clear_refs(&wf); w_flush(&wf); - if (wf.error != WFERR_OK) { - w_set_exception(&wf); - } + assert(!wf.error || PyErr_Occurred()); } typedef struct { @@ -2005,7 +1981,7 @@ _PyMarshal_WriteObjectToString(PyObject *x, int version, int allow_code) return NULL; wf.ptr = wf.buf = PyBytes_AS_STRING(wf.str); wf.end = wf.ptr + PyBytes_GET_SIZE(wf.str); - wf.error = WFERR_OK; + wf.error = false; wf.version = version; wf.allow_code = allow_code; if (w_init_refs(&wf, version)) { @@ -2019,9 +1995,9 @@ _PyMarshal_WriteObjectToString(PyObject *x, int version, int allow_code) if (_PyBytes_Resize(&wf.str, (Py_ssize_t)(wf.ptr - base)) < 0) return NULL; } - if (wf.error != WFERR_OK) { + if (wf.error) { + assert(PyErr_Occurred()); Py_XDECREF(wf.str); - w_set_exception(&wf); return NULL; } return wf.str; diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 6074d4439531f4..a9c98d73fd0f72 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -3080,7 +3080,18 @@ create_stdio(const PyConfig *config, PyObject* io, newline = "\n"; #endif - PyObject *encoding_str = PyUnicode_FromWideChar(encoding, -1); + PyObject *encoding_str; + if (encoding != NULL) { + encoding_str = PyUnicode_FromWideChar(encoding, -1); + } + else { + /* gh-86427: use the encoding of the device. */ + encoding_str = _Py_device_encoding(fd); + if (encoding_str == Py_None) { + Py_DECREF(encoding_str); + encoding_str = _Py_GetLocaleEncodingObject(); + } + } if (encoding_str == NULL) { Py_CLEAR(buf); goto error; diff --git a/Python/pystrhex.c b/Python/pystrhex.c index 8fb1fa36f85e73..ff6ee830fea7ab 100644 --- a/Python/pystrhex.c +++ b/Python/pystrhex.c @@ -168,15 +168,16 @@ _Py_strhex_impl(const char* argbuf, Py_ssize_t arglen, abs_bytes_per_sep = 0; } - PyObject *retval; + PyObject *retval = NULL; + PyBytesWriter *bytes_writer = NULL; Py_UCS1 *retbuf; if (return_bytes) { /* If _PyBytes_FromSize() were public we could avoid malloc+copy. */ - retval = PyBytes_FromStringAndSize(NULL, resultlen); - if (!retval) { + bytes_writer = PyBytesWriter_Create(resultlen); + if (!bytes_writer) { return NULL; } - retbuf = (Py_UCS1 *)PyBytes_AS_STRING(retval); + retbuf = PyBytesWriter_GetData(bytes_writer); } else { retval = PyUnicode_New(resultlen, 127); @@ -244,13 +245,15 @@ _Py_strhex_impl(const char* argbuf, Py_ssize_t arglen, } } + if (return_bytes) { + return PyBytesWriter_Finish(bytes_writer); + } + else { #ifdef Py_DEBUG - if (!return_bytes) { assert(_PyUnicode_CheckConsistency(retval, 1)); - } #endif - - return retval; + return retval; + } } PyObject * _Py_strhex(const char* argbuf, Py_ssize_t arglen)