Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Doc/conf.py
Original file line number Diff line number Diff line change
Expand Up @@ -635,6 +635,8 @@
"library/exceptions.rst": "builtins/exceptions.rst",
"library/threadsafety.rst": "builtins/threadsafety.rst",
"library/time-complexity.rst": "builtins/time-complexity.rst",
# Renamed to tkinter.dialogs.rst in GH-151656
"library/dialog.rst": "library/tkinter.dialogs.rst",
}

# Refuse to run the doctest builder under a mismatched Python
Expand Down
3 changes: 0 additions & 3 deletions Doc/tools/removed-ids.txt
Original file line number Diff line number Diff line change
Expand Up @@ -83,9 +83,6 @@ reference/expressions.html: generator.close
reference/expressions.html: generator.send
reference/expressions.html: generator.throw

# Renamed to library/tkinter.dialogs.html
library/dialog.html: (page missing)

# Obsolete sections in 'turtle' docs
library/turtle.html: changes-since-python-2-6
library/turtle.html: changes-since-python-3-0
17 changes: 17 additions & 0 deletions Include/internal/pycore_pystate.h
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,23 @@ extern void _PyThreadState_Detach(PyThreadState *tstate);
// to the "detached" state.
extern void _PyThreadState_Suspend(PyThreadState *tstate);

#ifdef Py_GIL_DISABLED
// Try to atomically transition a *different* thread's state from "detached"
// to "suspended". On success, the target thread cannot attach until
// _PyThreadState_ResumeDetached() is called, and the caller may safely
// perform operations that are normally only permitted for the owning thread
// (such as merging the biased reference counts of objects it owns).
//
// The caller must not run arbitrary Python code, allocate GC objects, or
// stop the world while holding the thread in the suspended state.
// Returns 1 on success, 0 if the thread was not in the "detached" state.
extern int _PyThreadState_TrySuspendDetached(PyThreadState *tstate);

// Undo a successful _PyThreadState_TrySuspendDetached(): switch the thread
// back to "detached" and wake it if it is waiting to attach.
extern void _PyThreadState_ResumeDetached(PyThreadState *tstate);
#endif

// Mark the thread state as "shutting down". This is used during interpreter
// and runtime finalization. The thread may no longer attach to the
// interpreter and will instead block via _PyThreadState_HangThread().
Expand Down
4,411 changes: 890 additions & 3,521 deletions Include/internal/pycore_unicodeobject_generated.h

Large diffs are not rendered by default.

29 changes: 26 additions & 3 deletions Lib/subprocess.py
Original file line number Diff line number Diff line change
Expand Up @@ -1629,21 +1629,31 @@ def _execute_child(self, args, executable, preexec_fn, close_fds,
assert not pass_fds, "pass_fds not supported on Windows."

if isinstance(args, str):
pass
# Filename is the program only. Later arguments can
# hold secrets. A leading quote ends at the next quote.
# Otherwise stop at the first space.
if args[:1] == '"':
end = args.find('"', 1)
orig_filename = args[1:end] if end != -1 else args
else:
orig_filename = args.split(' ', 1)[0]
elif isinstance(args, bytes):
if shell:
raise TypeError('bytes args is not allowed on Windows')
orig_filename = os.fsdecode(args)
args = list2cmdline([args])
elif isinstance(args, os.PathLike):
if shell:
raise TypeError('path-like args is not allowed when '
'shell is true')
orig_filename = os.fsdecode(args)
args = list2cmdline([args])
else:
args = list(args)
orig_filename = os.fsdecode(args[0]) if args else None
args = list2cmdline(args)

if executable is not None:
executable = os.fsdecode(executable)
orig_filename = executable = os.fsdecode(executable)

# Process startup details
if startupinfo is None:
Expand Down Expand Up @@ -1725,6 +1735,19 @@ def _execute_child(self, args, executable, preexec_fn, close_fds,
env,
cwd,
startupinfo)
except OSError as e:
# gh-119646: POSIX already puts the attempted path on
# OSError.filename. Windows CreateProcess did not, so
# failures (missing exe, WSL paths, invalid cwd) were
# reported without naming the command.
if e.filename is None:
# ERROR_DIRECTORY (267): CreateProcess rejected cwd.
if cwd is not None and e.winerror == 267:
name = cwd
else:
name = orig_filename
raise type(e)(e.errno, e.strerror, name, e.winerror) from None
raise
finally:
# Child is launched. Close the parent's copy of those pipe
# handles that only the child should have open. You need
Expand Down
18 changes: 17 additions & 1 deletion Lib/test/test_ctypes/test_find.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import os.path
import subprocess
import sys
import test.support
import unittest
Expand Down Expand Up @@ -78,9 +79,24 @@ def test_shell_injection(self):
@unittest.skipUnless(sys.platform.startswith('linux'),
'Test only valid for Linux')
class FindLibraryLinux(unittest.TestCase):
@classmethod
def setUpClass(cls):
try:
p = subprocess.run(['ld', '--version'],
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
text=True)
except OSError:
pass
else:
if p.stdout.startswith('mold '):
# The mold linker is known to be incompatible with
# the soft-deprecated ctypes.util.find_library
# (which uses `ld -t`).
raise unittest.SkipTest('Fails when ld is mold')

@thread_unsafe('uses setenv')
def test_find_on_libpath(self):
import subprocess
import tempfile

try:
Expand Down
4 changes: 4 additions & 0 deletions Lib/test/test_embed.py
Original file line number Diff line number Diff line change
Expand Up @@ -2114,6 +2114,10 @@ def test_thread_state_ensure_from_view(self):
def test_concurrent_finalization_stress(self):
self.run_embedded_interpreter("test_concurrent_finalization_stress")

def test_py_getenv(self):
# Test Py_GETENV() before init, when initialized, and after finalize
self.run_embedded_interpreter("test_py_getenv")


class MiscTests(EmbeddingTestsMixin, unittest.TestCase):
def test_unicode_id_init(self):
Expand Down
35 changes: 35 additions & 0 deletions Lib/test/test_free_threading/test_gc.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,9 @@
import time
from unittest import TestCase
import gc
import weakref

from test import support
from test.support import threading_helper


Expand Down Expand Up @@ -95,6 +97,39 @@ def evil():
thread.start()
thread.join()

def test_merge_brc_queue_of_detached_thread(self):
# GH-157838: objects queued for merging by a thread that is detached
# (blocked in a lock acquire, sleep, etc.) are merged and freed on its
# behalf instead of staying alive until it runs Python code again.
lock = threading.Lock()
lock.acquire()
ready = threading.Event()
objs = []

def worker():
# Objects owned by this thread; only the list holds a reference.
objs.extend(MyObj() for _ in range(100))
ready.set()
lock.acquire() # block while detached

thread = Thread(target=worker)
thread.start()
try:
ready.wait()
# The worker may not have detached yet when the first objects
# are dropped; keep trying until one is freed immediately.
for _ in support.sleeping_retry(support.SHORT_TIMEOUT, error=False):
obj = objs.pop()
wr = weakref.ref(obj)
del obj
if wr() is None:
break
else:
self.fail("object not freed while owning thread was detached")
finally:
lock.release()
thread.join()

def test_gc_callbacks_race_with_mutation(self):
def collect():
b.wait()
Expand Down
18 changes: 18 additions & 0 deletions Lib/test/test_memoryview.py
Original file line number Diff line number Diff line change
Expand Up @@ -931,6 +931,24 @@ def test_picklebuffer_reference_loop(self):
gc.collect()
self.assertIsNone(wr())

def test_overflows_in_floats(self):
half_data = array.array('e', [0.0])
float_data = array.array('f', [0.0])
complex_data = array.array('Zf', [123+321j])
half_view = memoryview(half_data)
float_view = memoryview(float_data)
complex_view = memoryview(complex_data)
with self.assertRaises(ValueError):
half_view[0] = 123456.0
with self.assertRaises(ValueError):
float_view[0] = 1e300
with self.assertRaises(ValueError):
complex_view[0] = 1e300
self.assertEqual(complex_view[0], 123+321j)
with self.assertRaises(ValueError):
complex_view[0] = 1e300j
self.assertEqual(complex_view[0], 123+321j)


@threading_helper.requires_working_threading()
@support.requires_resource("cpu")
Expand Down
51 changes: 47 additions & 4 deletions Lib/test/test_subprocess.py
Original file line number Diff line number Diff line change
Expand Up @@ -1797,13 +1797,28 @@ def test_failed_child_execute_fd_leak(self):
fds_after_exception = os.listdir(fd_directory)
self.assertEqual(fds_before_popen, fds_after_exception)

@unittest.skipIf(mswindows, "behavior currently not supported on Windows")
def test_file_not_found_includes_filename(self):
missing = (r'C:\opt\nonexistent_binary' if mswindows
else '/opt/nonexistent_binary')
with self.assertRaises(FileNotFoundError) as c:
subprocess.call(['/opt/nonexistent_binary', 'with', 'some', 'args'])
self.assertEqual(c.exception.filename, '/opt/nonexistent_binary')
subprocess.call([missing, 'with', 'some', 'args'])
self.assertEqual(c.exception.filename, missing)

def test_args_filter_iterable(self):
# gh-119646: Windows used to index args[0] before list2cmdline.
# test_faulthandler.test_sys_xoptions passes a filter() object.
args = filter(None, (sys.executable, "-c", "import sys; sys.exit(17)"))
self.assertEqual(subprocess.call(args), 17)

def test_file_not_found_includes_filename_from_iterable(self):
missing = (r'C:\opt\nonexistent_binary' if mswindows
else '/opt/nonexistent_binary')
args = filter(None, (missing, "with", "some", "args"))
with self.assertRaises(FileNotFoundError) as c:
subprocess.call(args)
self.assertEqual(c.exception.filename, missing)

@unittest.skipIf(mswindows, "behavior currently not supported on Windows")
@unittest.skipIf(mswindows, "Windows reports NotADirectoryError (WinError 267)")
def test_file_not_found_with_bad_cwd(self):
with self.assertRaises(FileNotFoundError) as c:
subprocess.Popen(['exit', '0'], cwd='/some/nonexistent/directory')
Expand Down Expand Up @@ -3718,6 +3733,34 @@ def test_vfork_used_when_expected(self):
@unittest.skipUnless(mswindows, "Windows specific tests")
class Win32ProcessTestCase(BaseTestCase):

def test_createprocess_bad_cwd_includes_filename(self):
# gh-119646: invalid cwd should appear on OSError.filename.
missing_cwd = r'C:\some\nonexistent\directory'
with self.assertRaises(OSError) as c:
subprocess.Popen([sys.executable, '-c', 'pass'], cwd=missing_cwd)
self.assertEqual(c.exception.filename, missing_cwd)
self.assertEqual(c.exception.winerror, 267)

def test_command_string_filename_omits_later_args(self):
# gh-119646: a command-line string must not put later arguments
# on OSError.filename. Those arguments can hold secrets.
missing = r'C:\opt\nonexistent_binary'
secret = 'NOT-A-REAL-SECRET'
quoted = r'C:\Program Files\nonexistent_binary'
cases = [
(missing, missing),
(f'{missing} --token {secret}', missing),
(f'"{missing}" --token {secret}', missing),
(f'"{quoted}" --token {secret}', quoted),
]
for command, expected in cases:
with self.subTest(command=command):
with self.assertRaises(FileNotFoundError) as c:
subprocess.call(command)
self.assertEqual(c.exception.filename, expected)
self.assertNotIn(secret, c.exception.filename or '')
self.assertNotIn(secret, str(c.exception))

def test_startupinfo(self):
# startupinfo argument
# We uses hardcoded constants, because we do not want to
Expand Down
1 change: 1 addition & 0 deletions Makefile.pre.in
Original file line number Diff line number Diff line change
Expand Up @@ -631,6 +631,7 @@ LIBEXPAT_HEADERS= \
Modules/expat/expat_config.h \
Modules/expat/expat_external.h \
Modules/expat/fallthrough.h \
Modules/expat/hash_table.h \
Modules/expat/iasciitab.h \
Modules/expat/internal.h \
Modules/expat/latin1tab.h \
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Raise :exc:`ValueError`'s for overflows, while trying to change
:class:`memoryview` elements with ``'f'`` and ``'Zf'`` format codes. Patch
by Sergey B Kirpichev.
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Merge biased reference counts on behalf of threads that are detached instead of waiting for them to attach again, in the free-threaded build.
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
On Windows, :exc:`OSError` from :mod:`subprocess` now includes the attempted
executable or working directory in ``filename``.
Original file line number Diff line number Diff line change
@@ -1 +1 @@
Update bundled `libexpat <https://libexpat.github.io/>`_ to version 2.8.4.
Update bundled `libexpat <https://libexpat.github.io/>`_ to version 2.8.5.
Loading
Loading