Merge branch '3.13' into gh-156466-3.13

Author: iritkatriel

Doc/library/imaplib.rst

@@ -618,6 +618,20 @@ An :class:`IMAP4` instance has the following methods:
 
 The following attributes are defined on instances of :class:`IMAP4`:
 
+.. attribute:: IMAP4.capabilities
+
+   A tuple of the capabilities advertised by the server, in upper case.
+
+   It is set when the connection is established,
+   and refreshed after a successful :meth:`~IMAP4.login`,
+   :meth:`~IMAP4.authenticate` or :meth:`~IMAP4.starttls`,
+   because the server can advertise different capabilities
+   in different connection states.
+
+   .. versionchanged:: 3.13.15
+      Refreshed after :meth:`~IMAP4.login` and :meth:`~IMAP4.authenticate`.
+
+
 .. attribute:: IMAP4.PROTOCOL_VERSION
 
    The most recent supported protocol in the ``CAPABILITY`` response from the

Lib/imaplib.py

@@ -270,7 +270,8 @@ def _connect(self):
                 self._encoding, 'replace')
             raise self.error('invalid greeting: ' + greeting)
 
-        self._refresh_capabilities()
+        # The greeting is not a response to a command.
+        self._refresh_capabilities(consume=True)
         if __debug__:
             if self.debug >= 3:
                 self._mesg('CAPABILITIES: %r' % (self.capabilities,))
@@ -1142,10 +1143,14 @@ def _get_capabilities(self):
         self.capabilities = tuple(dat.split())
 
 
-    def _refresh_capabilities(self):
+    def _refresh_capabilities(self, consume=False):
         # Use a CAPABILITY response sent by the server, or ask for it.
+        # Unless it is consumed, the response can still be read with
+        # response('CAPABILITY').
         if 'CAPABILITY' in self.untagged_responses:
-            dat = self.untagged_responses.pop('CAPABILITY')[-1]
+            dat = self.untagged_responses['CAPABILITY'][-1]
+            if consume:
+                del self.untagged_responses['CAPABILITY']
             self.capabilities = tuple(str(dat, self._encoding).upper().split())
         else:
             self._get_capabilities()

Lib/inspect.py

@@ -3422,6 +3422,10 @@ def _main():
     import argparse
     import importlib
 
+    # The printed text can contain characters unencodable in the encoding
+    # of stdout, e.g. undecodable bytes of a file name.
+    sys.stdout.reconfigure(errors='backslashreplace')
+
     parser = argparse.ArgumentParser()
     parser.add_argument(
         'object',

Lib/ntpath.py

@@ -626,12 +626,23 @@ def _getfinalpathname_nonstrict(path, ignored_error=OSError):
         allowed_winerror = 1, 2, 3, 5, 21, 32, 50, 53, 65, 67, 87, 123, 161, 1005, 1920, 1921
 
         # Non-strict algorithm is to find as much of the target directory
-        # as we can and join the rest.
+        # as we can and join the rest.  join() is not used, because the tail
+        # can contain a colon and be mistaken for a drive (gh-102475).
+        if isinstance(path, bytes):
+            sep = b'\\'
+        else:
+            sep = '\\'
+
+        def join(path, tail):
+            if path[-1:] == sep or not tail:
+                return path + tail
+            return path + sep + tail
+
         tail = path[:0]
         while path:
             try:
                 path = _getfinalpathname(path)
-                return join(path, tail) if tail else path
+                return join(path, tail)
             except ignored_error as ex:
                 if ex.winerror not in allowed_winerror:
                     raise
@@ -642,7 +653,7 @@ def _getfinalpathname_nonstrict(path, ignored_error=OSError):
                     new_path = _readlink_deep(path,
                                               ignored_error=ignored_error)
                     if new_path != path:
-                        return join(new_path, tail) if tail else new_path
+                        return join(new_path, tail)
                 except ignored_error:
                     # If we fail to readlink(), let's keep traversing
                     pass
@@ -657,7 +668,7 @@ def _getfinalpathname_nonstrict(path, ignored_error=OSError):
                     path, name = split(path)
                 if path and not name:
                     return path + tail
-                tail = join(name, tail) if tail else name
+                tail = join(name, tail)
         return tail
 
     def realpath(path, *, strict=False):
@@ -666,7 +677,7 @@ def realpath(path, *, strict=False):
             prefix = b'\\\\?\\'
             unc_prefix = b'\\\\?\\UNC\\'
             new_unc_prefix = b'\\\\'
-            cwd = os.getcwdb()
+            colon_sep = b':\\'
             # bpo-38081: Special case for realpath(b'nul')
             devnull = b'nul'
             if normcase(path) == devnull:
@@ -675,7 +686,7 @@ def realpath(path, *, strict=False):
             prefix = '\\\\?\\'
             unc_prefix = '\\\\?\\UNC\\'
             new_unc_prefix = '\\\\'
-            cwd = os.getcwd()
+            colon_sep = ':\\'
             # bpo-38081: Special case for realpath('nul')
             devnull = 'nul'
             if normcase(path) == devnull:
@@ -691,7 +702,9 @@ def realpath(path, *, strict=False):
             ignored_error = OSError
 
         if not had_prefix and not isabs(path):
-            path = join(cwd, path)
+            # abspath() is used instead of join(cwd, path), because the path
+            # can be relative to another drive (gh-102475).
+            path = abspath(path)
         try:
             path = _getfinalpathname(path)
             initial_winerror = 0
@@ -711,25 +724,29 @@ def realpath(path, *, strict=False):
         # strip off that prefix unless it was already provided on the original
         # path.
         if not had_prefix and path.startswith(prefix):
-            # For UNC paths, the prefix will actually be \\?\UNC\
-            # Handle that case as well.
+            # For UNC drives, the path starts with \\?\UNC\.
             if path.startswith(unc_prefix):
                 spath = new_unc_prefix + path[len(unc_prefix):]
-            else:
+            # For drive-letter drives, the path starts with \\?\<letter>:\.
+            elif path.startswith(colon_sep, len(prefix) + 1):
                 spath = path[len(prefix):]
-            # Ensure that the non-prefixed path resolves to the same path
-            try:
-                if _getfinalpathname(spath) == path:
-                    path = spath
-            except ValueError as ex:
-                # Unexpected, as an invalid path should not have gained a prefix
-                # at any point, but we ignore this error just in case.
-                pass
-            except OSError as ex:
-                # If the path does not exist and originally did not exist, then
-                # strip the prefix anyway.
-                if ex.winerror == initial_winerror:
-                    path = spath
+            # For all others, e.g. volume GUID paths, it cannot be stripped.
+            else:
+                spath = None
+            if spath is not None:
+                # Ensure that the non-prefixed path resolves to the same path
+                try:
+                    if _getfinalpathname(spath) == path:
+                        path = spath
+                except ValueError:
+                    # Unexpected, as an invalid path should not have gained a
+                    # prefix at any point, but we ignore this error just in case.
+                    pass
+                except OSError as ex:
+                    # If the path does not exist and originally did not exist,
+                    # then strip the prefix anyway.
+                    if ex.winerror == initial_winerror:
+                        path = spath
         return path
 
 

Lib/test/test_cmd_line.py

@@ -960,6 +960,41 @@ def test_python_gil(self):
                 self.assertEqual(proc.stdout.rstrip(), expected)
                 self.assertEqual(proc.stderr, '')
 
+    @unittest.skipUnless(support.MS_WINDOWS, 'Test only applicable on Windows')
+    def test_python_legacy_windows_stdio_encoding(self):
+        # gh-86427: In the legacy mode the encoding of the standard streams
+        # is the encoding of the console.
+        import ctypes
+        kernel32 = ctypes.WinDLL('kernel32', use_last_error=True)
+        try:
+            fin = open('CONIN$')
+        except OSError:
+            self.skipTest('no console')
+        # We cannot use PIPE, because the standard streams should be
+        # connected to the console.  So we use the exit code.
+        code = ("import sys; sys.exit(sys.stdin.encoding != 'cp850' or "
+                "sys.stdout.encoding != 'cp850')")
+        env = os.environ.copy()
+        env['PYTHONLEGACYWINDOWSSTDIO'] = '1'
+        env['PYTHONUTF8'] = '0'
+        env.pop('PYTHONIOENCODING', None)
+        old_cp = kernel32.GetConsoleCP()
+        old_output_cp = kernel32.GetConsoleOutputCP()
+        with fin, open('CONOUT$', 'w') as fout:
+            try:
+                if not kernel32.SetConsoleCP(850):
+                    self.skipTest('cannot set the console input code page')
+                if not kernel32.SetConsoleOutputCP(850):
+                    self.skipTest('cannot set the console output code page')
+                proc = subprocess.run([sys.executable, '-c', code], env=env,
+                                      stdin=fin, stdout=fout,
+                                      stderr=subprocess.DEVNULL)
+            finally:
+                kernel32.SetConsoleCP(old_cp)
+                kernel32.SetConsoleOutputCP(old_output_cp)
+        support.skip_on_low_desktop_heap_memory_subprocess(proc.returncode)
+        self.assertEqual(proc.returncode, 0)
+
     @unittest.skipUnless(sys.platform == 'win32',
                          'bpo-32457 only applies on Windows')
     def test_argv0_normalization(self):

Lib/test/test_curses.py

@@ -434,6 +434,26 @@ def test_output_string_embedded_null_chars(self):
                 self.assertRaises(ValueError, stdscr.insstr, arg)
                 self.assertRaises(ValueError, stdscr.insnstr, arg, 1)
 
+    def test_output_string_attr_restored(self):
+        # A write with an attr restores the window rendition afterwards,
+        # whether it succeeded or failed.
+        win = curses.newwin(2, 10, 0, 0)
+        def current_attrs():
+            # Write a cell with the window's current rendition and read it
+            # back, so that the rendition itself is checked.
+            win.addstr(1, 0, ' ')
+            return win.inch(1, 0) & curses.A_ATTRIBUTES
+        for func, args in [(win.addstr, ('x',)), (win.addnstr, ('x', 1)),
+                           (win.insstr, ('x',)), (win.insnstr, ('x', 1))]:
+            with self.subTest(func.__qualname__):
+                win.attrset(curses.A_UNDERLINE)
+                # y=100 is outside the window, so the write fails.
+                self.assertRaises(curses.error, func, 100, 0, *args,
+                                  curses.A_BOLD)
+                self.assertEqual(current_attrs(), curses.A_UNDERLINE)
+                func(0, 0, *args, curses.A_BOLD)
+                self.assertEqual(current_attrs(), curses.A_UNDERLINE)
+
     def test_add_string_behavior(self):
         # addstr() advances the cursor past the written text; addnstr()
         # writes at most n characters.

Lib/test/test_imaplib.py

@@ -785,6 +785,8 @@ def cmd_ENABLE(self, tag, args):
         client.login('user', 'pass')
         self.assertIn('ENABLE', client.capabilities)
         self.assertIn('UTF8=ACCEPT', client.capabilities)
+        self.assertEqual(client.response('CAPABILITY'),
+                         ('CAPABILITY', [b'IMAP4rev1 ENABLE UTF8=ACCEPT']))
         typ, _ = client.enable('UTF8=ACCEPT')
         self.assertEqual(typ, 'OK')
 
@@ -803,6 +805,8 @@ def cmd_AUTHENTICATE(self, tag, args):
         self.assertNotIn('ENABLE', client.capabilities)
         client.authenticate('MYAUTH', lambda x: b'fake')
         self.assertIn('ENABLE', client.capabilities)
+        self.assertEqual(client.response('CAPABILITY'),
+                         ('CAPABILITY', [b'IMAP4rev1 ENABLE']))
 
     def test_greeting_capabilities(self):
         # Capabilities advertised in the greeting are used directly,
@@ -816,6 +820,8 @@ def cmd_CAPABILITY(self, tag, args):
         client, server = self._setup(GreetingHandler)
         self.assertEqual(client.capabilities, ('IMAP4REV1', 'ENABLE'))
         self.assertFalse(getattr(server, 'capability_queried', False))
+        # The greeting is not a response to a command, so it is consumed.
+        self.assertEqual(client.response('CAPABILITY'), ('CAPABILITY', [None]))
 
     def test_login_requery_capabilities(self):
         # If the server does not advertise capabilities after login,

Lib/test/test_inspect/test_inspect.py

@@ -39,7 +39,7 @@
 from test.support import cpython_only, import_helper, suppress_immortalization
 from test.support import MISSING_C_DOCSTRINGS, ALWAYS_EQ
 from test.support.import_helper import DirsOnSysPath, ready_to_import
-from test.support.os_helper import TESTFN, temp_cwd
+from test.support.os_helper import TESTFN, TESTFN_UNDECODABLE, temp_cwd
 from test.support.script_helper import assert_python_ok, assert_python_failure, kill_python
 from test.support import has_subprocess_support, SuppressCrashReport
 from test import support
@@ -6556,6 +6556,25 @@ def test_builtins(self):
         lines = err.decode().splitlines()
         self.assertEqual(lines, ["Can't get info for builtin modules."])
 
+    @unittest.skipUnless(TESTFN_UNDECODABLE,
+                         'requires undecodable file names')
+    def test_details_undecodable_path(self):
+        # gh-69370: the path of the module is not encodable in the encoding
+        # of stdout.
+        with temp_cwd() as test_dir:
+            subdir = os.path.join(os.fsencode(test_dir), TESTFN_UNDECODABLE)
+            try:
+                os.mkdir(subdir)
+            except OSError:
+                self.skipTest('undecodable paths are not supported')
+            with open(os.path.join(subdir, b'undecodable_mod.py'), 'w') as f:
+                f.write('"""Module docstring."""\n')
+            rc, out, err = assert_python_ok('-X', 'utf8=0', '-m', 'inspect',
+                                            '--details', 'undecodable_mod',
+                                            PYTHONPATH=os.fsdecode(subdir))
+        self.assertIn(b'Target: undecodable_mod', out)
+        self.assertEqual(err, b'')
+
     def test_details(self):
         module = importlib.import_module('unittest')
         args = support.optim_args_from_interpreter_flags()

Lib/test/test_ntpath.py

@@ -10,6 +10,7 @@
 from test import support
 from test.support import os_helper, is_emscripten
 from test.support.os_helper import FakePath
+from test.support.testcase import ExtraAssertions
 from test import test_genericpath
 from tempfile import TemporaryFile
 
@@ -82,7 +83,7 @@ def _parameterize(*parameters):
     return support.subTests('kwargs', parameters, _do_cleanups=True)
 
 
-class NtpathTestCase(unittest.TestCase):
+class NtpathTestCase(unittest.TestCase, ExtraAssertions):
     def assertPathEqual(self, path1, path2):
         if path1 == path2 or _norm(path1) == _norm(path2):
             return
@@ -1409,6 +1410,62 @@ def test_isjunction(self):
                 self.assertFalse(ntpath.isjunction('tmpdir'))
                 self.assertPathEqual(ntpath.realpath('testjunc'), ntpath.realpath('tmpdir'))
 
+    @unittest.skipIf(sys.platform != 'win32', "Can only test on win32.")
+    def test_realpath_drive_like_names(self):
+        # gh-102475: the unresolved tail is appended, not joined, so a name
+        # which looks like a drive does not reset the path.
+        drive = ntpath.splitroot(os.getcwd())[0]
+        for path, expected in [
+            ('C:/spam:eggs', 'C:\\spam:eggs'),
+            ('C:/nonexistent/spam:eggs', 'C:\\nonexistent\\spam:eggs'),
+            ('C:/spam:eggs/ham', 'C:\\spam:eggs\\ham'),
+            ('C:/nonexistent/spam:eggs/ham', 'C:\\nonexistent\\spam:eggs\\ham'),
+        ]:
+            with self.subTest(path=path):
+                self.assertEqual(ntpath.realpath(path), expected)
+                self.assertEqual(ntpath.realpath(os.fsencode(path)),
+                                 os.fsencode(expected))
+
+    @unittest.skipIf(sys.platform != 'win32', "Can only test on win32.")
+    def test_realpath_drive_relative(self):
+        # gh-102475: the working directory of a drive which does not exist
+        # is its root directory.
+        for drive in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ':
+            if not ntpath.exists(drive + ':'):
+                break
+        else:
+            raise unittest.SkipTest('all drives exist')
+        self.assertEqual(ntpath.realpath(drive + ':spam'),
+                         drive + ':\\spam')
+        self.assertEqual(ntpath.realpath(drive + ':'), drive + ':\\')
+
+    @unittest.skipIf(sys.platform != 'win32', "Can only test junctions with creation on win32.")
+    def test_realpath_volume_guid_path(self):
+        # gh-89760: the \\?\ prefix cannot be stripped from a volume GUID path.
+        # Find a volume which is not mounted as a drive.
+        for volume in os.listvolumes():
+            if not os.listmounts(volume):
+                break
+        else:
+            raise unittest.SkipTest('no volume without a mount point')
+
+        with os_helper.temp_dir() as d:
+            with os_helper.change_cwd(d):
+                # _winapi.CreateJunction() adds the \\??\\ prefix to a path
+                # which already has a prefix.
+                try:
+                    subprocess.run(['cmd', '/c', 'mklink', '/j',
+                                    'testjunc', volume],
+                                   check=True, capture_output=True)
+                except (OSError, subprocess.CalledProcessError):
+                    raise unittest.SkipTest('creating the test junction failed')
+
+                for path in 'testjunc', 'testjunc/spam', 'testjunc/spam/eggs':
+                    with self.subTest(path=path):
+                        realpath = ntpath.realpath(path)
+                        self.assertStartsWith(realpath, '\\\\?\\Volume{')
+                        self.assertTrue(ntpath.isabs(realpath), realpath)
+
     def test_isfile_invalid_paths(self):
         isfile = ntpath.isfile
         self.assertIs(isfile('/tmp\udfffabcds'), False)