gh-69057: Add startline and endline attributes to the tokens returned by shlex

Doc/library/shlex.rst

@@ -25,6 +25,9 @@ The :mod:`!shlex` module defines the following functions:
    in POSIX mode by default, but uses non-POSIX mode if the *posix* argument is
    false.
 
+   The returned tokens have attributes ``startline`` giving the line
+   number of the first character of the token, and ``endline`` giving
+   the line number of the last character of the token.
    .. versionchanged:: 3.12
       Passing ``None`` for *s* argument now raises an exception, rather than
       reading :data:`sys.stdin`.
@@ -165,6 +168,10 @@ A :class:`~shlex.shlex` instance has the following methods:
    encounters an immediate end-of-file, :attr:`eof` is returned (the empty
    string (``''``) in non-POSIX mode, and ``None`` in POSIX mode).
 
+   Tokens read from the input stream have attributes ``startline`` giving the
+   line number of the first character of the token, and ``endline`` giving the
+   line number of the last character of the token.
+
 
 .. method:: shlex.push_token(str)
 
@@ -177,6 +184,10 @@ A :class:`~shlex.shlex` instance has the following methods:
    requests.  (This is not ordinarily a useful entry point, and is documented here
    only for the sake of completeness.)
 
+   The returned token has attributes ``startline`` giving the line
+   number of the first character of the token, and ``endline`` giving
+   the line number of the last character of the token.
+
 
 .. method:: shlex.sourcehook(filename)
 

Lib/shlex.py

@@ -12,6 +12,18 @@
 
 __all__ = ["shlex", "split", "quote", "join"]
 
+
+class Token(str):
+    "A string with startline and endline attributes."
+    __slots__ = ['startline', 'endline']
+
+    def __new__(cls, token, startline, endline):
+        self = super().__new__(cls, token)
+        self.startline = startline
+        self.endline = endline
+        return self
+
+
 class shlex:
     "A lexical analyzer class for simple shell-like syntaxes."
     def __init__(self, instream=None, infile=None, posix=False,
@@ -131,6 +143,18 @@ def get_token(self):
     def read_token(self):
         quoted = False
         escapedstate = ' '
+        startline = endline = self.lineno
+
+        def start_token(s):
+            self.token = s
+            nonlocal startline, endline
+            startline = endline = self.lineno
+
+        def continue_token(s):
+            self.token += s
+            nonlocal endline
+            endline = self.lineno
+
         while True:
             if self.punctuation_chars and self._pushback_chars:
                 nextchar = self._pushback_chars.pop()
@@ -142,7 +166,7 @@ def read_token(self):
                 print("shlex: in state %r I see character: %r" % (self.state,
                                                                   nextchar))
             if self.state is None:
-                self.token = ''        # past end of file
+                start_token('')        # past end of file
                 break
             elif self.state == ' ':
                 if not nextchar:
@@ -162,20 +186,20 @@ def read_token(self):
                     escapedstate = 'a'
                     self.state = nextchar
                 elif nextchar in self.wordchars:
-                    self.token = nextchar
+                    start_token(nextchar)
                     self.state = 'a'
                 elif nextchar in self.punctuation_chars:
-                    self.token = nextchar
+                    start_token(nextchar)
                     self.state = 'c'
                 elif nextchar in self.quotes:
                     if not self.posix:
-                        self.token = nextchar
+                        start_token(nextchar)
                     self.state = nextchar
                 elif self.whitespace_split:
-                    self.token = nextchar
+                    start_token(nextchar)
                     self.state = 'a'
                 else:
-                    self.token = nextchar
+                    start_token(nextchar)
                     if self.token or (self.posix and quoted):
                         break   # emit current token
                     else:
@@ -189,7 +213,7 @@ def read_token(self):
                     raise ValueError("No closing quotation")
                 if nextchar == self.state:
                     if not self.posix:
-                        self.token += nextchar
+                        continue_token(nextchar)
                         self.state = ' '
                         break
                     else:
@@ -199,7 +223,7 @@ def read_token(self):
                     escapedstate = self.state
                     self.state = nextchar
                 else:
-                    self.token += nextchar
+                    continue_token(nextchar)
             elif self.state in self.escape:
                 if not nextchar:      # end of file
                     if self.debug >= 2:
@@ -211,7 +235,7 @@ def read_token(self):
                 if (escapedstate in self.quotes and
                         nextchar != self.state and nextchar != escapedstate):
                     self.token += self.state
-                self.token += nextchar
+                continue_token(nextchar)
                 self.state = escapedstate
             elif self.state in ('a', 'c'):
                 if not nextchar:
@@ -236,7 +260,7 @@ def read_token(self):
                             continue
                 elif self.state == 'c':
                     if nextchar in self.punctuation_chars:
-                        self.token += nextchar
+                        continue_token(nextchar)
                     else:
                         if nextchar not in self.whitespace:
                             self._pushback_chars.append(nextchar)
@@ -250,7 +274,7 @@ def read_token(self):
                 elif (nextchar in self.wordchars or nextchar in self.quotes
                       or (self.whitespace_split and
                           nextchar not in self.punctuation_chars)):
-                    self.token += nextchar
+                    continue_token(nextchar)
                 else:
                     if self.punctuation_chars:
                         self._pushback_chars.append(nextchar)
@@ -267,6 +291,8 @@ def read_token(self):
         self.token = ''
         if self.posix and not quoted and result == '':
             result = None
+        else:
+            result = Token(result, startline, endline)
         if self.debug > 1:
             if result:
                 print("shlex: raw token=" + repr(result))

Lib/test/test_shlex.py

@@ -379,6 +379,13 @@ def testPunctuationCharsReadOnly(self):
         with self.assertRaises(AttributeError):
             shlex_instance.punctuation_chars = False
 
+    def testLineNumbers(self):
+        """Test that tokens have correct line number attributes."""
+        source = '1\n2 "2\n3" 3\n4 "4\n5\n6"'
+        expected = [(1, 1), (2, 2), (2, 3), (3, 3), (4, 4), (4, 6)]
+        found = [(t.startline, t.endline) for t in shlex.split(source)]
+        self.assertEqual(expected, found)
+
     def testLinenoAfterNewLine(self):
         s = shlex.shlex("line 1\nline 2")
         self.assertEqual(s.lineno, 1)  # before consumption