bpo-24869: Add startline and endline attributes to the tokens returned by shlex.

Author: gareth-rees

Doc/library/shlex.rst

@@ -30,6 +30,10 @@ The :mod:`shlex` module defines the following functions:
    in POSIX mode by default, but uses non-POSIX mode if the *posix* argument is
    false.
 
+   The returned tokens have attributes ``startline`` giving the line
+   number of the first character of the token, and ``endline`` giving
+   the line number of the last character of the token.
+
    .. note::
 
       Since the :func:`split` function instantiates a :class:`~shlex.shlex`
@@ -124,6 +128,10 @@ A :class:`~shlex.shlex` instance has the following methods:
    encounters an immediate end-of-file, :attr:`eof` is returned (the empty
    string (``''``) in non-POSIX mode, and ``None`` in POSIX mode).
 
+   Tokens read from the input stream have attributes ``startline`` giving the
+   line number of the first character of the token, and ``endline`` giving the
+   line number of the last character of the token.
+
 
 .. method:: shlex.push_token(str)
 
@@ -136,6 +144,10 @@ A :class:`~shlex.shlex` instance has the following methods:
    requests.  (This is not ordinarily a useful entry point, and is documented here
    only for the sake of completeness.)
 
+   The returned token has attributes ``startline`` giving the line
+   number of the first character of the token, and ``endline`` giving
+   the line number of the last character of the token.
+
 
 .. method:: shlex.sourcehook(filename)
 

Lib/shlex.py

@@ -16,6 +16,18 @@
 
 __all__ = ["shlex", "split", "quote"]
 
+
+class Token(str):
+    "A string with startline and endline attributes."
+    __slots__ = ['startline', 'endline']
+
+    def __new__(cls, token, startline, endline):
+        self = super().__new__(cls, token)
+        self.startline = startline
+        self.endline = endline
+        return self
+
+
 class shlex:
     "A lexical analyzer class for simple shell-like syntaxes."
     def __init__(self, instream=None, infile=None, posix=False,
@@ -129,6 +141,18 @@ def get_token(self):
     def read_token(self):
         quoted = False
         escapedstate = ' '
+        startline = endline = self.lineno
+
+        def start_token(s):
+            self.token = s
+            nonlocal startline, endline
+            startline = endline = self.lineno
+
+        def continue_token(s):
+            self.token += s
+            nonlocal endline
+            endline = self.lineno
+
         while True:
             if self.punctuation_chars and self._pushback_chars:
                 nextchar = self._pushback_chars.pop()
@@ -140,7 +164,7 @@ def read_token(self):
                 print("shlex: in state %r I see character: %r" % (self.state,
                                                                   nextchar))
             if self.state is None:
-                self.token = ''        # past end of file
+                start_token('')        # past end of file
                 break
             elif self.state == ' ':
                 if not nextchar:
@@ -160,20 +184,20 @@ def read_token(self):
                     escapedstate = 'a'
                     self.state = nextchar
                 elif nextchar in self.wordchars:
-                    self.token = nextchar
+                    start_token(nextchar)
                     self.state = 'a'
                 elif nextchar in self.punctuation_chars:
-                    self.token = nextchar
+                    start_token(nextchar)
                     self.state = 'c'
                 elif nextchar in self.quotes:
                     if not self.posix:
-                        self.token = nextchar
+                        start_token(nextchar)
                     self.state = nextchar
                 elif self.whitespace_split:
-                    self.token = nextchar
+                    start_token(nextchar)
                     self.state = 'a'
                 else:
-                    self.token = nextchar
+                    start_token(nextchar)
                     if self.token or (self.posix and quoted):
                         break   # emit current token
                     else:
@@ -187,7 +211,7 @@ def read_token(self):
                     raise ValueError("No closing quotation")
                 if nextchar == self.state:
                     if not self.posix:
-                        self.token += nextchar
+                        continue_token(nextchar)
                         self.state = ' '
                         break
                     else:
@@ -197,7 +221,7 @@ def read_token(self):
                     escapedstate = self.state
                     self.state = nextchar
                 else:
-                    self.token += nextchar
+                    continue_token(nextchar)
             elif self.state in self.escape:
                 if not nextchar:      # end of file
                     if self.debug >= 2:
@@ -209,7 +233,7 @@ def read_token(self):
                 if (escapedstate in self.quotes and
                         nextchar != self.state and nextchar != escapedstate):
                     self.token += self.state
-                self.token += nextchar
+                continue_token(nextchar)
                 self.state = escapedstate
             elif self.state in ('a', 'c'):
                 if not nextchar:
@@ -234,7 +258,7 @@ def read_token(self):
                             continue
                 elif self.state == 'c':
                     if nextchar in self.punctuation_chars:
-                        self.token += nextchar
+                        continue_token(nextchar)
                     else:
                         if nextchar not in self.whitespace:
                             self._pushback_chars.append(nextchar)
@@ -247,7 +271,7 @@ def read_token(self):
                     self.state = nextchar
                 elif (nextchar in self.wordchars or nextchar in self.quotes
                       or self.whitespace_split):
-                    self.token += nextchar
+                    continue_token(nextchar)
                 else:
                     if self.punctuation_chars:
                         self._pushback_chars.append(nextchar)
@@ -264,6 +288,8 @@ def read_token(self):
         self.token = ''
         if self.posix and not quoted and result == '':
             result = None
+        else:
+            result = Token(result, startline, endline)
         if self.debug > 1:
             if result:
                 print("shlex: raw token=" + repr(result))

Lib/test/test_shlex.py

@@ -308,6 +308,13 @@ def testQuote(self):
             self.assertEqual(shlex.quote("test%s'name'" % u),
                              "'test%s'\"'\"'name'\"'\"''" % u)
 
+    def testLineNumbers(self):
+        """Test that tokens have correct line number attributes."""
+        source = '1\n2 "2\n3" 3\n4 "4\n5\n6"'
+        expected = [(1, 1), (2, 2), (2, 3), (3, 3), (4, 4), (4, 6)]
+        found = [(t.startline, t.endline) for t in shlex.split(source)]
+        self.assertEqual(expected, found)
+
 # Allow this test to be used with old shlex.py
 if not getattr(shlex, "split", None):
     for methname in dir(ShlexTest):