diff --git a/Doc/library/shlex.rst b/Doc/library/shlex.rst index 63deb4c7f7645c..8b0ed514f44fcd 100644 --- a/Doc/library/shlex.rst +++ b/Doc/library/shlex.rst @@ -25,6 +25,9 @@ The :mod:`!shlex` module defines the following functions: in POSIX mode by default, but uses non-POSIX mode if the *posix* argument is false. + The returned tokens have attributes ``startline`` giving the line + number of the first character of the token, and ``endline`` giving + the line number of the last character of the token. .. versionchanged:: 3.12 Passing ``None`` for *s* argument now raises an exception, rather than reading :data:`sys.stdin`. @@ -165,6 +168,10 @@ A :class:`~shlex.shlex` instance has the following methods: encounters an immediate end-of-file, :attr:`eof` is returned (the empty string (``''``) in non-POSIX mode, and ``None`` in POSIX mode). + Tokens read from the input stream have attributes ``startline`` giving the + line number of the first character of the token, and ``endline`` giving the + line number of the last character of the token. + .. method:: shlex.push_token(str) @@ -177,6 +184,10 @@ A :class:`~shlex.shlex` instance has the following methods: requests. (This is not ordinarily a useful entry point, and is documented here only for the sake of completeness.) + The returned token has attributes ``startline`` giving the line + number of the first character of the token, and ``endline`` giving + the line number of the last character of the token. + .. method:: shlex.sourcehook(filename) diff --git a/Lib/shlex.py b/Lib/shlex.py index c7ffc918d53961..76a18fb7ed0512 100644 --- a/Lib/shlex.py +++ b/Lib/shlex.py @@ -12,6 +12,18 @@ __all__ = ["shlex", "split", "quote", "join"] + +class Token(str): + "A string with startline and endline attributes." + __slots__ = ['startline', 'endline'] + + def __new__(cls, token, startline, endline): + self = super().__new__(cls, token) + self.startline = startline + self.endline = endline + return self + + class shlex: "A lexical analyzer class for simple shell-like syntaxes." def __init__(self, instream=None, infile=None, posix=False, @@ -131,6 +143,18 @@ def get_token(self): def read_token(self): quoted = False escapedstate = ' ' + startline = endline = self.lineno + + def start_token(s): + self.token = s + nonlocal startline, endline + startline = endline = self.lineno + + def continue_token(s): + self.token += s + nonlocal endline + endline = self.lineno + while True: if self.punctuation_chars and self._pushback_chars: nextchar = self._pushback_chars.pop() @@ -142,7 +166,7 @@ def read_token(self): print("shlex: in state %r I see character: %r" % (self.state, nextchar)) if self.state is None: - self.token = '' # past end of file + start_token('') # past end of file break elif self.state == ' ': if not nextchar: @@ -162,20 +186,20 @@ def read_token(self): escapedstate = 'a' self.state = nextchar elif nextchar in self.wordchars: - self.token = nextchar + start_token(nextchar) self.state = 'a' elif nextchar in self.punctuation_chars: - self.token = nextchar + start_token(nextchar) self.state = 'c' elif nextchar in self.quotes: if not self.posix: - self.token = nextchar + start_token(nextchar) self.state = nextchar elif self.whitespace_split: - self.token = nextchar + start_token(nextchar) self.state = 'a' else: - self.token = nextchar + start_token(nextchar) if self.token or (self.posix and quoted): break # emit current token else: @@ -189,7 +213,7 @@ def read_token(self): raise ValueError("No closing quotation") if nextchar == self.state: if not self.posix: - self.token += nextchar + continue_token(nextchar) self.state = ' ' break else: @@ -199,7 +223,7 @@ def read_token(self): escapedstate = self.state self.state = nextchar else: - self.token += nextchar + continue_token(nextchar) elif self.state in self.escape: if not nextchar: # end of file if self.debug >= 2: @@ -211,7 +235,7 @@ def read_token(self): if (escapedstate in self.quotes and nextchar != self.state and nextchar != escapedstate): self.token += self.state - self.token += nextchar + continue_token(nextchar) self.state = escapedstate elif self.state in ('a', 'c'): if not nextchar: @@ -236,7 +260,7 @@ def read_token(self): continue elif self.state == 'c': if nextchar in self.punctuation_chars: - self.token += nextchar + continue_token(nextchar) else: if nextchar not in self.whitespace: self._pushback_chars.append(nextchar) @@ -250,7 +274,7 @@ def read_token(self): elif (nextchar in self.wordchars or nextchar in self.quotes or (self.whitespace_split and nextchar not in self.punctuation_chars)): - self.token += nextchar + continue_token(nextchar) else: if self.punctuation_chars: self._pushback_chars.append(nextchar) @@ -267,6 +291,8 @@ def read_token(self): self.token = '' if self.posix and not quoted and result == '': result = None + else: + result = Token(result, startline, endline) if self.debug > 1: if result: print("shlex: raw token=" + repr(result)) diff --git a/Lib/test/test_shlex.py b/Lib/test/test_shlex.py index 4c0cd88bbcda14..2002fac6054063 100644 --- a/Lib/test/test_shlex.py +++ b/Lib/test/test_shlex.py @@ -379,6 +379,13 @@ def testPunctuationCharsReadOnly(self): with self.assertRaises(AttributeError): shlex_instance.punctuation_chars = False + def testLineNumbers(self): + """Test that tokens have correct line number attributes.""" + source = '1\n2 "2\n3" 3\n4 "4\n5\n6"' + expected = [(1, 1), (2, 2), (2, 3), (3, 3), (4, 4), (4, 6)] + found = [(t.startline, t.endline) for t in shlex.split(source)] + self.assertEqual(expected, found) + def testLinenoAfterNewLine(self): s = shlex.shlex("line 1\nline 2") self.assertEqual(s.lineno, 1) # before consumption