From 21e9d189b773a4dc37d44fa105603a38a9792a22 Mon Sep 17 00:00:00 2001 From: Gareth Rees Date: Fri, 21 Jul 2017 11:18:56 +0100 Subject: [PATCH] bpo-24869: Add startline and endline attributes to the tokens returned by shlex. --- Doc/library/shlex.rst | 12 +++++++++++ Lib/shlex.py | 48 ++++++++++++++++++++++++++++++++---------- Lib/test/test_shlex.py | 7 ++++++ 3 files changed, 56 insertions(+), 11 deletions(-) diff --git a/Doc/library/shlex.rst b/Doc/library/shlex.rst index fb335c690068165..119e6d9511b56d3 100644 --- a/Doc/library/shlex.rst +++ b/Doc/library/shlex.rst @@ -30,6 +30,10 @@ The :mod:`shlex` module defines the following functions: in POSIX mode by default, but uses non-POSIX mode if the *posix* argument is false. + The returned tokens have attributes ``startline`` giving the line + number of the first character of the token, and ``endline`` giving + the line number of the last character of the token. + .. note:: Since the :func:`split` function instantiates a :class:`~shlex.shlex` @@ -124,6 +128,10 @@ A :class:`~shlex.shlex` instance has the following methods: encounters an immediate end-of-file, :attr:`eof` is returned (the empty string (``''``) in non-POSIX mode, and ``None`` in POSIX mode). + Tokens read from the input stream have attributes ``startline`` giving the + line number of the first character of the token, and ``endline`` giving the + line number of the last character of the token. + .. method:: shlex.push_token(str) @@ -136,6 +144,10 @@ A :class:`~shlex.shlex` instance has the following methods: requests. (This is not ordinarily a useful entry point, and is documented here only for the sake of completeness.) + The returned token has attributes ``startline`` giving the line + number of the first character of the token, and ``endline`` giving + the line number of the last character of the token. + .. method:: shlex.sourcehook(filename) diff --git a/Lib/shlex.py b/Lib/shlex.py index 2c9786c517a350f..c94da1f97d3a069 100644 --- a/Lib/shlex.py +++ b/Lib/shlex.py @@ -16,6 +16,18 @@ __all__ = ["shlex", "split", "quote"] + +class Token(str): + "A string with startline and endline attributes." + __slots__ = ['startline', 'endline'] + + def __new__(cls, token, startline, endline): + self = super().__new__(cls, token) + self.startline = startline + self.endline = endline + return self + + class shlex: "A lexical analyzer class for simple shell-like syntaxes." def __init__(self, instream=None, infile=None, posix=False, @@ -129,6 +141,18 @@ def get_token(self): def read_token(self): quoted = False escapedstate = ' ' + startline = endline = self.lineno + + def start_token(s): + self.token = s + nonlocal startline, endline + startline = endline = self.lineno + + def continue_token(s): + self.token += s + nonlocal endline + endline = self.lineno + while True: if self.punctuation_chars and self._pushback_chars: nextchar = self._pushback_chars.pop() @@ -140,7 +164,7 @@ def read_token(self): print("shlex: in state %r I see character: %r" % (self.state, nextchar)) if self.state is None: - self.token = '' # past end of file + start_token('') # past end of file break elif self.state == ' ': if not nextchar: @@ -160,20 +184,20 @@ def read_token(self): escapedstate = 'a' self.state = nextchar elif nextchar in self.wordchars: - self.token = nextchar + start_token(nextchar) self.state = 'a' elif nextchar in self.punctuation_chars: - self.token = nextchar + start_token(nextchar) self.state = 'c' elif nextchar in self.quotes: if not self.posix: - self.token = nextchar + start_token(nextchar) self.state = nextchar elif self.whitespace_split: - self.token = nextchar + start_token(nextchar) self.state = 'a' else: - self.token = nextchar + start_token(nextchar) if self.token or (self.posix and quoted): break # emit current token else: @@ -187,7 +211,7 @@ def read_token(self): raise ValueError("No closing quotation") if nextchar == self.state: if not self.posix: - self.token += nextchar + continue_token(nextchar) self.state = ' ' break else: @@ -197,7 +221,7 @@ def read_token(self): escapedstate = self.state self.state = nextchar else: - self.token += nextchar + continue_token(nextchar) elif self.state in self.escape: if not nextchar: # end of file if self.debug >= 2: @@ -209,7 +233,7 @@ def read_token(self): if (escapedstate in self.quotes and nextchar != self.state and nextchar != escapedstate): self.token += self.state - self.token += nextchar + continue_token(nextchar) self.state = escapedstate elif self.state in ('a', 'c'): if not nextchar: @@ -234,7 +258,7 @@ def read_token(self): continue elif self.state == 'c': if nextchar in self.punctuation_chars: - self.token += nextchar + continue_token(nextchar) else: if nextchar not in self.whitespace: self._pushback_chars.append(nextchar) @@ -247,7 +271,7 @@ def read_token(self): self.state = nextchar elif (nextchar in self.wordchars or nextchar in self.quotes or self.whitespace_split): - self.token += nextchar + continue_token(nextchar) else: if self.punctuation_chars: self._pushback_chars.append(nextchar) @@ -264,6 +288,8 @@ def read_token(self): self.token = '' if self.posix and not quoted and result == '': result = None + else: + result = Token(result, startline, endline) if self.debug > 1: if result: print("shlex: raw token=" + repr(result)) diff --git a/Lib/test/test_shlex.py b/Lib/test/test_shlex.py index fd35788e81b2720..76d0c28f5acf1a3 100644 --- a/Lib/test/test_shlex.py +++ b/Lib/test/test_shlex.py @@ -308,6 +308,13 @@ def testQuote(self): self.assertEqual(shlex.quote("test%s'name'" % u), "'test%s'\"'\"'name'\"'\"''" % u) + def testLineNumbers(self): + """Test that tokens have correct line number attributes.""" + source = '1\n2 "2\n3" 3\n4 "4\n5\n6"' + expected = [(1, 1), (2, 2), (2, 3), (3, 3), (4, 4), (4, 6)] + found = [(t.startline, t.endline) for t in shlex.split(source)] + self.assertEqual(expected, found) + # Allow this test to be used with old shlex.py if not getattr(shlex, "split", None): for methname in dir(ShlexTest):