Issue #16121: Fix line number accounting in shlex

diff --git a/Lib/shlex.py b/Lib/shlex.py
index e7c8accd427590b035bdf47f82d005daec879c63..6114d2df014d121eb490ec45c434e896118ca6e4 100644 (file)
--- a/Lib/shlex.py
+++ b/Lib/shlex.py
@@ -48,6 +48,7 @@ class shlex:
         self.state = ' '
         self.pushback = deque()
         self.lineno = 1
+        self._lines_found = 0
         self.debug = 0
         self.token = ''
         self.filestack = deque()
@@ -118,12 +119,23 @@ class shlex:
         return raw
 
     def read_token(self):
+        if self._lines_found:
+            self.lineno += self._lines_found
+            self._lines_found = 0
+
+        i = 0
         quoted = False
         escapedstate = ' '
         while True:
+            i += 1
             nextchar = self.instream.read(1)
             if nextchar == '\n':
-                self.lineno = self.lineno + 1
+                # In case newline is the first character increment lineno
+                if i == 1:
+                    self.lineno += 1
+                else:
+                    self._lines_found += 1
+
             if self.debug >= 3:
                 print "shlex: in state", repr(self.state), \
                       "I see character:", repr(nextchar)
@@ -143,6 +155,7 @@ class shlex:
                         continue
                 elif nextchar in self.commenters:
                     self.instream.readline()
+                    # Not considered a token so incrementing lineno directly
                     self.lineno = self.lineno + 1
                 elif self.posix and nextchar in self.escape:
                     escapedstate = 'a'
@@ -210,6 +223,7 @@ class shlex:
                         continue
                 elif nextchar in self.commenters:
                     self.instream.readline()
+                    # Not considered a token so incrementing lineno directly
                     self.lineno = self.lineno + 1
                     if self.posix:
                         self.state = ' '

diff --git a/Lib/test/test_shlex.py b/Lib/test/test_shlex.py
index 6c35f491b5bbf99a0baf7075e21223341d52e542..2325651d5844221354e7c0c1a8bfce1ae8b23172 100644 (file)
--- a/Lib/test/test_shlex.py
+++ b/Lib/test/test_shlex.py
@@ -178,6 +178,15 @@ class ShlexTest(unittest.TestCase):
                              "%s: %s != %s" %
                              (self.data[i][0], l, self.data[i][1:]))
 
+    def testLineNumbers(self):
+        data = '"a \n b \n c"\n"x"\n"y"'
+        for is_posix in (True, False):
+            s = shlex.shlex(data, posix=is_posix)
+            for i in (1, 4, 5):
+                s.read_token()
+                self.assertEqual(s.lineno, i)
+
+
 # Allow this test to be used with old shlex.py
 if not getattr(shlex, "split", None):
     for methname in dir(ShlexTest):

diff --git a/Misc/ACKS b/Misc/ACKS
index 1c453f0c741aafc26b32f7dd45a8e75c9d09215d..fc4fcd2a373229409ae34ee40561a8c1b5ffce8f 100644 (file)
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -717,6 +717,7 @@ Samuel Nicolary
 Gustavo Niemeyer
 Oscar Nierstrasz
 Hrvoje Niksic
+Birk Nilson
 Gregory Nofi
 Jesse Noller
 Bill Noon

diff --git a/Misc/NEWS b/Misc/NEWS
index a2f397ca9c7fb3bea8336c5aee04d9a367429b37..93173dfd65bb7623010d5830a4413775e46a50c9 100644 (file)
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -208,6 +208,9 @@ Core and Builtins
 Library
 -------
 
+- Issue #16121: Fix line number accounting in shlex. Patch by Birk
+  Nilson.
+
 - Issue #14720: sqlite3: Convert datetime microseconds correctly.
   Patch by Lowe Thiderman.