Issue #8260: The read(), readline() and readlines() methods of

codecs.StreamReader returned incomplete data when were called after
readline() or read(size).  Based on patch by Amaury Forgeot d'Arc.

diff --git a/Lib/codecs.py b/Lib/codecs.py
index 2e2e7555a4829724fb6e72de10a210048de10781..c2065dafa27ceb87377717c163fc6331c8b5d110 100644 (file)
--- a/Lib/codecs.py
+++ b/Lib/codecs.py
@@ -475,15 +475,12 @@ class StreamReader(Codec):
         # read until we get the required number of characters (if available)
         while True:
             # can the request be satisfied from the character buffer?
-            if chars < 0:
-                if size < 0:
-                    if self.charbuffer:
-                        break
-                elif len(self.charbuffer) >= size:
-                    break
-            else:
+            if chars >= 0:
                 if len(self.charbuffer) >= chars:
                     break
+            elif size >= 0:
+                if len(self.charbuffer) >= size:
+                    break
             # we need more data
             if size < 0:
                 newdata = self.stream.read()
@@ -491,6 +488,8 @@ class StreamReader(Codec):
                 newdata = self.stream.read(size)
             # decode bytes (those remaining from the last call included)
             data = self.bytebuffer + newdata
+            if not data:
+                break
             try:
                 newchars, decodedbytes = self.decode(data, self.errors)
             except UnicodeDecodeError as exc:

diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py
index a32ce76bc8d1271e65148954c19c133f2c91904d..3950c3bc2080086d4422877eac0a8311cd3e39df 100644 (file)
--- a/Lib/test/test_codecs.py
+++ b/Lib/test/test_codecs.py
@@ -175,6 +175,40 @@ class ReadTest(MixInCheckStateHandling):
                         size*"a",
                     )
 
+    def test_mixed_readline_and_read(self):
+        lines = ["Humpty Dumpty sat on a wall,\n",
+                 "Humpty Dumpty had a great fall.\r\n",
+                 "All the king's horses and all the king's men\r",
+                 "Couldn't put Humpty together again."]
+        data = ''.join(lines)
+        def getreader():
+            stream = io.BytesIO(data.encode(self.encoding))
+            return codecs.getreader(self.encoding)(stream)
+
+        # Issue #8260: Test readline() followed by read()
+        f = getreader()
+        self.assertEqual(f.readline(), lines[0])
+        self.assertEqual(f.read(), ''.join(lines[1:]))
+        self.assertEqual(f.read(), '')
+
+        # Issue #16636: Test readline() followed by readlines()
+        f = getreader()
+        self.assertEqual(f.readline(), lines[0])
+        self.assertEqual(f.readlines(), lines[1:])
+        self.assertEqual(f.read(), '')
+
+        # Test read() followed by read()
+        f = getreader()
+        self.assertEqual(f.read(size=40, chars=5), data[:5])
+        self.assertEqual(f.read(), data[5:])
+        self.assertEqual(f.read(), '')
+
+        # Issue #12446: Test read() followed by readlines()
+        f = getreader()
+        self.assertEqual(f.read(size=40, chars=5), data[:5])
+        self.assertEqual(f.readlines(), [lines[0][5:]] + lines[1:])
+        self.assertEqual(f.read(), '')
+
     def test_bug1175396(self):
         s = [
             '<%!--===================================================\r\n',
@@ -2370,8 +2404,6 @@ class TransformCodecTest(unittest.TestCase):
 
     def test_readline(self):
         for encoding in bytes_transform_encodings:
-            if encoding in ['uu_codec', 'zlib_codec']:
-                continue
             with self.subTest(encoding=encoding):
                 sin = codecs.encode(b"\x80", encoding)
                 reader = codecs.getreader(encoding)(io.BytesIO(sin))

diff --git a/Misc/NEWS b/Misc/NEWS
index 25167357181b218149a9e3e54c69276ceaea4fef..9c0ed5a651515e5a3f1874bbd14fe5a62cf0fb6c 100644 (file)
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -36,6 +36,10 @@ Core and Builtins
 Library
 -------
 
+- Issue #8260: The read(), readline() and readlines() methods of
+  codecs.StreamReader returned incomplete data when were called after
+  readline() or read(size).  Based on patch by Amaury Forgeot d'Arc.
+
 - Issue #20105: the codec exception chaining now correctly sets the
   traceback of the original exception as its __traceback__ attribute.