#14645: Generator now emits correct linesep for all parts.

Previously the parts of the message retained whatever linesep they had on
read, which means if the messages weren't read in univeral newline mode, the
line endings could well be inconsistent.  In general sending it via smtplib
would result in them getting fixed, but it is better to generate them
correctly to begin with.  Also, the new send_message method of smtplib does
not do the fixup, so that method is producing rfc-invalid output without this
fix.

diff --git a/Lib/email/generator.py b/Lib/email/generator.py
index d6acde36c181d8b3f4141ebc8cdbea27e9ab3322..cba0382fedb38a9371bb92087a7f199ede71152b 100644 (file)
--- a/Lib/email/generator.py
+++ b/Lib/email/generator.py
@@ -119,6 +119,19 @@ class Generator:
         # BytesGenerator overrides this to encode strings to bytes.
         return s
 
+    def _write_lines(self, lines):
+        # We have to transform the line endings.
+        if not lines:
+            return
+        lines = lines.splitlines(True)
+        for line in lines[:-1]:
+            self.write(line.rstrip('\r\n'))
+            self.write(self._NL)
+        laststripped = lines[-1].rstrip('\r\n')
+        self.write(laststripped)
+        if len(lines[-1])!=len(laststripped):
+            self.write(self._NL)
+
     def _write(self, msg):
         # We can't write the headers yet because of the following scenario:
         # say a multipart message includes the boundary string somewhere in
@@ -198,7 +211,7 @@ class Generator:
                 payload = msg.get_payload()
         if self._mangle_from_:
             payload = fcre.sub('>From ', payload)
-        self.write(payload)
+        self._write_lines(payload)
 
     # Default body handler
     _writeBody = _handle_text
@@ -237,7 +250,8 @@ class Generator:
                 preamble = fcre.sub('>From ', msg.preamble)
             else:
                 preamble = msg.preamble
-            self.write(preamble + self._NL)
+            self._write_lines(preamble)
+            self.write(self._NL)
         # dash-boundary transport-padding CRLF
         self.write('--' + boundary + self._NL)
         # body-part
@@ -259,7 +273,7 @@ class Generator:
                 epilogue = fcre.sub('>From ', msg.epilogue)
             else:
                 epilogue = msg.epilogue
-            self.write(epilogue)
+            self._write_lines(epilogue)
 
     def _handle_multipart_signed(self, msg):
         # The contents of signed parts has to stay unmodified in order to keep
@@ -393,7 +407,7 @@ class BytesGenerator(Generator):
         if _has_surrogates(msg._payload):
             if self._mangle_from_:
                 msg._payload = fcre.sub(">From ", msg._payload)
-            self.write(msg._payload)
+            self._write_lines(msg._payload)
         else:
             super(BytesGenerator,self)._handle_text(msg)
 

diff --git a/Lib/email/test/test_email.py b/Lib/email/test/test_email.py
index daed3b0d63f6d7a9fb78efab95030e4fa5d839e8..5e61a7b97a25ec4dfa10b857b1ae1bf3a6ea3253 100644 (file)
--- a/Lib/email/test/test_email.py
+++ b/Lib/email/test/test_email.py
@@ -68,6 +68,7 @@ class TestEmailBase(unittest.TestCase):
         with openfile(findfile(filename)) as fp:
             return email.message_from_file(fp)
 
+    maxDiff = None
 
 
 # Test various aspects of the Message class's API
@@ -2907,6 +2908,40 @@ multipart/report
             email.utils.make_msgid(domain='testdomain-string')[-19:],
             '@testdomain-string>')
 
+    def test_Generator_linend(self):
+        # Issue 14645.
+        with openfile('msg_26.txt', newline='\n') as f:
+            msgtxt = f.read()
+        msgtxt_nl = msgtxt.replace('\r\n', '\n')
+        msg = email.message_from_string(msgtxt)
+        s = StringIO()
+        g = email.generator.Generator(s)
+        g.flatten(msg)
+        self.assertEqual(s.getvalue(), msgtxt_nl)
+
+    def test_BytesGenerator_linend(self):
+        # Issue 14645.
+        with openfile('msg_26.txt', newline='\n') as f:
+            msgtxt = f.read()
+        msgtxt_nl = msgtxt.replace('\r\n', '\n')
+        msg = email.message_from_string(msgtxt_nl)
+        s = BytesIO()
+        g = email.generator.BytesGenerator(s)
+        g.flatten(msg, linesep='\r\n')
+        self.assertEqual(s.getvalue().decode('ascii'), msgtxt)
+
+    def test_BytesGenerator_linend_with_non_ascii(self):
+        # Issue 14645.
+        with openfile('msg_26.txt', 'rb') as f:
+            msgtxt = f.read()
+        msgtxt = msgtxt.replace(b'with attachment', b'fo\xf6')
+        msgtxt_nl = msgtxt.replace(b'\r\n', b'\n')
+        msg = email.message_from_bytes(msgtxt_nl)
+        s = BytesIO()
+        g = email.generator.BytesGenerator(s)
+        g.flatten(msg, linesep='\r\n')
+        self.assertEqual(s.getvalue(), msgtxt)
+
 
 # Test the iterator/generators
 class TestIterators(TestEmailBase):

diff --git a/Misc/NEWS b/Misc/NEWS
index 95dea0aac8cbd6adc615187777dc76fc98601a8c..0c43f4652296423d3fd626ab50b3bfebbc854ff4 100644 (file)
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -233,6 +233,11 @@ Core and Builtins
 Library
 -------
 
+- Issue #14645: The email generator classes now produce output using the
+  specified linesep throughout.  Previously if the prolog, epilog, or
+  body were stored with a different linesep, that linesep was used.  This
+  fix corrects an RFC non-compliance issue with smtplib.send_message.
+
 - Issue #17278: Fix a crash in heapq.heappush() and heapq.heappop() when
   the list is being resized concurrently.