From a33df31629f2f6ed85890baa9b4e71c30efa95a9 Mon Sep 17 00:00:00 2001 From: R David Murray Date: Mon, 11 May 2015 12:11:40 -0400 Subject: [PATCH] #21795: advertise 8BITMIME if decode_data is False. Patch by Milan Oberkirch, with a few updates. This changeset also tweaks the smtpd and whatsnew docs for smtpd into what should be the final form for the 3.5 release. --- Doc/library/smtpd.rst | 58 ++++++++++-------- Doc/whatsnew/3.5.rst | 22 +++++-- Lib/smtpd.py | 104 ++++++++++++++++++--------------- Lib/test/test_smtpd.py | 129 ++++++++++++++++++++++++++++++++++------- Misc/NEWS | 3 + 5 files changed, 219 insertions(+), 97 deletions(-) diff --git a/Doc/library/smtpd.rst b/Doc/library/smtpd.rst index 3e0c6fbff5..575dcec994 100644 --- a/Doc/library/smtpd.rst +++ b/Doc/library/smtpd.rst @@ -40,20 +40,27 @@ SMTPServer Objects accepted in a ``DATA`` command. A value of ``None`` or ``0`` means no limit. - *enable_SMTPUTF8* determins whether the ``SMTPUTF8`` extension (as defined - in :RFC:`6531`) should be enabled. The default is ``False``. If - *enable_SMTPUTF* is set to ``True``, the :meth:`process_smtputf8_message` - method must be defined. A :exc:`ValueError` is raised if both - *enable_SMTPUTF8* and *decode_data* are set to ``True`` at the same time. + *map* is the socket map to use for connections (an initially empty + dictionary is a suitable value). If not specified the :mod:`asyncore` + global socket map is used. - A dictionary can be specified in *map* to avoid using a global socket map. + *enable_SMTPUTF8* determins whether the ``SMTPUTF8`` extension (as defined + in :RFC:`6531`) should be enabled. The default is ``False``. If set to + ``True``, *decode_data* must be ``False`` (otherwise an error is raised). + When ``True``, ``SMTPUTF8`` is accepted as a parameter to the ``MAIL`` + command and when present is passed to :meth:`process_message` in the + ``kwargs['mail_options']`` list. *decode_data* specifies whether the data portion of the SMTP transaction should be decoded using UTF-8. The default is ``True`` for backward - compatibility reasons, but will change to ``False`` in Python 3.6. Specify - the keyword value explicitly to avoid the :exc:`DeprecationWarning`. + compatibility reasons, but will change to ``False`` in Python 3.6; specify + the keyword value explicitly to avoid the :exc:`DeprecationWarning`. When + *decode_data* is set to ``False`` the server advertises the ``8BITMIME`` + extension (:rfc:`6152`), accepts the ``BODY=8BITMIME`` parameter to + the ``MAIL`` command, and when present passes it to :meth:`process_message` + in the ``kwargs['mail_options']`` list. - .. method:: process_message(peer, mailfrom, rcpttos, data) + .. method:: process_message(peer, mailfrom, rcpttos, data, **kwargs) Raise a :exc:`NotImplementedError` exception. Override this in subclasses to do something useful with this message. Whatever was passed in the @@ -67,34 +74,39 @@ SMTPServer Objects argument will be a unicode string. If it is set to ``False``, it will be a bytes object. - Return ``None`` to request a normal ``250 Ok`` response; otherwise - return the desired response string in :RFC:`5321` format. + *kwargs* is a dictionary containing additional information. It is empty + unless at least one of ``decode_data=False`` or ``enable_SMTPUTF8=True`` + was given as an init parameter, in which case it contains the following + keys: + + *mail_options*: + a list of all received parameters to the ``MAIL`` + command (the elements are uppercase strings; example: + ``['BODY=8BITMIME', 'SMTPUTF8']``). - .. method:: process_smtputf8_message(peer, mailfrom, rcpttos, data) + *rcpt_options*: + same as *mail_options* but for the ``RCPT`` command. + Currently no ``RCPT TO`` options are supported, so for now + this will always be an empty list. - Raise a :exc:`NotImplementedError` exception. Override this in - subclasses to do something useful with messages when *enable_SMTPUTF8* - has been set to ``True`` and the SMTP client requested ``SMTPUTF8``, - since this method is called rather than :meth:`process_message` when the - client actively requests ``SMTPUTF8``. The *data* argument will always - be a bytes object, and any non-``None`` return value should conform to - :rfc:`6531`; otherwise, the API is the same as for - :meth:`process_message`. + Return ``None`` to request a normal ``250 Ok`` response; otherwise + return the desired response string in :RFC:`5321` format. .. attribute:: channel_class Override this in subclasses to use a custom :class:`SMTPChannel` for managing SMTP clients. - .. versionchanged:: 3.4 - The *map* argument was added. + .. versionadded:: 3.4 + The *map* constructor argument. .. versionchanged:: 3.5 *localaddr* and *remoteaddr* may now contain IPv6 addresses. .. versionadded:: 3.5 the *decode_data* and *enable_SMTPUTF8* constructor arguments, and the - :meth:`process_smtputf8_message` method. + *kwargs* argument to :meth:`process_message` when one or more of these is + specified. DebuggingServer Objects diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst index 02a7065a37..86febb0691 100644 --- a/Doc/whatsnew/3.5.rst +++ b/Doc/whatsnew/3.5.rst @@ -468,16 +468,28 @@ smtpd transaction is decoded using the ``utf-8`` codec or is instead provided to :meth:`~smtpd.SMTPServer.process_message` as a byte string. The default is ``True`` for backward compatibility reasons, but will change to ``False`` - in Python 3.6. (Contributed by Maciej Szulik in :issue:`19662`.) + in Python 3.6. If *decode_data* is set to ``False``, the + :meth:`~smtpd.SMTPServer.process_message` method must be prepared to accept + keyword arguments. (Contributed by Maciej Szulik in :issue:`19662`.) + +* :class:`~smtpd.SMTPServer` now advertises the ``8BITMIME`` extension + (:rfc:`6152`) if if *decode_data* has been set ``True``. If the client + specifies ``BODY=8BITMIME`` on the ``MAIL`` command, it is passed to + :meth:`~smtpd.SMTPServer.process_message` via the ``mail_options`` keyword. + (Contributed by Milan Oberkirch and R. David Murray in :issue:`21795`.) + +* :class:`~smtpd.SMTPServer` now supports the ``SMTPUTF8`` extension + (:rfc:`6531`: Internationalized Email). If the client specified ``SMTPUTF8 + BODY=8BITMIME`` on the ``MAIL`` command, they are passed to + :meth:`~smtpd.SMTPServer.process_message` via the ``mail_options`` keyword. + It is the responsibility of the :meth:`~smtpd.SMTPServer.process_message` + method to correctly handle the ``SMTPUTF8`` data. (Contributed by Milan + Oberkirch in :issue:`21725`.) * It is now possible to provide, directly or via name resolution, IPv6 addresses in the :class:`~smtpd.SMTPServer` constructor, and have it successfully connect. (Contributed by Milan Oberkirch in :issue:`14758`.) -* :mod:`~smtpd.SMTPServer` now supports :rfc:`6531` via the *enable_SMTPUTF8* - constructor argument and a user-provided - :meth:`~smtpd.SMTPServer.process_smtputf8_message` method. - smtplib ------- diff --git a/Lib/smtpd.py b/Lib/smtpd.py index dd410b8e16..ff86e7d206 100755 --- a/Lib/smtpd.py +++ b/Lib/smtpd.py @@ -381,10 +381,13 @@ class SMTPChannel(asynchat.async_chat): data.append(text) self.received_data = self._newline.join(data) args = (self.peer, self.mailfrom, self.rcpttos, self.received_data) - if self.require_SMTPUTF8: - status = self.smtp_server.process_smtputf8_message(*args) - else: - status = self.smtp_server.process_message(*args) + kwargs = {} + if not self._decode_data: + kwargs = { + 'mail_options': self.mail_options, + 'rcpt_options': self.rcpt_options, + } + status = self.smtp_server.process_message(*args, **kwargs) self._set_post_data_state() if not status: self.push('250 OK') @@ -419,8 +422,9 @@ class SMTPChannel(asynchat.async_chat): if self.data_size_limit: self.push('250-SIZE %s' % self.data_size_limit) self.command_size_limits['MAIL'] += 26 - if self.enable_SMTPUTF8: + if not self._decode_data: self.push('250-8BITMIME') + if self.enable_SMTPUTF8: self.push('250-SMTPUTF8') self.command_size_limits['MAIL'] += 10 self.push('250 HELP') @@ -454,11 +458,15 @@ class SMTPChannel(asynchat.async_chat): return address.addr_spec, rest def _getparams(self, params): - # Return any parameters that appear to be syntactically valid according - # to RFC 1869, ignore all others. (Postel rule: accept what we can.) - params = [param.split('=', 1) if '=' in param else (param, True) - for param in params.split()] - return {k: v for k, v in params if k.isalnum()} + # Return params as dictionary. Return None if not all parameters + # appear to be syntactically valid according to RFC 1869. + result = {} + for param in params: + param, eq, value = param.partition('=') + if not param.isalnum() or eq and not value: + return None + result[param] = value if eq else True + return result def smtp_HELP(self, arg): if arg: @@ -508,7 +516,7 @@ class SMTPChannel(asynchat.async_chat): def smtp_MAIL(self, arg): if not self.seen_greeting: - self.push('503 Error: send HELO first'); + self.push('503 Error: send HELO first') return print('===> MAIL', arg, file=DEBUGSTREAM) syntaxerr = '501 Syntax: MAIL FROM:
' @@ -528,18 +536,23 @@ class SMTPChannel(asynchat.async_chat): if self.mailfrom: self.push('503 Error: nested MAIL command') return - params = self._getparams(params.upper()) + self.mail_options = params.upper().split() + params = self._getparams(self.mail_options) if params is None: self.push(syntaxerr) return - body = params.pop('BODY', '7BIT') - if self.enable_SMTPUTF8 and params.pop('SMTPUTF8', False): - if body != '8BITMIME': - self.push('501 Syntax: MAIL FROM:
' - ' [BODY=8BITMIME SMTPUTF8]') + if not self._decode_data: + body = params.pop('BODY', '7BIT') + if body not in ['7BIT', '8BITMIME']: + self.push('501 Error: BODY can only be one of 7BIT, 8BITMIME') return - else: + if self.enable_SMTPUTF8: + smtputf8 = params.pop('SMTPUTF8', False) + if smtputf8 is True: self.require_SMTPUTF8 = True + elif smtputf8 is not False: + self.push('501 Error: SMTPUTF8 takes no arguments') + return size = params.pop('SIZE', None) if size: if not size.isdigit(): @@ -574,16 +587,16 @@ class SMTPChannel(asynchat.async_chat): if not address: self.push(syntaxerr) return - if params: - if self.extended_smtp: - params = self._getparams(params.upper()) - if params is None: - self.push(syntaxerr) - return - else: - self.push(syntaxerr) - return - if params and len(params.keys()) > 0: + if not self.extended_smtp and params: + self.push(syntaxerr) + return + self.rcpt_options = params.upper().split() + params = self._getparams(self.rcpt_options) + if params is None: + self.push(syntaxerr) + return + # XXX currently there are no options we recognize. + if len(params.keys()) > 0: self.push('555 RCPT TO parameters not recognized or not implemented') return self.rcpttos.append(address) @@ -667,7 +680,7 @@ class SMTPServer(asyncore.dispatcher): self._decode_data) # API for "doing something useful with the message" - def process_message(self, peer, mailfrom, rcpttos, data): + def process_message(self, peer, mailfrom, rcpttos, data, **kwargs): """Override this abstract method to handle messages from the client. peer is a tuple containing (ipaddr, port) of the client that made the @@ -685,21 +698,16 @@ class SMTPServer(asyncore.dispatcher): containing a `.' followed by other text has had the leading dot removed. - This function should return None for a normal `250 Ok' response; - otherwise, it should return the desired response string in RFC 821 - format. - - """ - raise NotImplementedError - - # API for processing messeges needing Unicode support (RFC 6531, RFC 6532). - def process_smtputf8_message(self, peer, mailfrom, rcpttos, data): - """Same as ``process_message`` but for messages for which the client - has sent the SMTPUTF8 parameter with the MAIL command (see the - enable_SMTPUTF8 parameter of the constructor). + kwargs is a dictionary containing additional information. It is empty + unless decode_data=False or enable_SMTPUTF8=True was given as init + parameter, in which case ut will contain the following keys: + 'mail_options': list of parameters to the mail command. All + elements are uppercase strings. Example: + ['BODY=8BITMIME', 'SMTPUTF8']. + 'rcpt_options': same, for the rcpt command. This function should return None for a normal `250 Ok' response; - otherwise, it should return the desired response string in RFC 6531 + otherwise, it should return the desired response string in RFC 821 format. """ @@ -725,13 +733,13 @@ class DebuggingServer(SMTPServer): line = repr(line) print(line) - def process_message(self, peer, mailfrom, rcpttos, data): + def process_message(self, peer, mailfrom, rcpttos, data, **kwargs): print('---------- MESSAGE FOLLOWS ----------') - self._print_message_content(peer, data) - print('------------ END MESSAGE ------------') - - def process_smtputf8_message(self, peer, mailfrom, rcpttos, data): - print('----- SMTPUTF8 MESSAGE FOLLOWS ------') + if kwargs: + if kwargs.get('mail_options'): + print('mail options: %s' % kwargs['mail_options']) + if kwargs.get('rcpt_options'): + print('rcpt options: %s\n' % kwargs['rcpt_options']) self._print_message_content(peer, data) print('------------ END MESSAGE ------------') diff --git a/Lib/test/test_smtpd.py b/Lib/test/test_smtpd.py index 6eb47f1dee..1aa55d2144 100644 --- a/Lib/test/test_smtpd.py +++ b/Lib/test/test_smtpd.py @@ -16,13 +16,12 @@ class DummyServer(smtpd.SMTPServer): else: self.return_status = b'return status' - def process_message(self, peer, mailfrom, rcpttos, data): + def process_message(self, peer, mailfrom, rcpttos, data, **kw): self.messages.append((peer, mailfrom, rcpttos, data)) if data == self.return_status: return '250 Okish' - - def process_smtputf8_message(self, *args, **kwargs): - return '250 SMTPUTF8 message okish' + if 'mail_options' in kw and 'SMTPUTF8' in kw['mail_options']: + return '250 SMTPUTF8 message okish' class DummyDispatcherBroken(Exception): @@ -54,22 +53,6 @@ class SMTPDServerTest(unittest.TestCase): write_line(b'DATA') self.assertRaises(NotImplementedError, write_line, b'spam\r\n.\r\n') - def test_process_smtputf8_message_unimplemented(self): - server = smtpd.SMTPServer((support.HOST, 0), ('b', 0), - enable_SMTPUTF8=True) - conn, addr = server.accept() - channel = smtpd.SMTPChannel(server, conn, addr, enable_SMTPUTF8=True) - - def write_line(line): - channel.socket.queue_recv(line) - channel.handle_read() - - write_line(b'EHLO example') - write_line(b'MAIL From: BODY=8BITMIME SMTPUTF8') - write_line(b'RCPT To: ') - write_line(b'DATA') - self.assertRaises(NotImplementedError, write_line, b'spam\r\n.\r\n') - def test_decode_data_default_warns(self): with self.assertWarns(DeprecationWarning): smtpd.SMTPServer((support.HOST, 0), ('b', 0)) @@ -168,7 +151,8 @@ class DebuggingServerTest(unittest.TestCase): enable_SMTPUTF8=True) stdout = s.getvalue() self.assertEqual(stdout, textwrap.dedent("""\ - ----- SMTPUTF8 MESSAGE FOLLOWS ------ + ---------- MESSAGE FOLLOWS ---------- + mail options: ['BODY=8BITMIME', 'SMTPUTF8'] b'From: test' b'X-Peer: peer-address' b'' @@ -201,6 +185,109 @@ class TestFamilyDetection(unittest.TestCase): self.assertEqual(server.socket.family, socket.AF_INET) +class TestRcptOptionParsing(unittest.TestCase): + error_response = (b'555 RCPT TO parameters not recognized or not ' + b'implemented\r\n') + + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + self.old_debugstream = smtpd.DEBUGSTREAM + self.debug = smtpd.DEBUGSTREAM = io.StringIO() + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + smtpd.DEBUGSTREAM = self.old_debugstream + + def write_line(self, channel, line): + channel.socket.queue_recv(line) + channel.handle_read() + + def test_params_rejected(self): + server = DummyServer((support.HOST, 0), ('b', 0), decode_data=False) + conn, addr = server.accept() + channel = smtpd.SMTPChannel(server, conn, addr, decode_data=False) + self.write_line(channel, b'EHLO example') + self.write_line(channel, b'MAIL from: size=20') + self.write_line(channel, b'RCPT to: foo=bar') + self.assertEqual(channel.socket.last, self.error_response) + + def test_nothing_accepted(self): + server = DummyServer((support.HOST, 0), ('b', 0), decode_data=False) + conn, addr = server.accept() + channel = smtpd.SMTPChannel(server, conn, addr, decode_data=False) + self.write_line(channel, b'EHLO example') + self.write_line(channel, b'MAIL from: size=20') + self.write_line(channel, b'RCPT to: ') + self.assertEqual(channel.socket.last, b'250 OK\r\n') + + +class TestMailOptionParsing(unittest.TestCase): + error_response = (b'555 MAIL FROM parameters not recognized or not ' + b'implemented\r\n') + + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + self.old_debugstream = smtpd.DEBUGSTREAM + self.debug = smtpd.DEBUGSTREAM = io.StringIO() + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + smtpd.DEBUGSTREAM = self.old_debugstream + + def write_line(self, channel, line): + channel.socket.queue_recv(line) + channel.handle_read() + + def test_with_decode_data_true(self): + server = DummyServer((support.HOST, 0), ('b', 0), decode_data=True) + conn, addr = server.accept() + channel = smtpd.SMTPChannel(server, conn, addr, decode_data=True) + self.write_line(channel, b'EHLO example') + for line in [ + b'MAIL from: size=20 SMTPUTF8', + b'MAIL from: size=20 SMTPUTF8 BODY=8BITMIME', + b'MAIL from: size=20 BODY=UNKNOWN', + b'MAIL from: size=20 body=8bitmime', + ]: + self.write_line(channel, line) + self.assertEqual(channel.socket.last, self.error_response) + self.write_line(channel, b'MAIL from: size=20') + self.assertEqual(channel.socket.last, b'250 OK\r\n') + + def test_with_decode_data_false(self): + server = DummyServer((support.HOST, 0), ('b', 0), decode_data=False) + conn, addr = server.accept() + channel = smtpd.SMTPChannel(server, conn, addr, decode_data=False) + self.write_line(channel, b'EHLO example') + for line in [ + b'MAIL from: size=20 SMTPUTF8', + b'MAIL from: size=20 SMTPUTF8 BODY=8BITMIME', + ]: + self.write_line(channel, line) + self.assertEqual(channel.socket.last, self.error_response) + self.write_line( + channel, + b'MAIL from: size=20 SMTPUTF8 BODY=UNKNOWN') + self.assertEqual( + channel.socket.last, + b'501 Error: BODY can only be one of 7BIT, 8BITMIME\r\n') + self.write_line( + channel, b'MAIL from: size=20 body=8bitmime') + self.assertEqual(channel.socket.last, b'250 OK\r\n') + + def test_with_enable_smtputf8_true(self): + server = DummyServer((support.HOST, 0), ('b', 0), enable_SMTPUTF8=True) + conn, addr = server.accept() + channel = smtpd.SMTPChannel(server, conn, addr, enable_SMTPUTF8=True) + self.write_line(channel, b'EHLO example') + self.write_line( + channel, + b'MAIL from: size=20 body=8bitmime smtputf8') + self.assertEqual(channel.socket.last, b'250 OK\r\n') + + class SMTPDChannelTest(unittest.TestCase): def setUp(self): smtpd.socket = asyncore.socket = mock_socket diff --git a/Misc/NEWS b/Misc/NEWS index b18c5d7d74..cc2ff1dfde 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -38,6 +38,9 @@ Core and Builtins Library ------- +- Issue #21795: smtpd now supports the 8BITMIME extension whenever + the new *decode_data* constructor argument is set to False. + - Issue #21800: imaplib now supports RFC 5161 (enable), RFC 6855 (utf8/internationalized email) and automatically encodes non-ASCII usernames and passwords to UTF8. -- 2.40.0