* bpo-21315: Fix parsing of encoded words with missing leading ws.
Because of missing leading whitespace, encoded word would get parsed as
unstructured token. This patch fixes that by looking for encoded words when
splitting tokens with whitespace.
Missing trailing whitespace around encoded word now register a defect
instead.
Original patch suggestion by David R. Murray on bpo-21315.
def quote_string(value):
return '"'+str(value).replace('\\', '\\\\').replace('"', r'\"')+'"'
+# Match a RFC 2047 word, looks like =?utf-8?q?someword?=
+rfc2047_matcher = re.compile(r'''
+ =\? # literal =?
+ [^?]* # charset
+ \? # literal ?
+ [qQbB] # literal 'q' or 'b', case insensitive
+ \? # literal ?
+ .*? # encoded word
+ \?= # literal ?=
+''', re.VERBOSE | re.MULTILINE)
+
+
#
# TokenList and its subclasses
#
_validate_xtext(vtext)
ew.append(vtext)
text = ''.join(remainder)
+ # Encoded words should be followed by a WS
+ if value and value[0] not in WSP:
+ ew.defects.append(errors.InvalidHeaderDefect(
+ "missing trailing whitespace after encoded-word"))
return ew, value
def get_unstructured(value):
unstructured.append(token)
continue
tok, *remainder = _wsp_splitter(value, 1)
+ # Split in the middle of an atom if there is a rfc2047 encoded word
+ # which does not have WSP on both sides. The defect will be registered
+ # the next time through the loop.
+ if rfc2047_matcher.search(tok):
+ tok, *remainder = value.partition('=?')
vtext = ValueTerminal(tok, 'vtext')
_validate_xtext(vtext)
unstructured.append(vtext)
'=?us-ascii?q?first?==?utf-8?q?second?=',
'first',
'first',
- [],
+ [errors.InvalidHeaderDefect],
'=?utf-8?q?second?=')
def test_get_encoded_word_sets_extra_attributes(self):
'=?utf-8?q?foo?==?utf-8?q?bar?=',
'foobar',
'foobar',
+ [errors.InvalidHeaderDefect,
+ errors.InvalidHeaderDefect],
+ '')
+
+ def test_get_unstructured_ew_without_leading_whitespace(self):
+ self._test_get_x(
+ self._get_unst,
+ 'nowhitespace=?utf-8?q?somevalue?=',
+ 'nowhitespacesomevalue',
+ 'nowhitespacesomevalue',
+ [errors.InvalidHeaderDefect],
+ '')
+
+ def test_get_unstructured_ew_without_trailing_whitespace(self):
+ self._test_get_x(
+ self._get_unst,
+ '=?utf-8?q?somevalue?=nowhitespace',
+ 'somevaluenowhitespace',
+ 'somevaluenowhitespace',
[errors.InvalidHeaderDefect],
'')
'"=?utf-8?Q?not_really_valid?="',
'"not really valid"',
'not really valid',
- [errors.InvalidHeaderDefect],
+ [errors.InvalidHeaderDefect,
+ errors.InvalidHeaderDefect],
'')
# get_comment
'rfc2047_atom_in_quoted_string_is_decoded':
('"=?utf-8?q?=C3=89ric?=" <foo@example.com>',
- [errors.InvalidHeaderDefect],
+ [errors.InvalidHeaderDefect,
+ errors.InvalidHeaderDefect],
'Éric <foo@example.com>',
'Éric',
'foo@example.com',
--- /dev/null
+Email headers containing RFC2047 encoded words are parsed despite the missing
+whitespace, and a defect registered. Also missing trailing whitespace after
+encoded words is now registered as a defect.
+