in regular expressions.
place it at the beginning of the set. For example, both ``[()[\]{}]`` and
``[]()[{}]`` will both match a parenthesis.
+ * Support of nested sets and set operations as in `Unicode Technical
+ Standard #18`_ might be added in the future. This would change the
+ syntax, so to facilitate this change a :exc:`FutureWarning` will be raised
+ in ambiguous cases for the time being.
+ That include sets starting with a literal ``'['`` or containing literal
+ character sequences ``'--'``, ``'&&'``, ``'~~'``, and ``'||'``. To
+ avoid a warning escape them with a backslash.
+
+ .. _Unicode Technical Standard #18: https://unicode.org/reports/tr18/
+
+ .. versionchanged:: 3.7
+ :exc:`FutureWarning` is raised if a character set contains constructs
+ that will change semantically in the future.
+
``|``
``A|B``, where *A* and *B* can be arbitrary REs, creates a regular expression that
will match either *A* or *B*. An arbitrary number of REs can be separated by the
>>> legal_chars = string.ascii_lowercase + string.digits + "!#$%&'*+-.^_`|~:"
>>> print('[%s]+' % re.escape(legal_chars))
- [abcdefghijklmnopqrstuvwxyz0123456789!\#\$%&'\*\+\-\.\^_`\|~:]+
+ [abcdefghijklmnopqrstuvwxyz0123456789!\#\$%\&'\*\+\-\.\^_`\|\~:]+
>>> operators = ['+', '-', '*', '/', '**']
>>> print('|'.join(map(re.escape, sorted(operators, reverse=True))))
whatsnew/3.2,,:location,zope9-location = ${zope9:location}
whatsnew/3.2,,:prefix,zope-conf = ${custom:prefix}/etc/zope.conf
library/re,,`,!#$%&'*+-.^_`|~:
-library/re,,`,!\#\$%&'\*\+\-\.\^_`\|~:
+library/re,,`,!\#\$%\&'\*\+\-\.\^_`\|\~:
library/tarfile,,:xz,'x:xz'
library/xml.etree.elementtree,,:sometag,prefix:sometag
library/xml.etree.elementtree,,:fictional,"<actors xmlns:fictional=""http://characters.example.com"""
argument ``os.scandir`` instead of ``os.listdir`` when listing the direcory
is failed.
+* Support of nested sets and set operations in regular expressions as in
+ `Unicode Technical Standard #18`_ might be added in the future. This would
+ change the syntax, so to facilitate this change a :exc:`FutureWarning` will
+ be raised in ambiguous cases for the time being.
+ That include sets starting with a literal ``'['`` or containing literal
+ character sequences ``'--'``, ``'&&'``, ``'~~'``, and ``'||'``. To
+ avoid a warning escape them with a backslash.
+ (Contributed by Serhiy Storchaka in :issue:`30349`.)
+
+.. _Unicode Technical Standard #18: https://unicode.org/reports/tr18/
+
Changes in the C API
--------------------
_wsp_splitter = re.compile(r'([{}]+)'.format(''.join(WSP))).split
_non_atom_end_matcher = re.compile(r"[^{}]+".format(
- ''.join(ATOM_ENDS).replace('\\','\\\\').replace(']',r'\]'))).match
+ re.escape(''.join(ATOM_ENDS)))).match
_non_printable_finder = re.compile(r"[\x00-\x20\x7F]").findall
_non_token_end_matcher = re.compile(r"[^{}]+".format(
- ''.join(TOKEN_ENDS).replace('\\','\\\\').replace(']',r'\]'))).match
+ re.escape(''.join(TOKEN_ENDS)))).match
_non_attribute_end_matcher = re.compile(r"[^{}]+".format(
- ''.join(ATTRIBUTE_ENDS).replace('\\','\\\\').replace(']',r'\]'))).match
+ re.escape(''.join(ATTRIBUTE_ENDS)))).match
_non_extended_attribute_end_matcher = re.compile(r"[^{}]+".format(
- ''.join(EXTENDED_ATTRIBUTE_ENDS).replace(
- '\\','\\\\').replace(']',r'\]'))).match
+ re.escape(''.join(EXTENDED_ATTRIBUTE_ENDS)))).match
def _validate_xtext(xtext):
"""If input token contains ASCII non-printables, register a defect."""
# SPECIAL_CHARS
# closing ')', '}' and ']'
# '-' (a range in character set)
+# '&', '~', (extended character set operations)
# '#' (comment) and WHITESPACE (ignored) in verbose mode
-_special_chars_map = {i: '\\' + chr(i) for i in b'()[]{}?*+-|^$\\.# \t\n\r\v\f'}
+_special_chars_map = {i: '\\' + chr(i) for i in b'()[]{}?*+-|^$\\.&~# \t\n\r\v\f'}
def escape(pattern):
"""
setappend = set.append
## if sourcematch(":"):
## pass # handle character classes
+ if source.next == '[':
+ import warnings
+ warnings.warn(
+ 'Possible nested set at position %d' % source.tell(),
+ FutureWarning, stacklevel=nested + 6
+ )
negate = sourcematch("^")
# check remaining characters
while True:
elif this[0] == "\\":
code1 = _class_escape(source, this)
else:
+ if set and this in '-&~|' and source.next == this:
+ import warnings
+ warnings.warn(
+ 'Possible set %s at position %d' % (
+ 'difference' if this == '-' else
+ 'intersection' if this == '&' else
+ 'symmetric difference' if this == '~' else
+ 'union',
+ source.tell() - 1),
+ FutureWarning, stacklevel=nested + 6
+ )
code1 = LITERAL, _ord(this)
if sourcematch("-"):
# potential range
if that[0] == "\\":
code2 = _class_escape(source, that)
else:
+ if that == '-':
+ import warnings
+ warnings.warn(
+ 'Possible set difference at position %d' % (
+ source.tell() - 2),
+ FutureWarning, stacklevel=nested + 6
+ )
code2 = LITERAL, _ord(that)
if code1[0] != LITERAL or code2[0] != LITERAL:
msg = "bad character range %s-%s" % (this, that)
self.assertEqual(re.search(r"\s([^a])", " b").group(1), "b")
self.assertEqual(re.search(r"\s([^a]*)", " bb").group(1), "bb")
+ def test_possible_set_operations(self):
+ s = bytes(range(128)).decode()
+ with self.assertWarns(FutureWarning):
+ p = re.compile(r'[0-9--1]')
+ self.assertEqual(p.findall(s), list('-./0123456789'))
+ self.assertEqual(re.findall(r'[--1]', s), list('-./01'))
+ with self.assertWarns(FutureWarning):
+ p = re.compile(r'[%--1]')
+ self.assertEqual(p.findall(s), list("%&'()*+,-1"))
+ with self.assertWarns(FutureWarning):
+ p = re.compile(r'[%--]')
+ self.assertEqual(p.findall(s), list("%&'()*+,-"))
+
+ with self.assertWarns(FutureWarning):
+ p = re.compile(r'[0-9&&1]')
+ self.assertEqual(p.findall(s), list('&0123456789'))
+ with self.assertWarns(FutureWarning):
+ p = re.compile(r'[\d&&1]')
+ self.assertEqual(p.findall(s), list('&0123456789'))
+ self.assertEqual(re.findall(r'[&&1]', s), list('&1'))
+
+ with self.assertWarns(FutureWarning):
+ p = re.compile(r'[0-9||a]')
+ self.assertEqual(p.findall(s), list('0123456789a|'))
+ with self.assertWarns(FutureWarning):
+ p = re.compile(r'[\d||a]')
+ self.assertEqual(p.findall(s), list('0123456789a|'))
+ self.assertEqual(re.findall(r'[||1]', s), list('1|'))
+
+ with self.assertWarns(FutureWarning):
+ p = re.compile(r'[0-9~~1]')
+ self.assertEqual(p.findall(s), list('0123456789~'))
+ with self.assertWarns(FutureWarning):
+ p = re.compile(r'[\d~~1]')
+ self.assertEqual(p.findall(s), list('0123456789~'))
+ self.assertEqual(re.findall(r'[~~1]', s), list('1~'))
+
+ with self.assertWarns(FutureWarning):
+ p = re.compile(r'[[0-9]|]')
+ self.assertEqual(p.findall(s), list('0123456789[]'))
+
+ with self.assertWarns(FutureWarning):
+ p = re.compile(r'[[:digit:]|]')
+ self.assertEqual(p.findall(s), list(':[]dgit'))
+
def test_search_coverage(self):
self.assertEqual(re.search(r"\s(b)", " b").group(1), "b")
self.assertEqual(re.search(r"a\s", "a ").group(0), "a ")
self.assertEqual(m.group(), match)
self.assertEqual(m.span(), span)
- LITERAL_CHARS = string.ascii_letters + string.digits + '!"%&\',/:;<=>@_`~'
+ LITERAL_CHARS = string.ascii_letters + string.digits + '!"%\',/:;<=>@_`'
def test_re_escape(self):
p = ''.join(chr(i) for i in range(256))
--- /dev/null
+FutureWarning is now emitted if a regular expression contains character set
+constructs that will change semantically in the future (nested sets and set
+operations).
projects / python / commitdiff