bpo-36878: Only allow text after `# type: ignore` if first character ASCII (GH-13504)

This disallows things like `# type: ignoreé`, which seems wrong.

Also switch to using Py_ISALNUM for the alnum check, for consistency
with other code (and maybe correctness re: locale issues?).

https://bugs.python.org/issue36878

diff --git a/Lib/test/test_type_comments.py b/Lib/test/test_type_comments.py
index c62894fa4255d64ae917130e041543b81184fbb7..83d8717247aab098ee5f140cc491d5d547601f7b 100644 (file)
--- a/Lib/test/test_type_comments.py
+++ b/Lib/test/test_type_comments.py
@@ -334,6 +334,7 @@ class TypeCommentTests(unittest.TestCase):
         check_both_ways("try:  # type: int\n  pass\nfinally:\n  pass\n")
         check_both_ways("try:\n  pass\nfinally:  # type: int\n  pass\n")
         check_both_ways("pass  # type: ignorewhatever\n")
+        check_both_ways("pass  # type: ignoreé\n")
 
     def test_func_type_input(self):
 

diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-05-22-11-16-16.bpo-36878.QwLa3P.rst b/Misc/NEWS.d/next/Core and Builtins/2019-05-22-11-16-16.bpo-36878.QwLa3P.rst
new file mode 100644 (file)
index 0000000..2d9f014
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-05-22-11-16-16.bpo-36878.QwLa3P.rst
@@ -0,0 +1,2 @@
+Only accept text after `# type: ignore` if the first character is ASCII.
+This is to disallow things like `# type: ignoreé`.

diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 9b269afc429bc105eac4f667d93a5f243d19583e..c2ec659fed888f769febaa302b90a4d63d009e97 100644 (file)
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -1275,10 +1275,11 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
                 type_start = p;
 
                 /* A TYPE_IGNORE is "type: ignore" followed by the end of the token
-                 * or anything non-alphanumeric. */
+                 * or anything ASCII and non-alphanumeric. */
                 is_type_ignore = (
                     tok->cur >= ignore_end && memcmp(p, "ignore", 6) == 0
-                    && !(tok->cur > ignore_end && isalnum(p[6])));
+                    && !(tok->cur > ignore_end
+                         && ((unsigned char)ignore_end[0] >= 128 || Py_ISALNUM(ignore_end[0]))));
 
                 if (is_type_ignore) {
                     *p_start = (char *) ignore_end;