GH-13238 made extra text after a # type: ignore accepted by the parser.
This finishes the job and actually plumbs the extra text through the
parser and makes it available in the AST.
union {
struct {
int lineno;
+ string tag;
} TypeIgnore;
} v;
#define withitem(a0, a1, a2) _Py_withitem(a0, a1, a2)
withitem_ty _Py_withitem(expr_ty context_expr, expr_ty optional_vars, PyArena
*arena);
-#define TypeIgnore(a0, a1) _Py_TypeIgnore(a0, a1)
-type_ignore_ty _Py_TypeIgnore(int lineno, PyArena *arena);
+#define TypeIgnore(a0, a1, a2) _Py_TypeIgnore(a0, a1, a2)
+type_ignore_ty _Py_TypeIgnore(int lineno, string tag, PyArena *arena);
PyObject* PyAST_mod2obj(mod_ty t);
mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode);
def test_ignores(self):
for tree in self.parse_all(ignores):
- self.assertEqual([ti.lineno for ti in tree.type_ignores], [2, 5, 8, 9, 10, 11])
+ self.assertEqual(
+ [(ti.lineno, ti.tag) for ti in tree.type_ignores],
+ [
+ (2, ''),
+ (5, ''),
+ (8, '[excuse]'),
+ (9, '=excuse'),
+ (10, ' [excuse]'),
+ (11, ' whatever'),
+ ])
tree = self.classic_parse(ignores)
self.assertEqual(tree.type_ignores, [])
Andreas Stührk
Colin Su
Pal Subbiah
+Michael J. Sullivan
Nathan Sullivan
Mark Summerfield
Reuben Sumner
--- /dev/null
+Store text appearing after a `# type: ignore` comment in the AST. For
+example a type ignore like `# type: ignore[E1000]` will have the string
+`"[E1000]"` stored in its AST node.
withitem = (expr context_expr, expr? optional_vars)
- type_ignore = TypeIgnore(int lineno)
+ type_ignore = TypeIgnore(int lineno, string tag)
}
-
static int initerr(perrdetail *err_ret, PyObject * filename);
typedef struct {
- int *items;
+ struct {
+ int lineno;
+ char *comment;
+ } *items;
size_t size;
size_t num_items;
-} growable_int_array;
+} growable_comment_array;
static int
-growable_int_array_init(growable_int_array *arr, size_t initial_size) {
+growable_comment_array_init(growable_comment_array *arr, size_t initial_size) {
assert(initial_size > 0);
arr->items = malloc(initial_size * sizeof(*arr->items));
arr->size = initial_size;
}
static int
-growable_int_array_add(growable_int_array *arr, int item) {
+growable_comment_array_add(growable_comment_array *arr, int lineno, char *comment) {
if (arr->num_items >= arr->size) {
arr->size *= 2;
arr->items = realloc(arr->items, arr->size * sizeof(*arr->items));
}
}
- arr->items[arr->num_items] = item;
+ arr->items[arr->num_items].lineno = lineno;
+ arr->items[arr->num_items].comment = comment;
arr->num_items++;
return 1;
}
static void
-growable_int_array_deallocate(growable_int_array *arr) {
+growable_comment_array_deallocate(growable_comment_array *arr) {
+ for (unsigned i = 0; i < arr->num_items; i++) {
+ PyObject_FREE(arr->items[i].comment);
+ }
free(arr->items);
}
node *n;
int started = 0;
int col_offset, end_col_offset;
- growable_int_array type_ignores;
+ growable_comment_array type_ignores;
- if (!growable_int_array_init(&type_ignores, 10)) {
+ if (!growable_comment_array_init(&type_ignores, 10)) {
err_ret->error = E_NOMEM;
PyTokenizer_Free(tok);
return NULL;
}
if (type == TYPE_IGNORE) {
- PyObject_FREE(str);
- if (!growable_int_array_add(&type_ignores, tok->lineno)) {
+ if (!growable_comment_array_add(&type_ignores, tok->lineno, str)) {
err_ret->error = E_NOMEM;
break;
}
REQ(ch, ENDMARKER);
for (i = 0; i < type_ignores.num_items; i++) {
- PyNode_AddChild(ch, TYPE_IGNORE, NULL,
- type_ignores.items[i], 0,
- type_ignores.items[i], 0);
+ int res = PyNode_AddChild(ch, TYPE_IGNORE, type_ignores.items[i].comment,
+ type_ignores.items[i].lineno, 0,
+ type_ignores.items[i].lineno, 0);
+ if (res != 0) {
+ err_ret->error = res;
+ PyNode_Free(n);
+ n = NULL;
+ break;
+ }
+ type_ignores.items[i].comment = NULL;
}
}
is a single statement by looking at what is left in the
buffer after parsing. Trailing whitespace and comments
are OK. */
- if (start == single_input) {
+ if (err_ret->error == E_DONE && start == single_input) {
char *cur = tok->cur;
char c = *tok->cur;
else
n = NULL;
- growable_int_array_deallocate(&type_ignores);
+ growable_comment_array_deallocate(&type_ignores);
#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
*flags = ps->p_flags;
/* This is a type comment if we matched all of type_comment_prefix. */
if (!*prefix) {
int is_type_ignore = 1;
+ const char *ignore_end = p + 6;
tok_backup(tok, c); /* don't eat the newline or EOF */
type_start = p;
/* A TYPE_IGNORE is "type: ignore" followed by the end of the token
* or anything non-alphanumeric. */
is_type_ignore = (
- tok->cur >= p + 6 && memcmp(p, "ignore", 6) == 0
- && !(tok->cur > p + 6 && isalnum(p[6])));
+ tok->cur >= ignore_end && memcmp(p, "ignore", 6) == 0
+ && !(tok->cur > ignore_end && isalnum(p[6])));
if (is_type_ignore) {
+ *p_start = (char *) ignore_end;
+ *p_end = tok->cur;
+
/* If this type ignore is the only thing on the line, consume the newline also. */
if (blankline) {
tok_nextc(tok);
static PyTypeObject *type_ignore_type;
static PyObject* ast2obj_type_ignore(void*);
static PyTypeObject *TypeIgnore_type;
+_Py_IDENTIFIER(tag);
static char *TypeIgnore_fields[]={
"lineno",
+ "tag",
};
if (!type_ignore_type) return 0;
if (!add_attributes(type_ignore_type, NULL, 0)) return 0;
TypeIgnore_type = make_type("TypeIgnore", type_ignore_type,
- TypeIgnore_fields, 1);
+ TypeIgnore_fields, 2);
if (!TypeIgnore_type) return 0;
initialized = 1;
return 1;
}
type_ignore_ty
-TypeIgnore(int lineno, PyArena *arena)
+TypeIgnore(int lineno, string tag, PyArena *arena)
{
type_ignore_ty p;
+ if (!tag) {
+ PyErr_SetString(PyExc_ValueError,
+ "field tag is required for TypeIgnore");
+ return NULL;
+ }
p = (type_ignore_ty)PyArena_Malloc(arena, sizeof(*p));
if (!p)
return NULL;
p->kind = TypeIgnore_kind;
p->v.TypeIgnore.lineno = lineno;
+ p->v.TypeIgnore.tag = tag;
return p;
}
if (_PyObject_SetAttrId(result, &PyId_lineno, value) == -1)
goto failed;
Py_DECREF(value);
+ value = ast2obj_string(o->v.TypeIgnore.tag);
+ if (!value) goto failed;
+ if (_PyObject_SetAttrId(result, &PyId_tag, value) == -1)
+ goto failed;
+ Py_DECREF(value);
break;
}
return result;
}
if (isinstance) {
int lineno;
+ string tag;
if (_PyObject_LookupAttrId(obj, &PyId_lineno, &tmp) < 0) {
return 1;
if (res != 0) goto failed;
Py_CLEAR(tmp);
}
- *out = TypeIgnore(lineno, arena);
+ if (_PyObject_LookupAttrId(obj, &PyId_tag, &tmp) < 0) {
+ return 1;
+ }
+ if (tmp == NULL) {
+ PyErr_SetString(PyExc_TypeError, "required field \"tag\" missing from TypeIgnore");
+ return 1;
+ }
+ else {
+ int res;
+ res = obj2ast_string(tmp, &tag, arena);
+ if (res != 0) goto failed;
+ Py_CLEAR(tmp);
+ }
+ *out = TypeIgnore(lineno, tag, arena);
if (*out == NULL) goto failed;
return 0;
}
goto out;
for (i = 0; i < num; i++) {
- type_ignore_ty ti = TypeIgnore(LINENO(CHILD(ch, i)), arena);
+ string type_comment = new_type_comment(STR(CHILD(ch, i)), &c);
+ if (!type_comment)
+ goto out;
+ type_ignore_ty ti = TypeIgnore(LINENO(CHILD(ch, i)), type_comment, arena);
if (!ti)
goto out;
asdl_seq_SET(type_ignores, i, ti);