Each instance of a concrete class has one attribute for each child node,
of the type as defined in the grammar. For example, \code{_ast.BinOp}
-instances have an attribute \code{left} of type \code{_ast.expr}.
+instances have an attribute \code{left} of type \code{_ast.expr}.
+Instances of \code{_ast.expr} and \code{_ast.stmt} subclasses also
+have lineno and col_offset attributes. The lineno is the line number
+of source text (1 indexed so the first line is line 1) and the
+col_offset is the utf8 byte offset of the first token that generated
+the node. The utf8 offset is recorded because the parser uses utf8
+internally.
If these attributes are marked as optional in the grammar (using a
question mark), the value might be \code{None}. If the attributes
} v;
int lineno;
+ int col_offset;
};
struct _expr {
} v;
int lineno;
+ int col_offset;
};
struct _slice {
mod_ty Expression(expr_ty body, PyArena *arena);
mod_ty Suite(asdl_seq * body, PyArena *arena);
stmt_ty FunctionDef(identifier name, arguments_ty args, asdl_seq * body,
- asdl_seq * decorators, int lineno, PyArena *arena);
+ asdl_seq * decorators, int lineno, int col_offset, PyArena
+ *arena);
stmt_ty ClassDef(identifier name, asdl_seq * bases, asdl_seq * body, int
- lineno, PyArena *arena);
-stmt_ty Return(expr_ty value, int lineno, PyArena *arena);
-stmt_ty Delete(asdl_seq * targets, int lineno, PyArena *arena);
-stmt_ty Assign(asdl_seq * targets, expr_ty value, int lineno, PyArena *arena);
+ lineno, int col_offset, PyArena *arena);
+stmt_ty Return(expr_ty value, int lineno, int col_offset, PyArena *arena);
+stmt_ty Delete(asdl_seq * targets, int lineno, int col_offset, PyArena *arena);
+stmt_ty Assign(asdl_seq * targets, expr_ty value, int lineno, int col_offset,
+ PyArena *arena);
stmt_ty AugAssign(expr_ty target, operator_ty op, expr_ty value, int lineno,
- PyArena *arena);
-stmt_ty Print(expr_ty dest, asdl_seq * values, bool nl, int lineno, PyArena
- *arena);
+ int col_offset, PyArena *arena);
+stmt_ty Print(expr_ty dest, asdl_seq * values, bool nl, int lineno, int
+ col_offset, PyArena *arena);
stmt_ty For(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * orelse,
- int lineno, PyArena *arena);
-stmt_ty While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno,
- PyArena *arena);
-stmt_ty If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno,
- PyArena *arena);
+ int lineno, int col_offset, PyArena *arena);
+stmt_ty While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, int
+ col_offset, PyArena *arena);
+stmt_ty If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, int
+ col_offset, PyArena *arena);
stmt_ty With(expr_ty context_expr, expr_ty optional_vars, asdl_seq * body, int
- lineno, PyArena *arena);
-stmt_ty Raise(expr_ty type, expr_ty inst, expr_ty tback, int lineno, PyArena
- *arena);
+ lineno, int col_offset, PyArena *arena);
+stmt_ty Raise(expr_ty type, expr_ty inst, expr_ty tback, int lineno, int
+ col_offset, PyArena *arena);
stmt_ty TryExcept(asdl_seq * body, asdl_seq * handlers, asdl_seq * orelse, int
- lineno, PyArena *arena);
-stmt_ty TryFinally(asdl_seq * body, asdl_seq * finalbody, int lineno, PyArena
- *arena);
-stmt_ty Assert(expr_ty test, expr_ty msg, int lineno, PyArena *arena);
-stmt_ty Import(asdl_seq * names, int lineno, PyArena *arena);
+ lineno, int col_offset, PyArena *arena);
+stmt_ty TryFinally(asdl_seq * body, asdl_seq * finalbody, int lineno, int
+ col_offset, PyArena *arena);
+stmt_ty Assert(expr_ty test, expr_ty msg, int lineno, int col_offset, PyArena
+ *arena);
+stmt_ty Import(asdl_seq * names, int lineno, int col_offset, PyArena *arena);
stmt_ty ImportFrom(identifier module, asdl_seq * names, int level, int lineno,
- PyArena *arena);
-stmt_ty Exec(expr_ty body, expr_ty globals, expr_ty locals, int lineno, PyArena
- *arena);
-stmt_ty Global(asdl_seq * names, int lineno, PyArena *arena);
-stmt_ty Expr(expr_ty value, int lineno, PyArena *arena);
-stmt_ty Pass(int lineno, PyArena *arena);
-stmt_ty Break(int lineno, PyArena *arena);
-stmt_ty Continue(int lineno, PyArena *arena);
-expr_ty BoolOp(boolop_ty op, asdl_seq * values, int lineno, PyArena *arena);
-expr_ty BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno, PyArena
- *arena);
-expr_ty UnaryOp(unaryop_ty op, expr_ty operand, int lineno, PyArena *arena);
-expr_ty Lambda(arguments_ty args, expr_ty body, int lineno, PyArena *arena);
-expr_ty IfExp(expr_ty test, expr_ty body, expr_ty orelse, int lineno, PyArena
- *arena);
-expr_ty Dict(asdl_seq * keys, asdl_seq * values, int lineno, PyArena *arena);
-expr_ty ListComp(expr_ty elt, asdl_seq * generators, int lineno, PyArena
- *arena);
-expr_ty GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, PyArena
- *arena);
-expr_ty Yield(expr_ty value, int lineno, PyArena *arena);
+ int col_offset, PyArena *arena);
+stmt_ty Exec(expr_ty body, expr_ty globals, expr_ty locals, int lineno, int
+ col_offset, PyArena *arena);
+stmt_ty Global(asdl_seq * names, int lineno, int col_offset, PyArena *arena);
+stmt_ty Expr(expr_ty value, int lineno, int col_offset, PyArena *arena);
+stmt_ty Pass(int lineno, int col_offset, PyArena *arena);
+stmt_ty Break(int lineno, int col_offset, PyArena *arena);
+stmt_ty Continue(int lineno, int col_offset, PyArena *arena);
+expr_ty BoolOp(boolop_ty op, asdl_seq * values, int lineno, int col_offset,
+ PyArena *arena);
+expr_ty BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno, int
+ col_offset, PyArena *arena);
+expr_ty UnaryOp(unaryop_ty op, expr_ty operand, int lineno, int col_offset,
+ PyArena *arena);
+expr_ty Lambda(arguments_ty args, expr_ty body, int lineno, int col_offset,
+ PyArena *arena);
+expr_ty IfExp(expr_ty test, expr_ty body, expr_ty orelse, int lineno, int
+ col_offset, PyArena *arena);
+expr_ty Dict(asdl_seq * keys, asdl_seq * values, int lineno, int col_offset,
+ PyArena *arena);
+expr_ty ListComp(expr_ty elt, asdl_seq * generators, int lineno, int
+ col_offset, PyArena *arena);
+expr_ty GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int
+ col_offset, PyArena *arena);
+expr_ty Yield(expr_ty value, int lineno, int col_offset, PyArena *arena);
expr_ty Compare(expr_ty left, asdl_seq * ops, asdl_seq * comparators, int
- lineno, PyArena *arena);
+ lineno, int col_offset, PyArena *arena);
expr_ty Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, expr_ty
- starargs, expr_ty kwargs, int lineno, PyArena *arena);
-expr_ty Repr(expr_ty value, int lineno, PyArena *arena);
-expr_ty Num(object n, int lineno, PyArena *arena);
-expr_ty Str(string s, int lineno, PyArena *arena);
+ starargs, expr_ty kwargs, int lineno, int col_offset, PyArena
+ *arena);
+expr_ty Repr(expr_ty value, int lineno, int col_offset, PyArena *arena);
+expr_ty Num(object n, int lineno, int col_offset, PyArena *arena);
+expr_ty Str(string s, int lineno, int col_offset, PyArena *arena);
expr_ty Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int
- lineno, PyArena *arena);
+ lineno, int col_offset, PyArena *arena);
expr_ty Subscript(expr_ty value, slice_ty slice, expr_context_ty ctx, int
- lineno, PyArena *arena);
-expr_ty Name(identifier id, expr_context_ty ctx, int lineno, PyArena *arena);
-expr_ty List(asdl_seq * elts, expr_context_ty ctx, int lineno, PyArena *arena);
-expr_ty Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno, PyArena *arena);
+ lineno, int col_offset, PyArena *arena);
+expr_ty Name(identifier id, expr_context_ty ctx, int lineno, int col_offset,
+ PyArena *arena);
+expr_ty List(asdl_seq * elts, expr_context_ty ctx, int lineno, int col_offset,
+ PyArena *arena);
+expr_ty Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno, int col_offset,
+ PyArena *arena);
slice_ty Ellipsis(PyArena *arena);
slice_ty Slice(expr_ty lower, expr_ty upper, expr_ty step, PyArena *arena);
slice_ty ExtSlice(asdl_seq * dims, PyArena *arena);
short n_type;
char *n_str;
int n_lineno;
+ int n_col_offset;
int n_nchildren;
struct _node *n_child;
} node;
PyAPI_FUNC(node *) PyNode_New(int type);
PyAPI_FUNC(int) PyNode_AddChild(node *n, int type,
- char *str, int lineno);
+ char *str, int lineno, int col_offset);
PyAPI_FUNC(void) PyNode_Free(node *n);
/* Node access functions */
import sys, itertools
+import _ast
def to_tuple(t):
if t is None or isinstance(t, (basestring, int, long, complex)):
elif isinstance(t, list):
return [to_tuple(e) for e in t]
result = [t.__class__.__name__]
+ if hasattr(t, 'lineno') and hasattr(t, 'col_offset'):
+ result.append((t.lineno, t.col_offset))
if t._fields is None:
return tuple(result)
for f in t._fields:
# List
"[1,2,3]",
# Tuple
- "1,2,3"
+ "1,2,3",
+ # Combination
+ "a.b.c.d(a.b[1:2])",
+
]
# TODO: expr_context, slice, boolop, operator, unaryop, cmpop, comprehension
print "run_tests()"
raise SystemExit
+def test_order(ast_node, parent_pos):
+
+ if not isinstance(ast_node, _ast.AST) or ast_node._fields == None:
+ return
+ if isinstance(ast_node, (_ast.expr, _ast.stmt)):
+ node_pos = (ast_node.lineno, ast_node.col_offset)
+ assert node_pos >= parent_pos, (node_pos, parent_pos)
+ parent_pos = (ast_node.lineno, ast_node.col_offset)
+ for name in ast_node._fields:
+ value = getattr(ast_node, name)
+ if isinstance(value, list):
+ for child in value:
+ test_order(child, parent_pos)
+ elif value != None:
+ test_order(value, parent_pos)
+
def run_tests():
for input, output, kind in ((exec_tests, exec_results, "exec"),
(single_tests, single_results, "single"),
(eval_tests, eval_results, "eval")):
for i, o in itertools.izip(input, output):
- assert to_tuple(compile(i, "?", kind, 0x400)) == o
+ ast_tree = compile(i, "?", kind, 0x400)
+ assert to_tuple(ast_tree) == o
+ test_order(ast_tree, (0, 0))
#### EVERYTHING BELOW IS GENERATED #####
exec_results = [
-('Module', [('FunctionDef', 'f', ('arguments', [], None, None, []), [('Pass',)], [])]),
-('Module', [('ClassDef', 'C', [], [('Pass',)])]),
-('Module', [('FunctionDef', 'f', ('arguments', [], None, None, []), [('Return', ('Num', 1))], [])]),
-('Module', [('Delete', [('Name', 'v', ('Del',))])]),
-('Module', [('Assign', [('Name', 'v', ('Store',))], ('Num', 1))]),
-('Module', [('AugAssign', ('Name', 'v', ('Load',)), ('Add',), ('Num', 1))]),
-('Module', [('Print', ('Name', 'f', ('Load',)), [('Num', 1)], False)]),
-('Module', [('For', ('Name', 'v', ('Store',)), ('Name', 'v', ('Load',)), [('Pass',)], [])]),
-('Module', [('While', ('Name', 'v', ('Load',)), [('Pass',)], [])]),
-('Module', [('If', ('Name', 'v', ('Load',)), [('Pass',)], [])]),
-('Module', [('Raise', ('Name', 'Exception', ('Load',)), ('Str', 'string'), None)]),
-('Module', [('TryExcept', [('Pass',)], [('excepthandler', ('Name', 'Exception', ('Load',)), None, [('Pass',)])], [])]),
-('Module', [('TryFinally', [('Pass',)], [('Pass',)])]),
-('Module', [('Assert', ('Name', 'v', ('Load',)), None)]),
-('Module', [('Import', [('alias', 'sys', None)])]),
-('Module', [('ImportFrom', 'sys', [('alias', 'v', None)], 0)]),
-('Module', [('Exec', ('Str', 'v'), None, None)]),
-('Module', [('Global', ['v'])]),
-('Module', [('Expr', ('Num', 1))]),
-('Module', [('Pass',)]),
-('Module', [('Break',)]),
-('Module', [('Continue',)]),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, []), [('Pass', (1, 9))], [])]),
+('Module', [('ClassDef', (1, 0), 'C', [], [('Pass', (1, 8))])]),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, []), [('Return', (1, 8), ('Num', (1, 15), 1))], [])]),
+('Module', [('Delete', (1, 0), [('Name', (1, 4), 'v', ('Del',))])]),
+('Module', [('Assign', (1, 0), [('Name', (1, 0), 'v', ('Store',))], ('Num', (1, 4), 1))]),
+('Module', [('AugAssign', (1, 0), ('Name', (1, 0), 'v', ('Load',)), ('Add',), ('Num', (1, 5), 1))]),
+('Module', [('Print', (1, 0), ('Name', (1, 8), 'f', ('Load',)), [('Num', (1, 11), 1)], False)]),
+('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Pass', (1, 11))], [])]),
+('Module', [('While', (1, 0), ('Name', (1, 6), 'v', ('Load',)), [('Pass', (1, 8))], [])]),
+('Module', [('If', (1, 0), ('Name', (1, 3), 'v', ('Load',)), [('Pass', (1, 5))], [])]),
+('Module', [('Raise', (1, 0), ('Name', (1, 6), 'Exception', ('Load',)), ('Str', (1, 17), 'string'), None)]),
+('Module', [('TryExcept', (1, 0), [('Pass', (2, 2))], [('excepthandler', ('Name', (3, 7), 'Exception', ('Load',)), None, [('Pass', (4, 2))])], [])]),
+('Module', [('TryFinally', (1, 0), [('Pass', (2, 2))], [('Pass', (4, 2))])]),
+('Module', [('Assert', (1, 0), ('Name', (1, 7), 'v', ('Load',)), None)]),
+('Module', [('Import', (1, 0), [('alias', 'sys', None)])]),
+('Module', [('ImportFrom', (1, 0), 'sys', [('alias', 'v', None)], 0)]),
+('Module', [('Exec', (1, 0), ('Str', (1, 5), 'v'), None, None)]),
+('Module', [('Global', (1, 0), ['v'])]),
+('Module', [('Expr', (1, 0), ('Num', (1, 0), 1))]),
+('Module', [('Pass', (1, 0))]),
+('Module', [('Break', (1, 0))]),
+('Module', [('Continue', (1, 0))]),
]
single_results = [
-('Interactive', [('Expr', ('BinOp', ('Num', 1), ('Add',), ('Num', 2)))]),
+('Interactive', [('Expr', (1, 0), ('BinOp', (1, 0), ('Num', (1, 0), 1), ('Add',), ('Num', (1, 2), 2)))]),
]
eval_results = [
-('Expression', ('BoolOp', ('And',), [('Name', 'a', ('Load',)), ('Name', 'b', ('Load',))])),
-('Expression', ('BinOp', ('Name', 'a', ('Load',)), ('Add',), ('Name', 'b', ('Load',)))),
-('Expression', ('UnaryOp', ('Not',), ('Name', 'v', ('Load',)))),
-('Expression', ('Lambda', ('arguments', [], None, None, []), ('Name', 'None', ('Load',)))),
-('Expression', ('Dict', [('Num', 1)], [('Num', 2)])),
-('Expression', ('ListComp', ('Name', 'a', ('Load',)), [('comprehension', ('Name', 'b', ('Store',)), ('Name', 'c', ('Load',)), [('Name', 'd', ('Load',))])])),
-('Expression', ('GeneratorExp', ('Name', 'a', ('Load',)), [('comprehension', ('Name', 'b', ('Store',)), ('Name', 'c', ('Load',)), [('Name', 'd', ('Load',))])])),
-('Expression', ('Compare', ('Num', 1), [('Lt',), ('Lt',)], [('Num', 2), ('Num', 3)])),
-('Expression', ('Call', ('Name', 'f', ('Load',)), [('Num', 1), ('Num', 2)], [('keyword', 'c', ('Num', 3))], ('Name', 'd', ('Load',)), ('Name', 'e', ('Load',)))),
-('Expression', ('Repr', ('Name', 'v', ('Load',)))),
-('Expression', ('Num', 10L)),
-('Expression', ('Str', 'string')),
-('Expression', ('Attribute', ('Name', 'a', ('Load',)), 'b', ('Load',))),
-('Expression', ('Subscript', ('Name', 'a', ('Load',)), ('Slice', ('Name', 'b', ('Load',)), ('Name', 'c', ('Load',)), None), ('Load',))),
-('Expression', ('Name', 'v', ('Load',))),
-('Expression', ('List', [('Num', 1), ('Num', 2), ('Num', 3)], ('Load',))),
-('Expression', ('Tuple', [('Num', 1), ('Num', 2), ('Num', 3)], ('Load',))),
+('Expression', ('BoolOp', (1, 0), ('And',), [('Name', (1, 0), 'a', ('Load',)), ('Name', (1, 6), 'b', ('Load',))])),
+('Expression', ('BinOp', (1, 0), ('Name', (1, 0), 'a', ('Load',)), ('Add',), ('Name', (1, 4), 'b', ('Load',)))),
+('Expression', ('UnaryOp', (1, 0), ('Not',), ('Name', (1, 4), 'v', ('Load',)))),
+('Expression', ('Lambda', (1, 0), ('arguments', [], None, None, []), ('Name', (1, 7), 'None', ('Load',)))),
+('Expression', ('Dict', (1, 0), [('Num', (1, 2), 1)], [('Num', (1, 4), 2)])),
+('Expression', ('ListComp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])),
+('Expression', ('GeneratorExp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])),
+('Expression', ('Compare', (1, 0), ('Num', (1, 0), 1), [('Lt',), ('Lt',)], [('Num', (1, 4), 2), ('Num', (1, 8), 3)])),
+('Expression', ('Call', (1, 0), ('Name', (1, 0), 'f', ('Load',)), [('Num', (1, 2), 1), ('Num', (1, 4), 2)], [('keyword', 'c', ('Num', (1, 8), 3))], ('Name', (1, 11), 'd', ('Load',)), ('Name', (1, 15), 'e', ('Load',)))),
+('Expression', ('Repr', (1, 0), ('Name', (1, 1), 'v', ('Load',)))),
+('Expression', ('Num', (1, 0), 10L)),
+('Expression', ('Str', (1, 0), 'string')),
+('Expression', ('Attribute', (1, 0), ('Name', (1, 0), 'a', ('Load',)), 'b', ('Load',))),
+('Expression', ('Subscript', (1, 0), ('Name', (1, 0), 'a', ('Load',)), ('Slice', ('Name', (1, 2), 'b', ('Load',)), ('Name', (1, 4), 'c', ('Load',)), None), ('Load',))),
+('Expression', ('Name', (1, 0), 'v', ('Load',))),
+('Expression', ('List', (1, 0), [('Num', (1, 1), 1), ('Num', (1, 3), 2), ('Num', (1, 5), 3)], ('Load',))),
+('Expression', ('Tuple', (1, 0), [('Num', (1, 0), 1), ('Num', (1, 2), 2), ('Num', (1, 4), 3)], ('Load',))),
+('Expression', ('Call', (1, 0), ('Attribute', (1, 0), ('Attribute', (1, 0), ('Attribute', (1, 0), ('Name', (1, 0), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',)), 'd', ('Load',)), [('Subscript', (1, 8), ('Attribute', (1, 8), ('Name', (1, 8), 'a', ('Load',)), 'b', ('Load',)), ('Slice', ('Num', (1, 12), 1), ('Num', (1, 14), 2), None), ('Load',))], [], None, None)),
]
run_tests()
Py_XDECREF(elem);
return (0);
}
- err = PyNode_AddChild(root, type, strn, *line_num);
+ err = PyNode_AddChild(root, type, strn, *line_num, 0);
if (err == E_NOMEM) {
PyMem_DEL(strn);
return (node *) PyErr_NoMemory();
| Pass | Break | Continue
-- XXX Jython will be different
- attributes (int lineno)
+ -- col_offset is the byte offset in the utf8 string the parser uses
+ attributes (int lineno, int col_offset)
-- BoolOp() can use left & right?
expr = BoolOp(boolop op, expr* values)
| List(expr* elts, expr_context ctx)
| Tuple(expr *elts, expr_context ctx)
- attributes (int lineno)
+ -- col_offset is the byte offset in the utf8 string the parser uses
+ attributes (int lineno, int col_offset)
expr_context = Load | Store | Del | AugLoad | AugStore | Param
if id.value != "attributes":
raise ASDLSyntaxError(id.lineno,
msg="expected attributes, found %s" % id)
+ if attributes:
+ attributes.reverse()
return Sum(sum, attributes)
def p_product(self, (_0, fields, _1)):
int
-PyNode_AddChild(register node *n1, int type, char *str, int lineno)
+PyNode_AddChild(register node *n1, int type, char *str, int lineno, int col_offset)
{
const int nch = n1->n_nchildren;
int current_capacity;
n->n_type = type;
n->n_str = str;
n->n_lineno = lineno;
+ n->n_col_offset = col_offset;
n->n_nchildren = 0;
n->n_child = NULL;
return 0;
/* PARSER STACK OPERATIONS */
static int
-shift(register stack *s, int type, char *str, int newstate, int lineno)
+shift(register stack *s, int type, char *str, int newstate, int lineno, int col_offset)
{
int err;
assert(!s_empty(s));
- err = PyNode_AddChild(s->s_top->s_parent, type, str, lineno);
+ err = PyNode_AddChild(s->s_top->s_parent, type, str, lineno, col_offset);
if (err)
return err;
s->s_top->s_state = newstate;
}
static int
-push(register stack *s, int type, dfa *d, int newstate, int lineno)
+push(register stack *s, int type, dfa *d, int newstate, int lineno, int col_offset)
{
int err;
register node *n;
n = s->s_top->s_parent;
assert(!s_empty(s));
- err = PyNode_AddChild(n, type, (char *)NULL, lineno);
+ err = PyNode_AddChild(n, type, (char *)NULL, lineno, col_offset);
if (err)
return err;
s->s_top->s_state = newstate;
int
PyParser_AddToken(register parser_state *ps, register int type, char *str,
- int lineno, int *expected_ret)
+ int lineno, int col_offset, int *expected_ret)
{
register int ilabel;
int err;
dfa *d1 = PyGrammar_FindDFA(
ps->p_grammar, nt);
if ((err = push(&ps->p_stack, nt, d1,
- arrow, lineno)) > 0) {
+ arrow, lineno, col_offset)) > 0) {
D(printf(" MemError: push\n"));
return err;
}
/* Shift the token */
if ((err = shift(&ps->p_stack, type, str,
- x, lineno)) > 0) {
+ x, lineno, col_offset)) > 0) {
D(printf(" MemError: shift.\n"));
return err;
}
parser_state *PyParser_New(grammar *g, int start);
void PyParser_Delete(parser_state *ps);
-int PyParser_AddToken(parser_state *ps, int type, char *str, int lineno,
+int PyParser_AddToken(parser_state *ps, int type, char *str, int lineno, int col_offset,
int *expected_ret);
void PyGrammar_AddAccelerators(grammar *g);
int type;
size_t len;
char *str;
+ int col_offset;
type = PyTokenizer_Get(tok, &a, &b);
if (type == ERRORTOKEN) {
len == 4 && str[0] == 'w' && strcmp(str, "with") == 0)
handling_with = 1;
#endif
-
+ if (a >= tok->line_start)
+ col_offset = a - tok->line_start;
+ else
+ col_offset = -1;
+
if ((err_ret->error =
- PyParser_AddToken(ps, (int)type, str, tok->lineno,
+ PyParser_AddToken(ps, (int)type, str, tok->lineno, col_offset,
&(err_ret->expected))) != E_OK) {
if (err_ret->error != E_DONE)
PyObject_FREE(str);
}
if (tok->start == NULL)
tok->buf = tok->cur;
+ tok->line_start = tok->cur;
tok->lineno++;
tok->inp = end;
return Py_CHARMASK(*tok->cur++);
}
tok->buf = buf;
tok->cur = tok->buf + oldlen;
+ tok->line_start = tok->cur;
strcpy(tok->buf + oldlen, new);
PyMem_FREE(new);
tok->inp = tok->buf + newlen;
if (tok->buf != NULL)
PyMem_DEL(tok->buf);
tok->buf = new;
+ tok->line_start = tok->buf;
tok->cur = tok->buf;
+ tok->line_start = tok->buf;
tok->inp = strchr(tok->buf, '\0');
tok->end = tok->inp + 1;
}
done = tok->inp[-1] == '\n';
}
tok->cur = tok->buf + cur;
+ tok->line_start = tok->cur;
/* replace "\r\n" with "\n" */
/* For Mac we leave the \r, giving a syntax error */
pt = tok->inp - 2;
int read_coding_spec; /* whether 'coding:...' has been read */
char *encoding;
int cont_line; /* whether we are in a continuation line. */
+ const char* line_start; /* pointer to start of current line */
#ifndef PGEN
PyObject *decoding_readline; /* codecs.open(...).readline */
PyObject *decoding_buffer;
static PyTypeObject *stmt_type;
static char *stmt_attributes[] = {
"lineno",
+ "col_offset",
};
static PyObject* ast2obj_stmt(void*);
static PyTypeObject *FunctionDef_type;
static PyTypeObject *expr_type;
static char *expr_attributes[] = {
"lineno",
+ "col_offset",
};
static PyObject* ast2obj_expr(void*);
static PyTypeObject *BoolOp_type;
if (!Suite_type) return 0;
stmt_type = make_type("stmt", AST_type, NULL, 0);
if (!stmt_type) return 0;
- if (!add_attributes(stmt_type, stmt_attributes, 1)) return 0;
+ if (!add_attributes(stmt_type, stmt_attributes, 2)) return 0;
FunctionDef_type = make_type("FunctionDef", stmt_type,
FunctionDef_fields, 4);
if (!FunctionDef_type) return 0;
if (!Continue_type) return 0;
expr_type = make_type("expr", AST_type, NULL, 0);
if (!expr_type) return 0;
- if (!add_attributes(expr_type, expr_attributes, 1)) return 0;
+ if (!add_attributes(expr_type, expr_attributes, 2)) return 0;
BoolOp_type = make_type("BoolOp", expr_type, BoolOp_fields, 2);
if (!BoolOp_type) return 0;
BinOp_type = make_type("BinOp", expr_type, BinOp_fields, 3);
stmt_ty
FunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq *
- decorators, int lineno, PyArena *arena)
+ decorators, int lineno, int col_offset, PyArena *arena)
{
stmt_ty p;
if (!name) {
p->v.FunctionDef.body = body;
p->v.FunctionDef.decorators = decorators;
p->lineno = lineno;
+ p->col_offset = col_offset;
return p;
}
stmt_ty
-ClassDef(identifier name, asdl_seq * bases, asdl_seq * body, int lineno,
- PyArena *arena)
+ClassDef(identifier name, asdl_seq * bases, asdl_seq * body, int lineno, int
+ col_offset, PyArena *arena)
{
stmt_ty p;
if (!name) {
p->v.ClassDef.bases = bases;
p->v.ClassDef.body = body;
p->lineno = lineno;
+ p->col_offset = col_offset;
return p;
}
stmt_ty
-Return(expr_ty value, int lineno, PyArena *arena)
+Return(expr_ty value, int lineno, int col_offset, PyArena *arena)
{
stmt_ty p;
p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
p->kind = Return_kind;
p->v.Return.value = value;
p->lineno = lineno;
+ p->col_offset = col_offset;
return p;
}
stmt_ty
-Delete(asdl_seq * targets, int lineno, PyArena *arena)
+Delete(asdl_seq * targets, int lineno, int col_offset, PyArena *arena)
{
stmt_ty p;
p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
p->kind = Delete_kind;
p->v.Delete.targets = targets;
p->lineno = lineno;
+ p->col_offset = col_offset;
return p;
}
stmt_ty
-Assign(asdl_seq * targets, expr_ty value, int lineno, PyArena *arena)
+Assign(asdl_seq * targets, expr_ty value, int lineno, int col_offset, PyArena
+ *arena)
{
stmt_ty p;
if (!value) {
p->v.Assign.targets = targets;
p->v.Assign.value = value;
p->lineno = lineno;
+ p->col_offset = col_offset;
return p;
}
stmt_ty
-AugAssign(expr_ty target, operator_ty op, expr_ty value, int lineno, PyArena
- *arena)
+AugAssign(expr_ty target, operator_ty op, expr_ty value, int lineno, int
+ col_offset, PyArena *arena)
{
stmt_ty p;
if (!target) {
p->v.AugAssign.op = op;
p->v.AugAssign.value = value;
p->lineno = lineno;
+ p->col_offset = col_offset;
return p;
}
stmt_ty
-Print(expr_ty dest, asdl_seq * values, bool nl, int lineno, PyArena *arena)
+Print(expr_ty dest, asdl_seq * values, bool nl, int lineno, int col_offset,
+ PyArena *arena)
{
stmt_ty p;
p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
p->v.Print.values = values;
p->v.Print.nl = nl;
p->lineno = lineno;
+ p->col_offset = col_offset;
return p;
}
stmt_ty
For(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * orelse, int
- lineno, PyArena *arena)
+ lineno, int col_offset, PyArena *arena)
{
stmt_ty p;
if (!target) {
p->v.For.body = body;
p->v.For.orelse = orelse;
p->lineno = lineno;
+ p->col_offset = col_offset;
return p;
}
stmt_ty
-While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, PyArena
- *arena)
+While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, int
+ col_offset, PyArena *arena)
{
stmt_ty p;
if (!test) {
p->v.While.body = body;
p->v.While.orelse = orelse;
p->lineno = lineno;
+ p->col_offset = col_offset;
return p;
}
stmt_ty
-If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, PyArena *arena)
+If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, int
+ col_offset, PyArena *arena)
{
stmt_ty p;
if (!test) {
p->v.If.body = body;
p->v.If.orelse = orelse;
p->lineno = lineno;
+ p->col_offset = col_offset;
return p;
}
stmt_ty
With(expr_ty context_expr, expr_ty optional_vars, asdl_seq * body, int lineno,
- PyArena *arena)
+ int col_offset, PyArena *arena)
{
stmt_ty p;
if (!context_expr) {
p->v.With.optional_vars = optional_vars;
p->v.With.body = body;
p->lineno = lineno;
+ p->col_offset = col_offset;
return p;
}
stmt_ty
-Raise(expr_ty type, expr_ty inst, expr_ty tback, int lineno, PyArena *arena)
+Raise(expr_ty type, expr_ty inst, expr_ty tback, int lineno, int col_offset,
+ PyArena *arena)
{
stmt_ty p;
p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
p->v.Raise.inst = inst;
p->v.Raise.tback = tback;
p->lineno = lineno;
+ p->col_offset = col_offset;
return p;
}
stmt_ty
TryExcept(asdl_seq * body, asdl_seq * handlers, asdl_seq * orelse, int lineno,
- PyArena *arena)
+ int col_offset, PyArena *arena)
{
stmt_ty p;
p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
p->v.TryExcept.handlers = handlers;
p->v.TryExcept.orelse = orelse;
p->lineno = lineno;
+ p->col_offset = col_offset;
return p;
}
stmt_ty
-TryFinally(asdl_seq * body, asdl_seq * finalbody, int lineno, PyArena *arena)
+TryFinally(asdl_seq * body, asdl_seq * finalbody, int lineno, int col_offset,
+ PyArena *arena)
{
stmt_ty p;
p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
p->v.TryFinally.body = body;
p->v.TryFinally.finalbody = finalbody;
p->lineno = lineno;
+ p->col_offset = col_offset;
return p;
}
stmt_ty
-Assert(expr_ty test, expr_ty msg, int lineno, PyArena *arena)
+Assert(expr_ty test, expr_ty msg, int lineno, int col_offset, PyArena *arena)
{
stmt_ty p;
if (!test) {
p->v.Assert.test = test;
p->v.Assert.msg = msg;
p->lineno = lineno;
+ p->col_offset = col_offset;
return p;
}
stmt_ty
-Import(asdl_seq * names, int lineno, PyArena *arena)
+Import(asdl_seq * names, int lineno, int col_offset, PyArena *arena)
{
stmt_ty p;
p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
p->kind = Import_kind;
p->v.Import.names = names;
p->lineno = lineno;
+ p->col_offset = col_offset;
return p;
}
stmt_ty
-ImportFrom(identifier module, asdl_seq * names, int level, int lineno, PyArena
- *arena)
+ImportFrom(identifier module, asdl_seq * names, int level, int lineno, int
+ col_offset, PyArena *arena)
{
stmt_ty p;
if (!module) {
p->v.ImportFrom.names = names;
p->v.ImportFrom.level = level;
p->lineno = lineno;
+ p->col_offset = col_offset;
return p;
}
stmt_ty
-Exec(expr_ty body, expr_ty globals, expr_ty locals, int lineno, PyArena *arena)
+Exec(expr_ty body, expr_ty globals, expr_ty locals, int lineno, int col_offset,
+ PyArena *arena)
{
stmt_ty p;
if (!body) {
p->v.Exec.globals = globals;
p->v.Exec.locals = locals;
p->lineno = lineno;
+ p->col_offset = col_offset;
return p;
}
stmt_ty
-Global(asdl_seq * names, int lineno, PyArena *arena)
+Global(asdl_seq * names, int lineno, int col_offset, PyArena *arena)
{
stmt_ty p;
p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
p->kind = Global_kind;
p->v.Global.names = names;
p->lineno = lineno;
+ p->col_offset = col_offset;
return p;
}
stmt_ty
-Expr(expr_ty value, int lineno, PyArena *arena)
+Expr(expr_ty value, int lineno, int col_offset, PyArena *arena)
{
stmt_ty p;
if (!value) {
p->kind = Expr_kind;
p->v.Expr.value = value;
p->lineno = lineno;
+ p->col_offset = col_offset;
return p;
}
stmt_ty
-Pass(int lineno, PyArena *arena)
+Pass(int lineno, int col_offset, PyArena *arena)
{
stmt_ty p;
p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
}
p->kind = Pass_kind;
p->lineno = lineno;
+ p->col_offset = col_offset;
return p;
}
stmt_ty
-Break(int lineno, PyArena *arena)
+Break(int lineno, int col_offset, PyArena *arena)
{
stmt_ty p;
p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
}
p->kind = Break_kind;
p->lineno = lineno;
+ p->col_offset = col_offset;
return p;
}
stmt_ty
-Continue(int lineno, PyArena *arena)
+Continue(int lineno, int col_offset, PyArena *arena)
{
stmt_ty p;
p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
}
p->kind = Continue_kind;
p->lineno = lineno;
+ p->col_offset = col_offset;
return p;
}
expr_ty
-BoolOp(boolop_ty op, asdl_seq * values, int lineno, PyArena *arena)
+BoolOp(boolop_ty op, asdl_seq * values, int lineno, int col_offset, PyArena
+ *arena)
{
expr_ty p;
if (!op) {
p->v.BoolOp.op = op;
p->v.BoolOp.values = values;
p->lineno = lineno;
+ p->col_offset = col_offset;
return p;
}
expr_ty
-BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno, PyArena *arena)
+BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno, int col_offset,
+ PyArena *arena)
{
expr_ty p;
if (!left) {
p->v.BinOp.op = op;
p->v.BinOp.right = right;
p->lineno = lineno;
+ p->col_offset = col_offset;
return p;
}
expr_ty
-UnaryOp(unaryop_ty op, expr_ty operand, int lineno, PyArena *arena)
+UnaryOp(unaryop_ty op, expr_ty operand, int lineno, int col_offset, PyArena
+ *arena)
{
expr_ty p;
if (!op) {
p->v.UnaryOp.op = op;
p->v.UnaryOp.operand = operand;
p->lineno = lineno;
+ p->col_offset = col_offset;
return p;
}
expr_ty
-Lambda(arguments_ty args, expr_ty body, int lineno, PyArena *arena)
+Lambda(arguments_ty args, expr_ty body, int lineno, int col_offset, PyArena
+ *arena)
{
expr_ty p;
if (!args) {
p->v.Lambda.args = args;
p->v.Lambda.body = body;
p->lineno = lineno;
+ p->col_offset = col_offset;
return p;
}
expr_ty
-IfExp(expr_ty test, expr_ty body, expr_ty orelse, int lineno, PyArena *arena)
+IfExp(expr_ty test, expr_ty body, expr_ty orelse, int lineno, int col_offset,
+ PyArena *arena)
{
expr_ty p;
if (!test) {
p->v.IfExp.body = body;
p->v.IfExp.orelse = orelse;
p->lineno = lineno;
+ p->col_offset = col_offset;
return p;
}
expr_ty
-Dict(asdl_seq * keys, asdl_seq * values, int lineno, PyArena *arena)
+Dict(asdl_seq * keys, asdl_seq * values, int lineno, int col_offset, PyArena
+ *arena)
{
expr_ty p;
p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
p->v.Dict.keys = keys;
p->v.Dict.values = values;
p->lineno = lineno;
+ p->col_offset = col_offset;
return p;
}
expr_ty
-ListComp(expr_ty elt, asdl_seq * generators, int lineno, PyArena *arena)
+ListComp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset,
+ PyArena *arena)
{
expr_ty p;
if (!elt) {
p->v.ListComp.elt = elt;
p->v.ListComp.generators = generators;
p->lineno = lineno;
+ p->col_offset = col_offset;
return p;
}
expr_ty
-GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, PyArena *arena)
+GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset,
+ PyArena *arena)
{
expr_ty p;
if (!elt) {
p->v.GeneratorExp.elt = elt;
p->v.GeneratorExp.generators = generators;
p->lineno = lineno;
+ p->col_offset = col_offset;
return p;
}
expr_ty
-Yield(expr_ty value, int lineno, PyArena *arena)
+Yield(expr_ty value, int lineno, int col_offset, PyArena *arena)
{
expr_ty p;
p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
p->kind = Yield_kind;
p->v.Yield.value = value;
p->lineno = lineno;
+ p->col_offset = col_offset;
return p;
}
expr_ty
-Compare(expr_ty left, asdl_seq * ops, asdl_seq * comparators, int lineno,
- PyArena *arena)
+Compare(expr_ty left, asdl_seq * ops, asdl_seq * comparators, int lineno, int
+ col_offset, PyArena *arena)
{
expr_ty p;
if (!left) {
p->v.Compare.ops = ops;
p->v.Compare.comparators = comparators;
p->lineno = lineno;
+ p->col_offset = col_offset;
return p;
}
expr_ty
Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, expr_ty starargs,
- expr_ty kwargs, int lineno, PyArena *arena)
+ expr_ty kwargs, int lineno, int col_offset, PyArena *arena)
{
expr_ty p;
if (!func) {
p->v.Call.starargs = starargs;
p->v.Call.kwargs = kwargs;
p->lineno = lineno;
+ p->col_offset = col_offset;
return p;
}
expr_ty
-Repr(expr_ty value, int lineno, PyArena *arena)
+Repr(expr_ty value, int lineno, int col_offset, PyArena *arena)
{
expr_ty p;
if (!value) {
p->kind = Repr_kind;
p->v.Repr.value = value;
p->lineno = lineno;
+ p->col_offset = col_offset;
return p;
}
expr_ty
-Num(object n, int lineno, PyArena *arena)
+Num(object n, int lineno, int col_offset, PyArena *arena)
{
expr_ty p;
if (!n) {
p->kind = Num_kind;
p->v.Num.n = n;
p->lineno = lineno;
+ p->col_offset = col_offset;
return p;
}
expr_ty
-Str(string s, int lineno, PyArena *arena)
+Str(string s, int lineno, int col_offset, PyArena *arena)
{
expr_ty p;
if (!s) {
p->kind = Str_kind;
p->v.Str.s = s;
p->lineno = lineno;
+ p->col_offset = col_offset;
return p;
}
expr_ty
-Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int lineno,
- PyArena *arena)
+Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int lineno, int
+ col_offset, PyArena *arena)
{
expr_ty p;
if (!value) {
p->v.Attribute.attr = attr;
p->v.Attribute.ctx = ctx;
p->lineno = lineno;
+ p->col_offset = col_offset;
return p;
}
expr_ty
-Subscript(expr_ty value, slice_ty slice, expr_context_ty ctx, int lineno,
- PyArena *arena)
+Subscript(expr_ty value, slice_ty slice, expr_context_ty ctx, int lineno, int
+ col_offset, PyArena *arena)
{
expr_ty p;
if (!value) {
p->v.Subscript.slice = slice;
p->v.Subscript.ctx = ctx;
p->lineno = lineno;
+ p->col_offset = col_offset;
return p;
}
expr_ty
-Name(identifier id, expr_context_ty ctx, int lineno, PyArena *arena)
+Name(identifier id, expr_context_ty ctx, int lineno, int col_offset, PyArena
+ *arena)
{
expr_ty p;
if (!id) {
p->v.Name.id = id;
p->v.Name.ctx = ctx;
p->lineno = lineno;
+ p->col_offset = col_offset;
return p;
}
expr_ty
-List(asdl_seq * elts, expr_context_ty ctx, int lineno, PyArena *arena)
+List(asdl_seq * elts, expr_context_ty ctx, int lineno, int col_offset, PyArena
+ *arena)
{
expr_ty p;
if (!ctx) {
p->v.List.elts = elts;
p->v.List.ctx = ctx;
p->lineno = lineno;
+ p->col_offset = col_offset;
return p;
}
expr_ty
-Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno, PyArena *arena)
+Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno, int col_offset, PyArena
+ *arena)
{
expr_ty p;
if (!ctx) {
p->v.Tuple.elts = elts;
p->v.Tuple.ctx = ctx;
p->lineno = lineno;
+ p->col_offset = col_offset;
return p;
}
value = ast2obj_int(o->lineno);
if (!value) goto failed;
PyObject_SetAttrString(result, "lineno", value);
+ value = ast2obj_int(o->col_offset);
+ if (!value) goto failed;
+ PyObject_SetAttrString(result, "col_offset", value);
return result;
failed:
Py_XDECREF(value);
value = ast2obj_int(o->lineno);
if (!value) goto failed;
PyObject_SetAttrString(result, "lineno", value);
+ value = ast2obj_int(o->col_offset);
+ if (!value) goto failed;
+ PyObject_SetAttrString(result, "col_offset", value);
return result;
failed:
Py_XDECREF(value);
stmts = asdl_seq_new(1, arena);
if (!stmts)
goto error;
- asdl_seq_SET(stmts, 0, Pass(n->n_lineno, arena));
+ asdl_seq_SET(stmts, 0, Pass(n->n_lineno, n->n_col_offset, arena));
return Interactive(stmts, arena);
}
else {
ast_error(child, "assignment to None");
return NULL;
}
- arg = Name(NEW_IDENTIFIER(child), Store, LINENO(child),
+ arg = Name(NEW_IDENTIFIER(child), Store, LINENO(child), child->n_col_offset,
c->c_arena);
}
else {
asdl_seq_SET(args, i, arg);
}
- result = Tuple(args, Store, LINENO(n), c->c_arena);
+ result = Tuple(args, Store, LINENO(n), n->n_col_offset, c->c_arena);
if (!set_context(result, Store, n))
return NULL;
return result;
goto error;
}
name = Name(NEW_IDENTIFIER(CHILD(ch, 0)),
- Param, LINENO(ch), c->c_arena);
+ Param, LINENO(ch), ch->n_col_offset, c->c_arena);
if (!name)
goto error;
asdl_seq_SET(args, k++, name);
{
expr_ty e;
identifier id;
+ int lineno, col_offset;
int i;
REQ(n, dotted_name);
-
+
+ lineno = LINENO(n);
+ col_offset = n->n_col_offset;
+
id = NEW_IDENTIFIER(CHILD(n, 0));
if (!id)
return NULL;
- e = Name(id, Load, LINENO(n), c->c_arena);
+ e = Name(id, Load, lineno, col_offset, c->c_arena);
if (!e)
return NULL;
id = NEW_IDENTIFIER(CHILD(n, i));
if (!id)
return NULL;
- e = Attribute(e, id, Load, LINENO(CHILD(n, i)), c->c_arena);
+ e = Attribute(e, id, Load, lineno, col_offset, c->c_arena);
if (!e)
return NULL;
}
name_expr = NULL;
}
else if (NCH(n) == 5) { /* Call with no arguments */
- d = Call(name_expr, NULL, NULL, NULL, NULL, LINENO(n), c->c_arena);
+ d = Call(name_expr, NULL, NULL, NULL, NULL, LINENO(n), n->n_col_offset, c->c_arena);
if (!d)
return NULL;
name_expr = NULL;
if (!body)
return NULL;
- return FunctionDef(name, args, body, decorator_seq, LINENO(n), c->c_arena);
+ return FunctionDef(name, args, body, decorator_seq, LINENO(n), n->n_col_offset, c->c_arena);
}
static expr_ty
return NULL;
}
- return Lambda(args, expression, LINENO(n), c->c_arena);
+ return Lambda(args, expression, LINENO(n), n->n_col_offset, c->c_arena);
}
static expr_ty
orelse = ast_for_expr(c, CHILD(n, 4));
if (!orelse)
return NULL;
- return IfExp(expression, body, orelse, LINENO(n), c->c_arena);
+ return IfExp(expression, body, orelse, LINENO(n), n->n_col_offset, c->c_arena);
}
/* Count the number of 'for' loop in a list comprehension.
lc = comprehension(asdl_seq_GET(t, 0), expression, NULL,
c->c_arena);
else
- lc = comprehension(Tuple(t, Store, LINENO(ch), c->c_arena),
+ lc = comprehension(Tuple(t, Store, LINENO(ch), ch->n_col_offset, c->c_arena),
expression, NULL, c->c_arena);
if (!lc)
return NULL;
asdl_seq_SET(listcomps, i, lc);
}
- return ListComp(elt, listcomps, LINENO(n), c->c_arena);
+ return ListComp(elt, listcomps, LINENO(n), n->n_col_offset, c->c_arena);
}
/*
ge = comprehension(asdl_seq_GET(t, 0), expression,
NULL, c->c_arena);
else
- ge = comprehension(Tuple(t, Store, LINENO(ch), c->c_arena),
+ ge = comprehension(Tuple(t, Store, LINENO(ch), ch->n_col_offset, c->c_arena),
expression, NULL, c->c_arena);
if (!ge)
asdl_seq_SET(genexps, i, ge);
}
- return GeneratorExp(elt, genexps, LINENO(n), c->c_arena);
+ return GeneratorExp(elt, genexps, LINENO(n), n->n_col_offset, c->c_arena);
}
static expr_ty
case NAME:
/* All names start in Load context, but may later be
changed. */
- return Name(NEW_IDENTIFIER(ch), Load, LINENO(n), c->c_arena);
+ return Name(NEW_IDENTIFIER(ch), Load, LINENO(n), n->n_col_offset, c->c_arena);
case STRING: {
PyObject *str = parsestrplus(c, n);
if (!str)
return NULL;
PyArena_AddPyObject(c->c_arena, str);
- return Str(str, LINENO(n), c->c_arena);
+ return Str(str, LINENO(n), n->n_col_offset, c->c_arena);
}
case NUMBER: {
PyObject *pynum = parsenumber(STR(ch));
return NULL;
PyArena_AddPyObject(c->c_arena, pynum);
- return Num(pynum, LINENO(n), c->c_arena);
+ return Num(pynum, LINENO(n), n->n_col_offset, c->c_arena);
}
case LPAR: /* some parenthesized expressions */
ch = CHILD(n, 1);
if (TYPE(ch) == RPAR)
- return Tuple(NULL, Load, LINENO(n), c->c_arena);
+ return Tuple(NULL, Load, LINENO(n), n->n_col_offset, c->c_arena);
if (TYPE(ch) == yield_expr)
return ast_for_expr(c, ch);
ch = CHILD(n, 1);
if (TYPE(ch) == RSQB)
- return List(NULL, Load, LINENO(n), c->c_arena);
+ return List(NULL, Load, LINENO(n), n->n_col_offset, c->c_arena);
REQ(ch, listmaker);
if (NCH(ch) == 1 || TYPE(CHILD(ch, 1)) == COMMA) {
if (!elts)
return NULL;
- return List(elts, Load, LINENO(n), c->c_arena);
+ return List(elts, Load, LINENO(n), n->n_col_offset, c->c_arena);
}
else
return ast_for_listcomp(c, ch);
asdl_seq_SET(values, i / 4, expression);
}
- return Dict(keys, values, LINENO(n), c->c_arena);
+ return Dict(keys, values, LINENO(n), n->n_col_offset, c->c_arena);
}
case BACKQUOTE: { /* repr */
expr_ty expression = ast_for_testlist(c, CHILD(n, 1));
if (!expression)
return NULL;
- return Repr(expression, LINENO(n), c->c_arena);
+ return Repr(expression, LINENO(n), n->n_col_offset, c->c_arena);
}
default:
PyErr_Format(PyExc_SystemError, "unhandled atom %d", TYPE(ch));
if (!operator)
return NULL;
- result = BinOp(expr1, operator, expr2, LINENO(n), c->c_arena);
+ result = BinOp(expr1, operator, expr2, LINENO(n), n->n_col_offset, c->c_arena);
if (!result)
return NULL;
return NULL;
tmp_result = BinOp(result, operator, tmp,
- LINENO(next_oper), c->c_arena);
+ LINENO(next_oper), next_oper->n_col_offset, c->c_arena);
if (!tmp)
return NULL;
result = tmp_result;
REQ(n, trailer);
if (TYPE(CHILD(n, 0)) == LPAR) {
if (NCH(n) == 2)
- return Call(left_expr, NULL, NULL, NULL, NULL, LINENO(n), c->c_arena);
+ return Call(left_expr, NULL, NULL, NULL, NULL, LINENO(n), n->n_col_offset, c->c_arena);
else
return ast_for_call(c, CHILD(n, 1), left_expr);
}
else if (TYPE(CHILD(n, 0)) == DOT ) {
return Attribute(left_expr, NEW_IDENTIFIER(CHILD(n, 1)), Load,
- LINENO(n), c->c_arena);
+ LINENO(n), n->n_col_offset, c->c_arena);
}
else {
REQ(CHILD(n, 0), LSQB);
slice_ty slc = ast_for_slice(c, CHILD(n, 0));
if (!slc)
return NULL;
- return Subscript(left_expr, slc, Load, LINENO(n), c->c_arena);
+ return Subscript(left_expr, slc, Load, LINENO(n), n->n_col_offset, c->c_arena);
}
else {
/* The grammar is ambiguous here. The ambiguity is resolved
}
if (!simple) {
return Subscript(left_expr, ExtSlice(slices, c->c_arena),
- Load, LINENO(n), c->c_arena);
+ Load, LINENO(n), n->n_col_offset, c->c_arena);
}
/* extract Index values and put them in a Tuple */
elts = asdl_seq_new(asdl_seq_LEN(slices), c->c_arena);
assert(slc->kind == Index_kind && slc->v.Index.value);
asdl_seq_SET(elts, j, slc->v.Index.value);
}
- e = Tuple(elts, Load, LINENO(n), c->c_arena);
+ e = Tuple(elts, Load, LINENO(n), n->n_col_offset, c->c_arena);
if (!e)
return NULL;
return Subscript(left_expr, Index(e, c->c_arena),
- Load, LINENO(n), c->c_arena);
+ Load, LINENO(n), n->n_col_offset, c->c_arena);
}
}
}
tmp = ast_for_trailer(c, ch, e);
if (!tmp)
return NULL;
+ tmp->lineno = e->lineno;
+ tmp->col_offset = e->col_offset;
e = tmp;
}
if (TYPE(CHILD(n, NCH(n) - 1)) == factor) {
expr_ty f = ast_for_expr(c, CHILD(n, NCH(n) - 1));
if (!f)
return NULL;
- tmp = BinOp(e, Pow, f, LINENO(n), c->c_arena);
+ tmp = BinOp(e, Pow, f, LINENO(n), n->n_col_offset, c->c_arena);
if (!tmp)
return NULL;
e = tmp;
asdl_seq_SET(seq, i / 2, e);
}
if (!strcmp(STR(CHILD(n, 1)), "and"))
- return BoolOp(And, seq, LINENO(n), c->c_arena);
+ return BoolOp(And, seq, LINENO(n), n->n_col_offset, c->c_arena);
assert(!strcmp(STR(CHILD(n, 1)), "or"));
- return BoolOp(Or, seq, LINENO(n), c->c_arena);
+ return BoolOp(Or, seq, LINENO(n), n->n_col_offset, c->c_arena);
case not_test:
if (NCH(n) == 1) {
n = CHILD(n, 0);
if (!expression)
return NULL;
- return UnaryOp(Not, expression, LINENO(n), c->c_arena);
+ return UnaryOp(Not, expression, LINENO(n), n->n_col_offset, c->c_arena);
}
case comparison:
if (NCH(n) == 1) {
return NULL;
}
- return Compare(expression, ops, cmps, LINENO(n), c->c_arena);
+ return Compare(expression, ops, cmps, LINENO(n), n->n_col_offset, c->c_arena);
}
break;
if (!exp)
return NULL;
}
- return Yield(exp, LINENO(n), c->c_arena);
+ return Yield(exp, LINENO(n), n->n_col_offset, c->c_arena);
}
case factor: {
expr_ty expression;
switch (TYPE(CHILD(n, 0))) {
case PLUS:
- return UnaryOp(UAdd, expression, LINENO(n), c->c_arena);
+ return UnaryOp(UAdd, expression, LINENO(n), n->n_col_offset, c->c_arena);
case MINUS:
- return UnaryOp(USub, expression, LINENO(n), c->c_arena);
+ return UnaryOp(USub, expression, LINENO(n), n->n_col_offset, c->c_arena);
case TILDE:
- return UnaryOp(Invert, expression, LINENO(n), c->c_arena);
+ return UnaryOp(Invert, expression, LINENO(n), n->n_col_offset, c->c_arena);
}
PyErr_Format(PyExc_SystemError, "unhandled factor: %d",
TYPE(CHILD(n, 0)));
}
}
- return Call(func, args, keywords, vararg, kwarg, LINENO(n), c->c_arena);
+ return Call(func, args, keywords, vararg, kwarg, func->lineno, func->col_offset, c->c_arena);
}
static expr_ty
asdl_seq *tmp = seq_for_testlist(c, n);
if (!tmp)
return NULL;
- return Tuple(tmp, Load, LINENO(n), c->c_arena);
+ return Tuple(tmp, Load, LINENO(n), n->n_col_offset, c->c_arena);
}
}
if (!e)
return NULL;
- return Expr(e, LINENO(n), c->c_arena);
+ return Expr(e, LINENO(n), n->n_col_offset, c->c_arena);
}
else if (TYPE(CHILD(n, 1)) == augassign) {
expr_ty expr1, expr2;
if (TYPE(ch) == testlist)
expr1 = ast_for_testlist(c, ch);
else
- expr1 = Yield(ast_for_expr(c, CHILD(ch, 0)), LINENO(ch),
+ expr1 = Yield(ast_for_expr(c, CHILD(ch, 0)), LINENO(ch), n->n_col_offset,
c->c_arena);
if (!expr1)
if (TYPE(ch) == testlist)
expr2 = ast_for_testlist(c, ch);
else
- expr2 = Yield(ast_for_expr(c, ch), LINENO(ch), c->c_arena);
+ expr2 = Yield(ast_for_expr(c, ch), LINENO(ch), ch->n_col_offset, c->c_arena);
if (!expr2)
return NULL;
if (!operator)
return NULL;
- return AugAssign(expr1, operator, expr2, LINENO(n), c->c_arena);
+ return AugAssign(expr1, operator, expr2, LINENO(n), n->n_col_offset, c->c_arena);
}
else {
int i;
expression = ast_for_expr(c, value);
if (!expression)
return NULL;
- return Assign(targets, expression, LINENO(n), c->c_arena);
+ return Assign(targets, expression, LINENO(n), n->n_col_offset, c->c_arena);
}
}
asdl_seq_SET(seq, j, expression);
}
nl = (TYPE(CHILD(n, NCH(n) - 1)) == COMMA) ? false : true;
- return Print(dest, seq, nl, LINENO(n), c->c_arena);
+ return Print(dest, seq, nl, LINENO(n), n->n_col_offset, c->c_arena);
}
static asdl_seq *
expr_list = ast_for_exprlist(c, CHILD(n, 1), Del);
if (!expr_list)
return NULL;
- return Delete(expr_list, LINENO(n), c->c_arena);
+ return Delete(expr_list, LINENO(n), n->n_col_offset, c->c_arena);
}
static stmt_ty
ch = CHILD(n, 0);
switch (TYPE(ch)) {
case break_stmt:
- return Break(LINENO(n), c->c_arena);
+ return Break(LINENO(n), n->n_col_offset, c->c_arena);
case continue_stmt:
- return Continue(LINENO(n), c->c_arena);
+ return Continue(LINENO(n), n->n_col_offset, c->c_arena);
case yield_stmt: { /* will reduce to yield_expr */
expr_ty exp = ast_for_expr(c, CHILD(ch, 0));
if (!exp)
return NULL;
- return Expr(exp, LINENO(n), c->c_arena);
+ return Expr(exp, LINENO(n), n->n_col_offset, c->c_arena);
}
case return_stmt:
if (NCH(ch) == 1)
- return Return(NULL, LINENO(n), c->c_arena);
+ return Return(NULL, LINENO(n), n->n_col_offset, c->c_arena);
else {
expr_ty expression = ast_for_testlist(c, CHILD(ch, 1));
if (!expression)
return NULL;
- return Return(expression, LINENO(n), c->c_arena);
+ return Return(expression, LINENO(n), n->n_col_offset, c->c_arena);
}
case raise_stmt:
if (NCH(ch) == 1)
- return Raise(NULL, NULL, NULL, LINENO(n), c->c_arena);
+ return Raise(NULL, NULL, NULL, LINENO(n), n->n_col_offset, c->c_arena);
else if (NCH(ch) == 2) {
expr_ty expression = ast_for_expr(c, CHILD(ch, 1));
if (!expression)
return NULL;
- return Raise(expression, NULL, NULL, LINENO(n), c->c_arena);
+ return Raise(expression, NULL, NULL, LINENO(n), n->n_col_offset, c->c_arena);
}
else if (NCH(ch) == 4) {
expr_ty expr1, expr2;
if (!expr2)
return NULL;
- return Raise(expr1, expr2, NULL, LINENO(n), c->c_arena);
+ return Raise(expr1, expr2, NULL, LINENO(n), n->n_col_offset, c->c_arena);
}
else if (NCH(ch) == 6) {
expr_ty expr1, expr2, expr3;
if (!expr3)
return NULL;
- return Raise(expr1, expr2, expr3, LINENO(n), c->c_arena);
+ return Raise(expr1, expr2, expr3, LINENO(n), n->n_col_offset, c->c_arena);
}
default:
PyErr_Format(PyExc_SystemError,
import_from: 'from' ('.'* dotted_name | '.') 'import'
('*' | '(' import_as_names ')' | import_as_names)
*/
+ int lineno;
+ int col_offset;
int i;
asdl_seq *aliases;
REQ(n, import_stmt);
+ lineno = LINENO(n);
+ col_offset = n->n_col_offset;
n = CHILD(n, 0);
if (TYPE(n) == import_name) {
n = CHILD(n, 1);
return NULL;
asdl_seq_SET(aliases, i / 2, import_alias);
}
- return Import(aliases, LINENO(n), c->c_arena);
+ return Import(aliases, lineno, col_offset, c->c_arena);
}
else if (TYPE(n) == import_from) {
int n_children;
- int lineno = LINENO(n);
int idx, ndots = 0;
alias_ty mod = NULL;
identifier modname;
modname = mod->name;
else
modname = new_identifier("", c->c_arena);
- return ImportFrom(modname, aliases, ndots, lineno,
+ return ImportFrom(modname, aliases, ndots, lineno, col_offset,
c->c_arena);
}
PyErr_Format(PyExc_SystemError,
return NULL;
asdl_seq_SET(s, i / 2, name);
}
- return Global(s, LINENO(n), c->c_arena);
+ return Global(s, LINENO(n), n->n_col_offset, c->c_arena);
}
static stmt_ty
return NULL;
}
- return Exec(expr1, globals, locals, LINENO(n), c->c_arena);
+ return Exec(expr1, globals, locals, LINENO(n), n->n_col_offset, c->c_arena);
}
static stmt_ty
expr_ty expression = ast_for_expr(c, CHILD(n, 1));
if (!expression)
return NULL;
- return Assert(expression, NULL, LINENO(n), c->c_arena);
+ return Assert(expression, NULL, LINENO(n), n->n_col_offset, c->c_arena);
}
else if (NCH(n) == 4) {
expr_ty expr1, expr2;
if (!expr2)
return NULL;
- return Assert(expr1, expr2, LINENO(n), c->c_arena);
+ return Assert(expr1, expr2, LINENO(n), n->n_col_offset, c->c_arena);
}
PyErr_Format(PyExc_SystemError,
"improper number of parts to 'assert' statement: %d",
if (!suite_seq)
return NULL;
- return If(expression, suite_seq, NULL, LINENO(n), c->c_arena);
+ return If(expression, suite_seq, NULL, LINENO(n), n->n_col_offset, c->c_arena);
}
s = STR(CHILD(n, 4));
if (!seq2)
return NULL;
- return If(expression, seq1, seq2, LINENO(n), c->c_arena);
+ return If(expression, seq1, seq2, LINENO(n), n->n_col_offset, c->c_arena);
}
else if (s[2] == 'i') {
int i, n_elif, has_else = 0;
return NULL;
asdl_seq_SET(orelse, 0, If(expression, seq1, seq2,
- LINENO(CHILD(n, NCH(n) - 6)),
+ LINENO(CHILD(n, NCH(n) - 6)), CHILD(n, NCH(n) - 6)->n_col_offset,
c->c_arena));
/* the just-created orelse handled the last elif */
n_elif--;
asdl_seq_SET(new, 0,
If(expression, suite_seq, orelse,
- LINENO(CHILD(n, off)), c->c_arena));
+ LINENO(CHILD(n, off)), CHILD(n, off)->n_col_offset, c->c_arena));
orelse = new;
}
return If(ast_for_expr(c, CHILD(n, 1)),
ast_for_suite(c, CHILD(n, 3)),
- orelse, LINENO(n), c->c_arena);
+ orelse, LINENO(n), n->n_col_offset, c->c_arena);
}
PyErr_Format(PyExc_SystemError,
suite_seq = ast_for_suite(c, CHILD(n, 3));
if (!suite_seq)
return NULL;
- return While(expression, suite_seq, NULL, LINENO(n), c->c_arena);
+ return While(expression, suite_seq, NULL, LINENO(n), n->n_col_offset, c->c_arena);
}
else if (NCH(n) == 7) {
expr_ty expression;
if (!seq2)
return NULL;
- return While(expression, seq1, seq2, LINENO(n), c->c_arena);
+ return While(expression, seq1, seq2, LINENO(n), n->n_col_offset, c->c_arena);
}
PyErr_Format(PyExc_SystemError,
if (asdl_seq_LEN(_target) == 1)
target = asdl_seq_GET(_target, 0);
else
- target = Tuple(_target, Store, LINENO(n), c->c_arena);
+ target = Tuple(_target, Store, LINENO(n), n->n_col_offset, c->c_arena);
expression = ast_for_testlist(c, CHILD(n, 3));
if (!expression)
if (!suite_seq)
return NULL;
- return For(target, expression, suite_seq, seq, LINENO(n), c->c_arena);
+ return For(target, expression, suite_seq, seq, LINENO(n), n->n_col_offset, c->c_arena);
}
static excepthandler_ty
asdl_seq_SET(handlers, i, e);
}
- except_st = TryExcept(body, handlers, orelse, LINENO(n), c->c_arena);
+ except_st = TryExcept(body, handlers, orelse, LINENO(n), n->n_col_offset, c->c_arena);
if (!finally)
return except_st;
/* must be a try ... finally (except clauses are in body, if any exist) */
assert(finally != NULL);
- return TryFinally(body, finally, LINENO(n), c->c_arena);
+ return TryFinally(body, finally, LINENO(n), n->n_col_offset, c->c_arena);
}
static expr_ty
if (!suite_seq) {
return NULL;
}
- return With(context_expr, optional_vars, suite_seq, LINENO(n), c->c_arena);
+ return With(context_expr, optional_vars, suite_seq, LINENO(n),
+ n->n_col_offset, c->c_arena);
}
static stmt_ty
s = ast_for_suite(c, CHILD(n, 3));
if (!s)
return NULL;
- return ClassDef(NEW_IDENTIFIER(CHILD(n, 1)), NULL, s, LINENO(n),
+ return ClassDef(NEW_IDENTIFIER(CHILD(n, 1)), NULL, s, LINENO(n), n->n_col_offset,
c->c_arena);
}
/* check for empty base list */
s = ast_for_suite(c, CHILD(n,5));
if (!s)
return NULL;
- return ClassDef(NEW_IDENTIFIER(CHILD(n, 1)), NULL, s, LINENO(n),
+ return ClassDef(NEW_IDENTIFIER(CHILD(n, 1)), NULL, s, LINENO(n), n->n_col_offset,
c->c_arena);
}
s = ast_for_suite(c, CHILD(n, 6));
if (!s)
return NULL;
- return ClassDef(NEW_IDENTIFIER(CHILD(n, 1)), bases, s, LINENO(n),
+ return ClassDef(NEW_IDENTIFIER(CHILD(n, 1)), bases, s, LINENO(n), n->n_col_offset,
c->c_arena);
}
case del_stmt:
return ast_for_del_stmt(c, n);
case pass_stmt:
- return Pass(LINENO(n), c->c_arena);
+ return Pass(LINENO(n), n->n_col_offset, c->c_arena);
case flow_stmt:
return ast_for_flow_stmt(c, n);
case import_stmt:
switch (e->kind) {
case Attribute_kind:
auge = Attribute(e->v.Attribute.value, e->v.Attribute.attr,
- AugLoad, e->lineno, c->c_arena);
+ AugLoad, e->lineno, e->col_offset, c->c_arena);
if (auge == NULL)
return 0;
VISIT(c, expr, auge);
break;
case Subscript_kind:
auge = Subscript(e->v.Subscript.value, e->v.Subscript.slice,
- AugLoad, e->lineno, c->c_arena);
+ AugLoad, e->lineno, e->col_offset, c->c_arena);
if (auge == NULL)
return 0;
VISIT(c, expr, auge);
projects / python / commitdiff