Changes from Jonathan Riehl to allow his pgen extension (PEP 269) to

work.  This includes some more code that used to be part of pgen in
the main parser; I'm okay with that.  I'll see if the Windows build
needs work next.

diff --git a/Include/parsetok.h b/Include/parsetok.h
index 99a79b700914fc69c93899f4827e1d931331c04b..b78856685e985347441d44b3e2584ffefc013132 100644 (file)
--- a/Include/parsetok.h
+++ b/Include/parsetok.h
@@ -38,6 +38,10 @@ PyAPI_FUNC(node *) PyParser_ParseStringFlagsFilename(const char *,
                                              const char *,
                                              grammar *, int,
                                               perrdetail *, int);
+
+/* Note that he following function is defined in pythonrun.c not parsetok.c. */
+PyAPI_FUNC(void) PyParser_SetError(perrdetail *);
+
 #ifdef __cplusplus
 }
 #endif

diff --git a/Include/pgen.h b/Include/pgen.h
new file mode 100644 (file)
index 0000000..8a325ed
--- /dev/null
+++ b/Include/pgen.h
@@ -0,0 +1,18 @@
+#ifndef Py_PGEN_H
+#define Py_PGEN_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+/* Parser generator interface */
+
+extern grammar *meta_grammar(void);
+
+struct _node;
+extern grammar *pgen(struct _node *);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_PGEN_H */

diff --git a/Makefile.pre.in b/Makefile.pre.in
index 5313030935661066ab8170ed231dbc31ead09f81..d461346f124800e8169e9464a6e0345b8b8aa660 100644 (file)
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -194,7 +194,10 @@ POBJS=             \
                Parser/parser.o \
                Parser/parsetok.o \
                Parser/bitset.o \
-               Parser/metagrammar.o
+               Parser/metagrammar.o \
+               Parser/firstsets.o \
+               Parser/grammar.o \
+               Parser/pgen.o
 
 PARSER_OBJS=   $(POBJS) Parser/myreadline.o Parser/tokenizer.o
 
@@ -202,9 +205,6 @@ PGOBJS=             \
                Objects/obmalloc.o \
                Python/mysnprintf.o \
                Parser/tokenizer_pgen.o \
-               Parser/firstsets.o \
-               Parser/grammar.o \
-               Parser/pgen.o \
                Parser/printgrammar.o \
                Parser/pgenmain.o
 

diff --git a/Parser/grammar.c b/Parser/grammar.c
index 2cc15b581c58588f0e4373f7297b33ea8b940c28..1f0156248d5c1532c993af2881219d4d9889d562 100644 (file)
--- a/Parser/grammar.c
+++ b/Parser/grammar.c
@@ -38,7 +38,7 @@ adddfa(grammar *g, int type, char *name)
                Py_FatalError("no mem to resize dfa in adddfa");
        d = &g->g_dfa[g->g_ndfas++];
        d->d_type = type;
-       d->d_name = name;
+       d->d_name = strdup(name);
        d->d_nstates = 0;
        d->d_state = NULL;
        d->d_initial = -1;
@@ -98,7 +98,10 @@ addlabel(labellist *ll, int type, char *str)
                Py_FatalError("no mem to resize labellist in addlabel");
        lb = &ll->ll_label[ll->ll_nlabels++];
        lb->lb_type = type;
-       lb->lb_str = str; /* XXX strdup(str) ??? */
+       lb->lb_str = strdup(str);
+       if (Py_DebugFlag)
+               printf("Label @ %08x, %d: %s\n", (unsigned)ll, ll->ll_nlabels,
+                      PyGrammar_LabelRepr(lb));
        return lb - ll->ll_label;
 }
 
@@ -152,6 +155,7 @@ translabel(grammar *g, label *lb)
                                            lb->lb_str,
                                            g->g_dfa[i].d_type);
                                lb->lb_type = g->g_dfa[i].d_type;
+                               free(lb->lb_str);
                                lb->lb_str = NULL;
                                return;
                        }
@@ -162,6 +166,7 @@ translabel(grammar *g, label *lb)
                                        printf("Label %s is terminal %d.\n",
                                                lb->lb_str, i);
                                lb->lb_type = i;
+                               free(lb->lb_str);
                                lb->lb_str = NULL;
                                return;
                        }
@@ -173,18 +178,29 @@ translabel(grammar *g, label *lb)
        if (lb->lb_type == STRING) {
                if (isalpha((int)(lb->lb_str[1])) || lb->lb_str[1] == '_') {
                        char *p;
+                       char *src;
+                       char *dest;
+                       size_t name_len;
                        if (Py_DebugFlag)
                                printf("Label %s is a keyword\n", lb->lb_str);
                        lb->lb_type = NAME;
-                       lb->lb_str++;
-                       p = strchr(lb->lb_str, '\'');
+                       src = lb->lb_str + 1;
+                       p = strchr(src, '\'');
                        if (p)
-                               *p = '\0';
+                               name_len = p - src;
+                       else
+                               name_len = strlen(src);
+                       dest = malloc(name_len + 1);
+                       strncpy(dest, src, name_len);
+                       dest[name_len] = '\0';
+                       free(lb->lb_str);
+                       lb->lb_str = dest;
                }
                else if (lb->lb_str[2] == lb->lb_str[0]) {
                        int type = (int) PyToken_OneChar(lb->lb_str[1]);
                        if (type != OP) {
                                lb->lb_type = type;
+                               free(lb->lb_str);
                                lb->lb_str = NULL;
                        }
                        else
@@ -196,6 +212,7 @@ translabel(grammar *g, label *lb)
                                                   lb->lb_str[2]);
                        if (type != OP) {
                                lb->lb_type = type;
+                               free(lb->lb_str);
                                lb->lb_str = NULL;
                        }
                        else
@@ -208,6 +225,7 @@ translabel(grammar *g, label *lb)
                                                            lb->lb_str[3]);
                        if (type != OP) {
                                lb->lb_type = type;
+                               free(lb->lb_str);
                                lb->lb_str = NULL;
                        }
                        else

diff --git a/Parser/metagrammar.c b/Parser/metagrammar.c
index ae67eb1a02b7be88b6d3f9334c47a0fb274e0796..b61bc6d66b8501c3237f345f9283fbe87b37ffc9 100644 (file)
--- a/Parser/metagrammar.c
+++ b/Parser/metagrammar.c
@@ -151,3 +151,9 @@ meta_grammar(void)
 {
        return &_PyParser_Grammar;
 }
+
+grammar *
+Py_meta_grammar(void)
+{
+  return meta_grammar();
+}

diff --git a/Parser/pgen.c b/Parser/pgen.c
index f3fdb46d1e13cd94f28c6d664be87d66bd7503c8..453deb43f80b75a24bfe2bb22d0eae6299fbce7c 100644 (file)
--- a/Parser/pgen.c
+++ b/Parser/pgen.c
@@ -669,6 +669,11 @@ pgen(node *n)
        return g;
 }
 
+grammar *
+Py_pgen(node *n)
+{
+  return pgen(n);
+}
 
 /*