From b6afae71aaf6d2df76d0a0a77c8b630220a01ec1 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 20 Mar 2016 21:59:03 -0400 Subject: [PATCH] Use %option bison-bridge in psql/pgbench lexers. The point of this change is to use %pure-parser in pgbench's exprparse.y. The immediate reason is that it turns out very ancient versions of bison have a bug with the combination of a reentrant lexer and non-reentrant parser. We could consider dropping support for such ancient bisons; but considering that we might well need exprparse.y to be reentrant some day, it seems better to make it so right now than to move the portability goalposts. (AFAICT there's no particular performance consequence to this change, either, so there's no good reason not to do it.) Now, %pure-parser assumes that the called lexer is built with %option bison-bridge. Because we're assuming bitwise compatibility of yyscan_t (yyguts_t) data structures among all the psql/pgbench lexers, that requirement propagates back to psql's lexers as well. But it's just a few lines of change on that side too; and if psqlscan.l is to set the baseline for a possibly-large family of lexers, it should err on the side of including not omitting useful features. --- src/bin/pgbench/exprparse.y | 3 +++ src/bin/pgbench/exprscan.l | 13 ++++++++----- src/bin/pgbench/pgbench.h | 12 +++++++++--- src/bin/psql/psqlscan.l | 9 ++++++++- src/bin/psql/psqlscanslash.l | 14 +++++++++++--- 5 files changed, 39 insertions(+), 12 deletions(-) diff --git a/src/bin/pgbench/exprparse.y b/src/bin/pgbench/exprparse.y index 5d0d97272f..4948ff3b81 100644 --- a/src/bin/pgbench/exprparse.y +++ b/src/bin/pgbench/exprparse.y @@ -28,6 +28,7 @@ static PgBenchExpr *make_func(yyscan_t yyscanner, int fnumber, PgBenchExprList * %} +%pure-parser %expect 0 %name-prefix="expr_yy" @@ -263,5 +264,7 @@ make_func(yyscan_t yyscanner, int fnumber, PgBenchExprList *args) /* First, get rid of "#define yyscan_t" from pgbench.h */ #undef yyscan_t +/* ... and the yylval macro, which flex will have its own definition for */ +#undef yylval #include "exprscan.c" diff --git a/src/bin/pgbench/exprscan.l b/src/bin/pgbench/exprscan.l index d069c5b05b..825dacc0af 100644 --- a/src/bin/pgbench/exprscan.l +++ b/src/bin/pgbench/exprscan.l @@ -47,6 +47,7 @@ extern void expr_yyset_column(int column_no, yyscan_t yyscanner); /* Except for the prefix, these options should match psqlscan.l */ %option reentrant +%option bison-bridge %option 8bit %option never-interactive %option nodefault @@ -117,15 +118,15 @@ newline [\n] "," { return ','; } :{alnum}+ { - yylval.str = pg_strdup(yytext + 1); + yylval->str = pg_strdup(yytext + 1); return VARIABLE; } {digit}+ { - yylval.ival = strtoint64(yytext); + yylval->ival = strtoint64(yytext); return INTEGER; } {alpha}{alnum}* { - yylval.str = pg_strdup(yytext); + yylval->str = pg_strdup(yytext); return FUNCTION; } @@ -169,6 +170,7 @@ expr_yyerror_more(yyscan_t yyscanner, const char *message, const char *more) { PsqlScanState state = yyget_extra(yyscanner); int error_detection_offset = expr_scanner_offset(state) - 1; + YYSTYPE lval; char *full_line; size_t l; @@ -179,7 +181,7 @@ expr_yyerror_more(yyscan_t yyscanner, const char *message, const char *more) */ if (!last_was_newline) { - while (yylex(yyscanner)) + while (yylex(&lval, yyscanner)) /* skip */ ; } @@ -210,6 +212,7 @@ bool expr_lex_one_word(PsqlScanState state, PQExpBuffer word_buf, int *offset) { int lexresult; + YYSTYPE lval; /* Must be scanning already */ Assert(state->scanbufhandle != NULL); @@ -228,7 +231,7 @@ expr_lex_one_word(PsqlScanState state, PQExpBuffer word_buf, int *offset) state->start_state = INITIAL; /* And lex. */ - lexresult = yylex(state->scanner); + lexresult = yylex(&lval, state->scanner); /* * Save start offset of word, if any. We could do this more efficiently, diff --git a/src/bin/pgbench/pgbench.h b/src/bin/pgbench/pgbench.h index a9db9c5b83..101b0bdbf3 100644 --- a/src/bin/pgbench/pgbench.h +++ b/src/bin/pgbench/pgbench.h @@ -17,11 +17,17 @@ * This file is included outside exprscan.l, in places where we can't see * flex's definition of typedef yyscan_t. Fortunately, it's documented as * being "void *", so we can use a macro to keep the function declarations - * here looking like the definitions in exprscan.l. exprparse.y also - * uses this to be able to declare things as "yyscan_t". + * here looking like the definitions in exprscan.l. exprparse.y and + * pgbench.c also use this to be able to declare things as "yyscan_t". */ #define yyscan_t void * +/* + * Likewise, we can't see exprparse.y's definition of union YYSTYPE here, + * but for now there's no need to know what the union contents are. + */ +union YYSTYPE; + /* Types of expression nodes */ typedef enum PgBenchExprType { @@ -85,7 +91,7 @@ struct PgBenchExprList extern PgBenchExpr *expr_parse_result; extern int expr_yyparse(yyscan_t yyscanner); -extern int expr_yylex(yyscan_t yyscanner); +extern int expr_yylex(union YYSTYPE *lvalp, yyscan_t yyscanner); extern void expr_yyerror(yyscan_t yyscanner, const char *str) pg_attribute_noreturn(); extern void expr_yyerror_more(yyscan_t yyscanner, const char *str, const char *more) pg_attribute_noreturn(); diff --git a/src/bin/psql/psqlscan.l b/src/bin/psql/psqlscan.l index 26ad45c7f9..93c7355960 100644 --- a/src/bin/psql/psqlscan.l +++ b/src/bin/psql/psqlscan.l @@ -35,6 +35,12 @@ %{ #include "psqlscan_int.h" +/* + * We must have a typedef YYSTYPE for yylex's first argument, but this lexer + * doesn't presently make use of that argument, so just declare it as int. + */ +typedef int YYSTYPE; + /* * Set the type of yyextra; we use it as a pointer back to the containing * PsqlScanState. @@ -64,6 +70,7 @@ extern void psql_yyset_column(int column_no, yyscan_t yyscanner); %} %option reentrant +%option bison-bridge %option 8bit %option never-interactive %option nodefault @@ -1002,7 +1009,7 @@ psql_scan(PsqlScanState state, yy_switch_to_buffer(state->scanbufhandle, state->scanner); /* And lex. */ - lexresult = yylex(state->scanner); + lexresult = yylex(NULL, state->scanner); /* * Check termination state and return appropriate result info. diff --git a/src/bin/psql/psqlscanslash.l b/src/bin/psql/psqlscanslash.l index e3cef7c5c5..a89ce15ad4 100644 --- a/src/bin/psql/psqlscanslash.l +++ b/src/bin/psql/psqlscanslash.l @@ -26,6 +26,12 @@ %{ #include "psqlscan_int.h" +/* + * We must have a typedef YYSTYPE for yylex's first argument, but this lexer + * doesn't presently make use of that argument, so just declare it as int. + */ +typedef int YYSTYPE; + /* * Set the type of yyextra; we use it as a pointer back to the containing * PsqlScanState. @@ -62,7 +68,9 @@ extern void slash_yyset_column(int column_no, yyscan_t yyscanner); %} +/* Except for the prefix, these options should match psqlscan.l */ %option reentrant +%option bison-bridge %option 8bit %option never-interactive %option nodefault @@ -447,7 +455,7 @@ psql_scan_slash_command(PsqlScanState state) state->start_state = xslashcmd; /* And lex. */ - yylex(state->scanner); + yylex(NULL, state->scanner); /* There are no possible errors in this lex state... */ @@ -521,7 +529,7 @@ psql_scan_slash_option(PsqlScanState state, state->start_state = xslashargstart; /* And lex. */ - lexresult = yylex(state->scanner); + lexresult = yylex(NULL, state->scanner); /* Save final state for a moment... */ final_state = state->start_state; @@ -648,7 +656,7 @@ psql_scan_slash_command_end(PsqlScanState state) state->start_state = xslashend; /* And lex. */ - yylex(state->scanner); + yylex(NULL, state->scanner); /* There are no possible errors in this lex state... */ -- 2.40.0