From 204c1a336e56f9b110abae83efe6f1a843e39351 Mon Sep 17 00:00:00 2001 From: John Millaway Date: Sat, 7 Sep 2002 04:18:05 +0000 Subject: [PATCH] Members of struct yy_trans_info are now forced to be the same size. Added shared file tables_shared.c. Separated tables.h from flexdef.h Bulk of table deserialization code is done. --- Makefile.am | 44 ++++---- dfa.c | 1 + flex.skl | 263 ++++++++++++++++++++++++++++++++++++++++++++++-- flex.texi | 17 ++-- flexdef.h | 32 ------ gen.c | 36 +++++-- main.c | 3 +- misc.c | 1 + parse.y | 1 + tables.c | 70 ++++--------- tables.h | 84 ++++++++++++++++ tables_shared.c | 70 +++++++++++++ tables_shared.h | 116 +++++++++++++-------- 13 files changed, 568 insertions(+), 170 deletions(-) create mode 100644 tables.h create mode 100644 tables_shared.c diff --git a/Makefile.am b/Makefile.am index 4d80304..3ec2805 100644 --- a/Makefile.am +++ b/Makefile.am @@ -67,7 +67,8 @@ flex_SOURCES = \ options.c \ scanopt.c \ buf.c \ - tables.c + tables.c \ + tables_shared.c libfl_a_SOURCES = \ libmain.c \ @@ -78,6 +79,7 @@ noinst_HEADERS = \ version.h \ options.h \ scanopt.h \ + tables.h \ tables_shared.h include_HEADERS = \ @@ -134,25 +136,27 @@ $(man_MANS): $(srcdir)/main.c # Explicitly describe dependencies. # You can recreate this with `gcc -I. -MM *.c' -buf.o: buf.c flexdef.h config.h gettext.h tables_shared.h -ccl.o: ccl.c flexdef.h config.h gettext.h tables_shared.h -dfa.o: dfa.c flexdef.h config.h gettext.h tables_shared.h -ecs.o: ecs.c flexdef.h config.h gettext.h tables_shared.h -gen.o: gen.c flexdef.h config.h gettext.h tables_shared.h +buf.o: buf.c flexdef.h +ccl.o: ccl.c flexdef.h +dfa.o: dfa.c flexdef.h tables.h tables_shared.h +ecs.o: ecs.c flexdef.h +gen.o: gen.c flexdef.h tables.h tables_shared.h libmain.o: libmain.c libyywrap.o: libyywrap.c -main.o: main.c flexdef.h config.h gettext.h tables_shared.h version.h options.h scanopt.h -misc.o: misc.c flexdef.h config.h gettext.h tables_shared.h -nfa.o: nfa.c flexdef.h config.h gettext.h tables_shared.h -options.o: options.c options.h scanopt.h flexdef.h config.h gettext.h tables_shared.h -parse.o: parse.c flexdef.h config.h gettext.h tables_shared.h -scan.o: scan.c flexdef.h config.h gettext.h tables_shared.h parse.h -scanopt.o: scanopt.c scanopt.h flexdef.h config.h gettext.h tables_shared.h -skel.o: skel.c flexdef.h config.h gettext.h tables_shared.h -sym.o: sym.c flexdef.h config.h gettext.h tables_shared.h -tables.o: tables.c flexdef.h config.h gettext.h tables_shared.h -tblcmp.o: tblcmp.c flexdef.h config.h gettext.h tables_shared.h -yylex.o: yylex.c flexdef.h config.h gettext.h tables_shared.h parse.h +main.o: main.c flexdef.h version.h options.h scanopt.h tables.h \ + tables_shared.h +misc.o: misc.c flexdef.h tables.h tables_shared.h +nfa.o: nfa.c flexdef.h +options.o: options.c options.h scanopt.h flexdef.h +parse.o: parse.c flexdef.h tables.h tables_shared.h +scan.o: scan.c flexdef.h parse.h +scanopt.o: scanopt.c flexdef.h scanopt.h +skel.o: skel.c flexdef.h +sym.o: sym.c flexdef.h +tables.o: tables.c flexdef.h tables.h tables_shared.h +tables_shared.o: tables_shared.c flexdef.h tables.h tables_shared.h +tblcmp.o: tblcmp.c flexdef.h +yylex.o: yylex.c flexdef.h parse.h # Create a tags file. tags: @@ -174,8 +178,8 @@ ChangeLog: # Otherwise, indent overwrites your file even if it fails! indentfiles = buf.c ccl.c dfa.c ecs.c flexdef.h gen.c libmain.c \ libyywrap.c main.c misc.c nfa.c options.c options.h \ - scanopt.c scanopt.h sym.c tables.c tables_shared.h \ - tblcmp.c yylex.c + scanopt.c scanopt.h sym.c tables.c tables.h \ + tables_shared.c tables_shared.h tblcmp.c indent: if [ -f .indent.pro ] ; then \ diff --git a/dfa.c b/dfa.c index 627996e..ad76d03 100644 --- a/dfa.c +++ b/dfa.c @@ -30,6 +30,7 @@ /* PURPOSE. */ #include "flexdef.h" +#include "tables.h" /* declare functions that have forward references */ diff --git a/flex.skl b/flex.skl index ee337d8..8e14b36 100644 --- a/flex.skl +++ b/flex.skl @@ -30,7 +30,10 @@ #include #include #include +%t #include +#include +%t /* end standard C headers. */ %+ /* begin standard C++ headers. */ @@ -749,6 +752,24 @@ static int yy_top_state YY_PARAMS(( YY_PROTO_ONLY_ARG )); %t Tables API Structures and Prototypes m4_include(`tables_shared.h') + +/* Load the DFA tables from the given stream. */ +int yytables_load YY_PARAMS ((FILE * fp YY_PROTO_LAST_ARG)); + +/** Describes a mapping from a serialized table id to its deserialized state in + * this scanner. This is the bridge between our "generic" deserialization code + * and the specifics of this scanner. + */ + +struct yytbl_dmap { + enum yytbl_id dm_id;/**< table identifier */ + void **dm_arr; /**< address of pointer to store the deserialized table. */ + size_t dm_sz; /**< local sizeof() each element in table. */ +}; + +/** A {0,0,0}-terminated list of structs, forming the map */ +extern struct yytbl_dmap * yydmap; + %t End of Tables API Structures and Prototypes /* Default declaration of generated scanner - a define so the user can @@ -1748,14 +1769,6 @@ void yyFlexLexer::LexerError( yyconst char msg[] ) } %* -%t Tables API Routines -/* TODO: The Tables API is a work in progress. - * We need to generalize the input model for the tables so we - * don't duplicate code reading from a FILE*, int, or a void*. - */ - -%t End Tables API Routines - /* Redefine yyless() so it works in section 3 code. */ #undef yyless @@ -2048,6 +2061,240 @@ void yyfree YYFARGS1( void *,ptr) } #endif +%t Tables API Definitions +m4_include(`tables_shared.c') + +static int yytbl_read8 (void *v, FILE * fp) +{ + if (fread (v, sizeof (uint8_t), 1, fp) != 1) + return -1; + return 0; +} + +static int yytbl_read16 (void *v, FILE * fp) +{ + if (fread (v, sizeof (uint16_t), 1, fp) != 1) + return -1; + *((uint16_t *) v) = ntohs (*((uint16_t *) v)); + return 0; +} + +static int yytbl_read32 (void *v, FILE * fp) +{ + if (fread (v, sizeof (uint32_t), 1, fp) != 1) + return -1; + *((uint32_t *) v) = ntohl (*((uint32_t *) v)); + return 0; +} + +static int yytbl_hdr_read YYFARGS2 (struct yytbl_hdr *, th, FILE *, fp) +{ + int bytes; + memset (th, 0, sizeof (struct yytbl_hdr)); + + if (yytbl_read32 (&(th->th_magic), fp) != 0) + /* TODO: read error */ + return -1; + + if (th->th_magic != YYTBL_MAGIC) + /* TODO: bad magic number */ + return -1; + + if (yytbl_read32 (&(th->th_hsize), fp) != 0 + || yytbl_read32 (&(th->th_ssize), fp) != 0 + || yytbl_read16 (&(th->th_flags), fp) != 0) + /* TODO: read error */ + return -1; + + /* Sanity check on header size. Greater than 1k suggests some funny business. */ + if (th->th_hsize < 16 || th->th_hsize > 1024) + /* TODO: insane header size detected */ + return -1; + + /* Allocate enough space for the version and name fields */ + bytes = th->th_hsize - 14; + th->th_version = (char *) yyalloc (bytes YY_CALL_LAST_ARG); + + /* we read it all into th_version, and point th_name into that data */ + if (fread (th->th_version, 1, bytes, fp) != bytes) + /* TODO: read error */ + return -1; + + th->th_name = th->th_version + strlen (th->th_version) + 1; + return 0; +} + +/** lookup id in the dmap list. + * @param dmap pointer to first element in list + * @return NULL if not found. + */ +static struct yytbl_dmap *yytbl_dmap_lookup YYFARGS2 (struct yytbl_dmap *, dmap, + int, id) +{ + while (dmap->dm_id) + if (dmap->dm_id != id) + return dmap; + else + dmap++; + return NULL; +} + +/** Read a table while mapping its contents to the local array. + * @param dmap used to performing mapping + * @return 0 on success + */ +static int yytbl_data_load YYFARGS2 (struct yytbl_dmap *, dmap, FILE *, fp) +{ + struct yytbl_data td; + struct yytbl_dmap *transdmap; + int len, i, rv; + size_t bytes; + void *p; + + memset (&td, 0, sizeof (struct yytbl_data)); + + if (yytbl_read16 (&td.td_id, fp) != 0 + || yytbl_read16 (&td.td_flags, fp) != 0 + || yytbl_read32 (&td.td_hilen, fp) != 0 + || yytbl_read32 (&td.td_lolen, fp) != 0) + /* TODO: read error */ + return -1; + + if ((dmap = yytbl_dmap_lookup (dmap, td.td_id YY_CALL_LAST_ARG)) == NULL) + /* TODO: table id not found. This is bad. */ + return -1; + + /* Allocate space for table. */ + bytes = td.td_lolen * (td.td_hilen ? td.td_hilen : 1) * dmap->dm_sz; + *dmap->dm_arr = p = (void *) yyalloc (bytes YY_CALL_LAST_ARG); + + /* Lookup the map for the transition table so we have it in case we need it + * inside the loop below. This scanner might not even have a transition + * table, which is ok. + */ + transdmap = yytbl_dmap_lookup (dmap, YYT_ID_TRANSITION YY_CALL_LAST_ARG); + + /* read and map each element */ + len = yytbl_calc_total_len (&td); + for (i = 0; i < len; i++) { + int read_count = 1, j; + + /* If it's a struct, read 2 integers */ + if ((td.td_flags & YYTD_STRUCT) != 0) + read_count = 2; + + /* This loop executes at most 2 times. it is to handle YYTD_STRUCT */ + for (j = 0; j < read_count; j++, i++) { + uint32_t t32, t16, t8; + + /* read into t32 no matter what he real size is. */ + switch (YYTDFLAGS2BYTES (td.td_flags)) { + case sizeof (int32_t): + rv = yytbl_read32 (&t32, fp); + break; + case sizeof (int16_t): + rv = yytbl_read16 (&t16, fp); + t32 = t16; + break; + case sizeof (int8_t): + rv = yytbl_read8 (&t8, fp); + t32 = t8; + break; + default: /* TODO: invalid td_flags detected */ + return -1; + } + if (rv != 0) + /* TODO: read error */ + return -1; + + /* copy into the deserialized array... */ + + if ((td.td_flags & YYTD_STRUCT)) { + /* t32 is the j'th member of a two-element struct. */ + void *v; + + v = j == 0 ? &(((struct yy_trans_info *) p)->yy_verify) + : &(((struct yy_trans_info *) p)->yy_nxt); + + switch (dmap->dm_sz) { + case sizeof (int32_t): + ((int32_t *) v)[0] = (int32_t) t32; + break; + case sizeof (int16_t): + ((int16_t *) v)[0] = (int16_t) t32; + break; + case sizeof (int8_t): + ((int8_t *) v)[0] = (int8_t) t32; + break; + default: + break; + } + + /* if we're done with j, increment p */ + if (j == 1) + p = (struct yy_trans_info *) p + 1; + } + else if ((td.td_flags & YYTD_PTRANS)) { + + /* t32 is an index into the transition array. calculate the offset. */ + + if (!transdmap) + /* TODO: map for transition table not found. */ + return -1; + + ((struct yy_trans_info **) p)[0] = + &((*((struct yy_trans_info **) (transdmap->dm_arr)))[t32]); + p = (struct yy_trans_info **) p + 1; + } + else { + /* t32 is a plain int. copy data, then incrememnt p. */ + switch (dmap->dm_sz) { + case sizeof (int32_t): + ((int32_t *) p)[0] = (int32_t) t32; + p = ((int32_t *) p) + 1; + break; + case sizeof (int16_t): + ((int16_t *) p)[0] = (int16_t) t32; + p = ((int16_t *) p) + 1; + break; + case sizeof (int8_t): + ((int8_t *) p)[0] = (int8_t) t32; + p = ((int8_t *) p) + 1; + break; + default: + break; + } + } + } + + } + return 0; +} + +/* Load the DFA tables from the given stream. */ +int yytables_load YYFARGS1 (FILE *, fp) +{ + struct yytbl_hdr th; + + /* Keep trying until we find the right set of tables */ + for (;;) { + if (yytbl_hdr_read (&th, fp YY_CALL_LAST_ARG) != 0) + /* TODO: failed to read tables header */ + return -1; + + /* TODO: strcmp th_name with search key. For now, we just break out. */ + break; + } + + while (1) { + /* Load the data tables */ + //yytbl_data_load (fp YY_CALL_LAST_ARG); + } + + return 0; +} +%t End of Tables API Definitions + #if YY_MAIN int main YY_PARAMS((void)); diff --git a/flex.texi b/flex.texi index bee6a84..8f690d5 100644 --- a/flex.texi +++ b/flex.texi @@ -4761,18 +4761,21 @@ The data arrays are one-dimensional by default, but may be two dimensional as specified in the @code{td_hilen} field. @table @code -@item YYT_DATA8 (0x01) +@item YYTD_DATA8 (0x01) The data is serialized as an array of type int8. -@item YYT_DATA16 (0x02) +@item YYTD_DATA16 (0x02) The data is serialized as an array of type int16. -@item YYT_DATA32 (0x04) +@item YYTD_DATA32 (0x04) The data is serialized as an array of type int32. -@item YYT_PTRANS (0x08) +@item YYTD_PTRANS (0x08) The data is a list of indexes of entries in the expanded @code{yy_transition} array. Each index should be expanded to a pointer to the corresponding entry -in the @code{yy_transition} array. It is not guaranteed that the -@code{yy_transition} array has already been seen, but we can still calculate -the offsets. +in the @code{yy_transition} array. We count on the fact that the +@code{yy_transition} array has already been seen. +@item YYTD_STRUCT (0x10) +The data is a list of yy_trans_info structs, each of which consists of +two integers. There is no padding between struct elements. +The type of each member is determined by the above flags. @end table @item td_lolen diff --git a/flexdef.h b/flexdef.h index 2e2f58f..db0024c 100644 --- a/flexdef.h +++ b/flexdef.h @@ -1155,37 +1155,5 @@ extern jmp_buf flex_main_jmp_buf; /* Removes all \n and \r chars from tail of str. returns str. */ extern char *chomp (char *str); -/* Tables serialization API declarations. */ -#include "tables_shared.h" -struct yytbl_writer { - FILE *out; - uint32_t total_written; - /**< bytes written so far */ - fpos_t th_ssize_pos; - /**< position of th_ssize */ -}; - - -/* - * tablesext - if true, create external tables - * tablestoggle - if true, output external tables code while processing skel - * tablesfilename - filename for external tables - * tableswr - writer for external tables - */ -extern bool tablesext, tablestoggle; -extern char *tablesfilename; -extern struct yytbl_writer tableswr; - -int yytbl_writer_init (struct yytbl_writer *, FILE *); -int yytbl_hdr_init (struct yytbl_hdr *th, const char *version_str, - const char *name); -int yytbl_data_init (struct yytbl_data *tbl, enum yytbl_id id); -int yytbl_data_destroy (struct yytbl_data *td); -int yytbl_hdr_fwrite (struct yytbl_writer *wr, - const struct yytbl_hdr *th); -int yytbl_data_fwrite (struct yytbl_writer *wr, struct yytbl_data *td); -void yytbl_data_compress (struct yytbl_data *tbl); -struct yytbl_data *mkftbl (void); - #endif /* not defined FLEXDEF_H */ diff --git a/gen.c b/gen.c index 8a88514..9a78fd7 100644 --- a/gen.c +++ b/gen.c @@ -32,6 +32,7 @@ /* PURPOSE. */ #include "flexdef.h" +#include "tables.h" /* declare functions that have forward references */ @@ -92,7 +93,7 @@ struct yytbl_data *mkeoltbl (void) tbl = (struct yytbl_data *) calloc (1, sizeof (struct yytbl_data)); yytbl_data_init (tbl, YYT_ID_RULE_CAN_MATCH_EOL); - tbl->td_flags = YYT_DATA8; + tbl->td_flags = YYTD_DATA8; tbl->td_lolen = num_rules + 1; tbl->td_data = tdata = (int8_t *) calloc (tbl->td_lolen, sizeof (int8_t)); @@ -190,7 +191,7 @@ struct yytbl_data *mkctbl (void) tbl = (struct yytbl_data *) calloc (1, sizeof (struct yytbl_data)); yytbl_data_init (tbl, YYT_ID_TRANSITION); - tbl->td_flags |= YYT_DATA32; + tbl->td_flags = YYTD_DATA32 | YYTD_STRUCT; tbl->td_hilen = 0; tbl->td_lolen = tblend + numecs + 1; /* number of structs */ @@ -287,7 +288,7 @@ struct yytbl_data *mkssltbl (struct yytbl_data *trans_tbl) tbl = (struct yytbl_data *) calloc (1, sizeof (struct yytbl_data)); yytbl_data_init (tbl, YYT_ID_START_STATE_LIST); - tbl->td_flags = YYT_DATA32; + tbl->td_flags = YYTD_DATA32 | YYTD_PTRANS; tbl->td_hilen = 0; tbl->td_lolen = lastsc * 2 + 1; @@ -405,7 +406,7 @@ struct yytbl_data *mkecstbl (void) tbl = (struct yytbl_data *) calloc (1, sizeof (struct yytbl_data)); yytbl_data_init (tbl, YYT_ID_EC); - tbl->td_flags |= YYT_DATA32; + tbl->td_flags |= YYTD_DATA32; tbl->td_hilen = 0; tbl->td_lolen = csize; @@ -629,7 +630,7 @@ struct yytbl_data *mkftbl (void) tbl = (struct yytbl_data *) calloc (1, sizeof (struct yytbl_data)); yytbl_data_init (tbl, YYT_ID_ACCEPT); - tbl->td_flags |= YYT_DATA32; + tbl->td_flags |= YYTD_DATA32; tbl->td_hilen = 0; /* it's a one-dimensional array */ tbl->td_lolen = lastdfa + 1; @@ -1482,10 +1483,8 @@ void make_tables () indent_up (); indent_puts ("{"); - if (long_align) - indent_puts ("long yy_verify;"); - else - indent_puts ("short yy_verify;"); + /* We require that yy_verify and yy_nxt must be of the same size int. */ + indent_put2s ("%s yy_verify;", trans_offset_type); /* In cases where its sister yy_verify *is* a "yes, there is * a transition", yy_nxt is the offset (in records) to the @@ -1499,6 +1498,23 @@ void make_tables () indent_puts ("};"); indent_down (); } + else { + /* We generate a bogus 'struct yy_trans_info' data type + * so we can guarantee that it is always declared in the skel. + * This is so we can compile "sizeof(struct yy_trans_info)" + * in any scanner. + */ + indent_puts + ("/* This struct is not used in this scanner,"); + indent_puts (" but its presence is necessary. */"); + indent_puts ("struct yy_trans_info"); + indent_up (); + indent_puts ("{"); + indent_puts ("long yy_verify;"); + indent_puts ("long yy_nxt;"); + indent_puts ("};"); + indent_down (); + } if (fullspd) { genctbl (); @@ -1578,6 +1594,8 @@ void make_tables () sizeof (struct yytbl_data)); yytbl_data_init (yynultrans_tbl, YYT_ID_NUL_TRANS); + if (fullspd) + yynultrans_tbl->td_flags |= YYTD_PTRANS; yynultrans_tbl->td_lolen = lastdfa + 1; yynultrans_tbl->td_data = yynultrans_data = (int32_t *) calloc (yynultrans_tbl->td_lolen, diff --git a/main.c b/main.c index f248af5..9e8317c 100644 --- a/main.c +++ b/main.c @@ -35,6 +35,7 @@ #include "flexdef.h" #include "version.h" #include "options.h" +#include "tables.h" static char flex_version[] = FLEX_VERSION; @@ -350,7 +351,7 @@ void check_options () nbytes = strlen (prefix) + strlen ("tables") + 2; pname = (char *) calloc (nbytes, 1); sprintf (pname, "%stables", prefix); - yytbl_hdr_init (&hdr, "TODO", pname); + yytbl_hdr_init (&hdr, flex_version, pname); free (pname); if (yytbl_hdr_fwrite (&tableswr, &hdr) <= 0) diff --git a/misc.c b/misc.c index 05599c0..e2d4ea0 100644 --- a/misc.c +++ b/misc.c @@ -32,6 +32,7 @@ /* PURPOSE. */ #include "flexdef.h" +#include "tables.h" /* Append "#define defname value\n" to the running buffer. */ void action_define (defname, value) diff --git a/parse.y b/parse.y index 94a7075..340069b 100644 --- a/parse.y +++ b/parse.y @@ -70,6 +70,7 @@ #endif #include "flexdef.h" +#include "tables.h" /* The remainder of the alloca() cruft has to come after including flexdef.h, * so HAVE_ALLOCA_H is (possibly) defined. diff --git a/tables.c b/tables.c index f952f51..6be1dfb 100644 --- a/tables.c +++ b/tables.c @@ -1,4 +1,4 @@ -/* tables.h - tables serialization code +/* tables.c - tables serialization code * * Copyright (c) 1990 The Regents of the University of California. * All rights reserved. @@ -34,39 +34,31 @@ #include "flexdef.h" +#include "tables.h" /** Calculate (0-7) = number bytes needed to pad n to next 64-bit boundary. */ #define yypad64(n) ((8-((n)%8))%8) -/** Extract corresponding data size_t from td_flags */ -#define TFLAGS2BYTES(td_flags)\ - (((td_flags) & YYT_DATA8)\ - ? sizeof(int8_t)\ - :(((td_flags) & YYT_DATA16)\ - ? sizeof(int16_t)\ - :sizeof(int32_t))) - /** Convert size_t to t_flag. * @param n in {1,2,4} - * @return YYT_DATA*. + * @return YYTD_DATA*. */ #define BYTES2TFLAG(n)\ (((n) == sizeof(int8_t))\ - ? YYT_DATA8\ + ? YYTD_DATA8\ :(((n)== sizeof(int16_t))\ - ? YYT_DATA16\ - : YYT_DATA32)) + ? YYTD_DATA16\ + : YYTD_DATA32)) -/** Clear YYT_DATA* bit flags - * @return the flag with the YYT_DATA* bits cleared +/** Clear YYTD_DATA* bit flags + * @return the flag with the YYTD_DATA* bits cleared */ -#define TFLAGS_CLRDATA(flg) ((flg) & ~(YYT_DATA8 | YYT_DATA16 | YYT_DATA32)) +#define TFLAGS_CLRDATA(flg) ((flg) & ~(YYTD_DATA8 | YYTD_DATA16 | YYTD_DATA32)) int yytbl_fwrite32 (struct yytbl_writer *wr, uint32_t v); int yytbl_fwrite16 (struct yytbl_writer *wr, uint16_t v); int yytbl_fwrite8 (struct yytbl_writer *wr, uint8_t v); int yytbl_fwriten (struct yytbl_writer *wr, void *v, int32_t len); -static int32_t tbl_get_total_len (struct yytbl_data *tbl); static int32_t yytbl_data_geti (const struct yytbl_data *tbl, int i); @@ -92,7 +84,7 @@ int yytbl_hdr_init (struct yytbl_hdr *th, const char *version_str, { memset (th, 0, sizeof (struct yytbl_hdr)); - th->th_magic = 0xF13C57B1; + th->th_magic = YYTBL_MAGIC; th->th_hsize = 14 + strlen (version_str) + 1 + strlen (name) + 1; th->th_hsize += yypad64 (th->th_hsize); th->th_ssize = 0; // Not known at this point. @@ -112,7 +104,7 @@ int yytbl_data_init (struct yytbl_data *td, enum yytbl_id id) memset (td, 0, sizeof (struct yytbl_data)); td->td_id = id; - td->td_flags = YYT_DATA32; + td->td_flags = YYTD_DATA32; return 0; } @@ -217,9 +209,9 @@ int yytbl_data_fwrite (struct yytbl_writer *wr, struct yytbl_data *td) return -1; bwritten += rv; - total_len = tbl_get_total_len (td); + total_len = yytbl_calc_total_len (td); for (i = 0; i < total_len; i++) { - switch (TFLAGS2BYTES (td->td_flags)) { + switch (YYTDFLAGS2BYTES (td->td_flags)) { case sizeof (int8_t): rv = yytbl_fwrite8 (wr, yytbl_data_geti (td, i)); break; @@ -240,7 +232,7 @@ int yytbl_data_fwrite (struct yytbl_writer *wr, struct yytbl_data *td) } /* Sanity check */ - if (bwritten != (12 + total_len * TFLAGS2BYTES (td->td_flags))) { + if (bwritten != (12 + total_len * YYTDFLAGS2BYTES (td->td_flags))) { flex_die (_("insanity detected")); return -1; } @@ -338,26 +330,6 @@ int yytbl_fwrite8 (struct yytbl_writer *wr, uint8_t v) return bytes; } -/** Get the number of integers in this table. This is NOT the - * same thing as the number of elements. - * @param td the table - * @return the number of integers in the table - */ -static int32_t tbl_get_total_len (struct yytbl_data *tbl) -{ - - int32_t n; - - /* total number of ints */ - n = tbl->td_lolen; - if (tbl->td_hilen > 0) - n *= tbl->td_hilen; - - if (tbl->td_id == YYT_ID_TRANSITION) - n *= 2; - return n; -} - /** Extract data element [i][j] from array data tables. * @param tbl data table @@ -374,7 +346,7 @@ int32_t yytbl_data_getijk (const struct yytbl_data * tbl, int i, int j, k %= 2; lo = tbl->td_lolen; - switch (TFLAGS2BYTES (tbl->td_flags)) { + switch (YYTDFLAGS2BYTES (tbl->td_flags)) { case sizeof (int8_t): return ((int8_t *) (tbl->td_data))[(i * lo + j) * (k + 1) + k]; @@ -404,7 +376,7 @@ int32_t yytbl_data_getijk (const struct yytbl_data * tbl, int i, int j, static int32_t yytbl_data_geti (const struct yytbl_data *tbl, int i) { - switch (TFLAGS2BYTES (tbl->td_flags)) { + switch (YYTDFLAGS2BYTES (tbl->td_flags)) { case sizeof (int8_t): return ((int8_t *) (tbl->td_data))[i]; case sizeof (int16_t): @@ -429,7 +401,7 @@ static void yytbl_data_seti (const struct yytbl_data *tbl, int i, int32_t newval) { - switch (TFLAGS2BYTES (tbl->td_flags)) { + switch (YYTDFLAGS2BYTES (tbl->td_flags)) { case sizeof (int8_t): ((int8_t *) (tbl->td_data))[i] = (int8_t) newval; break; @@ -455,7 +427,7 @@ static size_t min_int_size (struct yytbl_data *tbl) uint32_t i, total_len; int32_t max = 0; - total_len = tbl_get_total_len (tbl); + total_len = yytbl_calc_total_len (tbl); for (i = 0; i < total_len; i++) { int32_t n; @@ -496,16 +468,16 @@ void yytbl_data_compress (struct yytbl_data *tbl) newsz = min_int_size (tbl); - if (newsz == TFLAGS2BYTES (tbl->td_flags)) + if (newsz == YYTDFLAGS2BYTES (tbl->td_flags)) /* No change in this table needed. */ return; - if (newsz > TFLAGS2BYTES (tbl->td_flags)) { + if (newsz > YYTDFLAGS2BYTES (tbl->td_flags)) { flex_die (_("detected negative compression")); return; } - total_len = tbl_get_total_len (tbl); + total_len = yytbl_calc_total_len (tbl); newtbl.td_data = calloc (total_len, newsz); newtbl.td_flags = TFLAGS_CLRDATA (newtbl.td_flags) | BYTES2TFLAG (newsz); diff --git a/tables.h b/tables.h new file mode 100644 index 0000000..1eec7a7 --- /dev/null +++ b/tables.h @@ -0,0 +1,84 @@ +/* tables.h - tables serialization code + * + * Copyright (c) 1990 The Regents of the University of California. + * All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * Vern Paxson. + * + * The United States Government has rights in this work pursuant + * to contract no. DE-AC03-76SF00098 between the United States + * Department of Energy and the University of California. + * + * This file is part of flex. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE. + */ + +#ifndef TABLES_H +#define TABLES_H + +#ifdef __cplusplus +/* *INDENT-OFF* */ +extern "C" { +/* *INDENT-ON* */ +#endif + +/* Tables serialization API declarations. */ +#include "tables_shared.h" +struct yytbl_writer { + FILE *out; + uint32_t total_written; + /**< bytes written so far */ + fpos_t th_ssize_pos; + /**< position of th_ssize */ +}; + +/* These are used by main.c, gen.c, etc. + * tablesext - if true, create external tables + * tablestoggle - if true, output external tables code while processing skel + * tablesfilename - filename for external tables + * tableswr - writer for external tables + * + */ +extern bool tablesext, tablestoggle; +extern char *tablesfilename; +extern struct yytbl_writer tableswr; + +int yytbl_writer_init (struct yytbl_writer *, FILE *); +int yytbl_hdr_init (struct yytbl_hdr *th, const char *version_str, + const char *name); +int yytbl_data_init (struct yytbl_data *tbl, enum yytbl_id id); +int yytbl_data_destroy (struct yytbl_data *td); +int yytbl_hdr_fwrite (struct yytbl_writer *wr, + const struct yytbl_hdr *th); +int yytbl_data_fwrite (struct yytbl_writer *wr, struct yytbl_data *td); +void yytbl_data_compress (struct yytbl_data *tbl); +struct yytbl_data *mkftbl (void); + + +#ifdef __cplusplus +/* *INDENT-OFF* */ +} +/* *INDENT-ON* */ +#endif +#endif + +/* vim:set expandtab cindent tabstop=4 softtabstop=4 shiftwidth=4 textwidth=0: */ diff --git a/tables_shared.c b/tables_shared.c new file mode 100644 index 0000000..5fbf886 --- /dev/null +++ b/tables_shared.c @@ -0,0 +1,70 @@ +#ifdef FLEX_SCANNER +/* +dnl tables_shared.c - tables serialization code +dnl +dnl Copyright (c) 1990 The Regents of the University of California. +dnl All rights reserved. +dnl +dnl This code is derived from software contributed to Berkeley by +dnl Vern Paxson. +dnl +dnl The United States Government has rights in this work pursuant +dnl to contract no. DE-AC03-76SF00098 between the United States +dnl Department of Energy and the University of California. +dnl +dnl This file is part of flex. +dnl +dnl Redistribution and use in source and binary forms, with or without +dnl modification, are permitted provided that the following conditions +dnl are met: +dnl +dnl 1. Redistributions of source code must retain the above copyright +dnl notice, this list of conditions and the following disclaimer. +dnl 2. Redistributions in binary form must reproduce the above copyright +dnl notice, this list of conditions and the following disclaimer in the +dnl documentation and/or other materials provided with the distribution. +dnl +dnl Neither the name of the University nor the names of its contributors +dnl may be used to endorse or promote products derived from this software +dnl without specific prior written permission. +dnl +dnl THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR +dnl IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED +dnl WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +dnl PURPOSE. +dnl +*/ + +/* This file is meant to be included in both the skeleton and the actual + * flex code (hence the name "_shared"). + */ +#ifndef yyskel_static +#define yyskel_static static +#endif +#else +#include "flexdef.h" +#include "tables.h" +#ifndef yyskel_static +#define yyskel_static +#endif +#endif + + +/** Get the number of integers in this table. This is NOT the + * same thing as the number of elements. + * @param td the table + * @return the number of integers in the table + */ +yyskel_static int32_t yytbl_calc_total_len (const struct yytbl_data *tbl) +{ + int32_t n; + + /* total number of ints */ + n = tbl->td_lolen; + if (tbl->td_hilen > 0) + n *= tbl->td_hilen; + + if (tbl->td_id == YYT_ID_TRANSITION) + n *= 2; + return n; +} diff --git a/tables_shared.h b/tables_shared.h index 848e3d0..571f4d8 100644 --- a/tables_shared.h +++ b/tables_shared.h @@ -1,40 +1,50 @@ -/* tables_shared.h - tables serialization header - * - * Copyright (c) 1990 The Regents of the University of California. - * All rights reserved. - * - * This code is derived from software contributed to Berkeley by - * Vern Paxson. - * - * The United States Government has rights in this work pursuant - * to contract no. DE-AC03-76SF00098 between the United States - * Department of Energy and the University of California. - * - * This file is part of flex. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * Neither the name of the University nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE. - */ +#ifdef FLEX_SCANNER +/* +dnl tables_shared.h - tables serialization header +dnl +dnl Copyright (c) 1990 The Regents of the University of California. +dnl All rights reserved. +dnl +dnl This code is derived from software contributed to Berkeley by +dnl Vern Paxson. +dnl +dnl The United States Government has rights in this work pursuant +dnl to contract no. DE-AC03-76SF00098 between the United States +dnl Department of Energy and the University of California. +dnl +dnl This file is part of flex. +dnl +dnl Redistribution and use in source and binary forms, with or without +dnl modification, are permitted provided that the following conditions +dnl are met: +dnl +dnl 1. Redistributions of source code must retain the above copyright +dnl notice, this list of conditions and the following disclaimer. +dnl 2. Redistributions in binary form must reproduce the above copyright +dnl notice, this list of conditions and the following disclaimer in the +dnl documentation and/or other materials provided with the distribution. +dnl +dnl Neither the name of the University nor the names of its contributors +dnl may be used to endorse or promote products derived from this software +dnl without specific prior written permission. +dnl +dnl THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR +dnl IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED +dnl WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +dnl PURPOSE. -/* This file is meant to be included in both the skeleton and the actual - * flex code (hence the name "_shared"). - */ +dnl +dnl This file is meant to be included in both the skeleton and the actual +dnl flex code (hence the name "_shared"). +*/ +#ifndef yyskel_static +#define yyskel_static static +#endif +#else +#ifndef yyskel_static +#define yyskel_static +#endif +#endif /* Structures and prototypes for serializing flex tables. The * binary format is documented in the manual. @@ -49,7 +59,9 @@ */ /** Magic number for serialized format. */ +#ifndef YYTBL_MAGIC #define YYTBL_MAGIC 0xF13C57B1 +#endif /** Possible values for t_id field. Each one corresponds to a * scanner table of the same name. @@ -61,21 +73,25 @@ enum yytbl_id { YYT_ID_DEF = 0x04, /**< 1-dim ints */ YYT_ID_EC = 0x05, /**< 1-dim ints */ YYT_ID_META = 0x06, /**< 1-dim ints */ - YYT_ID_NUL_TRANS = 0x07, /**< 1-dim ints */ - YYT_ID_NXT = 0x08, /**< may be 2 dimensional array */ + YYT_ID_NUL_TRANS = 0x07, /**< 1-dim ints, maybe indices */ + YYT_ID_NXT = 0x08, /**< may be 2 dimensional ints */ YYT_ID_RULE_CAN_MATCH_EOL = 0x09, /**< 1-dim ints */ - YYT_ID_START_STATE_LIST = 0x0A, /**< 1-dim ints */ + YYT_ID_START_STATE_LIST = 0x0A, /**< 1-dim indices into trans tbl */ YYT_ID_TRANSITION = 0x0B /**< structs */ }; /** bit flags for t_flags field of struct yytbl_data */ enum yytbl_flags { - YYT_DATA8 = 0x01, /**< data is an array of type int8_t */ - YYT_DATA16 = 0x02, /**< data is an array of type int16_t */ - YYT_DATA32 = 0x04, /**< data is an array of type int32_t */ - YYT_PTRANS = 0x08, /**< data is a list of indexes of entries + /* These first three are mutually exclusive */ + YYTD_DATA8 = 0x01, /**< data is an array of type int8_t */ + YYTD_DATA16 = 0x02, /**< data is an array of type int16_t */ + YYTD_DATA32 = 0x04, /**< data is an array of type int32_t */ + + /* These two are mutually exclusive. */ + YYTD_PTRANS = 0x08, /**< data is a list of indexes of entries into the expanded `yy_transition' array. See notes in manual. */ + YYTD_STRUCT = 0x10 /**< data consists of yy_trans_info structs */ }; /* The serialized tables header. */ @@ -90,11 +106,23 @@ struct yytbl_hdr { /** A single serialized table */ struct yytbl_data { - enum yytbl_id td_id; /**< table identifier */ + enum yytbl_id td_id; /**< uint16_t table identifier */ uint16_t td_flags; /**< how to interpret this data */ uint32_t td_hilen; /**< num elements in highest dimension array */ uint32_t td_lolen; /**< num elements in lowest dimension array */ void *td_data; /**< table data */ }; +/** Extract corresponding data size_t from td_flags */ +#ifndef YYTDFLAGS2BYTES +#define YYTDFLAGS2BYTES(td_flags)\ + (((td_flags) & YYTD_DATA8)\ + ? sizeof(int8_t)\ + :(((td_flags) & YYTD_DATA16)\ + ? sizeof(int16_t)\ + :sizeof(int32_t))) +#endif + +yyskel_static int32_t yytbl_calc_total_len (const struct yytbl_data *tbl); + /* vim:set noexpandtab cindent tabstop=8 softtabstop=0 shiftwidth=8 textwidth=0: */ -- 2.49.0