From: John Millaway <john43@users.sourceforge.net>
Date: Thu, 5 Sep 2002 04:24:40 +0000 (+0000)
Subject: Lots of work on tables serialization code.
X-Git-Tag: flex-2-5-19~10
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=d32f04f70eb5b1e588e62c0fbd6ffd25b4b723b7;p=flex

Lots of work on tables serialization code.
---

diff --git a/flexdef.h b/flexdef.h
index f5eb1cf..b7fdd64 100644
--- a/flexdef.h
+++ b/flexdef.h
@@ -1157,7 +1157,7 @@ extern char *chomp (char *str);
 
 /* Tables serialization API declarations. */
 #include "tables_shared.h"
-struct yytbl_data *yytbl_data_create (enum yytbl_id id);
+int     yytbl_data_init (struct yytbl_data *tbl, enum yytbl_id id);
 struct yytbl_data *mkftbl (void);
 
 
diff --git a/gen.c b/gen.c
index a79dc08..39b59a5 100644
--- a/gen.c
+++ b/gen.c
@@ -261,13 +261,14 @@ struct yytbl_data *mkecstbl (void)
 	struct yytbl_data *tbl = 0;
 	int32_t *tdata = 0;
 
-	tbl = yytbl_data_create (YYT_ID_EC);
-	tbl->t_flags |= YYT_DATA32;
-	tbl->t_hilen = 0;
-	tbl->t_lolen = csize;
+	tbl = (struct yytbl_data*)flex_alloc(sizeof(struct yytbl_data));
+	yytbl_data_init (tbl,YYT_ID_EC);
+	tbl->td_flags |= YYT_DATA32;
+	tbl->td_hilen = 0;
+	tbl->td_lolen = csize;
 
-	tbl->t_data = tdata =
-		(int32_t *) calloc (tbl->t_lolen, sizeof (int32_t));
+	tbl->td_data = tdata =
+		(int32_t *) calloc (tbl->td_lolen, sizeof (int32_t));
 
 	for (i = 1; i < csize; ++i) {
 		if (caseins && isupper (i))
@@ -483,13 +484,14 @@ struct yytbl_data *mkftbl (void)
 	struct yytbl_data *tbl;
 	int32_t *tdata = 0;
 
-	tbl = yytbl_data_create (YYT_ID_ACCEPT);
-	tbl->t_flags |= YYT_DATA32;
-	tbl->t_hilen = 0;	/* it's a one-dimensional array */
-	tbl->t_lolen = lastdfa + 1;
+	tbl = (struct yytbl_data*)flex_alloc(sizeof(struct yytbl_data));
+	yytbl_data_init (tbl,YYT_ID_ACCEPT);
+	tbl->td_flags |= YYT_DATA32;
+	tbl->td_hilen = 0;	/* it's a one-dimensional array */
+	tbl->td_lolen = lastdfa + 1;
 
-	tbl->t_data = tdata =
-		(int32_t *) calloc (tbl->t_lolen, sizeof (int32_t));
+	tbl->td_data = tdata =
+		(int32_t *) calloc (tbl->td_lolen, sizeof (int32_t));
 
 	dfaacc[end_of_buffer_state].dfaacc_state = end_of_buffer_action;
 
diff --git a/tables.c b/tables.c
index e0c2a51..a19bcfe 100644
--- a/tables.c
+++ b/tables.c
@@ -35,14 +35,33 @@
 
 #include "flexdef.h"
 
+/** Calculate (0-7) = number bytes needed to pad n to next 64-bit boundary. */
 #define yypad64(n) ((8-((n)%8))%8)
-#define TFLAGS2BYTES(flags)\
-        (((flags) & YYT_DATA8)\
+
+/** Extract corresponding data size_t from td_flags */
+#define TFLAGS2BYTES(td_flags)\
+        (((td_flags) & YYT_DATA8)\
             ? sizeof(int8_t)\
-            :(((flags) & YYT_DATA16)\
+            :(((td_flags) & YYT_DATA16)\
                 ? sizeof(int16_t)\
                 :sizeof(int32_t)))
 
+/** Convert size_t to t_flag.
+ *  @param n in {1,2,4}
+ *  @return YYT_DATA*. 
+ */
+#define BYTES2TFLAG(n)\
+    (((n) == sizeof(int8_t))\
+        ? YYT_DATA8\
+        :(((n)== sizeof(int16_t))\
+            ? YYT_DATA16\
+            : YYT_DATA32))
+
+/** Clear YYT_DATA* bit flags
+ * @return the flag with the YYT_DATA* bits cleared
+ */
+#define TFLAGS_CLRDATA(flg) ((flg) & ~(YYT_DATA8 | YYT_DATA16 | YYT_DATA32))
+
 int     yytbl_fwrite32 (FILE * out, uint32_t v);
 int     yytbl_fwrite16 (FILE * out, uint16_t v);
 int     yytbl_fwrite8 (FILE * out, uint8_t v);
@@ -68,18 +87,17 @@ void yytbl_hdr_init (struct yytbl_hdr *th, const char *version_str,
 }
 
 /** Allocate and initialize a table data structure.
- * @param id  the table identifier
- * @return a flex_alloc'd structure. Caller must flex_free it.
+ *  @param tbl a pointer to an uninitialized table
+ *  @param id  the table identifier
+ *  @return 0 on success
  */
-struct yytbl_data *yytbl_data_create (enum yytbl_id id)
+int yytbl_data_init (struct yytbl_data *td, enum yytbl_id id)
 {
-	struct yytbl_data *td;
 
-	td = (struct yytbl_data *) flex_alloc (sizeof (struct yytbl_data));
 	memset (td, 0, sizeof (struct yytbl_data));
-
-	td->t_id = id;
-	return td;
+	td->td_id = id;
+	td->td_flags = YYT_DATA32;
+	return 0;
 }
 
 /** write the header.
@@ -128,9 +146,9 @@ int yytbl_hdr_fwrite (FILE * out, const struct yytbl_hdr *th)
 }
 
 /** Write four bytes in network byte order
- * @param  out  the output stream
- * @param  v    a dword in host byte order
- * @return  -1 on error. number of bytes written on success.
+ *  @param  out  the output stream
+ *  @param  v    a dword in host byte order
+ *  @return  -1 on error. number of bytes written on success.
  */
 int yytbl_fwrite32 (FILE * out, uint32_t v)
 {
@@ -146,9 +164,9 @@ int yytbl_fwrite32 (FILE * out, uint32_t v)
 }
 
 /** Write two bytes in network byte order.
- * @param  out  the output stream
- * @param  v    a word in host byte order
- * @return  -1 on error. number of bytes written on success.
+ *  @param  out  the output stream
+ *  @param  v    a word in host byte order
+ *  @return  -1 on error. number of bytes written on success.
  */
 int yytbl_fwrite16 (FILE * out, uint16_t v)
 {
@@ -164,9 +182,9 @@ int yytbl_fwrite16 (FILE * out, uint16_t v)
 }
 
 /** Write a byte.
- * @param  out  the output stream
- * @param  v    the value to be written
- * @return  -1 on error. number of bytes written on success.
+ *  @param  out  the output stream
+ *  @param  v    the value to be written
+ *  @return  -1 on error. number of bytes written on success.
  */
 int yytbl_fwrite8 (FILE * out, uint8_t v)
 {
@@ -179,123 +197,180 @@ int yytbl_fwrite8 (FILE * out, uint8_t v)
 	return bytes;
 }
 
-/** Calculate the number of bytes  needed to hold the largest
- *  absolute value in this array.
- * @param arr  array
- * @param len  number of elements in array
- * @param sz   sizeof individual element
- * @return 1,2, or 4
+/** Get the number of integers in this table. This is NOT the
+ *  same thing as the number of elements.
+ *  @param td the table 
+ *  @return the number of integers in the table
  */
-static int min_int_size (const void *arr, int32_t len, int sz)
+static int32_t tbl_get_total_len (struct yytbl_data *tbl)
 {
-	int32_t curr, max = 0, i;
-
-	for (i = 0; i < len; i++) {
-		switch (sz) {
-		case 1:
-			curr = abs (((int8_t *) arr)[i]);
-			break;
-		case 2:
-			curr = abs (((int16_t *) arr)[i]);
-			break;
-		case 4:
-			curr = abs (((int32_t *) arr)[i]);
-			break;
-		default:
-			fprintf (stderr,
-				 "Illegal size (%d) in min_int_size\n",
-				 sz);
-			return 32;
-		}
-		if (curr > max)
-			max = curr;
-	}
-	if (max < INT8_MAX)
-		return sizeof (int8_t);
-	else if (max < INT16_MAX)
-		return sizeof (int16_t);
-	else
-		return sizeof (int32_t);
+
+	int32_t n;
+
+	/* total number of ints */
+	n = tbl->td_lolen;
+	if (tbl->td_hilen > 0)
+		n *= tbl->td_hilen;
+
+	if (tbl->td_id == YYT_ID_TRANSITION)
+		n *= 2;
+	return n;
 }
 
 
-/* Extract data element [i][j] from int array data tables. 
+/** Extract data element [i][j] from array data tables. 
  * @param tbl data table
- * @param i index into major array. i is zero for one-dimensional arrays.
- * @param j index into 
- * @return data[i][j]
+ * @param i index into higher dimension array. i should be zero for one-dimensional arrays.
+ * @param j index into lower dimension array.
+ * @param k index into struct, must be 0 or 1. Only valid for YYT_ID_TRANSITION table
+ * @return data[i][j + k]
  */
-static int32_t yytbl_data_geti (const struct yytbl_data *tbl, int i, int j)
+int32_t yytbl_data_getijk (const struct yytbl_data * tbl, int i, int j,
+			   int k)
 {
-	/* TODO */
+	int32_t lo;
+
+	k %= 2;
+	lo = tbl->td_lolen;
+
+	switch (TFLAGS2BYTES (tbl->td_flags)) {
+	case sizeof (int8_t):
+		return ((int8_t *) (tbl->td_data))[(i * lo + j) * (k + 1) +
+						   k];
+	case sizeof (int16_t):
+		return ((int16_t *) (tbl->td_data))[(i * lo + j) * (k +
+								    1) +
+						    k];
+	case sizeof (int32_t):
+		return ((int32_t *) (tbl->td_data))[(i * lo + j) * (k +
+								    1) +
+						    k];
+	default:		/* TODO: error. major foobar somewhere. */
+		break;
+	}
+
 	return 0;
 }
 
-/* Transform data to smallest possible of (int32, int16, int8).
- * For example, we may generate an int32 array due to user options
- * (e.g., %option align) but if the maximum value in that array
- * is 80, then we serialize it with 1 byte per int.
+/** Extract data element [i] from array data tables treated as a single flat array of integers.
+ * Be careful for 2-dimensional arrays or for YYT_ID_TRANSITION, which is an array
+ * of structs. 
+ * @param tbl data table
+ * @param i index into array.
+ * @return data[i]
+ */
+static int32_t yytbl_data_geti (const struct yytbl_data *tbl, int i)
+{
+
+	switch (TFLAGS2BYTES (tbl->td_flags)) {
+	case sizeof (int8_t):
+		return ((int8_t *) (tbl->td_data))[i];
+	case sizeof (int16_t):
+		return ((int16_t *) (tbl->td_data))[i];
+	case sizeof (int32_t):
+		return ((int32_t *) (tbl->td_data))[i];
+	default:		/* TODO: error. major foobar somewhere. */
+		break;
+	}
+	return 0;
+}
+
+/** Set data element [i] in array data tables treated as a single flat array of integers.
+ * Be careful for 2-dimensional arrays or for YYT_ID_TRANSITION, which is an array
+ * of structs. 
+ * @param tbl data table
+ * @param i index into array.
+ * @param newval new value for data[i]
+ */
+static void yytbl_data_seti (const struct yytbl_data *tbl, int i,
+			     int newval)
+{
+
+	switch (TFLAGS2BYTES (tbl->td_flags)) {
+	case sizeof (int8_t):
+		((int8_t *) (tbl->td_data))[i] = (int8_t) newval;
+	case sizeof (int16_t):
+		((int16_t *) (tbl->td_data))[i] = (int16_t) newval;
+	case sizeof (int32_t):
+		((int32_t *) (tbl->td_data))[i] = (int32_t) newval;
+	default:		/* TODO: error. major foobar somewhere. */
+		break;
+	}
+}
+
+/** Calculate the number of bytes  needed to hold the largest
+ *  absolute value in this data array.
+ *  @param tbl  the data table
+ *  @return sizeof(n) where n in {int8_t, int16_t, int32_t}
+ */
+static size_t min_int_size (struct yytbl_data *tbl)
+{
+	uint32_t i, total_len;
+	int32_t max = 0;
+
+	total_len = tbl_get_total_len (tbl);
+
+	for (i = 0; i < total_len; i++) {
+		int32_t n;
+
+		n = abs (yytbl_data_geti (tbl, i));
+
+		if (n > max)
+			max = n;
+	}
+
+	if (max <= INT8_MAX)
+		return sizeof (int8_t);
+	else if (max <= INT16_MAX)
+		return sizeof (int16_t);
+	else
+		return sizeof (int32_t);
+}
+
+/** Transform data to smallest possible of (int32, int16, int8).
+ * For example, we may have generated an int32 array due to user options
+ * (e.g., %option align), but if the maximum value in that array
+ * is 80 (for example), then we can serialize it with only 1 byte per int.
+ * This is NOT the same as compressed DFA tables. We're just trying
+ * to save storage space here.
  *
  * @param tbl the table to be compressed
  */
 void yytbl_data_compress (struct yytbl_data *tbl)
 {
-	int32_t i, sz;
-	void   *newdata = 0;
-
-	if (tbl->t_id != YYT_ID_TRANSITION
-	    && tbl->t_id != YYT_ID_START_STATE_LIST) {
-		if (tbl->t_hilen == 0) {
-			/* Data is a single-dimensional array of ints */
-			sz = min_int_size (tbl->t_data, tbl->t_lolen,
-					   TFLAGS2BYTES (tbl->t_flags));
-			if (sz == TFLAGS2BYTES (tbl->t_flags))
-				/* No change in this table needed. */
-				return;
-
-			if (sz > TFLAGS2BYTES (tbl->t_flags)) {
-				/* TODO: ERROR. The code is wrong somewhere. */
-				return;
-			}
-
-			newdata = flex_alloc (sz * tbl->t_lolen);
-			for (i = 0; i < tbl->t_lolen; i++) {
-				int32_t n;
-
-				n = yytbl_data_geti (tbl, 0, i);
-				switch (sz) {
-				case sizeof (int8_t):
-					((int8_t *) newdata)[i] =
-						(int8_t) n;
-					break;
-				case sizeof (int16_t):
-
-					((int16_t *) newdata)[i] =
-						(int16_t) n;
-					break;
-				case sizeof (int32_t):
-
-					((int32_t *) newdata)[i] =
-						(int32_t) n;
-					break;
-				default:	/* TODO: ERROR: unknown 'sz' */
-					break;
-				}
-			}
-			free (tbl->t_data);
-			tbl->t_data = newdata;
-
-		}
-		else {
-			/* Data is a two-dimensional array of ints */
-		}
-	}
-	else if (tbl->t_id == YYT_ID_TRANSITION) {
-		/* Data is an array of structs */
-	}
-	else if (tbl->t_id == YYT_ID_START_STATE_LIST) {
-		/* Data is an array of pointers */
+	int32_t i, newsz, total_len;
+	struct yytbl_data newtbl;
+
+	yytbl_data_init (&newtbl, tbl->td_id);
+	newtbl.td_hilen = tbl->td_hilen;
+	newtbl.td_lolen = tbl->td_lolen;
+	newtbl.td_flags = tbl->td_flags;
+
+	newsz = min_int_size (tbl);
+
+
+
+	if (newsz == TFLAGS2BYTES (tbl->td_flags))
+		/* No change in this table needed. */
+		return;
+
+	if (newsz > TFLAGS2BYTES (tbl->td_flags)) {
+		/* TODO: ERROR. The code is wrong somewhere. */
+		return;
 	}
+
+	total_len = tbl_get_total_len (tbl);
+	newtbl.td_data = flex_alloc (newsz * total_len);
+	newtbl.td_flags =
+		TFLAGS_CLRDATA (newtbl.td_flags) & BYTES2TFLAG (newsz);
+
+	for (i = 0; i < total_len; i++)
+		yytbl_data_seti (&newtbl, i, yytbl_data_geti (tbl, i));
+
+
+	/* Now copy over the old table */
+	free (tbl->td_data);
+	*tbl = newtbl;
 }
 
-/* vim:set expandtab cindent tabstop=4 softtabstop=4 shiftwidth=4 textwidth=0: */
+/* vim:set noexpandtab cindent tabstop=8 softtabstop=0 shiftwidth=8 textwidth=0: */
diff --git a/tables_shared.h b/tables_shared.h
index bf20668..848e3d0 100644
--- a/tables_shared.h
+++ b/tables_shared.h
@@ -55,17 +55,17 @@
  *  scanner table of the same name.
  */
 enum yytbl_id {
-	YYT_ID_ACCEPT = 0x01,
-	YYT_ID_BASE = 0x02,
-	YYT_ID_CHK = 0x03,
-	YYT_ID_DEF = 0x04,
-	YYT_ID_EC = 0x05,
-	YYT_ID_META = 0x06,
-	YYT_ID_NUL_TRANS = 0x07,
+	YYT_ID_ACCEPT = 0x01,		/**< 1-dim ints */
+	YYT_ID_BASE = 0x02,		/**< 1-dim ints */
+	YYT_ID_CHK = 0x03,		/**< 1-dim ints */
+	YYT_ID_DEF = 0x04,		/**< 1-dim ints */
+	YYT_ID_EC = 0x05,		/**< 1-dim ints */
+	YYT_ID_META = 0x06,		/**< 1-dim ints */
+	YYT_ID_NUL_TRANS = 0x07,	/**< 1-dim ints */
 	YYT_ID_NXT = 0x08,		/**< may be 2 dimensional array */
-	YYT_ID_RULE_CAN_MATCH_EOL = 0x09,
-	YYT_ID_START_STATE_LIST = 0x0A,	/**< array of pointers */
-	YYT_ID_TRANSITION = 0x0B	/**< array of structs */
+	YYT_ID_RULE_CAN_MATCH_EOL = 0x09, /**< 1-dim ints */
+	YYT_ID_START_STATE_LIST = 0x0A,	/**< 1-dim ints  */
+	YYT_ID_TRANSITION = 0x0B	/**< structs */
 };
 
 /** bit flags for t_flags field of struct yytbl_data */
@@ -90,9 +90,11 @@ struct yytbl_hdr {
 
 /** A single serialized table */
 struct yytbl_data {
-	enum yytbl_id t_id; /**< table identifier */
-	uint16_t t_flags;   /**< how to interpret this data */
-	uint32_t t_hilen;   /**< num elements in highest dimension array */
-	uint32_t t_lolen;   /**< num elements in lowest dimension array */
-	void   *t_data;	    /**< table data */
+	enum yytbl_id td_id; /**< table identifier */
+	uint16_t td_flags;   /**< how to interpret this data */
+	uint32_t td_hilen;   /**< num elements in highest dimension array */
+	uint32_t td_lolen;   /**< num elements in lowest dimension array */
+	void   *td_data;     /**< table data */
 };
+
+/* vim:set noexpandtab cindent tabstop=8 softtabstop=0 shiftwidth=8 textwidth=0: */