Fixed buffer overflow in reject state buffer.

Corrected documentation on the state buffer.

diff --git a/doc/flex.texi b/doc/flex.texi
index 377c345429f88ecdd3f206b9411558cfff46558e..ae744c3156c3efda3fd02970ecfe95abeb319153 100644 (file)
--- a/doc/flex.texi
+++ b/doc/flex.texi
@@ -4703,12 +4703,15 @@ character buffer.  This buffer is typically the largest chunk of dynamic memory
 flex consumes. This buffer will grow if necessary, doubling the size each time.
 Flex frees this memory when you call yylex_destroy().  The default size of this
 buffer (16384 bytes) is almost always too large.  The ideal size for this
-buffer is the length of the longest token expected.  Flex will allocate a few
-extra bytes for housekeeping.
-
-@item 16kb for the REJECT state. This will only be allocated if you use REJECT.
-The size is the same as the input buffer, so if you override the size of the
-input buffer, then you automatically override the size of this buffer as well.
+buffer is the length of the longest token expected, in bytes, plus a little more.  Flex will allocate a few
+extra bytes for housekeeping. Currently, to override the size of the input buffer
+you must @code{#define YY_BUF_SIZE} to whatever number of bytes you want. We don't plan
+to change this in the near future, but we reserve the right to do so if we ever add a more robust memory management
+API. 
+
+@item 64kb for the REJECT state. This will only be allocated if you use REJECT.
+The size is the large enough to hold the same number of states as characters in the input buffer. If you override the size of the
+input buffer (via @code{YY_BUF_SIZE}), then you automatically override the size of this buffer as well.
 
 @item 100 bytes for the start condition stack.
 Flex allocates memory for the start condition stack. This is the stack used

diff --git a/flex.skl b/flex.skl
index d0d83482d651989ee0c4cb051f398d84b12bfe32..683ead6eb9e6edd8696fab9c00cb40514665fd6f 100644 (file)
--- a/flex.skl
+++ b/flex.skl
@@ -418,6 +418,10 @@ m4_ifdef( [[M4_YY_NOT_IN_HEADER]],
 #ifndef YY_BUF_SIZE
 #define YY_BUF_SIZE 16384
 #endif
+
+/* The state buf must be large enough to hold one state per character in the main buffer.
+ */
+#define YY_STATE_BUF_SIZE   ((YY_BUF_SIZE + 2) * sizeof(yy_state_type))
 ]])
 
 
@@ -1228,8 +1232,9 @@ m4_ifdef( [[<M4_YY_BISON_LLOC>]],
 
 m4_ifdef( [[M4_YY_USES_REJECT]],
 [[
+        /* Create the reject buffer large enough to save one state per allowed character. */
         if ( ! YY_G(yy_state_buf) )
-            YY_G(yy_state_buf) = (yy_state_type *)yyalloc(YY_BUF_SIZE + 2  M4_YY_CALL_LAST_ARG);
+            YY_G(yy_state_buf) = (yy_state_type *)yyalloc(YY_STATE_BUF_SIZE  M4_YY_CALL_LAST_ARG);
 ]])
 
                if ( ! YY_G(yy_start) )
@@ -1449,7 +1454,7 @@ yyFlexLexer::yyFlexLexer( std::istream* arg_yyin, std::ostream* arg_yyout )
 
 m4_ifdef( [[M4_YY_USES_REJECT]],
 [[
-       yy_state_buf = new yy_state_type[YY_BUF_SIZE + 2];
+       yy_state_buf = new yy_state_type[YY_STATE_BUF_SIZE];
 ]],
 [[
        yy_state_buf = 0;