Repair merged.c and so, all Flex

diff --git a/src/ChangeLog b/src/ChangeLog
index 92b51a9db263535a116329800ff0a36e54eb1811..053553c79712c560f0527a7d29bf3d072a655a3d 100644 (file)
--- a/src/ChangeLog
+++ b/src/ChangeLog
@@ -1,3 +1,13 @@
+2008-02-22  François Pinard  <pinard@iro.umontreal.ca>
+
+       Repair all Flex steps, which were broken since version 3.6.
+       * mergelex.py: Provide a rather dumb YY_INPUT using get_byte,
+       and set yy_init to zero in all generated routines.
+       * recodext.h: Within Flex, use both get_byte_helper and
+       put_byte_helper, not only put_byte_helper.
+       * task.c (get_byte_helper) [!INLINE_HARDER]: Compile as well.
+       Properly declare the argument as a subtask, not as a task.
+
 2008-02-21  François Pinard  <pinard@iro.umontreal.ca>
 
        * Makefile.am (recode.1): Use --name calling help2man.

diff --git a/src/merged.c b/src/merged.c
index be3d0c792bb257f9bdc4ce12ba3268b9f0cf7c57..2b414ab7df8af25e3f14f6be997ad8e29815cffd 100644 (file)
--- a/src/merged.c
+++ b/src/merged.c
@@ -871,11 +871,12 @@ int yy_flex_debug = 0;
 #define YY_RESTORE_YY_MORE_OFFSET
 char *yytext;
 /* This file is generated automatically by `mergelex.py'.  */
-#define YY_NO_UNPUT
 #include "common.h"
 static RECODE_CONST_REQUEST request;
 static RECODE_SUBTASK subtask;
 
+#define YY_INPUT(buf, result, max_size)   {     int c = get_byte (subtask);     result = (c == EOF) ? YY_NULL : (buf[0] = c, 1);   }
+
 
 void texte_latin1_diaeresis PARAMS ((void));
 
@@ -3052,7 +3053,7 @@ transform_ascii_latin1 (RECODE_SUBTASK subtask_argument)
 {
   subtask = subtask_argument;
   request = subtask->task->request;
-  yy_init = 1;
+  yy_init = 0;
   yyin = subtask->input.file;
   yyout = subtask->output.file;
   BEGIN ascii_latin1;
@@ -3099,7 +3100,7 @@ transform_latex_latin1 (RECODE_SUBTASK subtask_argument)
 {
   subtask = subtask_argument;
   request = subtask->task->request;
-  yy_init = 1;
+  yy_init = 0;
   yyin = subtask->input.file;
   yyout = subtask->output.file;
   BEGIN latex_latin1;
@@ -3146,7 +3147,7 @@ transform_texte_latin1 (RECODE_SUBTASK subtask_argument)
 {
   subtask = subtask_argument;
   request = subtask->task->request;
-  yy_init = 1;
+  yy_init = 0;
   yyin = subtask->input.file;
   yyout = subtask->output.file;
   BEGIN texte_latin1;

diff --git a/src/mergelex.py b/src/mergelex.py
index f7f772d4f8a7b5faefc52fc82dfce8b4c51b8834..b35ef9558009eed725c2d5e84b2b2c65a228ce7c 100644 (file)
--- a/src/mergelex.py
+++ b/src/mergelex.py
@@ -23,21 +23,34 @@
 import re, sys
 
 # Initial comments.
-section0 = ["/* This file is generated automatically by `mergelex.py'.  */\n"]
+section0 = [
+"/* This file is generated automatically by `mergelex.py'.  */\n"]
+
 # Flex and C declarations.
-section1 = ["\n"
-            "%option noyywrap\n"
-            "%{\n"
-            '#define YY_NO_UNPUT\n'
-            '#include "common.h"\n'
-            "static RECODE_CONST_REQUEST request;\n"
-            "static RECODE_SUBTASK subtask;\n"
-            "%}\n"]
+section1 = '''\
+
+%option noyywrap
+%{
+#include "common.h"
+static RECODE_CONST_REQUEST request;
+static RECODE_SUBTASK subtask;
+
+#define YY_INPUT(buf, result, max_size) \
+  { \
+    int c = get_byte (subtask); \
+    result = (c == EOF) ? YY_NULL : (buf[0] = c, 1); \
+  }
+%}
+'''.splitlines(True)
+
 # Flex rules.
-section2 = ["%%\n"
-            "<<EOF>>                   { return 1; }\n"]
+section2 = '''\
+%%
+<<EOF>>                        { return 1; }
+'''.splitlines(True)
+
 # Rest of C code.
-section3 = ["%%\n"]
+section3 = ['%%\n']
 
 within_C_code = False
 definitions = {}
@@ -56,19 +69,20 @@ while True:
     if match:
         section = 1
         step_name = match.group(1)
-        section3.append(
-            '\n'
-            'static bool\n'
-            'transform_%s (RECODE_SUBTASK subtask_argument)\n'
-            '{\n'
-            '  subtask = subtask_argument;\n'
-            '  request = subtask->task->request;\n'
-            '  yy_init = 1;\n'
-            '  yyin = subtask->input.file;\n'
-            '  yyout = subtask->output.file;\n'
-            '  BEGIN %s;\n'
-            '  return yylex ();\n'
-            '}\n'
+        section3.append('''\
+
+static bool
+transform_%s (RECODE_SUBTASK subtask_argument)
+{
+  subtask = subtask_argument;
+  request = subtask->task->request;
+  yy_init = 0;
+  yyin = subtask->input.file;
+  yyout = subtask->output.file;
+  BEGIN %s;
+  return yylex ();
+}
+'''
             % (step_name, step_name))
         continue
 

diff --git a/src/recodext.h b/src/recodext.h
index 6995ca9352dffc91f7f601067f0570982157ca82..708a86c5747b2247969bc4e7c00838ba303d1e97 100644 (file)
--- a/src/recodext.h
+++ b/src/recodext.h
@@ -707,6 +707,13 @@ bool put_ucs4 PARAMS ((unsigned, RECODE_SUBTASK));
 /* Giving a name to the ASCII character assigned to position 0.  */
 #define NUL '\0'
 
+#ifdef FLEX_SCANNER
+# if !INLINE_HARDER
+#  undef USE_HELPERS
+#  define USE_HELPERS 1
+# endif
+#endif
+
 #if USE_HELPERS
 
 # define get_byte(Subtask) \
@@ -735,14 +742,6 @@ bool put_ucs4 PARAMS ((unsigned, RECODE_SUBTASK));
 
 #ifdef FLEX_SCANNER
 
-# if !INLINE_HARDER
-
-#  undef put_byte
-#  define put_byte(Byte, Subtask) \
-     put_byte_helper ((Byte), (Subtask))
-
-# endif
-
 # define PUT_NON_DIACRITIC_BYTE(Byte, Subtask) \
     if (request->diacritics_only)                              \
       ECHO;                                                    \

diff --git a/src/task.c b/src/task.c
index b4c4917f6f9ccebcc2fc2c3277b0a2a4322daba9..84882ffbf394cb59edef3ab4a345dbd9cf489bf3 100644 (file)
--- a/src/task.c
+++ b/src/task.c
@@ -74,34 +74,35 @@
 | Read one byte from the input text of TASK, or EOF is none remain.  |
 `-------------------------------------------------------------------*/
 
-/* This function is directly called by get_byte whenever USE_HELPERS is
-   defined.  Otherwise, get_byte does everything necessary and this routine
-   is not needed.  */
+/* This function is directly called by get_byte whenever USE_HELPERS
+   is defined, and also from within generated Flex code unless
+   INLINE_HARDER is defined.  Otherwise, get_byte does everything
+   necessary and this routine is not needed.  */
 
-#if USE_HELPERS
+#if USE_HELPERS || !INLINE_HARDER
 
 int
-get_byte_helper (RECODE_TASK task)
+get_byte_helper (RECODE_SUBTASK subtask)
 {
-  if (task->input.file)
-    return getc (task->input.file);
-  else if (task->input.cursor == task->input.limit)
+  if (subtask->input.file)
+    return getc (subtask->input.file);
+  else if (subtask->input.cursor == subtask->input.limit)
     return EOF;
   else
-    return (unsigned char) *task->input.cursor++;
+    return (unsigned char) *subtask->input.cursor++;
 }
 
-#endif /* USE_HELPERS */
+#endif /* USE_HELPERS || !INLINE_HARDER */
 
 /*-----------------------------------------.
 | Write BYTE on the output text for TASK.  |
 `-----------------------------------------*/
 
-/* This function is directly called by put_byte whenever USE_HELPERS is
-   defined.  It is also called when the output buffer needs to be
-   reallocated, which put_byte does not know how to handle itself.  Note
-   that when INLINE_HARDER is not defined, USE_HELPERS is implied for Flex
-   generated code.  */
+/* This function is directly called by put_byte whenever USE_HELPERS
+   is defined, and also from within generated Flex code unless
+   INLINE_HARDER is defined.  It is also called when the output
+   buffer needs to be reallocated, which put_byte does not know
+   how to handle itself.  */
 
 void
 put_byte_helper (int byte, RECODE_SUBTASK subtask)