for f in `find "$objdir" -type f -name "*.rst"`
do
- rst2html.py --link-stylesheet --stylesheet="css/default.css" "$f" > "${f%.rst}.html"
+ rst2html.py --stylesheet="$objdir/css/default.css" "$f" > "${f%.rst}.html"
+# rst2html.py --link-stylesheet --stylesheet="css/default.css" "$f" > "${f%.rst}.html"
done
--- /dev/null
+* 2015-02-23: 0.14
+ - Added generic input API (#21 "Support to configure how re2c code interfaced with the symbol buffer?")
+ - fixed #46 "re2c generates an infinite loop, depends on existence of previous parser"
+ - fixed #47 "Dot output label escaped characters"
+
+* 2014-08-22: 0.13.7.5
+
+ - Fixed `Gentoo bug with PHP lexer <https://bugs.gentoo.org/show_bug.cgi?id=518904>`_
+
+* 2014-07-29: 0.13.7.4
+
+ - Enabled ``make docs`` only if configured with ``--enable-docs``
+ - Disallowed to use yacc/byacc instead of bison to build parser
+ - Removed non-portable sed feature in script that runs tests
+
+* 2014-07-27: 0.13.7.3
+
+ - Fixed CXX warning
+ - Got rid of asciidoc build-time dependency
+
+* 2014-07-27: 0.13.7.2
+
+ - Included man page into dist, respect users CXXFLAGS.
+
+* 2014-07-26: 0.13.7.1
+
+ - Added missing files to tarball
+
+* 2014-07-25: 0.13.7
+
+ - Added UTF-8 support
+ - Added UTF-16 support
+ - Added default rule
+ - Added option to control ill-formed Unicode
+
+* 2013-07-04: 0.13.6
+
+ - Fixed #2535084 uint problem with Sun C 5.8
+ - #3308400: allow Yacc-style ``%{`` code brackets ``}%``
+ - #2506253: allow C++ ``//`` comments
+ - Fixed inplace configuration in ``-e`` mode.
+ - Applied #2482572 Typos in error messages.
+ - Applied #2482561 Error in manual section on ``-r`` mode.
+ - Fixed #2478216 Wrong ``start_label`` in ``-c`` mode.
+ - Fixed #2186718 Unescaped backslash in file name of ``#line`` directive.
+ - Fixed #2102138 Duplicate case labels on EBCDIC.
+ - Fixed #2088583 Compile problem on AIX.
+ - Fixed #2038610 Ebcdic problem.
+ - improve dot support: make char intervals (e.g. ``[A-Z]``) instead of one edge per char
+
+* 2008-05-25: 0.13.5
+
+ - Fixed #1952896 Segfault in ``re2c::Scanner::scan``.
+ - Fixed #1952842 Regression.
+
+* 2008-04-05: 0.13.4
+
+ - Added transparent handling of ``#line`` directives in input files.
+ - Added ``re2c:yyfill:check`` inplace configuration.
+ - Added ``re2c:define:YYSETSTATE:naked`` inplace configuration.
+ - Added ``re2c:flags:w`` and ``re2c:flags:u`` inplace configurations.
+ - Added the ability to add rules in ``use:re2c`` blocks.
+ - Changed ``-r`` flag to accept only ``rules:re2c`` and ``use:re2c`` blocks.
+
+* 2008-03-14: 0.13.3
+
+ - Added ``-r`` flag to allow reuse of scanner definitions.
+ - Added ``-F`` flag to support flex syntax in rules.
+ - Fixed SEGV in scanner that occurs with very large blocks.
+ - Fixed issue with unused ``yybm``.
+ - Partial support for flex syntax.
+ - Changed to allow ``/*`` comments with ``-c`` switch.
+ - Added flag ``-D/--emit-dot``.
+
+* 2008-02-14: 0.13.2
+
+ - Added flag ``--case-inverted``.
+ - Added flag ``--case-insensitive``.
+ - Added support for ``<!...>`` to enable rule setup.
+ - Added support for ``=>`` style rules.
+ - Added support for ``:=`` style rules.
+ - Added support for ``:=>`` style rules.
+ - Added ``re2c:cond:divider`` and ``re2c:cond:goto`` inplace configuration.
+ - Fixed code generation to emit space after ``if``.
+
+* 2007-08-24: 0.13.1
+
+ - Added custom build rules for Visual Studio 2005 (``re2c.rules``). (William Swanson)
+ - Fixed issue with some compilers.
+ - Fixed #1776177 Build on AIX.
+ - Fixed #1743180 ``fwrite`` with 0 length crashes on OS X.
+
+* 2007-06-24: 0.13.0
+
+ - Added ``-c`` and ``-t`` to generate scanners with (f)lex-like condition support.
+ - Fixed issue with short form of switches and parameter if not first switch.
+ - Fixed #1708378 segfault ``in actions.cc``.
+
+* 2007-08-24: 0.12.3
+
+ - Fixed issue with some compilers.
+ - Fixed #1776177 Build on AIX.
+ - Fixed #1743180 ``fwrite`` with 0 length crashes on OS X.
+
+* 2007-06-26: 0.12.2
+
+ - Fixed #1743180 ``fwrite`` with 0 length crashes on OS X.
+
+* 2007-05-23: 0.12.1
+
+ - Fixed #1711240 problem with ``"`` and ``7F`` on EBCDIC plattforms.
+
+* 2007-05-01: 0.12.0
+
+ - Re-release of 0.11.3 as new stable branch.
+ - Fixed issue with short form of switches and parameter if not first switch.
+ - Fixed #1708378 segfault in ``actions.cc``.
+ - re2c 0.12.0 has been tested with the following compilers:
+ + gcc version 4.1.2 (Gentoo 4.1.2)
+ + gcc version 4.1.2 20070302 (prerelease) (4.1.2-1mdv2007.1)
+ + gcc version 4.1.2 20061115 (prerelease) (Debian 4.1.1-21)
+ + gcc version 4.1.1 20070105 (Red Hat 4.1.1-51)
+ + gcc version 4.1.0 (SUSE Linux 10)
+ + gcc version 4.0.3 (4.0.3-0.20060215.2mdk for Mandriva Linux release 2006.1)
+ + gcc version 4.0.2 20050901 (prerelease) (SUSE Linux) (32 + 64 bit)
+ + MacPPC, gcc version 4.0.1 (Apple Computer, Inc. build 5367)
+ + MacIntel, gcc version 4.0.1 (Apple Computer, Inc. build 5250)
+ + gcc version 3.4.4 [FreeBSD] 20050518 (32 + 64 bit)
+ + gcc version 3.4.4 (cygming special) (gdc 0.12, using dmd 0.125)
+ + gcc version 3.4.2 [FreeBSD]
+ + gcc version 3.3.5 20050117 (prerelease) (SUSE Linux)
+ + gcc version 3.3.3 (PPC, 32 + 64 bit)
+ + Microsoft (R) C/C++ Optimizing Compiler Version 14.00.50727.762 for x64 (64 bit)
+ + Microsoft (R) 32-bit C/C++ Optimizing Compiler Version 14.00.50727.42 for 80x86 (Microsoft Visual C++ 2005)
+ + Microsoft (R) 32-bit C/C++ Optimizing Compiler Version 13.10.3077 for 80x86 (Mictosoft Visual C++ 2003)
+ + Microsoft (R) 32-bit C/C++ Optimizing Compiler Version 13.00.9466 for 80x86 (Microsoft Visual C++ 2002)
+ + Intel(R) C++ Compiler for 32-bit applications, Version 9.1 Build 20070322Z Package ID: W_CC_C_9.1.037
+ + Intel(R) C++ Compiler for Intel(R) EM64T-based applications, Version 9.1 (64 bit)
+ + icpcbin (ICC) 9.1 20070215
+ + CC: Sun C++ 5.8 2005/10/13 (``CXXFLAGS='-library=stlport4'``)
+ + MIPSpro Compilers: Version 7.4.4m (32 + 64 bit)
+ + aCC: HP C/aC++ B3910B A.06.15 [Mar 28 2007] (HP-UX IA64)
+
+* 2007-04-01: 0.11.3
+
+ - Added support for underscores in named definitions.
+ - Added new option ``--no-generation-date``.
+ - Fixed issue with long form of switches.
+
+* 2007-03-01: 0.11.2
+
+ - Added inplace configuration ``re2c:yyfill:parameter``.
+ - Added inplace configuration ``re2c:yych:conversion``.
+ - Fixed ``-u`` switch code generation.
+ - Added ability to avoid defines and overwrite generated variable names.
+
+* 2007-02-20: 0.11.1
+
+ - Applied #1647875 Add ``const`` to ``yybm`` vector.
+
+* 2007-01-01: 0.11.0
+
+ - Added ``-u`` switch to support unicode.
+
+* 2007-04-01: 0.10.8
+
+ - Fixed issue with long form of switches.
+
+* 2007-02-20: 0.10.7
+
+ - Applied #1647875 Add ``const`` to ``yybm`` vector.
+
+* 2006-08-05: 0.10.6
+
+ - Fixed #1529351 Segv bug on unterminated code blocks.
+ - Fixed #1528269 Invalid code generation.
+
+* 2006-06-11: 0.10.5
+
+ - Fixed long form of ``-1`` switch to ``--single-pass`` as noted in man page and help.
+ - Added MSVC 2003 project files and renamed old 2002 ones.
+
+* 2006-06-01: 0.10.4
+
+ - Fix whitespace in generated code.
+
+* 2006-05-14: 0.10.3
+
+ - Fixed issue with ``-wb`` and ``-ws``.
+ - Added ``-g`` switch to support gcc's computed goto's.
+ - Changed to use nested ``if``'s instead of ``switch(yyaccept)`` in ``-s`` mode.
+
+* 2006-05-01: 0.10.2
+
+ - Changed to generate ``YYMARKER`` only when needed or in single pass mode.
+ - Added ``-1`` switch to force single pass generation and make two pass the default.
+ - Fixed ``-i`` switch.
+ - Added configuration ``yyfill:enable`` to allow suppression of ``YYFILL()`` blocks.
+ - Added tutorial like lessons to re2c.
+ - Added ``/*!ignore:re2c */`` to support documenting of re2c source.
+ - Fixed issue with multiline re2c comments (``/*!max:re2c ... */`` and alike).
+ - Fixed generation of ``YYDEBUG()`` when using ``-d`` switch.
+ - Added ``/*!getstate:re2c */`` which triggers generation of the ``YYGETSTATE()`` block.
+ - Added configuration ``state:abort``.
+ - Changed to not generate ``yyNext`` unless configuration ``state:nextlabel`` is used.
+ - Changed to not generate ``yyaccept`` code unless needed.
+ - Changed to use ``if`` instead of ``switch`` expression when ``yyaccpt`` has only one case.
+ - Added docu, examples and tests to ``.src.zip`` package (0.10.1 zip was repackaged).
+ - Fixed #1479044 incorrect code generated when using ``-b``.
+ - Fixed #1472770 re2c creates an infinite loop.
+ - Fixed #1454253 Piece of code saving a backtracking point not generated.
+ - Fixed #1463639 Missing forward declaration.
+ - Implemented #1187127 savable state support for multiple re2c blocks.
+ - re2c 0.10.2 has been tested with the following compilers:
+ + gcc (GCC) 4.1.0 (Gentoo 4.1.0)
+ + gcc version 4.0.3 (4.0.3-0.20060215.2mdk for Mandriva Linux release 2006.1)
+ + gcc version 4.0.2 20050901 (prerelease) (SUSE Linux)
+ + gcc (GCC) 3.4.5 (Gentoo 3.4.5, ssp-3.4.5-1.0, pie-8.7.9)
+ + gcc version 3.4.4 [FreeBSD] 20050518
+ + gcc version 3.4.4 (cygming special) (gdc 0.12, using dmd 0.125)
+ + gcc version 3.4.2 20041017 (Red Hat 3.4.2-6.fc3)
+ + gcc-Version 3.3.5 (Debian 1:3.3.5-13)
+ + gcc-Version 3.3.0 (mips-sgi-irix6.5/3.3.0/specs)
+ + MIPSpro Compilers: Version 7.4.4m
+ + Microsoft (R) 32-bit C/C++ Optimizing Compiler Version 14.00.50727.42 for 80x86 (Microsoft Visual C++ 2005)
+ + Microsoft (R) 32-bit C/C++ Optimizing Compiler Version 13.10.3077 for 80x86 (Mictosoft Visual C++ 2003)
+ + Microsoft (R) 32-bit C/C++ Optimizing Compiler Version 13.00.9466 for 80x86 (Microsoft Visual C++ 2002)
+ + Intel(R) C++ Compiler for Intel(R) EM64T-based applications, Version 9.0 Build 20050430 Package ID: l_cc_p_9.0.021
+ + CC: Sun C++ 5.8 2005/10/13 (``CXXFLAGS='-library=stlport4'``)
+ + bison 2.1, 1.875d, 1.875b, 1.875
+
+* 2006-02-28: 0.10.1
+
+ - Added support for Solaris and native SUN compiler.
+ - Applied #1438160 expose ``YYCTXMARKER``.
+ - re2c 0.10.1 has been tested with the following compilers:
+ + gcc version 4.0.3 (4.0.3-0.20060215.2mdk for Mandriva Linux release 2006.1)
+ + gcc version 4.0.2 (4.0.2-1mdk for Mandriva Linux release 2006.1)
+ + gcc version 4.0.2 20050901 (prerelease) (SUSE Linux)
+ + gcc version 3.4.4 (cygming special) (gdc 0.12, using dmd 0.125)
+ + gcc-Version 3.3.5 (Debian 1:3.3.5-13)
+ + gcc-Version 3.3.0 (mips-sgi-irix6.5/3.3.0/specs)
+ + MIPSpro Compilers: Version 7.4.4m
+ + Microsoft (R) 32-bit C/C++ Optimizing Compiler Version 14.00.50727.42 for 80x86 (Microsoft Visual C 2005)
+ + Microsoft (R) 32-bit C/C++ Optimizing Compiler Version 13.00.9466 for 80x86 (Microsoft Visual C 2002)
+ + Intel(R) C++ Compiler for 32-bit applications, Version 9.0 Build 20051130Z Package ID: W_CC_C_9.0.028
+ + CC: Sun C++ 5.8 2005/10/13 (``CXXFLAGS='-compat5 -library=stlport4'``)
+ + bison 2.1, 1.875d, 1.875b, 1.875
+
+* 2006-02-18: 0.10.0
+
+ - Added make target ``zip`` to create windows source packages as zip files.
+ - Added ``re2c:startlabel`` configuration.
+ - Fixed code generation to not generate unreachable code for initial state.
+ - Added support for c/c++ compatible ``\u`` and ``\U`` unicode notation.
+ - Added ability to control indendation.
+ - Made scanner error out in case an ambiguous ``/*`` is found.
+ - Fixed indendation of generated code.
+ - Added support for DOS line endings.
+ - Added experimental unicode support.
+ - Added ``config_w32.h`` to build out of the box on windows (using msvc 2002+).
+ - Added Microsoft Visual C .NET 2005 build files.
+ - Applied #1411087 variable length trailing context.
+ - Applied #1408326 do not generate ``goto`` next state.
+ - Applied #1408282 ``CharSet`` initialization fix.
+ - Applied #1408278 ``readsome`` with MSVC.
+ - Applied #1307467 Unicode patch for 0.9.7.
+
+* 2005-12-28: 0.9.12
+
+ - Fixed bug #1390174 re2c cannot accept ``{0,}``.
+
+* 2005-12-18: 0.9.11
+
+ - Fixed #1313083 ``-e`` (EBCDIC cross compile) broken.
+ - Fixed #1297658 underestimation of ``n`` in ``YYFILL(n)``.
+ - Applied #1339483 Avoid rebuilds of re2c when running subtargets.
+ - Implemented #1335305 symbol table reimplementation, just slightly modifed.
+
+* 2005-09-04: 0.9.10
+
+ - Add ``-i`` switch to avoid generating ``#line`` information.
+ - Fixed bug #1251653 re2c generate some invalid ``#line`` on WIN32.
+
+* 2005-07-21: 0.9.9
+
+ - Implemented #1232777 negated char classes ``[^...]`` and the dot operator ``.``.
+ - Added hexadecimal character definitions.
+ - Added consistency check for octal character definitions.
+
+* 2005-06-26: 0.9.8
+
+ - Fixed code generation for ``-b`` switch.
+ - Added Microsoft Visual C .NET build files.
+
+* 2005-04-30: 0.9.7
+
+ - Applied #1181535 storable state patch.
+ - Added ``-d`` flag which outputs a debugable parser.
+ - Fixed generation of ``#line`` directives (according to ISO-C99).
+ - Fixed bug #1187785 Re2c fails to generate valid code.
+ - Fixed bug #1187452 unused variable ``yyaccept``.
+
+* 2005-04-14: 0.9.6
+
+ - Fix build with gcc >= 3.4.
+
+* 2005-04-08: 0.9.5
+
+ - Added ``/*!max:re2c */`` which emits ``#define YYMAXFILL <max>``
+ line. This allows to define buffers of the minimum required length.
+ Occurence must follow ``/*re2c */`` and cannot preceed it.
+ - Changed re2c to two pass generation to output warning free code.
+ - Fixed bug #1163046 re2c hangs when processing valid re-file.
+ - Fixed bug #1022799 re2c scanner has buffering bug.
+
+* 2005-03-12: 0.9.4
+
+ - Added ``--vernum`` support.
+ - Fixed bug #1054496 incorrect code generated with ``-b`` option.
+ - Fixed bug #1012748 re2c does not emit last line if ``\n`` missing.
+ - Fixed bug #999104 ``--output=output`` option does not work as documented.
+ - Fixed bug #999103 Invalid options prefixed with two dashes cause program crash.
+
+* 2004-05-26: 0.9.3
+
+ - Fixes one small possible bug in the generated output. ``ych`` instead of ``yych`` is output in certain circumstances.
+
+* 2004-05-26: 0.9.2
+
+ - Added ``-o`` option to specify the output file which also will set the ``#line`` directives to something useful.
+ - Print version to ``cout`` instead of ``cerr``.
+ - Added ``-h`` and ``--`` style options.
+ - Moved development to http://sourceforge.net/projects/re2c
+ - Fixed bug #960144 minor cosmetic problem.
+ - Fixed bug #953181 cannot compile with.
+ - Fixed bug #939277 Windows support.
+ - Fixed bug #914462 automake build patch
+ - Fixed bug #891940 braced quantifiers: ``{\d+(,|,\d+)?}`` style.
+ - Fixed bug #869298 Add case insensitive string literals.
+ - Fixed bug #869297 Input buffer overrun.
+
+* 2003-12-13: 0.9.1
+
+ - Removed rcs comments in source files.
+
+* 2003-12-09: re2c adopted
+ - Version 0.9.1 README::
+
+ Originally written by Peter Bumbulis (peter@csg.uwaterloo.ca)
+ Currently maintained by Brian Young (bayoung@acm.org)
+
+ The re2c distribution can be found at:
+ http://www.tildeslash.org/re2c/index.html
+
+ The source distribution is available from:
+ http://www.tildeslash.org/re2c/re2c-0.9.1.tar.gz
+
+ This distribution is a cleaned up version of the 0.5 release
+ maintained by me (Brian Young). Several bugs were fixed as well
+ as code cleanup for warning free compilation. It has been
+ developed and tested with egcs 1.0.2 and gcc 2.7.2.3 on Linux x86.
+ Peter Bumbulis' original release can be found at:
+ ftp://csg.uwaterloo.ca/pub/peter/re2c.0.5.tar.gz
+
+ re2c is a great tool for writing fast and flexible lexers.
+ It has served many people well for many years and it deserves
+ to be maintained more actively. re2c is on the order of 2-3
+ times faster than a flex based scanner, and its input model
+ is much more flexible.
+
+ Patches and requests for features will be entertained. Areas
+ of particular interest to me are porting (a Solaris and an NT
+ version will be forthcoming) and wide character support. Note
+ that the code is already quite portable and should be buildable
+ on any platform with minor makefile changes.
+
+ - Version 0.5 Peter's original ANNOUNCE and README::
+
+ re2c is a tool for generating C-based recognizers from regular
+ expressions. re2c-based scanners are efficient: for programming
+ languages, given similar specifications, an re2c-based scanner
+ is typically almost twice as fast as a flex-based scanner with
+ little or no increase in size (possibly a decrease on cisc
+ architectures). Indeed, re2c-based scanners are quite competitive
+ with hand-crafted ones.
+
+ Unlike flex, re2c does not generate complete scanners: the user
+ must supply some interface code. While this code is not bulky
+ (about 50-100 lines for a flex-like scanner; see the man page
+ and examples in the distribution) careful coding is required for
+ efficiency (and correctness). One advantage of this arrangement
+ is that the generated code is not tied to any particular input
+ model. For example, re2c generated code can be used to scan
+ data from a null-byte terminated buffer as illustrated below.
+
+ Given the following source:
+
+ #define NULL ((char*) 0)
+ char *scan(char *p) {
+ char *q;
+ #define YYCTYPE char
+ #define YYCURSOR p
+ #define YYLIMIT p
+ #define YYMARKER q
+ #define YYFILL(n)
+ /*!re2c
+ [0-9]+ {return YYCURSOR;}
+ [\000-\377] {return NULL;}
+ */
+ }
+
+ re2c will generate:
+
+ /* Generated by re2c on Sat Apr 16 11:40:58 1994 */
+ #line 1 "simple.re"
+ #define NULL ((char*) 0)
+ char *scan(char *p) {
+ char *q;
+ #define YYCTYPE char
+ #define YYCURSOR p
+ #define YYLIMIT p
+ #define YYMARKER q
+ #define YYFILL(n)
+ {
+ YYCTYPE yych;
+ unsigned int yyaccept;
+ goto yy0;
+ yy1: ++YYCURSOR;
+ yy0:
+ if((YYLIMIT - YYCURSOR) < 2) YYFILL(2);
+ yych = *YYCURSOR;
+ if(yych <= '/') goto yy4;
+ if(yych >= ':') goto yy4;
+ yy2: yych = *++YYCURSOR;
+ goto yy7;
+ yy3:
+ #line 10
+ {return YYCURSOR;}
+ yy4: yych = *++YYCURSOR;
+ yy5:
+ #line 11
+ {return NULL;}
+ yy6: ++YYCURSOR;
+ if(YYLIMIT == YYCURSOR) YYFILL(1);
+ yych = *YYCURSOR;
+ yy7: if(yych <= '/') goto yy3;
+ if(yych <= '9') goto yy6;
+ goto yy3;
+ }
+ #line 12
+
+ }
+
+ Note that most compilers will perform dead-code elimination to
+ remove all YYCURSOR, YYLIMIT comparisions.
+
+ re2c was developed for a particular project (constructing a fast
+ REXX scanner of all things!) and so while it has some rough edges,
+ it should be quite usable. More information about re2c can be
+ found in the (admittedly skimpy) man page; the algorithms and
+ heuristics used are described in an upcoming LOPLAS article
+ (included in the distribution). Probably the best way to find out
+ more about re2c is to try the supplied examples. re2c is written in
+ C++, and is currently being developed under Linux using gcc 2.5.8.
+
+ Peter
--------------------------------------------------------------------------------
-Authors
-~~~~~~~
-
-Originally written by Peter Bumbulis in 1994.
-Since then many people have contributed to re2c: Brian Young, Dan Nuffer, Marcus Börger,
-Hartmut Kaiser, Emmanuel Mogenet, Ulya Trofimovich (`let me know <skvadrik@gmail.com>`_ if I missed someone).
-
-Licence
-~~~~~~~
-
-re2c is distributed with no warranty whatever.
-The code is certain to contain errors.
-Neither the author nor any contributor takes responsibility for any consequences of its use.
-re2c is in the public domain. The data structures and algorithms used
-in re2c are all either taken from documents available to the general
-public or are inventions of the author. Programs generated by re2c may
-be distributed freely. re2c itself may be distributed freely, in source
-or binary, unchanged or modified. Distributors may charge whatever fees
-they can obtain for re2c. If you do make use of re2c, or incorporate it into a larger project an
-acknowledgement somewhere (documentation, research report, etc.) would
-be appreciated.
-
-Please send bug reports and feedback to `re2c-devel <re2c-devel@lists.sourceforge.net>`_ and
-`re2c-general <re2c-general@lists.sourceforge.net>`_ mailing lists.
+Please send feedback to `re2c-devel <re2c-devel@lists.sourceforge.net>`_ and
+`re2c-general <re2c-general@lists.sourceforge.net>`_ mailing lists
+or `report a bug <https://github.com/skvadrik/re2c/issues>`_.
+++ /dev/null
-====
-re2c
-====
-
------------------------------------------
-convert regular expressions to C/C++ code
------------------------------------------
-
-:Manual section: 1
-
-SYNOPSIS
---------
-
-``re2c [OPTIONS] FILE``
-
-DESCRIPTION
------------
-
-``re2c`` is a lexer generator for C/C++. It finds regular expression
-specifications inside of C/C++ comments and replaces them with a
-hard-coded DFA. The user must supply some interface code in order to
-control and customize the generated DFA.
-
-EXAMPLE
--------
-
-Given the following code:
-
-.. code-block:: c
-
- unsigned int stou (const char \* s)
- {
- # define YYCTYPE char const
- YYCTYPE * YYCURSOR = s;
- unsigned int result = 0;
- for (;;)
- {
- /*!re2c
- re2c:yyfill:enable = 0;
-
- "\x00" { return result; }
- [0-9] { result = result * 10 + c; continue; }
- */
- }
- }
-
-``re2c -is`` will generate:
-
-.. code-block:: c
-
- /* Generated by re2c 0.13.7.dev on Mon Jul 14 13:37:46 2014 */
- unsigned int stou (const char * s)
- {
- # define YYCTYPE char
- const YYCTYPE * YYCURSOR = s;
- unsigned int result = 0;
-
- for (;;)
- {
- {
- YYCTYPE yych;
-
- yych = *YYCURSOR;
- if (yych <= 0x00) goto yy3;
- if (yych <= '/') goto yy2;
- if (yych <= '9') goto yy5;
-
- yy2:
- yy3:
- ++YYCURSOR;
- { return result; }
- yy5:
- ++YYCURSOR;
- { result = result * 10 + c; continue; }
- }
- }
- }
-
-.. _OPTIONS:
-
-OPTIONS
--------
-
-.. include:: options.rst
-
-WARNINGS
---------
-
-``-W``
- Turn on all warnings.
-
-``-Werror``
- Turn warnings into errors. Note that this option along
- doesn't turn on any warnings, it only affects those warnings that have
- been turned on so far or will be turned on later.
-
-``-W<warning>``
- Turn on individual ``warning``.
-
-``-Wno-<warning>``
- Turn off individual ``warning``.
-
-``-Werror-<warning>``
- Turn on individual ``warning`` and treat it as error (this implies ``-W<warning>``).
-
-``-Wno-error-<warning>``
- Don't treat this particular ``warning`` as error. This doesn't turn off
- the warning itself.
-
-``-Wcondition-order``
- Warn if the generated program makes implicit
- assumptions about condition numbering. One should use either ``-t, --type-header`` option or
- ``/*!types:re2c*/`` directive to generate mapping of condition names to numbers and use
- autogenerated condition names.
-
-``-Wempty-character-class``
- Warn if regular expression contains empty
- character class. From the rational point of view trying to match empty
- character class makes no sense: it should always fail. However, for
- backwards compatibility reasons ``re2c`` allows empty character class and
- treats it as empty string. Use ``--empty-class`` option to change default
- behaviour.
-
-``-Wmatch-empty-string``
- Warn if regular expression in a rule is
- nullable (matches empty string). If DFA runs in a loop and empty match
- is unintentional (input position in not advanced manually), lexer may
- get stuck in eternal loop.
-
-``-Wswapped-range``
- Warn if range lower bound is greater that upper
- bound. Default ``re2c`` behaviour is to silently swap range bounds.
-
-``-Wundefined-control-flow``
- Warn if some input strings cause undefined
- control flow in lexer (the faulty patterns are reported). This is the
- most dangerous and common mistake. It can be easily fixed by adding
- default rule ``*`` (this rule has the lowest priority, matches any code unit and consumes
- exactly one code unit).
-
-``-Wuseless-escape``
- Warn if a symbol is escaped when it shouldn't be.
- By default re2c silently ignores escape, but this may as well indicate a
- typo or an error in escape sequence.
-
-INTERFACE CODE
---------------
-
-The user must supply interface code either in the form of C/C++ code
-(macros, functions, variables, etc.) or in the form of `INPLACE CONFIGURATIONS`_.
-Which symbols must be defined and which are optional
-depends on a particular use case.
-
-``YYCONDTYPE``
- In ``-c`` mode you can use ``-t`` to generate a file that
- contains the enumeration used as conditions. Each of the values refers
- to a condition of a rule set.
-
-``YYCTXMARKER``
- l-value of type ``YYCTYPE *``.
- The generated code saves trailing context backtracking information in
- ``YYCTXMARKER``. The user only needs to define this macro if a scanner
- specification uses trailing context in one or more of its regular
- expressions.
-
-``YYCTYPE``
- Type used to hold an input symbol (code unit). Usually
- ``char`` or ``unsigned char`` for ASCII, EBCDIC and UTF-8, *unsigned short*
- for UTF-16 or UCS-2 and ``unsigned int`` for UTF-32.
-
-``YYCURSOR``
- l-value of type ``YYCTYPE *`` that points to the current input symbol. The generated code advances
- ``YYCURSOR`` as symbols are matched. On entry, ``YYCURSOR`` is assumed to
- point to the first character of the current token. On exit, ``YYCURSOR``
- will point to the first character of the following token.
-
-``YYDEBUG (state, current)``
- This is only needed if the ``-d`` flag was
- specified. It allows to easily debug the generated parser by calling a
- user defined function for every state. The function should have the
- following signature: ``void YYDEBUG (int state, char current)``. The first
- parameter receives the state or -1 and the second parameter receives the
- input at the current cursor.
-
-``YYFILL (n)``
- The generated code "calls"" ``YYFILL (n)`` when the
- buffer needs (re)filling: at least ``n`` additional characters should be
- provided. ``YYFILL (n)`` should adjust ``YYCURSOR``, ``YYLIMIT``, ``YYMARKER``
- and ``YYCTXMARKER`` as needed. Note that for typical programming languages
- ``n`` will be the length of the longest keyword plus one. The user can
- place a comment of the form ``/*!max:re2c*/`` to insert ``YYMAXFILL`` definition that is set to the maximum
- length value.
-
-``YYGETCONDITION ()``
- This define is used to get the condition prior to
- entering the scanner code when using ``-c`` switch. The value must be
- initialized with a value from the enumeration ``YYCONDTYPE`` type.
-
-``YYGETSTATE ()``
- The user only needs to define this macro if the ``-f``
- flag was specified. In that case, the generated code "calls"
- ``YYGETSTATE ()`` at the very beginning of the scanner in order to obtain
- the saved state. ``YYGETSTATE ()`` must return a signed integer. The value
- must be either -1, indicating that the scanner is entered for the first
- time, or a value previously saved by ``YYSETSTATE (s)``. In the second
- case, the scanner will resume operations right after where the last
- ``YYFILL (n)`` was called.
-
-``YYLIMIT``
- Expression of type ``YYCTYPE *`` that marks the end of the buffer ``YYLIMIT[-1]``
- is the last character in the buffer). The generated code repeatedly
- compares ``YYCURSOR`` to ``YYLIMIT`` to determine when the buffer needs
- (re)filling.
-
-``YYMARKER``
- l-value of type ``YYCTYPE *``.
- The generated code saves backtracking information in ``YYMARKER``. Some
- easy scanners might not use this.
-
-``YYMAXFILL``
- This will be automatically defined by ``/*!max:re2c*/`` blocks as explained above.
-
-``YYSETCONDITION (c)``
- This define is used to set the condition in
- transition rules. This is only being used when ``-c`` is active and
- transition rules are being used.
-
-``YYSETSTATE (s)``
- The user only needs to define this macro if the ``-f``
- flag was specified. In that case, the generated code "calls"
- ``YYSETSTATE`` just before calling ``YYFILL (n)``. The parameter to
- ``YYSETSTATE`` is a signed integer that uniquely identifies the specific
- instance of ``YYFILL (n)`` that is about to be called. Should the user
- wish to save the state of the scanner and have ``YYFILL (n)`` return to
- the caller, all he has to do is store that unique identifer in a
- variable. Later, when the scannered is called again, it will call
- ``YYGETSTATE ()`` and resume execution right where it left off. The
- generated code will contain both ``YYSETSTATE (s)`` and ``YYGETSTATE`` even
- if ``YYFILL (n)`` is being disabled.
-
-SYNTAX
-------
-
-Code for ``re2c`` consists of a set of `RULES`_, `NAMED DEFINITIONS`_ and
-`INPLACE CONFIGURATIONS`_.
-
-.. _RULES:
-
-RULES
-~~~~~
-
-Rules consist of a regular expression (see `REGULAR EXPRESSIONS`_) along with a block of C/C++ code
-that is to be executed when the associated regular expression is
-matched. You can either start the code with an opening curly brace or
-the sequence ``:=``. When the code with a curly brace then ``re2c`` counts the brace depth
-and stops looking for code automatically. Otherwise curly braces are not
-allowed and ``re2c`` stops looking for code at the first line that does
-not begin with whitespace. If two or more rules overlap, the first rule
-is preferred.
-
- ``regular-expression { C/C++ code }``
-
- ``regular-expression := C/C++ code``
-
-There is one special rule: default rule ``*``
-
- ``* { C/C++ code }``
-
- ``* := C/C++ code``
-
-Note that default rule ``*`` differs from ``[^]``: default rule has the lowest priority,
-matches any code unit (either valid or invalid) and always consumes one character;
-while ``[^]`` matches any valid code point (not code unit) and can consume multiple
-code units. In fact, when variable-length encoding is used, ``*``
-is the only possible way to match invalid input character (see `ENCODINGS`_ for details).
-
-If ``-c`` is active then each regular expression is preceeded by a list
-of comma separated condition names. Besides normal naming rules there
-are two special cases: ``<*>`` (such rules are merged to all conditions)
-and ``<>`` (such the rule cannot have an associated regular expression,
-its code is merged to all actions). Non empty rules may further more specify the new
-condition. In that case ``re2c`` will generate the necessary code to
-change the condition automatically. Rules can use ``:=>`` as a shortcut
-to automatically generate code that not only sets the
-new condition state but also continues execution with the new state. A
-shortcut rule should not be used in a loop where there is code between
-the start of the loop and the ``re2c`` block unless ``re2c:cond:goto``
-is changed to ``continue``. If code is necessary before all rules (though not simple jumps) you
-can doso by using ``<!>`` pseudo-rules.
-
- ``<condition-list> regular-expression { C/C++ code }``
-
- ``<condition-list> regular-expression := C/C++ code``
-
- ``<condition-list> * { C/C++ code }``
-
- ``<condition-list> * := C/C++ code``
-
- ``<condition-list> regular-expression => condition { C/C++ code }``
-
- ``<condition-list> regular-expression => condition := C/C++ code``
-
- ``<condition-list> * => condition { C/C++ code }``
-
- ``<condition-list> * => condition := C/C++ code``
-
- ``<condition-list> regular-expression :=> condition``
-
-
- ``<*> regular-expression { C/C++ code }``
-
- ``<*> regular-expression := C/C++ code``
-
- ``<*> * { C/C++ code }``
-
- ``<*> * := C/C++ code``
-
- ``<*> regular-expression => condition { C/C++ code }``
-
- ``<*> regular-expression => condition := C/C++ code``
-
- ``<*> * => condition { C/C++ code }``
-
- ``<*> * => condition := C/C++ code``
-
- ``<*> regular-expression :=> condition``
-
-
- ``<> { C/C++ code }``
-
- ``<> := C/C++ code``
-
- ``<> => condition { C/C++ code }``
-
- ``<> => condition := C/C++ code``
-
- ``<> :=> condition``
-
- ``<> :=> condition``
-
-
- ``<! condition-list> { C/C++ code }``
-
- ``<! condition-list> := C/C++ code``
-
- ``<!> { C/C++ code }``
-
- ``<!> := C/C++ code``
-
-.. _NAMED DEFINITIONS:
-
-NAMED DEFINITIONS
-~~~~~~~~~~~~~~~~~
-
-Named definitions are of the form:
-
- ``name = regular-expression;``
-
-If ``-F`` is active, then named definitions are also of the form:
-
- ``name { regular-expression }``
-
-
-
-.. _INPLACE CONFIGURATIONS:
-
-INPLACE CONFIGURATIONS
-~~~~~~~~~~~~~~~~~~~~~~
-
-``re2c:condprefix = yyc;``
- Allows to specify the prefix used for
- condition labels. That is this text is prepended to any condition label
- in the generated output file.
-
-``re2c:condenumprefix = yyc;``
- Allows to specify the prefix used for
- condition values. That is this text is prepended to any condition enum
- value in the generated output file.
-
-``re2c:cond:divider = "/* *********************************** */";``
- Allows to customize the devider for condition blocks. You can use ``@@``
- to put the name of the condition or customize the placeholder using
- ``re2c:cond:divider@cond``.
-
-``re2c:cond:divider@cond = @@;``
- Specifies the placeholder that will be
- replaced with the condition name in ``re2c:cond:divider``.
-
-``re2c:cond:goto = "goto @@;";``
- Allows to customize the condition goto statements used with ``:=>`` style rules. You can use ``@@``
- to put the name of the condition or ustomize the placeholder using
- ``re2c:cond:goto@cond``. You can also change this to ``continue;``, which
- would allow you to continue with the next loop cycle including any code
- between loop start and re2c block.
-
-``re2c:cond:goto@cond = @@;``
- Spcifies the placeholder that will be replaced with the condition label in ``re2c:cond:goto``.
-
-``re2c:indent:top = 0;``
- Specifies the minimum number of indendation to
- use. Requires a numeric value greater than or equal zero.
-
-``re2c:indent:string = "\t";``
- Specifies the string to use for indendation. Requires a string that should
- contain only whitespace unless you need this for external tools. The easiest
- way to specify spaces is to enclude them in single or double quotes.
- If you do not want any indendation at all you can simply set this to "".
-
-``re2c:yych:conversion = 0;``
- When this setting is non zero, then ``re2c`` automatically generates
- conversion code whenever yych gets read. In this case the type must be
- defined using ``re2c:define:YYCTYPE``.
-
-``re2c:yych:emit = 1;``
- Generation of *yych* can be suppressed by setting this to 0.
-
-``re2c:yybm:hex = 0;``
- If set to zero then a decimal table is being used else a hexadecimal table will be generated.
-
-``re2c:yyfill:enable = 1;``
- Set this to zero to suppress generation of ``YYFILL (n)``. When using this be sure to verify that the generated
- scanner does not read behind input. Allowing this behavior might
- introduce sever security issues to you programs.
-
-``re2c:yyfill:check = 1;``
- This can be set 0 to suppress output of the
- pre condition using ``YYCURSOR`` and ``YYLIMIT`` which becomes usefull when
- ``YYLIMIT + YYMAXFILL`` is always accessible.
-
-``re2c:define:YYFILL = "YYFILL";``
- Substitution for ``YYFILL``. Note
- that by default ``re2c`` generates argument in braces and semicolon after
- ``YYFILL``. If you need to make ``YYFILL`` an arbitrary statement rather
- than a call, set ``re2c:define:YYFILL:naked`` to non-zero and use
- ``re2c:define:YYFILL@len`` to denote formal parameter inside of ``YYFILL``
- body.
-
-``re2c:define:YYFILL@len = "@@";``
- Any occurence of this text
- inside of ``YYFILL`` will be replaced with the actual argument.
-
-``re2c:yyfill:parameter = 1;``
- Controls argument in braces after
- ``YYFILL``. If zero, agrument is omitted. If non-zero, argument is
- generated unless ``re2c:define:YYFILL:naked`` is set to non-zero.
-
-``re2c:define:YYFILL:naked = 0;``
- Controls argument in braces and
- semicolon after ``YYFILL``. If zero, both agrument and semicolon are
- omitted. If non-zero, argument is generated unless
- ``re2c:yyfill:parameter`` is set to zero and semicolon is generated
- unconditionally.
-
-``re2c:startlabel = 0;``
- If set to a non zero integer then the start
- label of the next scanner blocks will be generated even if not used by
- the scanner itself. Otherwise the normal ``yy0`` like start label is only
- being generated if needed. If set to a text value then a label with that
- text will be generated regardless of whether the normal start label is
- being used or not. This setting is being reset to 0 after a start
- label has been generated.
-
-``re2c:labelprefix = "yy";``
- Allows to change the prefix of numbered
- labels. The default is ``yy`` and can be set any string that is a valid
- label.
-
-``re2c:state:abort = 0;``
- When not zero and switch ``-f`` is active then
- the ``YYGETSTATE`` block will contain a default case that aborts and a -1
- case is used for initialization.
-
-``re2c:state:nextlabel = 0;``
- Used when ``-f`` is active to control
- whether the ``YYGETSTATE`` block is followed by a ``yyNext:`` label line.
- Instead of using ``yyNext`` you can usually also use configuration
- ``startlabel`` to force a specific start label or default to ``yy0`` as
- start label. Instead of using a dedicated label it is often better to
- separate the ``YYGETSTATE`` code from the actual scanner code by placing a
- ``/*!getstate:re2c*/`` comment.
-
-``re2c:cgoto:threshold = 9;``
- When ``-g`` is active this value specifies
- the complexity threshold that triggers generation of jump tables rather
- than using nested if's and decision bitfields. The threshold is compared
- against a calculated estimation of if-s needed where every used bitmap
- divides the threshold by 2.
-
-``re2c:yych:conversion = 0;``
- When the input uses signed characters and
- ``-s`` or ``-b`` switches are in effect re2c allows to automatically convert
- to the unsigned character type that is then necessary for its internal
- single character. When this setting is zero or an empty string the
- conversion is disabled. Using a non zero number the conversion is taken
- from ``YYCTYPE``. If that is given by an inplace configuration that value
- is being used. Otherwise it will be ``(YYCTYPE)`` and changes to that
- configuration are no longer possible. When this setting is a string the
- braces must be specified. Now assuming your input is a ``char *``
- buffer and you are using above mentioned switches you can set
- ``YYCTYPE`` to ``unsigned char`` and this setting to either 1 or ``(unsigned char)``.
-
-``re2c:define:YYCONDTYPE = "YYCONDTYPE";``
- Enumeration used for condition support with ``-c`` mode.
-
-``re2c:define:YYCTXMARKER = "YYCTXMARKER";``
- Allows to overwrite the
- define ``YYCTXMARKER`` and thus avoiding it by setting the value to the
- actual code needed.
-
-``re2c:define:YYCTYPE = "YYCTYPE";``
- Allows to overwrite the define
- ``YYCTYPE`` and thus avoiding it by setting the value to the actual code
- needed.
-
-``re2c:define:YYCURSOR = "YYCURSOR";``
- Allows to overwrite the define
- ``YYCURSOR`` and thus avoiding it by setting the value to the actual code
- needed.
-
-``re2c:define:YYDEBUG = "YYDEBUG";``
- Allows to overwrite the define
- ``YYDEBUG`` and thus avoiding it by setting the value to the actual code
- needed.
-
-``re2c:define:YYGETCONDITION = "YYGETCONDITION";``
- Substitution for
- ``YYGETCONDITION``. Note that by default ``re2c`` generates braces after
- ``YYGETCONDITION``. Set ``re2c:define:YYGETCONDITION:naked`` to non-zero to
- omit braces.
-
-``re2c:define:YYGETCONDITION:naked = 0;``
- Controls braces after
- ``YYGETCONDITION``. If zero, braces are omitted. If non-zero, braces are
- generated.
-
-``re2c:define:YYSETCONDITION = "YYSETCONDITION";``
- Substitution for
- ``YYSETCONDITION``. Note that by default ``re2c`` generates argument in
- braces and semicolon after ``YYSETCONDITION``. If you need to make
- ``YYSETCONDITION`` an arbitrary statement rather than a call, set
- ``re2c:define:YYSETCONDITION:naked`` to non-zero and use
- ``re2c:define:YYSETCONDITION@cond`` to denote formal parameter inside of
- ``YYSETCONDITION`` body.
-
-``re2c:define:YYSETCONDITION@cond = "@@";``
- Any occurence of this
- text inside of ``YYSETCONDITION`` will be replaced with the actual
- argument.
-
-``re2c:define:YYSETCONDITION:naked = 0;``
- Controls argument in braces
- and semicolon after ``YYSETCONDITION``. If zero, both agrument and
- semicolon are omitted. If non-zero, both argument and semicolon are
- generated.
-
-``re2c:define:YYGETSTATE = "YYGETSTATE";``
- Substitution for
- ``YYGETSTATE``. Note that by default ``re2c`` generates braces after
- ``YYGETSTATE``. Set ``re2c:define:YYGETSTATE:naked`` to non-zero to omit
- braces.
-
-``re2c:define:YYGETSTATE:naked = 0;``
- Controls braces after
- ``YYGETSTATE``. If zero, braces are omitted. If non-zero, braces are
- generated.
-
-``re2c:define:YYSETSTATE = "YYSETSTATE";``
- Substitution for
- ``YYSETSTATE``. Note that by default ``re2c`` generates argument in braces
- and semicolon after ``YYSETSTATE``. If you need to make ``YYSETSTATE`` an
- arbitrary statement rather than a call, set
- ``re2c:define:YYSETSTATE:naked`` to non-zero and use
- ``re2c:define:YYSETSTATE@cond`` to denote formal parameter inside of
- ``YYSETSTATE`` body.
-
-``re2c:define:YYSETSTATE@state = "@@";``
- Any occurence of this text
- inside of ``YYSETSTATE`` will be replaced with the actual argument.
-
-``re2c:define:YYSETSTATE:naked = 0;``
- Controls argument in braces and
- semicolon after ``YYSETSTATE``. If zero, both agrument and semicolon are
- omitted. If non-zero, both argument and semicolon are generated.
-
-``re2c:define:YYLIMIT = "YYLIMIT";``
- Allows to overwrite the define
- ``YYLIMIT`` and thus avoiding it by setting the value to the actual code
- needed.
-
-``re2c:define:YYMARKER = "YYMARKER";``
- Allows to overwrite the define
- ``YYMARKER`` and thus avoiding it by setting the value to the actual code
- needed.
-
-``re2c:label:yyFillLabel = "yyFillLabel";``
- Allows to overwrite the name of the label ``yyFillLabel``.
-
-``re2c:label:yyNext = "yyNext";``
- Allows to overwrite the name of the label ``yyNext``.
-
-``re2c:variable:yyaccept = yyaccept;``
- Allows to overwrite the name of the variable ``yyaccept``.
-
-``re2c:variable:yybm = "yybm";``
- Allows to overwrite the name of the variable ``yybm``.
-
-``re2c:variable:yych = "yych";``
- Allows to overwrite the name of the variable ``yych``.
-
-``re2c:variable:yyctable = "yyctable";``
- When both ``-c`` and ``-g`` are active then ``re2c`` uses this variable to generate a static jump table
- for ``YYGETCONDITION``.
-
-``re2c:variable:yystable = "yystable";``
- Deprecated.
-
-``re2c:variable:yytarget = "yytarget";``
- Allows to overwrite the name of the variable ``yytarget``.
-
-
-
-.. _REGULAR EXPRESSIONS:
-
-REGULAR EXPRESSIONS
-~~~~~~~~~~~~~~~~~~~
-
-``"foo"``
- literal string ``"foo"``. ANSI-C escape sequences can be used.
-
-``'foo'``
- literal string ``"foo"`` (characters [a-zA-Z] treated
- case-insensitive). ANSI-C escape sequences can be used.
-
-``[xyz]``
- character class; in this case, regular expression matches either ``x``, ``y``, or ``z``.
-
-``[abj-oZ]``
- character class with a range in it; matches ``a``, ``b``, any letter from ``j`` through ``o`` or ``Z``.
-
-``[^class]``
- inverted character class.
-
-``r \ s``
- match any ``r`` which isn't ``s``. ``r`` and ``s`` must be regular expressions
- which can be expressed as character classes.
-
-``r*``
- zero or more occurences of ``r``.
-
-``r+``
- one or more occurences of ``r``.
-
-``r?``
- optional ``r``.
-
-``(r)``
- ``r``; parentheses are used to override precedence.
-
-``r s``
- ``r`` followed by ``s`` (concatenation).
-
-``r | s``
- either ``r`` or ``s`` (alternative).
-
-``r`` / ``s``
- ``r`` but only if it is followed by ``s``. Note that ``s`` is not
- part of the matched text. This type of regular expression is called
- "trailing context". Trailing context can only be the end of a rule
- and not part of a named definition.
-
-``r{n}``
- matches ``r`` exactly ``n`` times.
-
-``r{n,}``
- matches ``r`` at least ``n`` times.
-
-``r{n,m}``
- matches ``r`` at least ``n`` times, but not more than ``m`` times.
-
-``.``
- match any character except newline.
-
-``name``
- matches named definition as specified by ``name`` only if ``-F`` is
- off. If ``-F`` is active then this behaves like it was enclosed in double
- quotes and matches the string "name".
-
-Character classes and string literals may contain octal or hexadecimal
-character definitions and the following set of escape sequences:
-``\a``, ``\b``, ``\f``, ``\n``, ``\r``, ``\t``, ``\v``, ``\\``. An octal character is defined by a backslash
-followed by its three octal digits (e.g. ``\377``).
-Hexadecimal characters from 0 to 0xFF are defined by backslash, a lower
-cased ``x`` and two hexadecimal digits (e.g. ``\x12``). Hexadecimal characters from 0x100 to 0xFFFF are defined by backslash, a lower cased
-``\u`` or an upper cased ``\X`` and four hexadecimal digits (e.g. ``\u1234``).
-Hexadecimal characters from 0x10000 to 0xFFFFffff are defined by backslash, an upper cased ``\U``
-and eight hexadecimal digits (e.g. ``\U12345678``).
-
-The only portable "any" rule is the default rule ``*``.
-
-
-
-.. _SCANNER WITH STORABLE STATES:
-
-SCANNER WITH STORABLE STATES
-----------------------------
-
-When the ``-f`` flag is specified, ``re2c`` generates a scanner that can
-store its current state, return to the caller, and later resume
-operations exactly where it left off.
-
-The default operation of ``re2c`` is a
-"pull" model, where the scanner asks for extra input whenever it needs it. However, this mode of operation assumes that the scanner is the "owner"
-the parsing loop, and that may not always be convenient.
-
-Typically, if there is a preprocessor ahead of the scanner in the
-stream, or for that matter any other procedural source of data, the
-scanner cannot "ask" for more data unless both scanner and source
-live in a separate threads.
-
-The ``-f`` flag is useful for just this situation: it lets users design
-scanners that work in a "push" model, i.e. where data is fed to the
-scanner chunk by chunk. When the scanner runs out of data to consume, it
-just stores its state, and return to the caller. When more input data is
-fed to the scanner, it resumes operations exactly where it left off.
-
-Changes needed compared to the "pull" model:
-
-* User has to supply macros ``YYSETSTATE ()`` and ``YYGETSTATE (state)``.
-
-* The ``-f`` option inhibits declaration of ``yych`` and ``yyaccept``. So the
- user has to declare these. Also the user has to save and restore these.
- In the example ``examples/push_model/push.re`` these are declared as
- fields of the (C++) class of which the scanner is a method, so they do
- not need to be saved/restored explicitly. For C they could e.g. be made
- macros that select fields from a structure passed in as parameter.
- Alternatively, they could be declared as local variables, saved with
- ``YYFILL (n)`` when it decides to return and restored at entry to the
- function. Also, it could be more efficient to save the state from
- ``YYFILL (n)`` because ``YYSETSTATE (state)`` is called unconditionally.
- ``YYFILL (n)`` however does not get ``state`` as parameter, so we would have
- to store state in a local variable by ``YYSETSTATE (state)``.
-
-* Modify ``YYFILL (n)`` to return (from the function calling it) if more input is needed.
-
-* Modify caller to recognise if more input is needed and respond appropriately.
-
-* The generated code will contain a switch block that is used to
- restores the last state by jumping behind the corrspoding ``YYFILL (n)``
- call. This code is automatically generated in the epilog of the first ``/*!re2c */``
- block. It is possible to trigger generation of the ``YYGETSTATE ()``
- block earlier by placing a ``/*!getstate:re2c*/`` comment. This is especially useful when the scanner code should be
- wrapped inside a loop.
-
-Please see ``examples/push_model/push.re`` for "push" model scanner. The
-generated code can be tweaked using inplace configurations ``state:abort``
-and ``state:nextlabel``.
-
-
-
-SCANNER WITH CONDITION SUPPORT
-------------------------------
-
-You can preceed regular expressions with a list of condition names when
-using the ``-c`` switch. In this case ``re2c`` generates scanner blocks for
-each conditon. Where each of the generated blocks has its own
-precondition. The precondition is given by the interface define
-``YYGETCONDITON()`` and must be of type ``YYCONDTYPE``.
-
-There are two special rule types. First, the rules of the condition ``<*>``
-are merged to all conditions (note that they have lower priority than
-other rules of that condition). And second the empty condition list
-allows to provide a code block that does not have a scanner part.
-Meaning it does not allow any regular expression. The condition value
-referring to this special block is always the one with the enumeration
-value 0. This way the code of this special rule can be used to
-initialize a scanner. It is in no way necessary to have these rules: but
-sometimes it is helpful to have a dedicated uninitialized condition
-state.
-
-Non empty rules allow to specify the new condition, which makes them
-transition rules. Besides generating calls for the define
-``YYSETCONDTITION`` no other special code is generated.
-
-There is another kind of special rules that allow to prepend code to any
-code block of all rules of a certain set of conditions or to all code
-blocks to all rules. This can be helpful when some operation is common
-among rules. For instance this can be used to store the length of the
-scanned string. These special setup rules start with an exclamation mark
-followed by either a list of conditions ``<! condition, ... >`` or a star
-``<!*>``. When ``re2c`` generates the code for a rule whose state does not have a
-setup rule and a star'd setup rule is present, than that code will be
-used as setup code.
-
-
-
-.. _ENCODINGS:
-
-ENCODINGS
----------
-
-``re2c`` supports the following encodings: ASCII (default), EBCDIC (``-e``),
-UCS-2 (``-w``), UTF-16 (``-x``), UTF-32 (``-u``) and UTF-8 (``-8``).
-See also inplace configuration ``re2c:flags``.
-
-The following concepts should be clarified when talking about encoding.
-*Code point* is an abstract number, which represents single encoding
-symbol. *Code unit* is the smallest unit of memory, which is used in the
-encoded text (it corresponds to one character in the input stream). One
-or more code units can be needed to represent a single code point,
-depending on the encoding. In *fixed-length* encoding, each code point
-is represented with equal number of code units. In *variable-length*
-encoding, different code points can be represented with different number
-of code units.
-
-* ASCII is a fixed-length encoding. Its code space includes 0x100
- code points, from 0 to 0xFF. One code point is represented with exactly one
- 1-byte code unit, which has the same value as the code point. Size of
- ``YYCTYPE`` must be 1 byte.
-
-* EBCDIC is a fixed-length encoding. Its code space includes 0x100
- code points, from 0 to 0xFF. One code point is represented with exactly
- one 1-byte code unit, which has the same value as the code point. Size
- of ``YYCTYPE`` must be 1 byte.
-
-* UCS-2 is a fixed-length encoding. Its code space includes 0x10000
- code points, from 0 to 0xFFFF. One code point is represented with
- exactly one 2-byte code unit, which has the same value as the code
- point. Size of ``YYCTYPE`` must be 2 bytes.
-
-* UTF-16 is a variable-length encoding. Its code space includes all
- Unicode code points, from 0 to 0xD7FF and from 0xE000 to 0x10FFFF. One
- code point is represented with one or two 2-byte code units. Size of
- ``YYCTYPE`` must be 2 bytes.
-
-* UTF-32 is a fixed-length encoding. Its code space includes all
- Unicode code points, from 0 to 0xD7FF and from 0xE000 to 0x10FFFF. One
- code point is represented with exactly one 4-byte code unit. Size of
- ``YYCTYPE`` must be 4 bytes.
-
-* UTF-8 is a variable-length encoding. Its code space includes all
- Unicode code points, from 0 to 0xD7FF and from 0xE000 to 0x10FFFF. One
- code point is represented with sequence of one, two, three or four
- 1-byte code units. Size of ``YYCTYPE`` must be 1 byte.
-
-In Unicode, values from range 0xD800 to 0xDFFF (surrogates) are not
-valid Unicode code points, any encoded sequence of code units, that
-would map to Unicode code points in the range 0xD800-0xDFFF, is
-ill-formed. The user can control how ``re2c`` treats such ill-formed
-sequences with ``--encoding-policy <policy>`` flag (see `OPTIONS`_
-for full explanation).
-
-For some encodings, there are code units, that never occur in valid
-encoded stream (e.g. 0xFF byte in UTF-8). If the generated scanner must
-check for invalid input, the only true way to do so is to use default
-rule ``*``. Note, that full range rule ``[^]`` won't catch invalid code units when variable-length encoding is used
-(``[^]`` means "all valid code points", while default rule ``*`` means "all possible code units").
-
-
-
-GENERIC INPUT API
------------------
-
-``re2c`` usually operates on input using pointer-like primitives
-``YYCURSOR``, ``YYMARKER``, ``YYCTXMARKER`` and ``YYLIMIT``.
-
-Generic input API (enabled with ``--input custom`` switch) allows to
-customize input operations. In this mode, ``re2c`` will express all
-operations on input in terms of the following primitives:
-
- +---------------------+-----------------------------------------------------+
- | ``YYPEEK ()`` | get current input character |
- +---------------------+-----------------------------------------------------+
- | ``YYSKIP ()`` | advance to the next character |
- +---------------------+-----------------------------------------------------+
- | ``YYBACKUP ()`` | backup current input position |
- +---------------------+-----------------------------------------------------+
- | ``YYBACKUPCTX ()`` | backup current input position for trailing context |
- +---------------------+-----------------------------------------------------+
- | ``YYRESTORE ()`` | restore current input position |
- +---------------------+-----------------------------------------------------+
- | ``YYRESTORECTX ()`` | restore current input position for trailing context |
- +---------------------+-----------------------------------------------------+
- | ``YYLESSTHAN (n)`` | check if less than ``n`` input characters are left |
- +---------------------+-----------------------------------------------------+
-
-This `article <http://skvadrik.github.io/aleph_null/posts/re2c/2015-01-13-input_model.html>`_
-has more details, and you can find some usage examples
-`here <http://skvadrik.github.io/aleph_null/posts/re2c/2015-01-15-input_model_custom.html>`_.
-
-
-
-UNDERSTANDING RE2C
-------------------
-
-The subdirectory ``examples`` of the ``re2c`` distribution contains a few step
-by step examples to get you started with ``re2c``.
-
-
-
-BUGS
-----
-
-* Difference only works for character sets, and not in UTF-8 mode.
-* Some features don't work together (such as reusable rules, conditions, setup rules and default rules).
-
-
-
-SEE ALSO
---------
-
-flex(1), lex(1), `quex <http://quex.sourceforge.net>`_.
-
-
-
-AUTHORS
--------
-
-* Peter Bumbulis peter@csg.uwaterloo.ca
-* Brian Young bayoung@acm.org
-* Dan Nuffer nuffer@users.sourceforge.net
-* Marcus Boerger helly@users.sourceforge.net
-* Hartmut Kaiser hkaiser@users.sourceforge.net
-* Emmanuel Mogenet mgix@mgix.com (added storable state)
-* Ulya Trofimovich skvadrik@gmail.com
-
-VERSION INFORMATION
--------------------
-
-This manpage describes ``re2c`` version 0.14.1.dev, package date 15 Oct 2015.
+
+`[home] <index.html>`_
+
+--------------------------------------------------------------------------------
+
+* `About`_
+ - `Authors`_
+ - `License`_
+ - `Version`_
+* `Run`_
+ - `Synopsis`_
+ - `Options`_
+ - `Warnings`_
+* `Syntax`_
+ - `Rules`_
+ - `Definitions`_
+ - `Configurations`_
+ - `Regular expressions`_
+ - `Interface`_
+* `Features`_
+ - `Conditions`_
+ - `State`_
+ - `Reuse`_
+ - `Encodings`_
+ - `Generic interface`_
+* `Examples`_
+* `Changelog`_
+
+.. _About:
+
+About
+=====
+--------------------------------------------------------------------------------
+
+.. _Authors:
+
+Authors
+-------
+
+Originally written by Peter Bumbulis (peter@csg.uwaterloo.ca)
+and described in research article *"RE2C: a more versatile scanner generator"
+by Peter Bumbulis, Donald D. Cowan,
+1994,
+ACM Letters on Programming Languages and Systems (LOPLAS)*
+`[PDF] <1994_bumbulis_cowan_re2c_a_more_versatile_scanner_generator.pdf>`_.
+
+Since then many people have contributed to re2c:
+
+* Brian Young bayoung@acm.org
+* Dan Nuffer nuffer@users.sourceforge.net
+* Marcus Boerger helly@users.sourceforge.net
+* Hartmut Kaiser hkaiser@users.sourceforge.net
+* Emmanuel Mogenet mgix@mgix.com (added storable state)
+* Ulya Trofimovich skvadrik@gmail.com
+
+Let me know if I missed someone!
+
+.. _License:
+
+License
+-------
+
+re2c is distributed with no warranty whatever.
+The code is certain to contain errors.
+Neither the author nor any contributor takes responsibility for any consequences of its use.
+re2c is in the public domain. The data structures and algorithms used
+in re2c are all either taken from documents available to the general
+public or are inventions of the author. Programs generated by re2c may
+be distributed freely. re2c itself may be distributed freely, in source
+or binary, unchanged or modified. Distributors may charge whatever fees
+they can obtain for re2c. If you do make use of re2c, or incorporate it into a larger project an
+acknowledgement somewhere (documentation, research report, etc.) would
+be appreciated.
+
+.. _Version:
+
+Version
+-------
+
+This page describes ``re2c`` version 0.14.1.dev, package date 15 Oct 2015.
+
+.. _Run:
+
+Run
+===
+--------------------------------------------------------------------------------
+
+.. _Synopsis:
+
+Synopsis
+--------
+
+``re2c [OPTIONS] FILE [OPTIONS]``
+
+
+.. _Options:
+
+Options
+-------
+
+.. include:: options/options_list.rst
+
+.. _Warnings:
+
+Warnings
+--------
+
+.. include:: options/warnings_list.rst
+
+.. _Syntax:
+
+Syntax
+======
+--------------------------------------------------------------------------------
+
+Code for ``re2c`` consists of a set of `rules`_, `definitions`_ and
+`configurations`_.
+
+.. _Rules:
+
+Rules
+-----
+
+Rules consist of a `regular expressions`_ along with a block of C/C++ code
+that is to be executed when the associated regular expression is
+matched. You can either start the code with an opening curly brace or
+the sequence ``:=``. When the code with a curly brace then ``re2c`` counts the brace depth
+and stops looking for code automatically. Otherwise curly braces are not
+allowed and ``re2c`` stops looking for code at the first line that does
+not begin with whitespace. If two or more rules overlap, the first rule
+is preferred.
+
+ ``regular-expression { C/C++ code }``
+
+ ``regular-expression := C/C++ code``
+
+There is one special rule: default rule ``*``
+
+ ``* { C/C++ code }``
+
+ ``* := C/C++ code``
+
+Note that default rule ``*`` differs from ``[^]``: default rule has the lowest priority,
+matches any code unit (either valid or invalid) and always consumes one character;
+while ``[^]`` matches any valid code point (not code unit) and can consume multiple
+code units. In fact, when variable-length encoding is used, ``*``
+is the only possible way to match invalid input character (see `Encodings`_ for details).
+
+If ``-c`` is active then each regular expression is preceeded by a list
+of comma separated condition names. Besides normal naming rules there
+are two special cases: ``<*>`` (such rules are merged to all conditions)
+and ``<>`` (such the rule cannot have an associated regular expression,
+its code is merged to all actions). Non empty rules may further more specify the new
+condition. In that case ``re2c`` will generate the necessary code to
+change the condition automatically. Rules can use ``:=>`` as a shortcut
+to automatically generate code that not only sets the
+new condition state but also continues execution with the new state. A
+shortcut rule should not be used in a loop where there is code between
+the start of the loop and the ``re2c`` block unless ``re2c:cond:goto``
+is changed to ``continue``. If code is necessary before all rules (though not simple jumps) you
+can doso by using ``<!>`` pseudo-rules.
+
+ ``<condition-list> regular-expression { C/C++ code }``
+
+ ``<condition-list> regular-expression := C/C++ code``
+
+ ``<condition-list> * { C/C++ code }``
+
+ ``<condition-list> * := C/C++ code``
+
+ ``<condition-list> regular-expression => condition { C/C++ code }``
+
+ ``<condition-list> regular-expression => condition := C/C++ code``
+
+ ``<condition-list> * => condition { C/C++ code }``
+
+ ``<condition-list> * => condition := C/C++ code``
+
+ ``<condition-list> regular-expression :=> condition``
+
+
+ ``<*> regular-expression { C/C++ code }``
+
+ ``<*> regular-expression := C/C++ code``
+
+ ``<*> * { C/C++ code }``
+
+ ``<*> * := C/C++ code``
+
+ ``<*> regular-expression => condition { C/C++ code }``
+
+ ``<*> regular-expression => condition := C/C++ code``
+
+ ``<*> * => condition { C/C++ code }``
+
+ ``<*> * => condition := C/C++ code``
+
+ ``<*> regular-expression :=> condition``
+
+
+ ``<> { C/C++ code }``
+
+ ``<> := C/C++ code``
+
+ ``<> => condition { C/C++ code }``
+
+ ``<> => condition := C/C++ code``
+
+ ``<> :=> condition``
+
+ ``<> :=> condition``
+
+
+ ``<! condition-list> { C/C++ code }``
+
+ ``<! condition-list> := C/C++ code``
+
+ ``<!> { C/C++ code }``
+
+ ``<!> := C/C++ code``
+
+.. _Definitions:
+
+Definitions
+-----------
+
+Named definitions are of the form:
+
+ ``name = regular-expression;``
+
+If ``-F`` is active, then named definitions are also of the form:
+
+ ``name { regular-expression }``
+
+.. _Configurations:
+
+Configurations
+--------------
+
+``re2c:condprefix = yyc;``
+ Allows to specify the prefix used for
+ condition labels. That is this text is prepended to any condition label
+ in the generated output file.
+
+``re2c:condenumprefix = yyc;``
+ Allows to specify the prefix used for
+ condition values. That is this text is prepended to any condition enum
+ value in the generated output file.
+
+``re2c:cond:divider = "/* *********************************** */";``
+ Allows to customize the devider for condition blocks. You can use ``@@``
+ to put the name of the condition or customize the placeholder using
+ ``re2c:cond:divider@cond``.
+
+``re2c:cond:divider@cond = @@;``
+ Specifies the placeholder that will be
+ replaced with the condition name in ``re2c:cond:divider``.
+
+``re2c:cond:goto = "goto @@;";``
+ Allows to customize the condition goto statements used with ``:=>`` style rules. You can use ``@@``
+ to put the name of the condition or ustomize the placeholder using
+ ``re2c:cond:goto@cond``. You can also change this to ``continue;``, which
+ would allow you to continue with the next loop cycle including any code
+ between loop start and re2c block.
+
+``re2c:cond:goto@cond = @@;``
+ Spcifies the placeholder that will be replaced with the condition label in ``re2c:cond:goto``.
+
+``re2c:indent:top = 0;``
+ Specifies the minimum number of indendation to
+ use. Requires a numeric value greater than or equal zero.
+
+``re2c:indent:string = "\t";``
+ Specifies the string to use for indendation. Requires a string that should
+ contain only whitespace unless you need this for external tools. The easiest
+ way to specify spaces is to enclude them in single or double quotes.
+ If you do not want any indendation at all you can simply set this to "".
+
+``re2c:yych:conversion = 0;``
+ When this setting is non zero, then ``re2c`` automatically generates
+ conversion code whenever yych gets read. In this case the type must be
+ defined using ``re2c:define:YYCTYPE``.
+
+``re2c:yych:emit = 1;``
+ Generation of *yych* can be suppressed by setting this to 0.
+
+``re2c:yybm:hex = 0;``
+ If set to zero then a decimal table is being used else a hexadecimal table will be generated.
+
+``re2c:yyfill:enable = 1;``
+ Set this to zero to suppress generation of ``YYFILL (n)``. When using this be sure to verify that the generated
+ scanner does not read behind input. Allowing this behavior might
+ introduce sever security issues to you programs.
+
+``re2c:yyfill:check = 1;``
+ This can be set 0 to suppress output of the
+ pre condition using ``YYCURSOR`` and ``YYLIMIT`` which becomes usefull when
+ ``YYLIMIT + YYMAXFILL`` is always accessible.
+
+``re2c:define:YYFILL = "YYFILL";``
+ Substitution for ``YYFILL``. Note
+ that by default ``re2c`` generates argument in braces and semicolon after
+ ``YYFILL``. If you need to make ``YYFILL`` an arbitrary statement rather
+ than a call, set ``re2c:define:YYFILL:naked`` to non-zero and use
+ ``re2c:define:YYFILL@len`` to denote formal parameter inside of ``YYFILL``
+ body.
+
+``re2c:define:YYFILL@len = "@@";``
+ Any occurence of this text
+ inside of ``YYFILL`` will be replaced with the actual argument.
+
+``re2c:yyfill:parameter = 1;``
+ Controls argument in braces after
+ ``YYFILL``. If zero, agrument is omitted. If non-zero, argument is
+ generated unless ``re2c:define:YYFILL:naked`` is set to non-zero.
+
+``re2c:define:YYFILL:naked = 0;``
+ Controls argument in braces and
+ semicolon after ``YYFILL``. If zero, both agrument and semicolon are
+ omitted. If non-zero, argument is generated unless
+ ``re2c:yyfill:parameter`` is set to zero and semicolon is generated
+ unconditionally.
+
+``re2c:startlabel = 0;``
+ If set to a non zero integer then the start
+ label of the next scanner blocks will be generated even if not used by
+ the scanner itself. Otherwise the normal ``yy0`` like start label is only
+ being generated if needed. If set to a text value then a label with that
+ text will be generated regardless of whether the normal start label is
+ being used or not. This setting is being reset to 0 after a start
+ label has been generated.
+
+``re2c:labelprefix = "yy";``
+ Allows to change the prefix of numbered
+ labels. The default is ``yy`` and can be set any string that is a valid
+ label.
+
+``re2c:state:abort = 0;``
+ When not zero and switch ``-f`` is active then
+ the ``YYGETSTATE`` block will contain a default case that aborts and a -1
+ case is used for initialization.
+
+``re2c:state:nextlabel = 0;``
+ Used when ``-f`` is active to control
+ whether the ``YYGETSTATE`` block is followed by a ``yyNext:`` label line.
+ Instead of using ``yyNext`` you can usually also use configuration
+ ``startlabel`` to force a specific start label or default to ``yy0`` as
+ start label. Instead of using a dedicated label it is often better to
+ separate the ``YYGETSTATE`` code from the actual scanner code by placing a
+ ``/*!getstate:re2c*/`` comment.
+
+``re2c:cgoto:threshold = 9;``
+ When ``-g`` is active this value specifies
+ the complexity threshold that triggers generation of jump tables rather
+ than using nested if's and decision bitfields. The threshold is compared
+ against a calculated estimation of if-s needed where every used bitmap
+ divides the threshold by 2.
+
+``re2c:yych:conversion = 0;``
+ When the input uses signed characters and
+ ``-s`` or ``-b`` switches are in effect re2c allows to automatically convert
+ to the unsigned character type that is then necessary for its internal
+ single character. When this setting is zero or an empty string the
+ conversion is disabled. Using a non zero number the conversion is taken
+ from ``YYCTYPE``. If that is given by an inplace configuration that value
+ is being used. Otherwise it will be ``(YYCTYPE)`` and changes to that
+ configuration are no longer possible. When this setting is a string the
+ braces must be specified. Now assuming your input is a ``char *``
+ buffer and you are using above mentioned switches you can set
+ ``YYCTYPE`` to ``unsigned char`` and this setting to either 1 or ``(unsigned char)``.
+
+``re2c:define:YYCONDTYPE = "YYCONDTYPE";``
+ Enumeration used for condition support with ``-c`` mode.
+
+``re2c:define:YYCTXMARKER = "YYCTXMARKER";``
+ Allows to overwrite the
+ define ``YYCTXMARKER`` and thus avoiding it by setting the value to the
+ actual code needed.
+
+``re2c:define:YYCTYPE = "YYCTYPE";``
+ Allows to overwrite the define
+ ``YYCTYPE`` and thus avoiding it by setting the value to the actual code
+ needed.
+
+``re2c:define:YYCURSOR = "YYCURSOR";``
+ Allows to overwrite the define
+ ``YYCURSOR`` and thus avoiding it by setting the value to the actual code
+ needed.
+
+``re2c:define:YYDEBUG = "YYDEBUG";``
+ Allows to overwrite the define
+ ``YYDEBUG`` and thus avoiding it by setting the value to the actual code
+ needed.
+
+``re2c:define:YYGETCONDITION = "YYGETCONDITION";``
+ Substitution for
+ ``YYGETCONDITION``. Note that by default ``re2c`` generates braces after
+ ``YYGETCONDITION``. Set ``re2c:define:YYGETCONDITION:naked`` to non-zero to
+ omit braces.
+
+``re2c:define:YYGETCONDITION:naked = 0;``
+ Controls braces after
+ ``YYGETCONDITION``. If zero, braces are omitted. If non-zero, braces are
+ generated.
+
+``re2c:define:YYSETCONDITION = "YYSETCONDITION";``
+ Substitution for
+ ``YYSETCONDITION``. Note that by default ``re2c`` generates argument in
+ braces and semicolon after ``YYSETCONDITION``. If you need to make
+ ``YYSETCONDITION`` an arbitrary statement rather than a call, set
+ ``re2c:define:YYSETCONDITION:naked`` to non-zero and use
+ ``re2c:define:YYSETCONDITION@cond`` to denote formal parameter inside of
+ ``YYSETCONDITION`` body.
+
+``re2c:define:YYSETCONDITION@cond = "@@";``
+ Any occurence of this
+ text inside of ``YYSETCONDITION`` will be replaced with the actual
+ argument.
+
+``re2c:define:YYSETCONDITION:naked = 0;``
+ Controls argument in braces
+ and semicolon after ``YYSETCONDITION``. If zero, both agrument and
+ semicolon are omitted. If non-zero, both argument and semicolon are
+ generated.
+
+``re2c:define:YYGETSTATE = "YYGETSTATE";``
+ Substitution for
+ ``YYGETSTATE``. Note that by default ``re2c`` generates braces after
+ ``YYGETSTATE``. Set ``re2c:define:YYGETSTATE:naked`` to non-zero to omit
+ braces.
+
+``re2c:define:YYGETSTATE:naked = 0;``
+ Controls braces after
+ ``YYGETSTATE``. If zero, braces are omitted. If non-zero, braces are
+ generated.
+
+``re2c:define:YYSETSTATE = "YYSETSTATE";``
+ Substitution for
+ ``YYSETSTATE``. Note that by default ``re2c`` generates argument in braces
+ and semicolon after ``YYSETSTATE``. If you need to make ``YYSETSTATE`` an
+ arbitrary statement rather than a call, set
+ ``re2c:define:YYSETSTATE:naked`` to non-zero and use
+ ``re2c:define:YYSETSTATE@cond`` to denote formal parameter inside of
+ ``YYSETSTATE`` body.
+
+``re2c:define:YYSETSTATE@state = "@@";``
+ Any occurence of this text
+ inside of ``YYSETSTATE`` will be replaced with the actual argument.
+
+``re2c:define:YYSETSTATE:naked = 0;``
+ Controls argument in braces and
+ semicolon after ``YYSETSTATE``. If zero, both agrument and semicolon are
+ omitted. If non-zero, both argument and semicolon are generated.
+
+``re2c:define:YYLIMIT = "YYLIMIT";``
+ Allows to overwrite the define
+ ``YYLIMIT`` and thus avoiding it by setting the value to the actual code
+ needed.
+
+``re2c:define:YYMARKER = "YYMARKER";``
+ Allows to overwrite the define
+ ``YYMARKER`` and thus avoiding it by setting the value to the actual code
+ needed.
+
+``re2c:label:yyFillLabel = "yyFillLabel";``
+ Allows to overwrite the name of the label ``yyFillLabel``.
+
+``re2c:label:yyNext = "yyNext";``
+ Allows to overwrite the name of the label ``yyNext``.
+
+``re2c:variable:yyaccept = yyaccept;``
+ Allows to overwrite the name of the variable ``yyaccept``.
+
+``re2c:variable:yybm = "yybm";``
+ Allows to overwrite the name of the variable ``yybm``.
+
+``re2c:variable:yych = "yych";``
+ Allows to overwrite the name of the variable ``yych``.
+
+``re2c:variable:yyctable = "yyctable";``
+ When both ``-c`` and ``-g`` are active then ``re2c`` uses this variable to generate a static jump table
+ for ``YYGETCONDITION``.
+
+``re2c:variable:yystable = "yystable";``
+ Deprecated.
+
+``re2c:variable:yytarget = "yytarget";``
+ Allows to overwrite the name of the variable ``yytarget``.
+
+.. _Regular expressions:
+
+Regular expressions
+-------------------
+
+``"foo"``
+ literal string ``"foo"``. ANSI-C escape sequences can be used.
+
+``'foo'``
+ literal string ``"foo"`` (characters [a-zA-Z] treated
+ case-insensitive). ANSI-C escape sequences can be used.
+
+``[xyz]``
+ character class; in this case, regular expression matches either ``x``, ``y``, or ``z``.
+
+``[abj-oZ]``
+ character class with a range in it; matches ``a``, ``b``, any letter from ``j`` through ``o`` or ``Z``.
+
+``[^class]``
+ inverted character class.
+
+``r \ s``
+ match any ``r`` which isn't ``s``. ``r`` and ``s`` must be regular expressions
+ which can be expressed as character classes.
+
+``r*``
+ zero or more occurences of ``r``.
+
+``r+``
+ one or more occurences of ``r``.
+
+``r?``
+ optional ``r``.
+
+``(r)``
+ ``r``; parentheses are used to override precedence.
+
+``r s``
+ ``r`` followed by ``s`` (concatenation).
+
+``r | s``
+ either ``r`` or ``s`` (alternative).
+
+``r`` / ``s``
+ ``r`` but only if it is followed by ``s``. Note that ``s`` is not
+ part of the matched text. This type of regular expression is called
+ "trailing context". Trailing context can only be the end of a rule
+ and not part of a named definition.
+
+``r{n}``
+ matches ``r`` exactly ``n`` times.
+
+``r{n,}``
+ matches ``r`` at least ``n`` times.
+
+``r{n,m}``
+ matches ``r`` at least ``n`` times, but not more than ``m`` times.
+
+``.``
+ match any character except newline.
+
+``name``
+ matches named definition as specified by ``name`` only if ``-F`` is
+ off. If ``-F`` is active then this behaves like it was enclosed in double
+ quotes and matches the string "name".
+
+Character classes and string literals may contain octal or hexadecimal
+character definitions and the following set of escape sequences:
+``\a``, ``\b``, ``\f``, ``\n``, ``\r``, ``\t``, ``\v``, ``\\``. An octal character is defined by a backslash
+followed by its three octal digits (e.g. ``\377``).
+Hexadecimal characters from 0 to 0xFF are defined by backslash, a lower
+cased ``x`` and two hexadecimal digits (e.g. ``\x12``). Hexadecimal characters from 0x100 to 0xFFFF are defined by backslash, a lower cased
+``\u`` or an upper cased ``\X`` and four hexadecimal digits (e.g. ``\u1234``).
+Hexadecimal characters from 0x10000 to 0xFFFFffff are defined by backslash, an upper cased ``\U``
+and eight hexadecimal digits (e.g. ``\U12345678``).
+
+The only portable "any" rule is the default rule ``*``.
+
+.. _Interface:
+
+Interface
+---------
+
+The user must supply interface code either in the form of C/C++ code
+(macros, functions, variables, etc.) or in the form of `configurations`_.
+Which symbols must be defined and which are optional
+depends on a particular use case.
+
+``YYCONDTYPE``
+ In ``-c`` mode you can use ``-t`` to generate a file that
+ contains the enumeration used as conditions. Each of the values refers
+ to a condition of a rule set.
+
+``YYCTXMARKER``
+ l-value of type ``YYCTYPE *``.
+ The generated code saves trailing context backtracking information in
+ ``YYCTXMARKER``. The user only needs to define this macro if a scanner
+ specification uses trailing context in one or more of its regular
+ expressions.
+
+``YYCTYPE``
+ Type used to hold an input symbol (code unit). Usually
+ ``char`` or ``unsigned char`` for ASCII, EBCDIC and UTF-8, *unsigned short*
+ for UTF-16 or UCS-2 and ``unsigned int`` for UTF-32.
+
+``YYCURSOR``
+ l-value of type ``YYCTYPE *`` that points to the current input symbol. The generated code advances
+ ``YYCURSOR`` as symbols are matched. On entry, ``YYCURSOR`` is assumed to
+ point to the first character of the current token. On exit, ``YYCURSOR``
+ will point to the first character of the following token.
+
+``YYDEBUG (state, current)``
+ This is only needed if the ``-d`` flag was
+ specified. It allows to easily debug the generated parser by calling a
+ user defined function for every state. The function should have the
+ following signature: ``void YYDEBUG (int state, char current)``. The first
+ parameter receives the state or -1 and the second parameter receives the
+ input at the current cursor.
+
+``YYFILL (n)``
+ The generated code "calls"" ``YYFILL (n)`` when the
+ buffer needs (re)filling: at least ``n`` additional characters should be
+ provided. ``YYFILL (n)`` should adjust ``YYCURSOR``, ``YYLIMIT``, ``YYMARKER``
+ and ``YYCTXMARKER`` as needed. Note that for typical programming languages
+ ``n`` will be the length of the longest keyword plus one. The user can
+ place a comment of the form ``/*!max:re2c*/`` to insert ``YYMAXFILL`` definition that is set to the maximum
+ length value.
+
+``YYGETCONDITION ()``
+ This define is used to get the condition prior to
+ entering the scanner code when using ``-c`` switch. The value must be
+ initialized with a value from the enumeration ``YYCONDTYPE`` type.
+
+``YYGETSTATE ()``
+ The user only needs to define this macro if the ``-f``
+ flag was specified. In that case, the generated code "calls"
+ ``YYGETSTATE ()`` at the very beginning of the scanner in order to obtain
+ the saved state. ``YYGETSTATE ()`` must return a signed integer. The value
+ must be either -1, indicating that the scanner is entered for the first
+ time, or a value previously saved by ``YYSETSTATE (s)``. In the second
+ case, the scanner will resume operations right after where the last
+ ``YYFILL (n)`` was called.
+
+``YYLIMIT``
+ Expression of type ``YYCTYPE *`` that marks the end of the buffer ``YYLIMIT[-1]``
+ is the last character in the buffer). The generated code repeatedly
+ compares ``YYCURSOR`` to ``YYLIMIT`` to determine when the buffer needs
+ (re)filling.
+
+``YYMARKER``
+ l-value of type ``YYCTYPE *``.
+ The generated code saves backtracking information in ``YYMARKER``. Some
+ easy scanners might not use this.
+
+``YYMAXFILL``
+ This will be automatically defined by ``/*!max:re2c*/`` blocks as explained above.
+
+``YYSETCONDITION (c)``
+ This define is used to set the condition in
+ transition rules. This is only being used when ``-c`` is active and
+ transition rules are being used.
+
+``YYSETSTATE (s)``
+ The user only needs to define this macro if the ``-f``
+ flag was specified. In that case, the generated code "calls"
+ ``YYSETSTATE`` just before calling ``YYFILL (n)``. The parameter to
+ ``YYSETSTATE`` is a signed integer that uniquely identifies the specific
+ instance of ``YYFILL (n)`` that is about to be called. Should the user
+ wish to save the state of the scanner and have ``YYFILL (n)`` return to
+ the caller, all he has to do is store that unique identifer in a
+ variable. Later, when the scannered is called again, it will call
+ ``YYGETSTATE ()`` and resume execution right where it left off. The
+ generated code will contain both ``YYSETSTATE (s)`` and ``YYGETSTATE`` even
+ if ``YYFILL (n)`` is being disabled.
+
+.. _Features:
+
+Features
+========
+--------------------------------------------------------------------------------
+
+.. _Conditions:
+
+Conditions
+----------
+
+You can preceed regular expressions with a list of condition names when
+using the ``-c`` switch. In this case ``re2c`` generates scanner blocks for
+each conditon. Where each of the generated blocks has its own
+precondition. The precondition is given by the interface define
+``YYGETCONDITON()`` and must be of type ``YYCONDTYPE``.
+
+There are two special rule types. First, the rules of the condition ``<*>``
+are merged to all conditions (note that they have lower priority than
+other rules of that condition). And second the empty condition list
+allows to provide a code block that does not have a scanner part.
+Meaning it does not allow any regular expression. The condition value
+referring to this special block is always the one with the enumeration
+value 0. This way the code of this special rule can be used to
+initialize a scanner. It is in no way necessary to have these rules: but
+sometimes it is helpful to have a dedicated uninitialized condition
+state.
+
+Non empty rules allow to specify the new condition, which makes them
+transition rules. Besides generating calls for the define
+``YYSETCONDTITION`` no other special code is generated.
+
+There is another kind of special rules that allow to prepend code to any
+code block of all rules of a certain set of conditions or to all code
+blocks to all rules. This can be helpful when some operation is common
+among rules. For instance this can be used to store the length of the
+scanned string. These special setup rules start with an exclamation mark
+followed by either a list of conditions ``<! condition, ... >`` or a star
+``<!*>``. When ``re2c`` generates the code for a rule whose state does not have a
+setup rule and a star'd setup rule is present, than that code will be
+used as setup code.
+
+.. _State:
+
+State
+-----
+
+When the ``-f`` flag is specified, ``re2c`` generates a scanner that can
+store its current state, return to the caller, and later resume
+operations exactly where it left off.
+
+The default operation of ``re2c`` is a
+"pull" model, where the scanner asks for extra input whenever it needs it. However, this mode of operation assumes that the scanner is the "owner"
+the parsing loop, and that may not always be convenient.
+
+Typically, if there is a preprocessor ahead of the scanner in the
+stream, or for that matter any other procedural source of data, the
+scanner cannot "ask" for more data unless both scanner and source
+live in a separate threads.
+
+The ``-f`` flag is useful for just this situation: it lets users design
+scanners that work in a "push" model, i.e. where data is fed to the
+scanner chunk by chunk. When the scanner runs out of data to consume, it
+just stores its state, and return to the caller. When more input data is
+fed to the scanner, it resumes operations exactly where it left off.
+
+Changes needed compared to the "pull" model:
+
+* User has to supply macros ``YYSETSTATE ()`` and ``YYGETSTATE (state)``.
+
+* The ``-f`` option inhibits declaration of ``yych`` and ``yyaccept``. So the
+ user has to declare these. Also the user has to save and restore these.
+ In the example ``examples/push_model/push.re`` these are declared as
+ fields of the (C++) class of which the scanner is a method, so they do
+ not need to be saved/restored explicitly. For C they could e.g. be made
+ macros that select fields from a structure passed in as parameter.
+ Alternatively, they could be declared as local variables, saved with
+ ``YYFILL (n)`` when it decides to return and restored at entry to the
+ function. Also, it could be more efficient to save the state from
+ ``YYFILL (n)`` because ``YYSETSTATE (state)`` is called unconditionally.
+ ``YYFILL (n)`` however does not get ``state`` as parameter, so we would have
+ to store state in a local variable by ``YYSETSTATE (state)``.
+
+* Modify ``YYFILL (n)`` to return (from the function calling it) if more input is needed.
+
+* Modify caller to recognise if more input is needed and respond appropriately.
+
+* The generated code will contain a switch block that is used to
+ restores the last state by jumping behind the corrspoding ``YYFILL (n)``
+ call. This code is automatically generated in the epilog of the first ``/*!re2c */``
+ block. It is possible to trigger generation of the ``YYGETSTATE ()``
+ block earlier by placing a ``/*!getstate:re2c*/`` comment. This is especially useful when the scanner code should be
+ wrapped inside a loop.
+
+Please see ``examples/push_model/push.re`` for "push" model scanner. The
+generated code can be tweaked using inplace configurations ``state:abort``
+and ``state:nextlabel``.
+
+.. _Reuse:
+
+Reuse
+-----
+
+Reuse mode is controlled by ``-r --reusable`` option.
+Allows reuse of scanner definitions with ``/*!use:re2c */`` after ``/*!rules:re2c */``.
+In this mode no ``/*!re2c */`` block and exactly one ``/*!rules:re2c */`` must be present.
+The rules are being saved and used by every ``/*!use:re2c */`` block that follows.
+These blocks can contain inplace configurations, especially ``re2c:flags:e``,
+``re2c:flags:w``, ``re2c:flags:x``, ``re2c:flags:u`` and ``re2c:flags:8``.
+That way it is possible to create the same scanner multiple times for
+different character types, different input mechanisms or different output mechanisms.
+The ``/*!use:re2c */`` blocks can also contain additional rules that will be appended
+to the set of rules in ``/*!rules:re2c */``.
+
+.. _Encodings:
+
+Encodings
+---------
+
+``re2c`` supports the following encodings: ASCII (default), EBCDIC (``-e``),
+UCS-2 (``-w``), UTF-16 (``-x``), UTF-32 (``-u``) and UTF-8 (``-8``).
+See also inplace configuration ``re2c:flags``.
+
+The following concepts should be clarified when talking about encoding.
+*Code point* is an abstract number, which represents single encoding
+symbol. *Code unit* is the smallest unit of memory, which is used in the
+encoded text (it corresponds to one character in the input stream). One
+or more code units can be needed to represent a single code point,
+depending on the encoding. In *fixed-length* encoding, each code point
+is represented with equal number of code units. In *variable-length*
+encoding, different code points can be represented with different number
+of code units.
+
+* ASCII is a fixed-length encoding. Its code space includes 0x100
+ code points, from 0 to 0xFF. One code point is represented with exactly one
+ 1-byte code unit, which has the same value as the code point. Size of
+ ``YYCTYPE`` must be 1 byte.
+
+* EBCDIC is a fixed-length encoding. Its code space includes 0x100
+ code points, from 0 to 0xFF. One code point is represented with exactly
+ one 1-byte code unit, which has the same value as the code point. Size
+ of ``YYCTYPE`` must be 1 byte.
+
+* UCS-2 is a fixed-length encoding. Its code space includes 0x10000
+ code points, from 0 to 0xFFFF. One code point is represented with
+ exactly one 2-byte code unit, which has the same value as the code
+ point. Size of ``YYCTYPE`` must be 2 bytes.
+
+* UTF-16 is a variable-length encoding. Its code space includes all
+ Unicode code points, from 0 to 0xD7FF and from 0xE000 to 0x10FFFF. One
+ code point is represented with one or two 2-byte code units. Size of
+ ``YYCTYPE`` must be 2 bytes.
+
+* UTF-32 is a fixed-length encoding. Its code space includes all
+ Unicode code points, from 0 to 0xD7FF and from 0xE000 to 0x10FFFF. One
+ code point is represented with exactly one 4-byte code unit. Size of
+ ``YYCTYPE`` must be 4 bytes.
+
+* UTF-8 is a variable-length encoding. Its code space includes all
+ Unicode code points, from 0 to 0xD7FF and from 0xE000 to 0x10FFFF. One
+ code point is represented with sequence of one, two, three or four
+ 1-byte code units. Size of ``YYCTYPE`` must be 1 byte.
+
+In Unicode, values from range 0xD800 to 0xDFFF (surrogates) are not
+valid Unicode code points, any encoded sequence of code units, that
+would map to Unicode code points in the range 0xD800-0xDFFF, is
+ill-formed. The user can control how ``re2c`` treats such ill-formed
+sequences with ``--encoding-policy <policy>`` flag (see `Options`_
+for full explanation).
+
+For some encodings, there are code units, that never occur in valid
+encoded stream (e.g. 0xFF byte in UTF-8). If the generated scanner must
+check for invalid input, the only true way to do so is to use default
+rule ``*``. Note, that full range rule ``[^]`` won't catch invalid code units when variable-length encoding is used
+(``[^]`` means "all valid code points", while default rule ``*`` means "all possible code units").
+
+.. _Generic interface:
+
+Generic interface
+-----------------
+
+``re2c`` usually operates on input using pointer-like primitives
+``YYCURSOR``, ``YYMARKER``, ``YYCTXMARKER`` and ``YYLIMIT``.
+
+Generic input API (enabled with ``--input custom`` switch) allows to
+customize input operations. In this mode, ``re2c`` will express all
+operations on input in terms of the following primitives:
+
+ +---------------------+-----------------------------------------------------+
+ | ``YYPEEK ()`` | get current input character |
+ +---------------------+-----------------------------------------------------+
+ | ``YYSKIP ()`` | advance to the next character |
+ +---------------------+-----------------------------------------------------+
+ | ``YYBACKUP ()`` | backup current input position |
+ +---------------------+-----------------------------------------------------+
+ | ``YYBACKUPCTX ()`` | backup current input position for trailing context |
+ +---------------------+-----------------------------------------------------+
+ | ``YYRESTORE ()`` | restore current input position |
+ +---------------------+-----------------------------------------------------+
+ | ``YYRESTORECTX ()`` | restore current input position for trailing context |
+ +---------------------+-----------------------------------------------------+
+ | ``YYLESSTHAN (n)`` | check if less than ``n`` input characters are left |
+ +---------------------+-----------------------------------------------------+
+
+This `article <http://skvadrik.github.io/aleph_null/posts/re2c/2015-01-13-input_model.html>`_
+has more details, and you can find some usage examples
+`here <http://skvadrik.github.io/aleph_null/posts/re2c/2015-01-15-input_model_custom.html>`_.
+
+.. _Examples:
+
+Examples
+========
+
+--------------------------------------------------------------------------------
+
+The subdirectory ``examples`` of the ``re2c`` distribution contains a few step
+by step examples to get you started with ``re2c``.
+
+Given the following code:
+
+.. code-block:: c
+
+ unsigned int stou (const char \* s)
+ {
+ # define YYCTYPE char const
+ YYCTYPE * YYCURSOR = s;
+ unsigned int result = 0;
+ for (;;)
+ {
+ /*!re2c
+ re2c:yyfill:enable = 0;
+
+ "\x00" { return result; }
+ [0-9] { result = result * 10 + c; continue; }
+ */
+ }
+ }
+
+``re2c -is`` will generate:
+
+.. code-block:: c
+
+ /* Generated by re2c 0.13.7.dev on Mon Jul 14 13:37:46 2014 */
+ unsigned int stou (const char * s)
+ {
+ # define YYCTYPE char
+ const YYCTYPE * YYCURSOR = s;
+ unsigned int result = 0;
+
+ for (;;)
+ {
+ {
+ YYCTYPE yych;
+
+ yych = *YYCURSOR;
+ if (yych <= 0x00) goto yy3;
+ if (yych <= '/') goto yy2;
+ if (yych <= '9') goto yy5;
+
+ yy2:
+ yy3:
+ ++YYCURSOR;
+ { return result; }
+ yy5:
+ ++YYCURSOR;
+ { result = result * 10 + c; continue; }
+ }
+ }
+ }
+
+.. _Changelog:
+
+Changelog
+=========
+--------------------------------------------------------------------------------
+
+.. include:: changelog.rst
--- /dev/null
+``-W``
+ Turn on all warnings.
+
+``-Werror``
+ Turn warnings into errors. Note that this option along
+ doesn't turn on any warnings, it only affects those warnings that have
+ been turned on so far or will be turned on later.
+
+``-W<warning>``
+ Turn on individual ``warning``.
+
+``-Wno-<warning>``
+ Turn off individual ``warning``.
+
+``-Werror-<warning>``
+ Turn on individual ``warning`` and treat it as error (this implies ``-W<warning>``).
+
+``-Wno-error-<warning>``
+ Don't treat this particular ``warning`` as error. This doesn't turn off
+ the warning itself.
+
+``-Wcondition-order``
+ Warn if the generated program makes implicit
+ assumptions about condition numbering. One should use either ``-t, --type-header`` option or
+ ``/*!types:re2c*/`` directive to generate mapping of condition names to numbers and use
+ autogenerated condition names.
+
+``-Wempty-character-class``
+ Warn if regular expression contains empty
+ character class. From the rational point of view trying to match empty
+ character class makes no sense: it should always fail. However, for
+ backwards compatibility reasons ``re2c`` allows empty character class and
+ treats it as empty string. Use ``--empty-class`` option to change default
+ behaviour.
+
+``-Wmatch-empty-string``
+ Warn if regular expression in a rule is
+ nullable (matches empty string). If DFA runs in a loop and empty match
+ is unintentional (input position in not advanced manually), lexer may
+ get stuck in eternal loop.
+
+``-Wswapped-range``
+ Warn if range lower bound is greater that upper
+ bound. Default ``re2c`` behaviour is to silently swap range bounds.
+
+``-Wundefined-control-flow``
+ Warn if some input strings cause undefined
+ control flow in lexer (the faulty patterns are reported). This is the
+ most dangerous and common mistake. It can be easily fixed by adding
+ default rule ``*`` (this rule has the lowest priority, matches any code unit and consumes
+ exactly one code unit).
+
+``-Wuseless-escape``
+ Warn if a symbol is escaped when it shouldn't be.
+ By default re2c silently ignores escape, but this may as well indicate a
+ typo or an error in escape sequence.
projects / re2c / commitdiff