From af5b5438c41ae5556b97da925c45854e87fe65c5 Mon Sep 17 00:00:00 2001
From: helly <helly@642ea486-5414-0410-9d7f-a0204ed87703>
Date: Sun, 5 Nov 2006 00:48:30 +0000
Subject: [PATCH] - Added -u switch to support unicode.

---
 re2c/CHANGELOG           |   1 +
 re2c/actions.cc          |  19 +++-
 re2c/dfa.cc              |  24 ++++-
 re2c/globals.h           |   1 +
 re2c/htdocs/index.html   |   4 +
 re2c/main.cc             |  27 ++++-
 re2c/test/cvsignore.u.c  | 221 +++++++++++++++++++++++++++++++++++++++
 re2c/test/cvsignore.u.re |  63 +++++++++++
 8 files changed, 350 insertions(+), 10 deletions(-)
 create mode 100755 re2c/test/cvsignore.u.c
 create mode 100755 re2c/test/cvsignore.u.re

diff --git a/re2c/CHANGELOG b/re2c/CHANGELOG
index 8d0c51ed..1bdb684b 100644
--- a/re2c/CHANGELOG
+++ b/re2c/CHANGELOG
@@ -1,5 +1,6 @@
 Version 0.11.0 (????-??-??)
 ---------------------------
+- Added -u switch to support unicode.
 
 Version 0.10.6 (2006-08-05)
 ---------------------------
diff --git a/re2c/actions.cc b/re2c/actions.cc
index 8db3efe6..e0bacb44 100644
--- a/re2c/actions.cc
+++ b/re2c/actions.cc
@@ -524,7 +524,7 @@ uint Scanner::unescape(SubStr &s) const
 	static const char * oct = "01234567";
 
 	s.len--;
-	uint c;
+	uint c, ucb = 0;
 
 	if ((c = *s.str++) != '\\' || s.len == 0)
 	{
@@ -587,10 +587,20 @@ uint Scanner::unescape(SubStr &s) const
 				if (s.str[1] == '0')
 				{
 					l++;
-					if (s.str[2] == '0')
+					if (s.str[2] == '0' || (s.str[2] == '1' && uFlag))
 					{
 						l++;
-						if (s.str[3] == '0')
+						if (uFlag) {
+							const char *u3 = strchr(hex, tolower(s.str[2]));
+							const char *u4 = strchr(hex, tolower(s.str[3]));
+							if (u3 && u4)
+							{
+								ucb = (uint)((u3 - hex) << 20)
+							        + (uint)((u4 - hex) << 16);
+								l++;
+							}
+						}
+						else if (s.str[3] == '0')
 						{
 							l++;
 						}
@@ -641,7 +651,8 @@ uint Scanner::unescape(SubStr &s) const
 				uint v = (uint)((p1 - hex) << 12) 
 				       + (uint)((p2 - hex) <<  8)
 				       + (uint)((p3 - hex) <<  4)
-				       + (uint)((p4 - hex));
+				       + (uint)((p4 - hex))
+				       + ucb;
 	
 				if (v >= nRealChars)
 				{
diff --git a/re2c/dfa.cc b/re2c/dfa.cc
index dcd6ff6a..a1ea5d4b 100644
--- a/re2c/dfa.cc
+++ b/re2c/dfa.cc
@@ -28,7 +28,19 @@ void prtHex(std::ostream& o, uint c, bool useTalx)
 {
 	int oc = (int)(re2c::wFlag || !useTalx ? c : re2c::talx[c]);
 
-	if (re2c::wFlag)
+	if (re2c::uFlag)
+	{
+		o << "0x"
+		  << hexCh(oc >> 28)
+		  << hexCh(oc >> 24)
+		  << hexCh(oc >> 20)
+		  << hexCh(oc >> 16)
+		  << hexCh(oc >> 12)
+		  << hexCh(oc >>  8)
+		  << hexCh(oc >>  4)
+		  << hexCh(oc);
+	}
+	else if (re2c::wFlag)
 	{
 		o << "0x"
 		  << hexCh(oc >> 12)
@@ -92,6 +104,16 @@ void prtCh(std::ostream& o, uint c, bool useTalx)
 		{
 			o << (char) oc;
 		}
+		else if (re2c::uFlag)
+		{
+			o << "0x"
+			  << hexCh(oc >> 20)
+			  << hexCh(oc >> 16)
+			  << hexCh(oc >> 12)
+			  << hexCh(oc >>  8)
+			  << hexCh(oc >>  4)
+			  << hexCh(oc);
+		}
 		else if (re2c::wFlag)
 		{
 			o << "0x"
diff --git a/re2c/globals.h b/re2c/globals.h
index b8645c42..b4534ab8 100644
--- a/re2c/globals.h
+++ b/re2c/globals.h
@@ -21,6 +21,7 @@ extern bool fFlag;
 extern bool gFlag;
 extern bool iFlag;
 extern bool sFlag;
+extern bool uFlag;
 extern bool wFlag;
 
 extern bool bSinglePass;
diff --git a/re2c/htdocs/index.html b/re2c/htdocs/index.html
index 6ddfc56f..2075f376 100755
--- a/re2c/htdocs/index.html
+++ b/re2c/htdocs/index.html
@@ -79,6 +79,10 @@ fixes which were incorporated. <a href=
 </ul>
 <hr />
 <h1>Changelog</h1>
+<h2>200?-??-??: 0.11.0</h2>
+<ul>
+<li>Added -u switch to support unicode.</li>
+</ul>
 <h2>2006-08-05: 0.10.6</h2>
 <ul>
 <li>Fixed #1529351 Segv bug on unterminated code blocks.</li>
diff --git a/re2c/main.cc b/re2c/main.cc
index 27015694..f7ac6107 100644
--- a/re2c/main.cc
+++ b/re2c/main.cc
@@ -28,6 +28,7 @@ bool fFlag = false;
 bool gFlag = false;
 bool iFlag = false;
 bool sFlag = false;
+bool uFlag = false;
 bool wFlag = false;
 
 bool bSinglePass = false;
@@ -78,6 +79,7 @@ static const mbo_opt_struct OPTIONS[] =
 	mbo_opt_struct('i', 0, "no-debug-info"),
 	mbo_opt_struct('o', 1, "output"),
 	mbo_opt_struct('s', 0, "nested-ifs"),
+	mbo_opt_struct('u', 0, "unicode"),
 	mbo_opt_struct('v', 0, "version"),
 	mbo_opt_struct('V', 0, "vernum"),
 	mbo_opt_struct('w', 0, "wide-chars"),      
@@ -116,6 +118,8 @@ static void usage()
 	"-s     --nested-ifs     Generate nested ifs for some switches. Many compilers\n"
 	"                        need this assist to generate better code.\n"
 	"\n"
+	"-u     --unicode        Implies -w but supports the full Unicode character set.\n"
+	"\n"
 	"-v     --version        Show version information.\n"
 	"-V     --vernum         Show version as one number.\n"
 	"\n"
@@ -191,10 +195,6 @@ int main(int argc, char *argv[])
 			break;
 			
 			case '1':
-			if (bFlag) {
-				std::cerr << "re2c: error: cannot combine -1 and -f switch\n";
-				return 1;
-			}
 			bSinglePass = true;
 			break;
 
@@ -225,10 +225,22 @@ int main(int argc, char *argv[])
 			}
 			
 			case 'w':
-			nRealChars = (1<<16);
+			if (uFlag)
+			{
+				std::cerr << "re2c: error: cannot combine -w and -u switch\n";
+				return 1;
+			}
+			nRealChars = (1<<16); /* 0x10000 */
 			sFlag = true;
 			wFlag = true;
 			break;
+
+			case 'u':
+			nRealChars = 0x110000; /* 17 times w-Flag */
+			sFlag = true;
+			wFlag = true;
+			uFlag = true;
+			break;
 	  
 			case 'h':
 			case '?':
@@ -238,6 +250,11 @@ int main(int argc, char *argv[])
 		}
 	}
 
+	if (bFlag && bSinglePass) {
+		std::cerr << "re2c: error: cannot combine -1 and -f switch\n";
+		return 1;
+	}
+
 	if (wFlag && eFlag)
 	{
 		usage();
diff --git a/re2c/test/cvsignore.u.c b/re2c/test/cvsignore.u.c
new file mode 100755
index 00000000..5ec7a4ab
--- /dev/null
+++ b/re2c/test/cvsignore.u.c
@@ -0,0 +1,221 @@
+/* Generated by re2c */
+#line 1 "cvsignore.u.re"
+
+#define YYFILL(n) if (cursor >= limit) break;
+#define YYCTYPE unsigned short
+#define YYCURSOR cursor
+#define YYLIMIT limit
+#define YYMARKER marker
+
+#line 16 "cvsignore.u.re"
+
+
+#define APPEND(text) \
+	append(output, outsize, text, sizeof(text) - sizeof(YYCTYPE))
+
+inline void append(YYCTYPE *output, size_t & outsize, const YYCTYPE * text, size_t len)
+{
+	memcpy(output + outsize, text, len);
+	outsize += (len / sizeof(YYCTYPE));
+}
+
+void scan(YYCTYPE *pText, size_t *pSize, int *pbChanged)
+{
+	// rule
+	// scan lines
+	// find $ in lines
+	//   compact $<keyword>: .. $ to $<keyword>$
+  
+	YYCTYPE *output;
+	const YYCTYPE *cursor, *limit, *marker;
+
+	cursor = marker = output = *pText;
+
+	size_t insize = *pSize;
+	size_t outsize = 0;
+
+	limit = cursor + insize;
+
+	while(1) {
+loop:
+
+#line 43 "<stdout>"
+{
+	YYCTYPE yych;
+
+	if((YYLIMIT - YYCURSOR) < 11) YYFILL(11);
+	yych = *YYCURSOR;
+	if(yych != '$') goto yy4;
+	yych = *(YYMARKER = ++YYCURSOR);
+	switch(yych) {
+	case 'D':	goto yy10;
+	case 'I':	goto yy9;
+	case 'L':	goto yy8;
+	case 'R':	goto yy7;
+	case 'S':	goto yy5;
+	default:	goto yy3;
+	}
+yy3:
+#line 53 "cvsignore.u.re"
+	{ output[outsize++] = cursor[-1]; if (cursor >= limit) break; goto loop; }
+#line 62 "<stdout>"
+yy4:
+	yych = *++YYCURSOR;
+	goto yy3;
+yy5:
+	yych = *++YYCURSOR;
+	if(yych == 'o') goto yy44;
+yy6:
+	YYCURSOR = YYMARKER;
+	goto yy3;
+yy7:
+	yych = *++YYCURSOR;
+	if(yych == 'e') goto yy32;
+	goto yy6;
+yy8:
+	yych = *++YYCURSOR;
+	if(yych == 'o') goto yy25;
+	goto yy6;
+yy9:
+	yych = *++YYCURSOR;
+	if(yych == 'd') goto yy19;
+	goto yy6;
+yy10:
+	yych = *++YYCURSOR;
+	if(yych != 'a') goto yy6;
+	yych = *++YYCURSOR;
+	if(yych != 't') goto yy6;
+	yych = *++YYCURSOR;
+	if(yych != 'e') goto yy6;
+	yych = *++YYCURSOR;
+	if(yych == '$') goto yy15;
+	if(yych != ':') goto yy6;
+	yych = *++YYCURSOR;
+	if(yych == '$') goto yy6;
+	goto yy18;
+yy15:
+	++YYCURSOR;
+#line 48 "cvsignore.u.re"
+	{ APPEND(L"$" L"Date$"); goto loop; }
+#line 101 "<stdout>"
+yy17:
+	++YYCURSOR;
+	if(YYLIMIT == YYCURSOR) YYFILL(1);
+	yych = *YYCURSOR;
+yy18:
+	if(yych == 0x0000000A) goto yy6;
+	if(yych == '$') goto yy15;
+	goto yy17;
+yy19:
+	yych = *++YYCURSOR;
+	if(yych == '$') goto yy21;
+	if(yych != ':') goto yy6;
+	yych = *++YYCURSOR;
+	if(yych == '$') goto yy6;
+	goto yy24;
+yy21:
+	++YYCURSOR;
+#line 49 "cvsignore.u.re"
+	{ APPEND(L"$" L"Id$"); goto loop; }
+#line 121 "<stdout>"
+yy23:
+	++YYCURSOR;
+	if(YYLIMIT == YYCURSOR) YYFILL(1);
+	yych = *YYCURSOR;
+yy24:
+	if(yych == 0x0000000A) goto yy6;
+	if(yych == '$') goto yy21;
+	goto yy23;
+yy25:
+	yych = *++YYCURSOR;
+	if(yych != 'g') goto yy6;
+	yych = *++YYCURSOR;
+	if(yych == '$') goto yy28;
+	if(yych != ':') goto yy6;
+	yych = *++YYCURSOR;
+	if(yych == '$') goto yy6;
+	goto yy31;
+yy28:
+	++YYCURSOR;
+#line 50 "cvsignore.u.re"
+	{ APPEND(L"$" L"Log$"); goto loop; }
+#line 143 "<stdout>"
+yy30:
+	++YYCURSOR;
+	if(YYLIMIT == YYCURSOR) YYFILL(1);
+	yych = *YYCURSOR;
+yy31:
+	if(yych == 0x0000000A) goto yy6;
+	if(yych == '$') goto yy28;
+	goto yy30;
+yy32:
+	yych = *++YYCURSOR;
+	if(yych != 'v') goto yy6;
+	yych = *++YYCURSOR;
+	if(yych != 'i') goto yy6;
+	yych = *++YYCURSOR;
+	if(yych != 's') goto yy6;
+	yych = *++YYCURSOR;
+	if(yych != 'i') goto yy6;
+	yych = *++YYCURSOR;
+	if(yych != 'o') goto yy6;
+	yych = *++YYCURSOR;
+	if(yych != 'n') goto yy6;
+	yych = *++YYCURSOR;
+	if(yych == '$') goto yy40;
+	if(yych != ':') goto yy6;
+	yych = *++YYCURSOR;
+	if(yych == '$') goto yy6;
+	goto yy43;
+yy40:
+	++YYCURSOR;
+#line 51 "cvsignore.u.re"
+	{ APPEND(L"$" L"Revision$"); goto loop; }
+#line 175 "<stdout>"
+yy42:
+	++YYCURSOR;
+	if(YYLIMIT == YYCURSOR) YYFILL(1);
+	yych = *YYCURSOR;
+yy43:
+	if(yych == 0x0000000A) goto yy6;
+	if(yych == '$') goto yy40;
+	goto yy42;
+yy44:
+	yych = *++YYCURSOR;
+	if(yych != 'u') goto yy6;
+	yych = *++YYCURSOR;
+	if(yych != 'r') goto yy6;
+	yych = *++YYCURSOR;
+	if(yych != 'c') goto yy6;
+	yych = *++YYCURSOR;
+	if(yych != 'e') goto yy6;
+	yych = *++YYCURSOR;
+	if(yych == '$') goto yy50;
+	if(yych != ':') goto yy6;
+	yych = *++YYCURSOR;
+	if(yych == '$') goto yy6;
+	goto yy53;
+yy50:
+	++YYCURSOR;
+#line 52 "cvsignore.u.re"
+	{ APPEND(L"$" L"Source$"); goto loop; }
+#line 203 "<stdout>"
+yy52:
+	++YYCURSOR;
+	if(YYLIMIT == YYCURSOR) YYFILL(1);
+	yych = *YYCURSOR;
+yy53:
+	if(yych == 0x0000000A) goto yy6;
+	if(yych == '$') goto yy50;
+	goto yy52;
+}
+#line 55 "cvsignore.u.re"
+
+	}
+	output[outsize] = '\0';
+
+	// set the new size
+	*pSize = outsize;
+	
+	*pbChanged = (insize == outsize) ? 0 : 1;
+}
diff --git a/re2c/test/cvsignore.u.re b/re2c/test/cvsignore.u.re
new file mode 100755
index 00000000..442a45ed
--- /dev/null
+++ b/re2c/test/cvsignore.u.re
@@ -0,0 +1,63 @@
+
+#define YYFILL(n) if (cursor >= limit) break;
+#define YYCTYPE unsigned short
+#define YYCURSOR cursor
+#define YYLIMIT limit
+#define YYMARKER marker
+
+/*!re2c
+any     = (.|"\n");
+value	= (":" (.\"$")+)?;
+cvsdat	= "Date";
+cvsid	= "Id";
+cvslog	= "Log";
+cvsrev	= "Revision";
+cvssrc	= "Source";
+*/
+
+#define APPEND(text) \
+	append(output, outsize, text, sizeof(text) - sizeof(YYCTYPE))
+
+inline void append(YYCTYPE *output, size_t & outsize, const YYCTYPE * text, size_t len)
+{
+	memcpy(output + outsize, text, len);
+	outsize += (len / sizeof(YYCTYPE));
+}
+
+void scan(YYCTYPE *pText, size_t *pSize, int *pbChanged)
+{
+	// rule
+	// scan lines
+	// find $ in lines
+	//   compact $<keyword>: .. $ to $<keyword>$
+  
+	YYCTYPE *output;
+	const YYCTYPE *cursor, *limit, *marker;
+
+	cursor = marker = output = *pText;
+
+	size_t insize = *pSize;
+	size_t outsize = 0;
+
+	limit = cursor + insize;
+
+	while(1) {
+loop:
+/*!re2c
+
+"$" cvsdat value "$"	{ APPEND(L"$" L"Date$"); goto loop; }
+"$" cvsid  value "$"	{ APPEND(L"$" L"Id$"); goto loop; }
+"$" cvslog value "$"	{ APPEND(L"$" L"Log$"); goto loop; }
+"$" cvsrev value "$"	{ APPEND(L"$" L"Revision$"); goto loop; }
+"$" cvssrc value "$"	{ APPEND(L"$" L"Source$"); goto loop; }
+any						{ output[outsize++] = cursor[-1]; if (cursor >= limit) break; goto loop; }
+
+*/
+	}
+	output[outsize] = '\0';
+
+	// set the new size
+	*pSize = outsize;
+	
+	*pbChanged = (insize == outsize) ? 0 : 1;
+}
-- 
2.40.0