Added 38,642 missing characters to the Unicode database (first-last

ranges) -- but thanks to the 2.0 compression scheme, this doesn't add
a single byte to the resulting binaries (!)

Closes bug #117524

diff --git a/Lib/test/output/test_unicodedata b/Lib/test/output/test_unicodedata
index 6811fb409fda9c4bd19de65b72966ff5752888a3..3bd55757cc7660dc4a7810674b8a5847594a309a 100644 (file)
--- a/Lib/test/output/test_unicodedata
+++ b/Lib/test/output/test_unicodedata
@@ -1,5 +1,5 @@
 test_unicodedata
 Testing Unicode Database...
-Methods: 86793e1265f3cf5506e6ede8f69ab4deb973f3ea
-Functions: 5abd7e976848725e58f5834a0e5e37615f40d3a2
+Methods: 6c7a7c02657b69d0fdd7a7d174f573194bba2e18
+Functions: 41e1d4792185d6474a43c83ce4f593b1bdb01f8a
 API: ok

diff --git a/Modules/unicodedata_db.h b/Modules/unicodedata_db.h
index 7e15b728b9dbb734bfc19c1d79da44e4cd74e857..9f837d5594573035d22d73469f9f14c3b9fa7ac7 100644 (file)
--- a/Modules/unicodedata_db.h
+++ b/Modules/unicodedata_db.h
@@ -1,4 +1,4 @@
-/* this file was generated by ..\Tools\unicode\makeunicodedata.py 1.1 */
+/* this file was generated by tools\unicode\makeunicodedata.py 1.1 */
 
 /* a list of unique database records */
 const _PyUnicode_DatabaseRecord _PyUnicode_Database_Records[] = {
@@ -3634,7 +3634,7 @@ static const char *decomp_data[] = {
     "<narrow> 25CB",
     NULL
 };
-/* index tables used to find the right database record */
+/* index tables for the database records */
 #define SHIFT 5
 static unsigned char index1[] = {
     0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 8, 16, 17, 18, 18, 
@@ -3662,84 +3662,103 @@ static unsigned char index1[] = {
     35, 35, 35, 187, 169, 169, 155, 169, 169, 169, 169, 169, 169, 175, 188, 
     189, 190, 90, 103, 191, 90, 103, 192, 193, 194, 103, 103, 195, 128, 35, 
     35, 196, 197, 198, 199, 197, 200, 201, 202, 166, 166, 166, 203, 166, 166, 
-    204, 202, 205, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 206, 35, 35, 205, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    207, 35, 35, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    204, 202, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
     103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
-    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 208, 209, 210, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 205, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 205, 35, 35, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 206, 35, 35, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 207, 208, 209, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
     35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
     35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 35, 35, 35, 211, 35, 35, 212, 35, 35, 35, 35, 35, 35, 
+    35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 103, 103, 103, 103, 103, 103, 103, 210, 35, 35, 211, 35, 35, 35, 35, 
     35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 213, 212, 35, 35, 213, 212, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
+    35, 35, 35, 35, 212, 211, 35, 35, 212, 211, 35, 35, 35, 35, 35, 35, 35, 
     35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 213, 214, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
+    35, 35, 35, 35, 35, 212, 213, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
     35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
     35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
     35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
@@ -3750,10 +3769,10 @@ static unsigned char index1[] = {
     35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
     35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
     35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 
-    35, 35, 35, 35, 35, 215, 103, 103, 103, 103, 103, 103, 103, 103, 103, 
-    216, 35, 35, 35, 35, 35, 35, 217, 218, 219, 47, 47, 220, 221, 47, 47, 47, 
-    47, 47, 47, 47, 47, 47, 47, 222, 223, 47, 224, 47, 225, 226, 35, 227, 
-    228, 229, 47, 47, 47, 230, 231, 232, 233, 234, 235, 236, 237, 238, 
+    35, 35, 35, 35, 35, 35, 35, 214, 103, 103, 103, 103, 103, 103, 103, 103, 
+    103, 215, 35, 35, 35, 35, 35, 35, 216, 217, 218, 47, 47, 219, 220, 47, 
+    47, 47, 47, 47, 47, 47, 47, 47, 47, 221, 222, 47, 223, 47, 224, 225, 35, 
+    226, 227, 228, 47, 47, 47, 229, 230, 231, 232, 233, 234, 235, 236, 237, 
 };
 
 static unsigned char index2[] = {
@@ -4098,31 +4117,30 @@ static unsigned char index2[] = {
     42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 
     42, 0, 0, 0, 0, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 
     42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 
-    42, 42, 42, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-    0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+    42, 42, 42, 0, 0, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 
+    30, 30, 30, 30, 30, 30, 30, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 30, 30, 
+    30, 30, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
     0, 0, 0, 0, 0, 0, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 0, 
     0, 0, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 
     23, 0, 0, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 
     0, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 0, 23, 23, 23, 0, 23, 
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-    0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-    0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+    0, 30, 30, 30, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-    115, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+    0, 115, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 30, 30, 30, 30, 
-    30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-    0, 0, 0, 0, 0, 0, 0, 21, 21, 21, 21, 21, 21, 21, 0, 0, 0, 0, 0, 0, 0, 0, 
-    0, 0, 0, 0, 21, 21, 21, 21, 21, 0, 0, 0, 0, 0, 64, 117, 64, 64, 64, 64, 
-    64, 64, 64, 64, 64, 64, 11, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 
-    64, 64, 0, 64, 64, 64, 64, 64, 0, 64, 0, 64, 64, 0, 64, 64, 0, 64, 64, 
-    64, 64, 64, 64, 64, 64, 64, 64, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 30, 30, 30, 
+    30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+    0, 0, 0, 0, 0, 0, 0, 0, 21, 21, 21, 21, 21, 21, 21, 0, 0, 0, 0, 0, 0, 0, 
+    0, 0, 0, 0, 0, 21, 21, 21, 21, 21, 0, 0, 0, 0, 0, 64, 117, 64, 64, 64, 
+    64, 64, 64, 64, 64, 64, 64, 11, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 
+    64, 64, 64, 0, 64, 64, 64, 64, 64, 0, 64, 0, 64, 64, 0, 64, 64, 0, 64, 
+    64, 64, 64, 64, 64, 64, 64, 64, 64, 66, 66, 66, 66, 66, 66, 66, 66, 66, 
     66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 
-    66, 66, 66, 66, 66, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 66, 66, 66, 66, 
+    66, 66, 66, 66, 66, 66, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 66, 66, 66, 66, 
     66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 
     66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 
     66, 66, 94, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 66, 
@@ -4154,7 +4172,7 @@ static unsigned char index2[] = {
     0, 80, 80, 80, 23, 23, 0, 0, 
 };
 
-/* same, for the decomposition data */
+/* index tables for the decomposition data */
 #define DECOMP_SHIFT 5
 static unsigned char decomp_index1[] = {
     0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 0, 8, 9, 10, 11, 12, 0, 0, 0, 13, 14, 

diff --git a/Objects/unicodetype_db.h b/Objects/unicodetype_db.h
index 4d2fd7ddadbeb64326b69aaf83001d757f8aaed4..5d077ed702dea72e6c05574cab7a08c8526851ff 100644 (file)
--- a/Objects/unicodetype_db.h
+++ b/Objects/unicodetype_db.h
@@ -1,4 +1,4 @@
-/* this file was generated by Tools\unicode\makeunicodedata.py 1.1 */
+/* this file was generated by tools\unicode\makeunicodedata.py 1.1 */
 
 /* a list of unique character type descriptors */
 const _PyUnicode_TypeRecord _PyUnicode_TypeRecords[] = {
@@ -150,68 +150,83 @@ static unsigned char index1[] = {
     36, 36, 36, 36, 36, 36, 171, 23, 23, 152, 23, 23, 23, 23, 23, 23, 162, 
     172, 173, 174, 90, 48, 175, 90, 48, 176, 177, 178, 48, 48, 179, 127, 36, 
     36, 124, 23, 146, 180, 23, 181, 182, 183, 23, 23, 23, 184, 23, 23, 185, 
-    183, 186, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 187, 36, 36, 186, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 188, 36, 
+    183, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 186, 36, 36, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 187, 36, 
     36, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
     48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
-    48, 52, 189, 190, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 186, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
+    48, 52, 188, 189, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
     36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
     36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
+    36, 36, 36, 36, 36, 36, 36, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 190, 36, 36, 191, 
     36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
+    36, 36, 36, 36, 36, 36, 36, 36, 192, 191, 36, 36, 192, 191, 36, 36, 36, 
     36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
+    36, 36, 36, 36, 36, 36, 36, 36, 36, 192, 191, 36, 36, 36, 36, 36, 36, 36, 
     36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
     36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
     36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
@@ -222,25 +237,10 @@ static unsigned char index1[] = {
     36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
     36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
     36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 191, 36, 36, 192, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 193, 192, 36, 36, 193, 192, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 193, 192, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 
-    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 193, 48, 48, 48, 48, 48, 48, 
-    48, 48, 48, 194, 36, 36, 36, 36, 36, 36, 195, 196, 197, 48, 48, 198, 199, 
-    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 200, 201, 48, 202, 48, 203, 204, 
-    36, 151, 205, 206, 48, 48, 48, 207, 208, 2, 209, 210, 48, 211, 212, 213, 
+    36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 192, 48, 48, 48, 48, 48, 48, 
+    48, 48, 48, 193, 36, 36, 36, 36, 36, 36, 194, 195, 196, 48, 48, 197, 198, 
+    48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 199, 200, 48, 201, 48, 202, 203, 
+    36, 151, 204, 205, 48, 48, 48, 206, 207, 2, 208, 209, 48, 210, 211, 212, 
 };
 
 static unsigned char index2[] = {
@@ -531,20 +531,19 @@ static unsigned char index2[] = {
     1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
     1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 
-    0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-    0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 
+    0, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 
+    40, 40, 40, 40, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 40, 40, 40, 40, 40, 
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-    0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 
-    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 
-    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 
+    0, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 40, 40, 40, 0, 
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-    0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+    0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 40, 40, 40, 40, 40, 
-    40, 40, 40, 40, 40, 40, 40, 40, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-    0, 0, 0, 0, 0, 0, 16, 16, 16, 16, 16, 16, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
-    0, 0, 0, 16, 16, 16, 16, 16, 0, 0, 0, 0, 0, 40, 1, 40, 40, 40, 40, 40, 
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 40, 40, 40, 40, 
+    40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
+    0, 0, 0, 0, 0, 0, 0, 16, 16, 16, 16, 16, 16, 16, 0, 0, 0, 0, 0, 0, 0, 0, 
+    0, 0, 0, 0, 16, 16, 16, 16, 16, 0, 0, 0, 0, 0, 40, 1, 40, 40, 40, 40, 40, 
     40, 40, 40, 40, 40, 1, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 
     40, 0, 40, 40, 40, 40, 40, 0, 40, 0, 40, 40, 0, 40, 40, 0, 40, 40, 40, 
     40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 

diff --git a/Tools/unicode/makeunicodedata.py b/Tools/unicode/makeunicodedata.py
index 8c0c075815a1a1c0399da75fc16887a146774281..15841d7c519e9d4ef1441d23724c422787b2d8b4 100644 (file)
--- a/Tools/unicode/makeunicodedata.py
+++ b/Tools/unicode/makeunicodedata.py
@@ -9,6 +9,7 @@
 # 2000-09-25 fl   merged tim's splitbin fixes, separate decomposition table
 # 2000-09-25 fl   added character type table
 # 2000-09-26 fl   added LINEBREAK, DECIMAL, and DIGIT flags/fields
+# 2000-11-03 fl   expand first/last ranges
 #
 # written by Fredrik Lundh (fredrik@pythonware.com), September 2000
 #
@@ -39,10 +40,13 @@ SPACE_MASK = 0x20
 TITLE_MASK = 0x40
 UPPER_MASK = 0x80
 
-def maketables():
+def maketables(trace=0):
 
     unicode = UnicodeData(UNICODE_DATA)
 
+    print "--- Processing", UNICODE_DATA, "..."
+    print len(filter(None, unicode.table)), "characters"
+
     # extract unicode properties
     dummy = (0, 0, 0, 0)
     table = [dummy]
@@ -91,6 +95,11 @@ def maketables():
 
     FILE = "Modules/unicodedata_db.h"
 
+    print "--- Writing", FILE, "..."
+
+    print len(table), "unique properties"
+    print len(decomp_data), "unique decomposition entries"
+
     fp = open(FILE, "w")
     print >>fp, "/* this file was generated by %s %s */" % (SCRIPT, VERSION)
     print >>fp
@@ -125,7 +134,7 @@ def maketables():
     print >>fp, "};"
 
     # split record index table
-    index1, index2, shift = splitbins(index)
+    index1, index2, shift = splitbins(index, trace)
 
     print >>fp, "/* index tables for the database records */"
     print >>fp, "#define SHIFT", shift
@@ -133,7 +142,7 @@ def maketables():
     Array("index2", index2).dump(fp)
 
     # split decomposition index table
-    index1, index2, shift = splitbins(decomp_index)
+    index1, index2, shift = splitbins(decomp_index, trace)
 
     print >>fp, "/* index tables for the decomposition data */"
     print >>fp, "#define DECOMP_SHIFT", shift
@@ -200,12 +209,14 @@ def maketables():
                 table.append(item)
             index[char] = i
 
-    print len(table), "ctype entries"
-
     FILE = "Objects/unicodetype_db.h"
 
     fp = open(FILE, "w")
 
+    print "--- Writing", FILE, "..."
+
+    print len(table), "unique character type entries"
+
     print >>fp, "/* this file was generated by %s %s */" % (SCRIPT, VERSION)
     print >>fp
     print >>fp, "/* a list of unique character type descriptors */"
@@ -216,7 +227,7 @@ def maketables():
     print >>fp
 
     # split decomposition index table
-    index1, index2, shift = splitbins(index)
+    index1, index2, shift = splitbins(index, trace)
 
     print >>fp, "/* type indexes */"
     print >>fp, "#define SHIFT", shift
@@ -233,7 +244,7 @@ import string, sys
 
 class UnicodeData:
 
-    def __init__(self, filename):
+    def __init__(self, filename, expand=1):
         file = open(filename)
         table = [None] * 65536
         while 1:
@@ -244,6 +255,22 @@ class UnicodeData:
             char = string.atoi(s[0], 16)
             table[char] = s
 
+        # expand first-last ranges (ignore surrogates and private use)
+        if expand:
+            field = None
+            for i in range(0, 0xD800):
+                s = table[i]
+                if s:
+                    if s[1][-6:] == "First>":
+                        s[1] = ""
+                        field = s[:]
+                    elif s[1][-5:] == "Last>":
+                        s[1] = ""
+                        field = None
+                elif field:
+                    field[0] = hex(i)
+                    table[i] = field
+
         # public attributes
         self.filename = filename
         self.table = table
@@ -306,8 +333,9 @@ def splitbins(t, trace=0):
         t[i] == t2[(t1[i >> shift] << shift) + (i & mask)]
     where mask is a bitmask isolating the last "shift" bits.
 
-    If optional arg trace is true (default false), progress info is
-    printed to sys.stderr.
+    If optional arg trace is non-zero (default zero), progress info
+    is printed to sys.stderr.  The higher the value, the more info
+    you'll get.
     """
 
     import sys
@@ -341,7 +369,7 @@ def splitbins(t, trace=0):
             t1.append(index >> shift)
         # determine memory size
         b = len(t1)*getsize(t1) + len(t2)*getsize(t2)
-        if trace:
+        if trace > 1:
             dump(t1, t2, shift, b)
         if b < bytes:
             best = t1, t2, shift
@@ -358,4 +386,4 @@ def splitbins(t, trace=0):
     return best
 
 if __name__ == "__main__":
-    maketables()
+    maketables(1)