#include <iostream>
#include <vector>
+#include <algorithm>
#include "toolutil.h"
#include "uoptions.h"
#include "cmemory.h"
#include "unicode/uchar.h"
#include "unicode/errorcode.h"
#include "unicode/uniset.h"
+#include "unicode/uscript.h"
#include "unicode/putil.h"
#include "unicode/umutablecptrie.h"
#include "writesrc.h"
UCPTrieType trieType = UCPTRIE_TYPE_SMALL;
const char* destdir = "";
+// Mask constants for modified values in the Script CodePointTrie, values are logically 12-bits.
+int16_t DATAEXPORT_SCRIPT_X_WITH_COMMON = 0x0400;
+int16_t DATAEXPORT_SCRIPT_X_WITH_INHERITED = 0x0800;
+int16_t DATAEXPORT_SCRIPT_X_WITH_OTHER = 0x0c00;
+
+// TODO(ICU-21821): Replace this with a call to a library function
+int32_t scxCodePoints[] = {
+ 7415, 7377, 7380, 7387, 7390, 7391, 7394, 7395, 7396, 7397,
+ 7398, 7399, 7400, 7403, 7404, 7406, 7407, 7408, 7409, 113824,
+ 113825, 113826, 113827, 834, 837, 7616, 7617, 12294, 12350, 12351,
+ 12688, 12689, 12690, 12691, 12692, 12693, 12694, 12695, 12696, 12697,
+ 12698, 12699, 12700, 12701, 12702, 12703, 12736, 12737, 12738, 12739,
+ 12740, 12741, 12742, 12743, 12744, 12745, 12746, 12747, 12748, 12749,
+ 12750, 12751, 12752, 12753, 12754, 12755, 12756, 12757, 12758, 12759,
+ 12760, 12761, 12762, 12763, 12764, 12765, 12766, 12767, 12768, 12769,
+ 12770, 12771, 12832, 12833, 12834, 12835, 12836, 12837, 12838, 12839,
+ 12840, 12841, 12842, 12843, 12844, 12845, 12846, 12847, 12848, 12849,
+ 12850, 12851, 12852, 12853, 12854, 12855, 12856, 12857, 12858, 12859,
+ 12860, 12861, 12862, 12863, 12864, 12865, 12866, 12867, 12868, 12869,
+ 12870, 12871, 12928, 12929, 12930, 12931, 12932, 12933, 12934, 12935,
+ 12936, 12937, 12938, 12939, 12940, 12941, 12942, 12943, 12944, 12945,
+ 12946, 12947, 12948, 12949, 12950, 12951, 12952, 12953, 12954, 12955,
+ 12956, 12957, 12958, 12959, 12960, 12961, 12962, 12963, 12964, 12965,
+ 12966, 12967, 12968, 12969, 12970, 12971, 12972, 12973, 12974, 12975,
+ 12976, 12992, 12993, 12994, 12995, 12996, 12997, 12998, 12999, 13000,
+ 13001, 13002, 13003, 13055, 13144, 13145, 13146, 13147, 13148, 13149,
+ 13150, 13151, 13152, 13153, 13154, 13155, 13156, 13157, 13158, 13159,
+ 13160, 13161, 13162, 13163, 13164, 13165, 13166, 13167, 13168, 13179,
+ 13180, 13181, 13182, 13183, 13280, 13281, 13282, 13283, 13284, 13285,
+ 13286, 13287, 13288, 13289, 13290, 13291, 13292, 13293, 13294, 13295,
+ 13296, 13297, 13298, 13299, 13300, 13301, 13302, 13303, 13304, 13305,
+ 13306, 13307, 13308, 13309, 13310, 119648, 119649, 119650, 119651, 119652,
+ 119653, 119654, 119655, 119656, 119657, 119658, 119659, 119660, 119661, 119662,
+ 119663, 119664, 119665, 127568, 127569, 867, 868, 869, 870, 871,
+ 872, 873, 874, 875, 876, 877, 878, 879, 7418, 7674,
+ 66272, 66273, 66274, 66275, 66276, 66277, 66278, 66279, 66280, 66281,
+ 66282, 66283, 66284, 66285, 66286, 66287, 66288, 66289, 66290, 66291,
+ 66292, 66293, 66294, 66295, 66296, 66297, 66298, 66299, 1748, 64830,
+ 64831, 1611, 1612, 1613, 1614, 1615, 1616, 1617, 1618, 1619,
+ 1620, 1621, 1648, 65010, 65021, 7381, 7382, 7384, 7393, 7402,
+ 7405, 7413, 7414, 43249, 12330, 12331, 12332, 12333, 43471, 65794,
+ 65847, 65848, 65849, 65850, 65851, 65852, 65853, 65854, 65855, 1156,
+ 1159, 11843, 42607, 1157, 1158, 1155, 7672, 7379, 7411, 7416,
+ 7417, 7401, 7383, 7385, 7388, 7389, 7392, 43251, 4347, 3046,
+ 3047, 3048, 3049, 3050, 3051, 3052, 3053, 3054, 3055, 3056,
+ 3057, 3058, 3059, 70401, 70403, 70459, 70460, 73680, 73681, 73683,
+ 2790, 2791, 2792, 2793, 2794, 2795, 2796, 2797, 2798, 2799,
+ 2662, 2663, 2664, 2665, 2666, 2667, 2668, 2669, 2670, 2671,
+ 42752, 42753, 42754, 42755, 42756, 42757, 42758, 42759, 12337, 12338,
+ 12339, 12340, 12341, 12441, 12442, 12443, 12444, 12448, 12540, 65392,
+ 65438, 65439, 3302, 3303, 3304, 3305, 3306, 3307, 3308, 3309,
+ 3310, 3311, 8239, 68338, 6146, 6147, 6149, 1564, 1632, 1633,
+ 1634, 1635, 1636, 1637, 1638, 1639, 1640, 1641, 2534, 2535,
+ 2536, 2537, 2538, 2539, 2540, 2541, 2542, 2543, 4160, 4161,
+ 4162, 4163, 4164, 4165, 4166, 4167, 4168, 4169, 65792, 65793,
+ 65799, 65800, 65801, 65802, 65803, 65804, 65805, 65806, 65807, 65808,
+ 65809, 65810, 65811, 65812, 65813, 65814, 65815, 65816, 65817, 65818,
+ 65819, 65820, 65821, 65822, 65823, 65824, 65825, 65826, 65827, 65828,
+ 65829, 65830, 65831, 65832, 65833, 65834, 65835, 65836, 65837, 65838,
+ 65839, 65840, 65841, 65842, 65843, 7412, 8432, 12348, 12349, 43310,
+ 7376, 7378, 5941, 5942, 2406, 2407, 2408, 2409, 2410, 2411,
+ 2412, 2413, 2414, 2415, 12291, 12307, 12316, 12317, 12318, 12319,
+ 12336, 12343, 65093, 65094, 1548, 1563, 12289, 12290, 12296, 12297,
+ 12298, 12299, 12300, 12301, 12302, 12303, 12304, 12305, 12308, 12309,
+ 12310, 12311, 12312, 12313, 12314, 12315, 12539, 65377, 65378, 65379,
+ 65380, 65381, 7386, 1567, 7410, 1600, 43062, 43063, 43064, 43065,
+ 2386, 2385, 43059, 43060, 43061, 43056, 43057, 43058, 2404, 2405
+ };
+
void handleError(ErrorCode& status, const char* context) {
if (status.isFailure()) {
std::cerr << "Error: " << context << ": " << status.errorName() << std::endl;
usrc_writeUCPTrie(f, shortPropName, utrie.getAlias(), UPRV_TARGET_SYNTAX_TOML);
}
+void dumpScriptExtensions(FILE* f) {
+ IcuToolErrorCode status("icuexportdata: dumpScriptExtensions");
+
+ fputs("[[script_extensions]]\n", f);
+ const char* scxFullPropName = u_getPropertyName(UCHAR_SCRIPT_EXTENSIONS, U_LONG_PROPERTY_NAME);
+ const char* scxShortPropName = u_getPropertyName(UCHAR_SCRIPT_EXTENSIONS, U_SHORT_PROPERTY_NAME);
+ fprintf(f, "long_name = \"%s\"\n", scxFullPropName);
+ if (scxShortPropName) fprintf(f, "short_name = \"%s\"\n", scxShortPropName);
+
+ // We want to use 16 bits for our exported trie of sc/scx data because we
+ // need 12 bits to match the 12 bits of data stored for sc/scx in the trie
+ // in the uprops.icu data file.
+ UCPTrieValueWidth scWidth = UCPTRIE_VALUE_BITS_16;
+
+ // Create a mutable UCPTrie builder populated with Script property values data.
+ const UCPMap* scInvMap = u_getIntPropertyMap(UCHAR_SCRIPT, status);
+ handleError(status, scxFullPropName);
+ LocalUMutableCPTriePointer builder(umutablecptrie_fromUCPMap(scInvMap, status));
+ handleError(status, scxFullPropName);
+
+ // The values for the output scx companion array.
+ // Invariant is that all subvectors are distinct.
+ std::vector< std::vector<uint16_t> > outputDedupVec;
+
+ // The sc/scx companion array is an array of arrays (of script codes)
+ fputs("script_code_array = [\n", f);
+ for(const UChar32 cp : scxCodePoints) {
+ // Get the Script value
+ uint32_t scVal = umutablecptrie_get(builder.getAlias(), cp);
+ // Get the Script_Extensions value (array of Script codes)
+ const int32_t SCX_ARRAY_CAPACITY = 32;
+ UScriptCode scxValArray[SCX_ARRAY_CAPACITY];
+ int32_t numScripts = uscript_getScriptExtensions(cp, scxValArray, SCX_ARRAY_CAPACITY, status);
+ handleError(status, scxFullPropName);
+
+ // Convert the scx array into a vector
+ std::vector<uint16_t> scxValVec;
+ for(int i = 0; i < numScripts; i++) {
+ scxValVec.push_back(scxValArray[i]);
+ }
+ // Ensure that it is sorted
+ std::sort(scxValVec.begin(), scxValVec.end());
+ // Copy the Script value into the first position of the scx array only
+ // if we have the "other" case (Script value is not Common nor Inherited).
+ // This offers faster access when users want only the Script value.
+ if (scVal != USCRIPT_COMMON && scVal != USCRIPT_INHERITED) {
+ scxValVec.insert(scxValVec.begin(), scVal);
+ }
+
+ // See if there is already an scx value array matching the newly built one.
+ // If there is, then use its index.
+ // If not, then append the new value array.
+ bool isScxValUnique = true;
+ size_t outputIndex = 0;
+ for (outputIndex = 0; outputIndex < outputDedupVec.size(); outputIndex++) {
+ if (outputDedupVec[outputIndex] == scxValVec) {
+ isScxValUnique = false;
+ break;
+ }
+ }
+
+ if (isScxValUnique) {
+ outputDedupVec.push_back(scxValVec);
+ usrc_writeArray(f, " [", scxValVec.data(), 16, scxValVec.size(), " ", "],\n");
+ }
+
+ // We must update the value in the UCPTrie for the code point to contain:
+ // 9..0 the Script code in the lower 10 bits when 11..10 is 0, else it is
+ // the index into the companion array
+ // 11..10 the same higher-order 2 bits in the trie in uprops.icu indicating whether
+ // 3: other
+ // 2: Script=Inherited
+ // 1: Script=Common
+ // 0: Script=value in 9..0 (N/A because we are in this loop to create the companion array for non-0 cases)
+ uint16_t mask = 0;
+ if (scVal == USCRIPT_COMMON) {
+ mask = DATAEXPORT_SCRIPT_X_WITH_COMMON;
+ } else if (scVal == USCRIPT_INHERITED) {
+ mask = DATAEXPORT_SCRIPT_X_WITH_INHERITED;
+ } else {
+ mask = DATAEXPORT_SCRIPT_X_WITH_OTHER;
+ }
+
+ // The new trie value is the index into the new array with the high order bits set
+ uint32_t newScVal = outputIndex | mask;
+
+ // Update the code point in the mutable trie builder with the trie value
+ umutablecptrie_set(builder.getAlias(), cp, newScVal, status);
+ handleError(status, scxFullPropName);
+ }
+ fputs("]\n\n", f); // Print the TOML close delimiter for the outer array.
+
+ // Convert from mutable trie builder to immutable trie.
+ LocalUCPTriePointer utrie(umutablecptrie_buildImmutable(
+ builder.getAlias(),
+ trieType,
+ scWidth,
+ status));
+ handleError(status, scxFullPropName);
+
+ fputs("[script_extensions.code_point_trie]\n", f);
+ usrc_writeUCPTrie(f, scxShortPropName, utrie.getAlias(), UPRV_TARGET_SYNTAX_TOML);
+}
+
FILE* prepareOutputFile(const char* basename) {
IcuToolErrorCode status("icuexportdata");
CharString outFileName;
dumpBinaryProperty(propEnum, f);
} else if (UCHAR_INT_START <= propEnum && propEnum <= UCHAR_INT_LIMIT) {
dumpEnumeratedProperty(propEnum, f);
+ } else if (propEnum == UCHAR_SCRIPT_EXTENSIONS) {
+ dumpScriptExtensions(f);
} else {
std::cerr << "Don't know how to write property: " << propEnum << std::endl;
return U_INTERNAL_PROGRAM_ERROR;
projects / icu / commitdiff