+++ /dev/null
-// © 2016 and later: Unicode, Inc. and others.
-// License & terms of use: http://www.unicode.org/copyright.html#License
-/*
-**********************************************************************
-* Copyright (c) 2002-2010, International Business Machines
-* Corporation and others. All Rights Reserved.
-**********************************************************************
-* Author: Mark Davis
-**********************************************************************
-*/
-package com.ibm.icu.dev.test.cldr;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.io.StringWriter;
-import java.text.ParseException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.TreeSet;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import javax.xml.parsers.SAXParser;
-import javax.xml.parsers.SAXParserFactory;
-
-import org.junit.Ignore;
-import org.junit.Test;
-import org.xml.sax.Attributes;
-import org.xml.sax.SAXException;
-import org.xml.sax.helpers.DefaultHandler;
-
-import com.ibm.icu.dev.test.TestFmwk;
-import com.ibm.icu.text.DateFormat;
-import com.ibm.icu.text.NumberFormat;
-import com.ibm.icu.text.SimpleDateFormat;
-import com.ibm.icu.text.UTF16;
-import com.ibm.icu.text.UnicodeSet;
-import com.ibm.icu.util.Currency;
-import com.ibm.icu.util.TimeZone;
-import com.ibm.icu.util.ULocale;
-
-/**
- * This is a test file that takes in the CLDR XML test files and test against
- * ICU4J. This test file is used to verify that ICU4J is implemented correctly.
- * As it stands, the test generates all the errors to the console by logging it.
- * The logging is only possible if "-v" or verbose is set as an argument.
- * This will allow users to know what problems occurred within CLDR and ICU.
- * Collator was disabled in this test file and therefore will be skipped.
- *
- * Instructions:
- * 1) In order for this to work correctly, you must download the latest CLDR data
- * in the form of XML. You must also set the CLDR directory using:
- * -DCLDR_DIRECTORY=<top level of cldr>
- * 2) You may also consider increasing the memory using -Xmx512m.
- * 3) For speed purposes, you may consider creating a temporary directory for the
- * CLDR cache using:
- * -DCLDR_DTD_CACHE=<cldr cache directory>
- * 4) You may use other environment variables to narrow down your tests using:
- * -DXML_MATCH=".*"
- * -DXML_MATCH="de.*" (or whatever regex you want) to just test certain locales.
- * -DTEST_MATCH="zone.*" (or whatever regex you want) to just test collation, numbers, etc.
- * -DZONE_MATCH="(?!America/Argentina).*"
- * -DZONE_MATCH=".*Moscow.*" (to only test certain zones)
-
- * @author medavis
- * @author John Huan Vu (johnvu@us.ibm.com)
- */
-public class TestCLDRVsICU extends TestFmwk {
- static final boolean DEBUG = false;
-
- // ULocale uLocale = ULocale.ENGLISH;
- // Locale oLocale = Locale.ENGLISH; // TODO Drop once ICU4J has ULocale everywhere
- // static PrintWriter log;
- SAXParser SAX;
- static Matcher LOCALE_MATCH, TEST_MATCH, ZONE_MATCH;
- static String CLDR_DIRECTORY;
- static {
- System.out.println();
- LOCALE_MATCH = getEnvironmentRegex("XML_MATCH", ".*");
- TEST_MATCH = getEnvironmentRegex("TEST_MATCH", ".*");
- ZONE_MATCH = getEnvironmentRegex("ZONE_MATCH", ".*");
-
- // CLDR_DIRECTORY is where all the CLDR XML test files are located
- // WARNING: THIS IS TEMPORARY DIRECTORY UNTIL THE FILES ARE STRAIGHTENED OUT
- CLDR_DIRECTORY = getEnvironmentString("CLDR_DIRECTORY", "C:\\Unicode-CVS2\\cldr\\");
- System.out.println();
- }
-
- private static Matcher getEnvironmentRegex(String key, String defaultValue) {
- return Pattern.compile(getEnvironmentString(key, defaultValue)).matcher("");
- }
-
- private static String getEnvironmentString(String key, String defaultValue) {
- String temp = System.getProperty(key);
- if (temp == null)
- temp = defaultValue;
- else
- System.out.print("-D" + key + "=\"" + temp + "\" ");
- return temp;
- }
-
- Set allLocales = new TreeSet();
-
- // TODO(junit): seems to be failing with missing locales - maybe rewrite as parameterized
- @Ignore
- @Test
- public void TestFiles() throws SAXException, IOException {
- // only get ICU's locales
- Set s = new TreeSet();
- addLocales(NumberFormat.getAvailableULocales(), s);
- addLocales(DateFormat.getAvailableULocales(), s);
-
- // johnvu: Collator was originally disabled
- // addLocales(Collator.getAvailableULocales(), s);
-
- // filter, to make tracking down bugs easier
- for (Iterator it = s.iterator(); it.hasNext();) {
- String locale = (String) it.next();
- if (!LOCALE_MATCH.reset(locale).matches())
- continue;
- _test(locale);
- }
- }
-
- public void addLocales(ULocale[] list, Collection s) {
- for (int i = 0; i < list.length; ++i) {
- allLocales.add(list[i].toString());
- s.add(list[i].getLanguage());
- }
- }
-
- public String getLanguage(ULocale uLocale) {
- String result = uLocale.getLanguage();
- String script = uLocale.getScript();
- if (script.length() != 0)
- result += "_" + script;
- return result;
- }
-
- private void _test(String localeName) throws SAXException, IOException {
- // uLocale = new ULocale(localeName);
- // oLocale = uLocale.toLocale();
-
- File f = new File(CLDR_DIRECTORY, "test/" + localeName + ".xml");
- logln("Testing " + f.getCanonicalPath());
- SAX.parse(f, DEFAULT_HANDLER);
- }
-
- private static class ToHex {
- public String transliterate(String in) {
- StringBuilder sb = new StringBuilder();
- for (int i = 0; i < in.length(); ++i) {
- char c = in.charAt(i);
- sb.append("\\u");
- if (c < 1000) {
- sb.append('0');
- if (c < 100) {
- sb.append('0');
- if (c < 10) {
- sb.append('0');
- }
- }
- }
- sb.append(Integer.toHexString((int) c));
- }
- return sb.toString();
- }
- }
-
- // static Transliterator toUnicode = Transliterator.getInstance("any-hex");
- private static final ToHex toUnicode = new ToHex();
-
- static public String showString(String in) {
- return "\u00AB" + in + "\u00BB (" + toUnicode.transliterate(in) + ")";
- }
-
- // ============ SAX Handler Infrastructure ============
-
- abstract public class Handler {
- Map settings = new TreeMap();
- String name;
- List currentLocales = new ArrayList();
- int failures = 0;
-
- void setName(String name) {
- this.name = name;
- }
-
- void set(String attributeName, String attributeValue) {
- // if (DEBUG) logln(attributeName + " => " + attributeValue);
- settings.put(attributeName, attributeValue);
- }
-
- void checkResult(String value) {
- if (settings.get("draft").equals("unconfirmed") || settings.get("draft").equals("provisional")) {
- return; // skip draft
- }
- ULocale ul = new ULocale("xx");
- try {
- for (int i = 0; i < currentLocales.size(); ++i) {
- ul = (ULocale) currentLocales.get(i);
- // loglnSAX(" Checking " + ul + "(" + ul.getDisplayName(ULocale.ENGLISH) + ")" + " for " + name);
- handleResult(ul, value);
- if (failures != 0) {
- errln("\tTotal Failures: " + failures + "\t" + ul + "(" + ul.getDisplayName(ULocale.ENGLISH)
- + ")");
- failures = 0;
- }
- }
- } catch (Exception e) {
- StringWriter sw = new StringWriter();
- PrintWriter pw = new PrintWriter(sw);
- e.printStackTrace(pw);
- pw.flush();
- errln("Exception: Locale: " + ul + ",\tValue: <" + value + ">\r\n" + sw.toString());
- }
- }
-
- public void loglnSAX(String message) {
- String temp = message + "\t[" + name;
- for (Iterator it = settings.keySet().iterator(); it.hasNext();) {
- String attributeName = (String) it.next();
- String attributeValue = (String) settings.get(attributeName);
- temp += " " + attributeName + "=<" + attributeValue + ">";
- }
- logln(temp + "]");
- }
-
- int lookupValue(Object x, Object[] list) {
- for (int i = 0; i < list.length; ++i) {
- if (x.equals(list[i]))
- return i;
- }
- loglnSAX("Unknown String: " + x);
- return -1;
- }
-
- abstract void handleResult(ULocale currentLocale, String value) throws Exception;
-
- /**
- * @param attributes
- */
- public void setAttributes(Attributes attributes) {
- String localeList = attributes.getValue("locales");
- String[] currentLocaleString = new String[50];
- com.ibm.icu.impl.Utility.split(localeList, ' ', currentLocaleString);
- currentLocales.clear();
- for (int i = 0; i < currentLocaleString.length; ++i) {
- if (currentLocaleString[i].length() == 0)
- continue;
- if (allLocales.contains("")) {
- logln("Skipping locale, not in ICU4J: " + currentLocaleString[i]);
- continue;
- }
- currentLocales.add(new ULocale(currentLocaleString[i]));
- }
- if (DEBUG)
- logln("Setting locales: " + currentLocales);
- }
- }
-
- public Handler getHandler(String name, Attributes attributes) {
- if (DEBUG)
- logln("Creating Handler: " + name);
- Handler result = (Handler) RegisteredHandlers.get(name);
- if (result == null)
- logln("Unexpected test type: " + name);
- else {
- result.setAttributes(attributes);
- }
- return result;
- }
-
- public void addHandler(String name, Handler handler) {
- if (!TEST_MATCH.reset(name).matches())
- handler = new NullHandler();
- handler.setName(name);
- RegisteredHandlers.put(name, handler);
- }
-
- Map RegisteredHandlers = new HashMap();
-
- class NullHandler extends Handler {
- void handleResult(ULocale currentLocale, String value) throws Exception {
- }
- }
-
- // ============ Statics for Date/Number Support ============
-
- static TimeZone utc = TimeZone.getTimeZone("GMT");
- static DateFormat iso = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
- {
- iso.setTimeZone(utc);
- }
-
- static int[] DateFormatValues = { -1, DateFormat.SHORT, DateFormat.MEDIUM, DateFormat.LONG, DateFormat.FULL };
-
- // The following are different data format types that are part of the parameters in CLDR
- static String[] DateFormatNames = { "none", "short", "medium", "long", "full" };
-
- // The following are different number types that are part of the parameters in CLDR
- static String[] NumberNames = { "standard", "integer", "decimal", "percent", "scientific", "GBP" };
-
-
- // ============ Handler for Collation ============
- static UnicodeSet controlsAndSpace = new UnicodeSet("[:cc:]");
-
- static String remove(String in, UnicodeSet toRemove) {
- int cp;
- StringBuffer result = new StringBuffer();
- for (int i = 0; i < in.length(); i += UTF16.getCharCount(cp)) {
- cp = UTF16.charAt(in, i);
- if (!toRemove.contains(cp))
- UTF16.append(result, cp);
- }
- return result.toString();
- }
-
- {
- // johnvu: Collator was originally disabled
- // TODO (dougfelt) move this test
- /*
- addHandler("collation", new Handler() {
- public void handleResult(ULocale currentLocale, String value) {
- Collator col = Collator.getInstance(currentLocale);
- String lastLine = "";
- int count = 0;
- for (int pos = 0; pos < value.length();) {
- int nextPos = value.indexOf('\n', pos);
- if (nextPos < 0)
- nextPos = value.length();
- String line = value.substring(pos, nextPos);
- line = remove(line, controlsAndSpace); HACK for SAX
- if (line.trim().length() != 0) { HACK for SAX
- int comp = col.compare(lastLine, line);
- if (comp > 0) {
- failures++;
- errln("\tLine " + (count + 1) + "\tFailure: "
- + showString(lastLine) + " should be leq "
- + showString(line));
- } else if (DEBUG) {
- logln("OK: " + line);
- }
- lastLine = line;
- }
- pos = nextPos + 1;
- count++;
- }
- }
- });
- */
-
- // ============ Handler for Numbers ============
- addHandler("number", new Handler() {
- public void handleResult(ULocale locale, String result) {
- NumberFormat nf = null;
- double v = Double.NaN;
- for (Iterator it = settings.keySet().iterator(); it.hasNext();) {
- String attributeName = (String) it.next();
- String attributeValue = (String) settings.get(attributeName);
-
- // Checks if the attribute name is a draft and whether
- // or not it has been approved / contributed by CLDR yet
- // otherwise, skips it because it is most likely rejected by ICU
- if (attributeName.equals("draft")) {
- if (attributeValue.indexOf("approved") == -1 && attributeValue.indexOf("contributed") == -1) {
- break;
- }
- continue;
- }
-
- // Update the value to be checked
- if (attributeName.equals("input")) {
- v = Double.parseDouble(attributeValue);
- continue;
- }
-
- // At this point, it must be a numberType
- int index = lookupValue(attributeValue, NumberNames);
-
- if (DEBUG)
- logln("Getting number format for " + locale);
- switch (index) {
- case 0:
- nf = NumberFormat.getInstance(locale);
- break;
- case 1:
- nf = NumberFormat.getIntegerInstance(locale);
- break;
- case 2:
- nf = NumberFormat.getNumberInstance(locale);
- break;
- case 3:
- nf = NumberFormat.getPercentInstance(locale);
- break;
- case 4:
- nf = NumberFormat.getScientificInstance(locale);
- break;
- default:
- nf = NumberFormat.getCurrencyInstance(locale);
- nf.setCurrency(Currency.getInstance(attributeValue));
- break;
- }
- String temp = nf.format(v).trim();
- result = result.trim(); // HACK because of SAX
- if (!temp.equals(result)) {
- logln("Number: Locale: " + locale +
- "\n\tType: " + attributeValue +
- "\n\tDraft: " + settings.get("draft") +
- "\n\tCLDR: <" + result + ">" +
- "\n\tICU: <" + temp + ">");
- }
-
- }
- }
- });
-
- // ============ Handler for Dates ============
- addHandler("date", new Handler() {
- public void handleResult(ULocale locale, String result) throws ParseException {
- int dateFormat = 0;
- int timeFormat = 0;
- Date date = new Date();
- boolean approved = true;
-
- for (Iterator it = settings.keySet().iterator(); it.hasNext();) {
- String attributeName = (String) it.next();
- String attributeValue = (String) settings.get(attributeName);
-
- // Checks if the attribute name is a draft and whether
- // or not it has been approved / contributed by CLDR yet
- // otherwise, skips it because it is most likely rejected by ICU
- if (attributeName.equals("draft")) {
- if (attributeValue.indexOf("approved") == -1 && attributeValue.indexOf("contributed") == -1) {
- approved = false;
- break;
- }
- continue;
- }
-
- // Update the value to be checked
- if (attributeName.equals("input")) {
- date = iso.parse(attributeValue);
- continue;
- }
- // At this point, it must be either dateType or timeType
- int index = lookupValue(attributeValue, DateFormatNames);
- if (attributeName.equals("dateType"))
- dateFormat = index;
- else if (attributeName.equals("timeType"))
- timeFormat = index;
-
- }
-
- // The attribute value must be approved in order to be checked,
- // if it hasn't been approved, it shouldn't be checked if it
- // matches with ICU
- if (approved) {
- SimpleDateFormat dt = getDateFormat(locale, dateFormat, timeFormat);
- dt.setTimeZone(utc);
- String temp = dt.format(date).trim();
- result = result.trim(); // HACK because of SAX
- if (!temp.equals(result)) {
- logln("DateTime: Locale: " + locale +
- "\n\tDate: " + DateFormatNames[dateFormat] +
- "\n\tTime: " + DateFormatNames[timeFormat] +
- "\n\tDraft: " + settings.get("draft") +
- "\n\tCLDR: <" + result + "> " +
- "\n\tICU: <" + temp + ">");
- }
- }
- }
-
- private SimpleDateFormat getDateFormat(ULocale locale, int dateFormat, int timeFormat) {
- if (DEBUG)
- logln("Getting date/time format for " + locale);
- if (DEBUG && "ar_EG".equals(locale.toString())) {
- logln("debug here");
- }
- DateFormat dt;
- if (dateFormat == 0) {
- dt = DateFormat.getTimeInstance(DateFormatValues[timeFormat], locale);
- if (DEBUG)
- System.out.print("getTimeInstance");
- } else if (timeFormat == 0) {
- dt = DateFormat.getDateInstance(DateFormatValues[dateFormat], locale);
- if (DEBUG)
- System.out.print("getDateInstance");
- } else {
- dt = DateFormat.getDateTimeInstance(DateFormatValues[dateFormat], DateFormatValues[timeFormat],
- locale);
- if (DEBUG)
- System.out.print("getDateTimeInstance");
- }
- if (DEBUG)
- logln("\tinput:\t" + dateFormat + ", " + timeFormat + " => " + ((SimpleDateFormat) dt).toPattern());
- return (SimpleDateFormat) dt;
- }
- });
-
- // ============ Handler for Zones ============
- addHandler("zoneFields", new Handler() {
- String date = "";
- String zone = "";
- String parse = "";
- String pattern = "";
-
- public void handleResult(ULocale locale, String result) throws ParseException {
- for (Iterator it = settings.keySet().iterator(); it.hasNext();) {
- String attributeName = (String) it.next();
- String attributeValue = (String) settings.get(attributeName);
- if (attributeName.equals("date")) {
- date = attributeValue;
- } else if (attributeName.equals("field")) {
- pattern = attributeValue;
- } else if (attributeName.equals("zone")) {
- zone = attributeValue;
- } else if (attributeName.equals("parse")) {
- parse = attributeValue;
- }
- }
-
- if (!ZONE_MATCH.reset(zone).matches()) return;
- Date dateValue = iso.parse(date);
- SimpleDateFormat field = new SimpleDateFormat(pattern, locale);
- field.setTimeZone(TimeZone.getTimeZone(zone));
- String temp = field.format(dateValue).trim();
- // SKIP PARSE FOR NOW
- result = result.trim(); // HACK because of SAX
- if (!temp.equals(result)) {
- temp = field.format(dateValue).trim(); // call again for debugging
- logln("Zone Format: Locale: " + locale
- + "\n\tZone: " + zone
- + "\n\tDate: " + date
- + "\n\tField: " + pattern
- + "\n\tParse: " + parse
- + "\n\tDraft: " + settings.get("draft")
- + "\n\tCLDR: <" + result
- + ">\n\tICU: <" + temp + ">");
- }
- }
- });
- }
-
- // ============ Gorp for SAX ============
-
- {
- try {
- SAXParserFactory factory = SAXParserFactory.newInstance();
- factory.setValidating(true);
- SAX = factory.newSAXParser();
- } catch (Exception e) {
- throw new IllegalArgumentException("SAXParserFacotry was unable to start.");
- }
- }
-
- DefaultHandler DEFAULT_HANDLER = new DefaultHandler() {
- static final boolean DEBUG = false;
- StringBuffer lastChars = new StringBuffer();
- // boolean justPopped = false;
- Handler handler;
-
- public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException {
- // data.put(new ContextStack(contextStack), lastChars);
- // lastChars = "";
- try {
- if (qName.equals("cldrTest")) {
- // skip
- } else if (qName.equals("result") && handler != null) {
- for (int i = 0; i < attributes.getLength(); ++i) {
- handler.set(attributes.getQName(i), attributes.getValue(i));
- }
- } else {
- handler = getHandler(qName, attributes);
- // handler.set("locale", uLocale.toString());
- }
- // if (DEBUG) logln("startElement:\t" + contextStack);
- // justPopped = false;
- } catch (RuntimeException e) {
- e.printStackTrace();
- throw e;
- }
- }
-
- public void endElement(String uri, String localName, String qName) throws SAXException {
- try {
- // if (DEBUG) logln("endElement:\t" + contextStack);
- if (qName.equals("result") && handler != null) {
- handler.checkResult(lastChars.toString());
- } else if (qName.length() != 0) {
- // logln("Unexpected contents of: " + qName + ", <" + lastChars + ">");
- }
- lastChars.setLength(0);
- // justPopped = true;
- } catch (RuntimeException e) {
- e.printStackTrace();
- throw e;
- }
- }
-
- // Have to hack around the fact that the character data might be in pieces
- public void characters(char[] ch, int start, int length) throws SAXException {
- try {
- String value = new String(ch, start, length);
- if (DEBUG)
- logln("characters:\t" + value);
- lastChars.append(value);
- // justPopped = false;
- } catch (RuntimeException e) {
- e.printStackTrace();
- throw e;
- }
- }
-
- // just for debugging
-
- public void notationDecl(String name, String publicId, String systemId) throws SAXException {
- logln("notationDecl: " + name + ", " + publicId + ", " + systemId);
- }
-
- public void processingInstruction(String target, String data) throws SAXException {
- logln("processingInstruction: " + target + ", " + data);
- }
-
- public void skippedEntity(String name) throws SAXException {
- logln("skippedEntity: " + name);
- }
-
- public void unparsedEntityDecl(String name, String publicId, String systemId, String notationName)
- throws SAXException {
- logln("unparsedEntityDecl: " + name + ", " + publicId + ", " + systemId + ", " + notationName);
- }
- };
-}
projects / icu / commitdiff