From 4505512d3e0e532d8c04669f4de3a9853392abea Mon Sep 17 00:00:00 2001 From: Mark Davis Date: Fri, 23 Aug 2013 10:38:57 +0000 Subject: [PATCH] ICU-8474 Many changes to tests and code in order to properly separate DECIMAL from INTEGER isLimited and samples; fix samples/unbounded in CLDR, fix bounded, fix when to return null, check disallowed syntax etc. X-SVN-Rev: 34082 --- .../src/com/ibm/icu/text/PluralRules.java | 300 +++++++++++------- .../icu/dev/test/format/PluralRulesTest.java | 245 +++++++++++--- 2 files changed, 384 insertions(+), 161 deletions(-) diff --git a/icu4j/main/classes/core/src/com/ibm/icu/text/PluralRules.java b/icu4j/main/classes/core/src/com/ibm/icu/text/PluralRules.java index ff21ab22f8d..5851fa34ae7 100644 --- a/icu4j/main/classes/core/src/com/ibm/icu/text/PluralRules.java +++ b/icu4j/main/classes/core/src/com/ibm/icu/text/PluralRules.java @@ -18,11 +18,13 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashSet; +import java.util.Iterator; import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; import java.util.Set; import java.util.TreeSet; +import java.util.regex.Pattern; import com.ibm.icu.impl.PluralRulesLoader; import com.ibm.icu.impl.Utility; @@ -789,8 +791,8 @@ public class PluralRules implements Serializable, Comparable { } source = source.substring(7).trim(); // remove both - for (String range : source.split(",\\s*")) { - if (range.equals("…")) { + for (String range : COMMA_SEPARATED.split(source)) { + if (range.equals("…") || range.equals("...")) { bounded2 = false; haveBound = true; continue; @@ -798,15 +800,18 @@ public class PluralRules implements Serializable, Comparable { if (haveBound) { throw new IllegalArgumentException("Can only have … at the end of samples: " + range); } - String[] rangeParts = range.split("\\s*~\\s*"); + String[] rangeParts = TILDE_SEPARATED.split(range); switch (rangeParts.length) { case 1: FixedDecimal sample = new FixedDecimal(rangeParts[0]); + checkDecimal(sampleType2, sample); samples2.add(new FixedDecimalRange(sample, sample)); break; case 2: FixedDecimal start = new FixedDecimal(rangeParts[0]); FixedDecimal end = new FixedDecimal(rangeParts[1]); + checkDecimal(sampleType2, start); + checkDecimal(sampleType2, end); samples2.add(new FixedDecimalRange(start, end)); break; default: throw new IllegalArgumentException("Ill-formed number range: " + range); @@ -815,7 +820,13 @@ public class PluralRules implements Serializable, Comparable { return new FixedDecimalSamples(sampleType2, Collections.unmodifiableSet(samples2), bounded2); } - public void addSamples(Set result) { + private static void checkDecimal(SampleType sampleType2, FixedDecimal sample) { + if ((sampleType2 == SampleType.INTEGER) != (sample.getVisibleDecimalDigitCount() == 0)) { + throw new IllegalArgumentException("Ill-formed number range: " + sample); + } + } + + public Set addSamples(Set result) { for (FixedDecimalRange item : samples) { // we have to convert to longs so we don't get strange double issues long startDouble = item.start.getShiftedValue(); @@ -825,6 +836,7 @@ public class PluralRules implements Serializable, Comparable { result.add(d/(double)item.start.baseFactor); } } + return result; } @Override @@ -844,11 +856,11 @@ public class PluralRules implements Serializable, Comparable { } return b.toString(); } - + public Set getSamples() { return samples; } - + public void getStartEndSamples(Set target) { for (FixedDecimalRange item : samples) { target.add(item.start); @@ -876,7 +888,7 @@ public class PluralRules implements Serializable, Comparable { static class SimpleTokenizer { static final UnicodeSet BREAK_AND_IGNORE = new UnicodeSet(0x09, 0x0a, 0x0c, 0x0d, 0x20, 0x20).freeze(); - static final UnicodeSet BREAK_AND_KEEP = new UnicodeSet('!', '!', '%', '%', '=', '=', '≠', '≠').freeze(); + static final UnicodeSet BREAK_AND_KEEP = new UnicodeSet('!', '!', '%', '%', ',', ',', '.', '.', '=', '=').freeze(); static String[] split(String source) { int last = -1; List result = new ArrayList(); @@ -932,13 +944,11 @@ public class PluralRules implements Serializable, Comparable { private static Constraint parseConstraint(String description) throws ParseException { - description = description.trim().toLowerCase(Locale.ENGLISH); - Constraint result = null; - String[] or_together = Utility.splitString(description, "or"); + String[] or_together = OR_SEPARATED.split(description); for (int i = 0; i < or_together.length; ++i) { Constraint andConstraint = null; - String[] and_together = Utility.splitString(or_together[i], "and"); + String[] and_together = AND_SEPARATED.split(or_together[i]); for (int j = 0; j < and_together.length; ++j) { Constraint newConstraint = NO_CONSTRAINT; @@ -970,6 +980,9 @@ public class PluralRules implements Serializable, Comparable { if ("not".equals(t)) { inRange = !inRange; t = nextToken(tokens, x++, condition); + if ("=".equals(t)) { + throw unexpected(t, condition); + } } else if ("!".equals(t)) { inRange = !inRange; t = nextToken(tokens, x++, condition); @@ -979,6 +992,9 @@ public class PluralRules implements Serializable, Comparable { } if ("is".equals(t) || "in".equals(t) || "=".equals(t)) { hackForCompatibility = "is".equals(t); + if (hackForCompatibility && !inRange) { + throw unexpected(t, condition); + } t = nextToken(tokens, x++, condition); } else if ("within".equals(t)) { integersOnly = false; @@ -987,44 +1003,72 @@ public class PluralRules implements Serializable, Comparable { throw unexpected(t, condition); } if ("not".equals(t)) { - if (!inRange || !hackForCompatibility) { + if (!hackForCompatibility && !inRange) { throw unexpected(t, condition); } inRange = !inRange; t = nextToken(tokens, x++, condition); } - String[] range_list = Utility.splitString(t, ","); - vals = new long[range_list.length * 2]; - for (int k1 = 0, k2 = 0; k1 < range_list.length; ++k1, k2 += 2) { - String range = range_list[k1]; - String[] pair = Utility.splitString(range, ".."); - long low, high; - if (pair.length == 2) { - low = Long.parseLong(pair[0]); - high = Long.parseLong(pair[1]); - if (low > high) { - throw unexpected(range, condition); + List valueList = new ArrayList(); + + // the token t is always one item ahead + while (true) { + long low = Long.parseLong(t); + long high = low; + if (x < tokens.length) { + t = nextToken(tokens, x++, condition); + if (t.equals(".")) { + t = nextToken(tokens, x++, condition); + if (!t.equals(".")) { + throw unexpected(t, condition); + } + t = nextToken(tokens, x++, condition); + high = Long.parseLong(t); + if (x < tokens.length) { + t = nextToken(tokens, x++, condition); + if (!t.equals(",")) { // adjacent number: 1 2 + // no separator, fail + throw unexpected(t, condition); + } + } + } else if (!t.equals(",")) { // adjacent number: 1 2 + // no separator, fail + throw unexpected(t, condition); } - } else if (pair.length == 1) { - low = high = Long.parseLong(pair[0]); - } else { - throw unexpected(range, condition); } - if (mod != 0 && high >= mod) { - throw unexpected(range, condition); + // at this point, either we are out of tokens, or t is ',' + if (low > high) { + throw unexpected(low + "~" + high, condition); + } else if (mod != 0 && high >= mod) { + throw unexpected(high + ">mod=" + mod, condition); } - vals[k2] = low; - vals[k2+1] = high; + valueList.add(low); + valueList.add(high); lowBound = Math.min(lowBound, low); highBound = Math.max(highBound, high); + if (x >= tokens.length) { + break; + } + t = nextToken(tokens, x++, condition); + } + + if (t.equals(",")) { + throw unexpected(t, condition); } - if (vals.length == 2) { + + if (valueList.size() == 2) { vals = null; + } else { + vals = new long[valueList.size()]; + for (int k = 0; k < vals.length; ++k) { + vals[k] = valueList.get(k); + } } - if (x != tokens.length) { - throw unexpected(tokens[x], condition); + // Hack to exclude "is not 1,2" + if (lowBound != highBound && hackForCompatibility && !inRange) { + throw unexpected("is not ", condition); } newConstraint = @@ -1048,6 +1092,15 @@ public class PluralRules implements Serializable, Comparable { return result; } + static final Pattern AT_SEPARATED = Pattern.compile("\\s*\\Q\\E@\\s*"); + static final Pattern OR_SEPARATED = Pattern.compile("\\s*or\\s*"); + static final Pattern AND_SEPARATED = Pattern.compile("\\s*and\\s*"); + static final Pattern COMMA_SEPARATED = Pattern.compile("\\s*,\\s*"); + static final Pattern DOTDOT_SEPARATED = Pattern.compile("\\s*\\Q..\\E\\s*"); + static final Pattern TILDE_SEPARATED = Pattern.compile("\\s*~\\s*"); + static final Pattern SEMI_SEPARATED = Pattern.compile("\\s*;\\s*"); + + /* Returns a parse exception wrapping the token and context strings. */ private static ParseException unexpected(String token, String context) { return new ParseException("unexpected token '" + token + @@ -1074,6 +1127,9 @@ public class PluralRules implements Serializable, Comparable { if (description.length() == 0) { return DEFAULT_RULE; } + + description = description.toLowerCase(Locale.ENGLISH); + int x = description.indexOf(':'); if (x == -1) { throw new ParseException("missing ':' in rule description '" + @@ -1087,7 +1143,7 @@ public class PluralRules implements Serializable, Comparable { } description = description.substring(x+1).trim(); - String[] constraintOrSamples = description.split("\\s*@\\s*"); + String[] constraintOrSamples = AT_SEPARATED.split(description); boolean sampleFailure = false; FixedDecimalSamples integerSamples = null, decimalSamples = null; switch (constraintOrSamples.length) { @@ -1141,18 +1197,13 @@ public class PluralRules implements Serializable, Comparable { if (description.endsWith(";")) { description = description.substring(0,description.length()-1); } - String[] rules = Utility.split(description, ';'); - boolean haveOther = false; + String[] rules = SEMI_SEPARATED.split(description); for (int i = 0; i < rules.length; ++i) { Rule rule = parseRule(rules[i].trim()); - haveOther |= rule.keyword.equals("other"); result.hasExplicitBoundingInfo |= rule.integerSamples != null || rule.decimalSamples != null; result.addRule(rule); } - if (!haveOther) { - result.addRule(parseRule("other:")); - } - return result; + return result.finish(); } /* @@ -1182,38 +1233,6 @@ public class PluralRules implements Serializable, Comparable { this.operand = operand; } - // private void addRanges(Set toAddTo, int offset) { - // toAddTo.add(new FixedDecimal(lowerBound + offset)); - // if (upperBound != lowerBound) { - // toAddTo.add(new FixedDecimal(upperBound + offset)); - // } - // // if (range_list != null) { - // // // add from each range - // // for (int i = 0; i < range_list.length; i += 2) { - // // double lower = range_list[i]; - // // double upper = range_list[i+1]; - // // if (lower != lowerBound) { - // // toAddTo.add(new NumberInfo(lower + offset)); - // // } - // // if (upper != upperBound) { - // // toAddTo.add(new NumberInfo(upper + offset)); - // // } - // // } - // // } - // if (!integersOnly) { - // double average = (lowerBound + upperBound) / 2.0d; - // toAddTo.add(new FixedDecimal(average + offset)); - // // if (range_list != null) { - // // // we will just add one value from the middle - // // for (int i = 0; i < range_list.length; i += 2) { - // // double lower = range_list[i]; - // // double upper = range_list[i+1]; - // // toAddTo.add(new NumberInfo((lower + upper) / 2.0d + offset)); - // // } - // // } - // } - // } - public boolean isFulfilled(FixedDecimal number) { double n = number.get(operand); if ((integersOnly && (n - (long)n) != 0.0 @@ -1234,21 +1253,19 @@ public class PluralRules implements Serializable, Comparable { } public boolean isLimited(SampleType sampleType) { + boolean valueIsZero = lowerBound == upperBound && lowerBound == 0d; + boolean hasDecimals = + (operand == Operand.v || operand == Operand.w || operand == Operand.f || operand == Operand.t) + && inRange != valueIsZero; // either NOT f = zero or f = non-zero switch (sampleType) { case INTEGER: - boolean hasFraction = (operand == Operand.f || operand == Operand.t) - && !(lowerBound == upperBound || lowerBound == 0d); - - return hasFraction // will be empty + return hasDecimals // will be empty || (operand == Operand.n || operand == Operand.i || operand == Operand.j) && mod == 0 && inRange; case DECIMAL: - boolean noFraction = - (operand == Operand.v || operand == Operand.w || operand == Operand.f || operand == Operand.t) - && lowerBound == upperBound && lowerBound == 0d; - return (noFraction || operand == Operand.n || operand == Operand.j) + return (!hasDecimals || operand == Operand.n || operand == Operand.j) && (integersOnly || lowerBound == upperBound) && mod == 0 && inRange; @@ -1412,10 +1429,6 @@ public class PluralRules implements Serializable, Comparable { } } - /* - * Implementation of RuleList that is itself a node in a linked list. - * Immutable, but supports chaining with 'addRule'. - */ private static class RuleList implements Serializable { private boolean hasExplicitBoundingInfo = false; private static final long serialVersionUID = 1; @@ -1432,6 +1445,23 @@ public class PluralRules implements Serializable, Comparable { return this; } + public RuleList finish() throws ParseException { + // make sure that 'other' is present, and at the end. + Rule otherRule = null; + for (Iterator it = rules.iterator(); it.hasNext();) { + Rule rule = it.next(); + if ("other".equals(rule.getKeyword())) { + otherRule = rule; + it.remove(); + } + } + if (otherRule == null) { + otherRule = parseRule("other:"); // make sure we have always have an 'other' a rule + } + rules.add(otherRule); + return this; + } + private Rule selectRule(FixedDecimal n) { for (Rule rule : rules) { if (rule.appliesTo(n)) { @@ -1443,9 +1473,10 @@ public class PluralRules implements Serializable, Comparable { public String select(FixedDecimal n) { Rule r = selectRule(n); - if (r == null) { - return KEYWORD_OTHER; - } + // since we have explict 'other', we don't need this. + // if (r == null) { + // return KEYWORD_OTHER; + // } return r.getKeyword(); } @@ -1454,7 +1485,8 @@ public class PluralRules implements Serializable, Comparable { for (Rule rule : rules) { result.add(rule.getKeyword()); } - result.add(KEYWORD_OTHER); + // since we have explict 'other', we don't need this. + //result.add(KEYWORD_OTHER); return result; } @@ -1464,6 +1496,10 @@ public class PluralRules implements Serializable, Comparable { return mySamples == null ? true : mySamples.bounded; } + return computeLimited(keyword, sampleType); + } + + public boolean computeLimited(String keyword, SampleType sampleType) { // if all rules with this keyword are limited, it's limited, // and if there's no rule with this keyword, it's unlimited boolean result = false; @@ -1713,14 +1749,24 @@ public class PluralRules implements Serializable, Comparable { * @stable ICU 4.8 */ public Collection getAllKeywordValues(String keyword) { - if (!isLimited(keyword, SampleType.INTEGER) - || !isLimited(keyword, SampleType.DECIMAL)) { + return getAllKeywordValues(keyword, SampleType.INTEGER); + } + + /** + * Returns all the values that trigger this keyword, or null if the number of such + * values is unlimited. + * + * @param keyword the keyword + * @return the values that trigger this keyword, or null. The returned collection + * is immutable. It will be empty if the keyword is not defined. + * @stable ICU 4.8 + */ + public Collection getAllKeywordValues(String keyword, SampleType type) { + if (!isLimited(keyword, type)) { return null; } - TreeSet result = new TreeSet(); - result.addAll(getSamples(keyword, SampleType.INTEGER)); - result.addAll(getSamples(keyword, SampleType.DECIMAL)); - return Collections.unmodifiableSet(result); + Collection samples = getSamples(keyword, type); + return samples == null ? null : Collections.unmodifiableCollection(samples); } /** @@ -1734,17 +1780,16 @@ public class PluralRules implements Serializable, Comparable { * @stable ICU 4.8 */ public Collection getSamples(String keyword) { - Set result = new TreeSet(); - result.addAll(getSamples(keyword, SampleType.INTEGER)); - result.addAll(getSamples(keyword, SampleType.DECIMAL)); - return result.size() == 0 ? null : Collections.unmodifiableSet(result); + return getSamples(keyword, SampleType.INTEGER); } /** * Returns a list of values for which select() would return that keyword, - * or empty if the keyword is not defined. The returned collection is unmodifiable. + * or null if the keyword is not defined. + * The returned collection is unmodifiable. * The returned list is not complete, and there might be additional values that - * would return the keyword. + * would return the keyword. The keyword might be defined, and yet have an empty set of samples, + * IF there are samples for the other sampleType. * * @param keyword the keyword to test * @return a list of values matching the keyword. @@ -1753,16 +1798,14 @@ public class PluralRules implements Serializable, Comparable { */ public Collection getSamples(String keyword, SampleType sampleType) { if (!keywords.contains(keyword)) { - return Collections.emptySet(); + return null; } Set result = new TreeSet(); if (rules.hasExplicitBoundingInfo) { FixedDecimalSamples samples = rules.getDecimalSamples(keyword, sampleType); - if (samples != null) { - samples.addSamples(result); - } - return Collections.unmodifiableSet(result); + return samples == null ? Collections.unmodifiableSet(result) + : Collections.unmodifiableSet(samples.addSamples(result)); } // hack in case the rule is created without explicit samples @@ -1786,7 +1829,7 @@ public class PluralRules implements Serializable, Comparable { addSample(keyword, new FixedDecimal(1000000d, 1), maxCount, result); // hack for Welsh break; } - return Collections.unmodifiableSet(result); + return result.size() == 0 ? null : Collections.unmodifiableSet(result); } public boolean addSample(String keyword, Number sample, int maxCount, Set result) { @@ -1802,7 +1845,8 @@ public class PluralRules implements Serializable, Comparable { /** * Returns a list of values for which select() would return that keyword, - * or null if the keyword is not defined. The returned collection is unmodifiable. + * or null if the keyword is not defined or no samples are available. + * The returned collection is unmodifiable. * The returned list is not complete, and there might be additional values that * would return the keyword. * @@ -1888,7 +1932,7 @@ public class PluralRules implements Serializable, Comparable { */ INVALID, /** - * The keyword is valid, but unused (it is covered by the explicit values). + * The keyword is valid, but unused (it is covered by the explicit values, OR has no values for the given {@link SampleType}). * * @draft ICU 50 * @provisional This API might change or be removed in a future release. @@ -1930,11 +1974,31 @@ public class PluralRules implements Serializable, Comparable { * @param uniqueValue * If non null, set to the unique value. * @return the KeywordStatus - * @draft ICU 50 + * @draft ICU 52 * @provisional This API might change or be removed in a future release. */ public KeywordStatus getKeywordStatus(String keyword, int offset, Set explicits, Output uniqueValue) { + return getKeywordStatus(keyword, offset, explicits, uniqueValue, SampleType.INTEGER); + } + /** + * Find the status for the keyword, given a certain set of explicit values. + * + * @param keyword + * the particular keyword (call rules.getKeywords() to get the valid ones) + * @param offset + * the offset used, or 0.0d if not. Internally, the offset is subtracted from each explicit value before + * checking against the keyword values. + * @param explicits + * a set of Doubles that are used explicitly (eg [=0], "[=1]"). May be empty or null. + * @param uniqueValue + * If non null, set to the unique value. + * @return the KeywordStatus + * @draft ICU 50 + * @provisional This API might change or be removed in a future release. + */ + public KeywordStatus getKeywordStatus(String keyword, int offset, Set explicits, + Output uniqueValue, SampleType sampleType) { if (uniqueValue != null) { uniqueValue.value = null; } @@ -1943,10 +2007,12 @@ public class PluralRules implements Serializable, Comparable { return KeywordStatus.INVALID; } - Collection values = getAllKeywordValues(keyword); - if (values == null) { + if (!isLimited(keyword, sampleType)) { return KeywordStatus.UNBOUNDED; } + + Collection values = getSamples(keyword, sampleType); + int originalSize = values.size(); if (explicits == null) { @@ -2028,4 +2094,12 @@ public class PluralRules implements Serializable, Comparable { public boolean isLimited(String keyword, SampleType sampleType) { return rules.isLimited(keyword, sampleType); } + + /** + * @internal + * @deprecated internal + */ + public boolean computeLimited(String keyword, SampleType sampleType) { + return rules.computeLimited(keyword, sampleType); + } } diff --git a/icu4j/main/tests/core/src/com/ibm/icu/dev/test/format/PluralRulesTest.java b/icu4j/main/tests/core/src/com/ibm/icu/dev/test/format/PluralRulesTest.java index df2a6a3b236..2e5ecc14e9d 100644 --- a/icu4j/main/tests/core/src/com/ibm/icu/dev/test/format/PluralRulesTest.java +++ b/icu4j/main/tests/core/src/com/ibm/icu/dev/test/format/PluralRulesTest.java @@ -57,19 +57,99 @@ public class PluralRulesTest extends TestFmwk { new PluralRulesTest().run(args); } - public void testNewSamples() { - String description = "one: n is 3 or f is 5 @integer 3,19, @decimal 3.50 ~ 3.53, …; other: @decimal 99~103, 999, …"; + public void testSyntaxRestrictions() { + Object[][] shouldFail = { + {"a:n in 3..10,13..19"}, + + // = and != always work + {"a:n=1"}, + {"a:n=1,3"}, + {"a:n!=1"}, + {"a:n!=1,3"}, + + // with spacing + {"a: n = 1"}, + {"a: n = 1, 3"}, + {"a: n != 1"}, + {"a: n != 1, 3"}, + {"a: n ! = 1"}, + {"a: n ! = 1, 3"}, + {"a: n = 1 , 3"}, + {"a: n != 1 , 3"}, + {"a: n ! = 1 , 3"}, + {"a: n = 1 .. 3"}, + {"a: n != 1 .. 3"}, + {"a: n ! = 1 .. 3"}, + + // more complicated + {"a:n in 3 .. 10 , 13 .. 19"}, + + // singles have special exceptions + {"a: n is 1"}, + {"a: n is not 1"}, + {"a: n not is 1", ParseException.class}, // hacked to fail + {"a: n in 1"}, + {"a: n not in 1"}, + + // multiples also have special exceptions + // TODO enable the following once there is an update to CLDR + // {"a: n is 1,3", ParseException.class}, + {"a: n is not 1,3", ParseException.class}, // hacked to fail + {"a: n not is 1,3", ParseException.class}, // hacked to fail + {"a: n in 1,3"}, + {"a: n not in 1,3"}, + + // disallow not with = + {"a: n not= 1", ParseException.class}, // hacked to fail + {"a: n not= 1,3", ParseException.class}, // hacked to fail + + // disallow double negatives + {"a: n ! is not 1", ParseException.class}, + {"a: n ! is not 1", ParseException.class}, + {"a: n not not in 1", ParseException.class}, + {"a: n is not not 1", NumberFormatException.class}, + + // disallow screwy cases + {null, NullPointerException.class}, + {"djkl;", ParseException.class}, + {"a: n = 1 .", ParseException.class}, + {"a: n = 1 ..", ParseException.class}, + {"a: n = 1 2", ParseException.class}, + {"a: n = 1 ,", ParseException.class}, + {"a:n in 3 .. 10 , 13 .. 19 ,", ParseException.class}, + }; + for (Object[] shouldFailTest : shouldFail) { + String rules = (String) shouldFailTest[0]; + Class exception = shouldFailTest.length < 2 ? null : (Class) shouldFailTest[1]; + Class actualException = null; + try { + PluralRules test = PluralRules.parseDescription(rules); + } catch (Exception e) { + actualException = e.getClass(); + } + assertEquals("Exception " + rules, exception, actualException); + } + } + public void testSamples() { + String description = "one: n is 3 or f is 5 @integer 3,19, @decimal 3.50 ~ 3.53, …; other: @decimal 99.0~99.2, 999.0, …"; PluralRules test = PluralRules.createRules(description); checkNewSamples(description, test, "one", PluralRules.SampleType.INTEGER, "@integer 3, 19", true, new FixedDecimal(3)); checkNewSamples(description, test, "one", PluralRules.SampleType.DECIMAL, "@decimal 3.50~3.53, …", false, new FixedDecimal(3.5,2)); - Collection oldSamples = test.getSamples("one"); - assertEquals("getSamples; " + "one" + "; " + description, new TreeSet(Arrays.asList(3d, 19d, 3.5d, 3.51d, 3.52d, 3.53d)), oldSamples); + checkOldSamples(description, test, "one", SampleType.INTEGER, 3d, 19d); + checkOldSamples(description, test, "one", SampleType.DECIMAL, 3.5d, 3.51d, 3.52d, 3.53d); checkNewSamples(description, test, "other", PluralRules.SampleType.INTEGER, "", true, null); - checkNewSamples(description, test, "other", PluralRules.SampleType.DECIMAL, "@decimal 99~103, 999, …", false, new FixedDecimal(99d)); - Collection oldSamples2 = test.getSamples("other"); - assertEquals("getSamples; " + "other" + "; " + description, new TreeSet(Arrays.asList(99d, 100d, 101d, 102d, 103d, 999d)), oldSamples2); + checkNewSamples(description, test, "other", PluralRules.SampleType.DECIMAL, "@decimal 99.0~99.2, 999.0, …", false, new FixedDecimal(99d,1)); + checkOldSamples(description, test, "other", SampleType.INTEGER); + checkOldSamples(description, test, "other", SampleType.DECIMAL, 99d, 99.1, 99.2d, 999d); + } + + public void checkOldSamples(String description, PluralRules rules, String keyword, SampleType sampleType, Double... expected) { + Collection oldSamples = rules.getSamples(keyword, sampleType); + if (!assertEquals("getOldSamples; " + keyword + "; " + description, new HashSet(Arrays.asList(expected)), oldSamples)) { + rules.getSamples(keyword, sampleType); + } } public void checkNewSamples(String description, PluralRules test, String keyword, SampleType sampleType, @@ -269,7 +349,7 @@ public class PluralRulesTest extends TestFmwk { } } - private void checkCategoriesAndExpected(String title, String categoriesAndExpected, PluralRules rules) { + private void checkCategoriesAndExpected(String title1, String categoriesAndExpected, PluralRules rules) { for (String categoryAndExpected : categoriesAndExpected.split("\\s*;\\s*")) { String[] categoryFromExpected = categoryAndExpected.split("\\s*:\\s*"); String expected = categoryFromExpected[0]; @@ -285,7 +365,8 @@ public class PluralRulesTest extends TestFmwk { fractionaldigits = Integer.parseInt(value.substring(decimalPos)); } String result = rules.select(number, countVisibleFractionDigits, fractionaldigits); - assertEquals("testing <" + title + "> with <" + value + ">", expected, result); + ULocale locale = null; + assertEquals(getAssertMessage(title1, locale, rules, expected) + "; value: " + value, expected, result); } } } @@ -294,9 +375,9 @@ public class PluralRulesTest extends TestFmwk { // once we add fractions, we had to retract the "test all possibilities" for equality, // so we only have a limited set of equality tests now. { "c: n%11!=5", "c: n mod 11 is not 5" }, - { "c: n not is 7", "c: n != 7" }, + { "c: n is not 7", "c: n != 7" }, { "a:n in 2;", "a: n = 2" }, - { "b:n not in 5;", "b: n not = 5" }, + { "b:n not in 5;", "b: n != 5" }, // { "a: n is 5", // "a: n in 2..6 and n not in 2..4 and n is not 6" }, @@ -496,10 +577,16 @@ public class PluralRulesTest extends TestFmwk { for (String keyword : keywords) { Collection list = rules.getSamples(keyword); logln("keyword: " + keyword + ", samples: " + list); - - assertNotNull("keyword: " + keyword + ", rules: " + rules + ": list is not null", list); - if (list != null) { - if (!assertTrue(locale + "Testing getSamples.isEmpty for " + keyword + ", in " + rules.getRules(keyword), !list.isEmpty())) { + // with fractions, the samples can be empty and thus the list null. In that case, however, there will be FixedDecimal values. + // So patch the test for that. + if (list.size() == 0) { + // when the samples (meaning integer samples) are null, then then integerSamples must be, and the decimalSamples must not be + FixedDecimalSamples integerSamples = rules.getDecimalSamples(keyword, SampleType.INTEGER); + FixedDecimalSamples decimalSamples = rules.getDecimalSamples(keyword, SampleType.DECIMAL); + assertTrue(getAssertMessage("List is not null", locale, rules, keyword), + integerSamples == null && decimalSamples != null && decimalSamples.samples.size() != 0); + } else { + if (!assertTrue(getAssertMessage("Test getSamples.isEmpty", locale, rules, keyword), !list.isEmpty())) { int debugHere = 0; rules.getSamples(keyword); } @@ -507,14 +594,35 @@ public class PluralRulesTest extends TestFmwk { // hack until we remove j } else { for (double value : list) { - assertEquals(locale + " value " + value + " matching keyword", keyword, rules.select(value)); + assertEquals(getAssertMessage("Match keyword", locale, rules, keyword) + "; value '" + value + "'", keyword, rules.select(value)); } } } } - assertNull("list is null", rules.getSamples("@#$%^&*")); + assertNull(locale + ", list is null", rules.getSamples("@#$%^&*")); + assertNull(locale + ", list is null", rules.getSamples("@#$%^&*", SampleType.DECIMAL)); + } + } + + public String getAssertMessage(String message, ULocale locale, PluralRules rules, String keyword) { + String ruleString = ""; + if (keyword != null) { + if (keyword.equals("other")) { + for (String keyword2 : rules.getKeywords()) { + ruleString += " NOR " + rules.getRules(keyword2).split("@")[0]; + } + } else { + String rule = rules.getRules(keyword); + ruleString = rule == null ? null : rule.split("@")[0]; + } + ruleString = "; rule: '" + keyword + ": " + ruleString + "'"; + // !keyword.equals("other") ? "'; keyword: '" + keyword + "'; rule: '" + rules.getRules(keyword) + "'" + // : "'; keyword: '" + keyword + "'; rules: '" + rules.toString() + "'"; } + return message + + (locale == null ? "" : "; locale: '" + locale + "'") + + ruleString; } /** @@ -524,22 +632,22 @@ public class PluralRulesTest extends TestFmwk { public void TestGetAllKeywordValues() { // data is pairs of strings, the rule, and the expected values as arguments String[] data = { - "a: n mod 3 is 0", "a: null", + "other: ; a: n mod 3 is 0", "a: null", "a: n in 2..5 and n within 5..8", "a: 5", "a: n in 2..5", "a: 2,3,4,5; other: null", "a: n not in 2..5", "a: null; other: null", - "a: n within 2..5", "a: null; other: null", + "a: n within 2..5", "a: 2,3,4,5; other: null", "a: n not within 2..5", "a: null; other: null", - "a: n in 2..5 or n within 6..8", "a: null", // ignore 'other' here on out, always null - "a: n in 2..5 and n within 6..8", "a:", + "a: n in 2..5 or n within 6..8", "a: 2,3,4,5,6,7,8", // ignore 'other' here on out, always null + "a: n in 2..5 and n within 6..8", "a: null", // we no longer support 'degenerate' rules // "a: n within 2..5 and n within 6..8", "a:", // our sampling catches these // "a: n within 2..5 and n within 5..8", "a: 5", // '' // "a: n within 1..2 and n within 2..3 or n within 3..4 and n within 4..5", "a: 2,4", // "a: n mod 3 is 0 and n within 0..5", "a: 0,3", - "a: n within 1..2 and n within 2..3 or n within 3..4 and n within 4..5 or n within 5..6 and n within 6..7", "a: null", // but not this... + "a: n within 1..2 and n within 2..3 or n within 3..4 and n within 4..5 or n within 5..6 and n within 6..7", "a: 2,4,6", // but not this... "a: n mod 3 is 0 and n within 1..2", "a: null", - "a: n mod 3 is 0 and n within 0..6", "a: null", // similarly with mod, we don't catch... + "a: n mod 3 is 0 and n within 0..6", "a: 0,3,6", "a: n mod 3 is 0 and n in 3..12", "a: 3,6,9,12", "a: n in 2,4..6 and n is not 5", "a: 2,4,6", }; @@ -571,7 +679,7 @@ public class PluralRulesTest extends TestFmwk { } Collection results = p.getAllKeywordValues(keyword); - assertEquals(keyword + " in " + ruleDescription, values, results); + assertEquals(keyword + " in " + ruleDescription, values, results == null ? null : new HashSet(results)); if (results != null) { try { @@ -590,37 +698,73 @@ public class PluralRulesTest extends TestFmwk { assertEquals("PluralRules(en-ordinal).select(2)", "two", pr.select(2)); } + public void TestLimitedAndSamplesConsistency() { + for (ULocale locale : PluralRules.getAvailableULocales()) { + ULocale loc2 = PluralRules.getFunctionalEquivalent(locale, null); + if (!loc2.equals(locale)) { + continue; // only need "unique" rules + } + for (PluralType type : PluralType.values()) { + PluralRules rules = PluralRules.forLocale(locale, type); + for (SampleType sampleType : SampleType.values()) { + for (String keyword : rules.getKeywords()) { + boolean isLimited = rules.isLimited(keyword, sampleType); + boolean computeLimited = rules.computeLimited(keyword, sampleType); + if (!keyword.equals("other")) { + assertEquals(getAssertMessage("computeLimited == isLimited", locale, rules, keyword), computeLimited, isLimited); + } + Collection samples = rules.getSamples(keyword, sampleType); + FixedDecimalSamples decimalSamples = rules.getDecimalSamples(keyword, sampleType); + assertNotNull(getAssertMessage("Samples must not be null", locale, rules, keyword), samples); + //assertNotNull(getAssertMessage("Decimal samples must be null if unlimited", locale, rules, keyword), decimalSamples); + } + } + } + } + } + public void TestKeywords() { Set possibleKeywords = new LinkedHashSet(Arrays.asList("zero", "one", "two", "few", "many", "other")); - Object[][] tests = { + Object[][][] tests = { // format is locale, explicits, then triples of keyword, status, unique value. - {"en", null, - "one", KeywordStatus.UNIQUE, 1.0d, - "other", KeywordStatus.UNBOUNDED, null}, - {"pl", null, - "one", KeywordStatus.UNIQUE, 1.0d, - "few", KeywordStatus.UNBOUNDED, null, - "many", KeywordStatus.UNBOUNDED, null, - "other", KeywordStatus.UNBOUNDED, null}, - {"en", new HashSet(Arrays.asList(1.0d)), // check that 1 is suppressed - "one", KeywordStatus.SUPPRESSED, null, - "other", KeywordStatus.UNBOUNDED, null}, + {{"en", null}, + {"one", KeywordStatus.UNIQUE, 1.0d}, + {"other", KeywordStatus.UNBOUNDED, null} + }, + {{"pl", null}, + {"one", KeywordStatus.UNIQUE, 1.0d}, + {"few", KeywordStatus.UNBOUNDED, null}, + {"many", KeywordStatus.UNBOUNDED, null}, + {"other", KeywordStatus.SUPPRESSED, null, KeywordStatus.UNBOUNDED, null} // note that it is suppressed in INTEGER but not DECIMAL + }, + {{"en", new HashSet(Arrays.asList(1.0d))}, // check that 1 is suppressed + {"one", KeywordStatus.SUPPRESSED, null}, + {"other", KeywordStatus.UNBOUNDED, null} + }, }; Output uniqueValue = new Output(); - for (Object[] test : tests) { - ULocale locale = new ULocale((String) test[0]); + for (Object[][] test : tests) { + ULocale locale = new ULocale((String) test[0][0]); // NumberType numberType = (NumberType) test[1]; - Set explicits = (Set) test[1]; + Set explicits = (Set) test[0][1]; PluralRules pluralRules = factory.forLocale(locale); LinkedHashSet remaining = new LinkedHashSet(possibleKeywords); - for (int i = 2; i < test.length; i += 3) { - String keyword = (String) test[i]; - KeywordStatus statusExpected = (KeywordStatus) test[i+1]; - Double uniqueExpected = test[i+2] == null ? null : (Double) test[i+2]; + for (int i = 1; i < test.length; ++i) { + Object[] row = test[i]; + String keyword = (String) row[0]; + KeywordStatus statusExpected = (KeywordStatus) row[1]; + Double uniqueExpected = (Double) row[2]; remaining.remove(keyword); KeywordStatus status = pluralRules.getKeywordStatus(keyword, 0, explicits, uniqueValue); - assertEquals("Keyword Status for " + locale + ", " + keyword, statusExpected, status); - assertEquals("Unique Value for " + locale + ", " + keyword, uniqueExpected, uniqueValue.value); + assertEquals(getAssertMessage("Unique Value", locale, pluralRules, keyword), uniqueExpected, uniqueValue.value); + assertEquals(getAssertMessage("Keyword Status", locale, pluralRules, keyword), statusExpected, status); + if (row.length > 3) { + statusExpected = (KeywordStatus) row[3]; + uniqueExpected = (Double) row[4]; + status = pluralRules.getKeywordStatus(keyword, 0, explicits, uniqueValue, SampleType.DECIMAL); + assertEquals(getAssertMessage("Unique Value - decimal", locale, pluralRules, keyword), uniqueExpected, uniqueValue.value); + assertEquals(getAssertMessage("Keyword Status - decimal", locale, pluralRules, keyword), statusExpected, status); + } } for (String keyword : remaining) { KeywordStatus status = pluralRules.getKeywordStatus(keyword, 0, null, uniqueValue); @@ -747,13 +891,18 @@ public class PluralRulesTest extends TestFmwk { // [one, other] "fil,tl; one: 0, 1; other: 0.0, 0.00, 0.03, 0.1, 0.3, 0.30, 1.99, 2, 2.0, 2.00, 2.01, 2.1, 2.10, 3", "ca,de,en,et,fi,gl,it,nl,sw,ur,yi; one: 1; other: 0, 0.0, 0.00, 0.01, 0.1, 0.10, 1.0, 1.00, 1.03, 1.3, 1.30, 1.99, 2, 3", - "da,sv; one: 0.01, 0.1, 1; other: 0, 0.0, 0.00, 0.10, 1.0, 1.00, 1.03, 1.3, 1.30, 1.99, 2, 3", - "is; one: 0.1, 0.31, 1, 31; other: 0, 0.0, 0.00, 1.0, 1.00, 1.11, 1.99, 2, 11, 111, 311", + // danish is now: one: n is 1 or t is not 0 and i is 0,1 @integer 1 @decimal 0.1~0.8 + "da; one: 0.01, 0.1, 1, 0.10, 1.0, 1.00, 1.03, 1.3, 1.30, 1.99; other: 0, 0.0, 0.00, 2, 2.2, 2.9, 3", + // swedish is now: one: i is 1 and v is 0 @integer 1 + "sv; one: 1; other: 0.01, 0.1, 0, 0.0, 0.00, 0.10, 1.0, 1.00, 1.03, 1.3, 1.30, 1.99, 2, 3", + // icelandic is now: one: t is 0 and i mod 10 is 1 and i mod 100 is not 11 or t is not 0 + "is; one: 0.1, 0.31, 1, 31, 1.0, 1.00, 1.11, 1.99; other: 0, 0.0, 0.00, 2, 11, 111, 311", "mk; one: 0.1, 0.31, 1, 11, 31; other: 0, 0.0, 0.00, 1.0, 1.00, 1.03, 1.3, 1.30, 1.99, 2, 3", "ak,bh,guw,ln,mg,nso,pa,ti,wa; one: 0, 0.0, 0.00, 1; other: 0.03, 0.1, 0.3, 0.30, 1.99, 2, 2.0, 2.00, 2.01, 2.1, 2.10, 3", "tzm; one: 0, 0.0, 0.00, 1, 11, 99; other: 0.03, 0.1, 0.3, 0.30, 1.99, 2, 2.0, 2.00, 2.11, 3", "af,asa,ast,az,bem,bez,bg,brx,cgg,chr,ckb,dv,ee,el,eo,es,eu,fo,fur,fy,gsw,ha,haw,hu,jgo,jmc,ka,kaj,kcg,kk,kkj,kl,ks,ksb,ku,ky,lb,lg,mas,mgo,ml,mn,nah,nb,nd,ne,nn,nnh,no,nr,ny,nyn,om,or,os,pap,ps,rm,rof,rwk,saq,seh,sn,so,sq,ss,ssy,st,syr,ta,te,teo,tig,tk,tn,tr,ts,ve,vo,vun,wae,xh,xog; one: 1, 1.0, 1.00; other: 0, 0.0, 0.00, 0.01, 0.1, 0.10, 1.03, 1.3, 1.30, 1.99, 2, 3", - "pt; one: 0.01, 0.1, 1, 1.0, 1.00; other: 0, 0.0, 0.00, 0.10, 1.03, 1.3, 1.30, 1.99, 2, 3", + // pt is now: one: i is 1 and v is 0 or f is 1 + "pt; one: 0.01, 0.1, 1; other: 0, 0.0, 0.00, 0.10, 1.03, 1.3, 1.30, 1.99, 2, 3, 1.0, 1.00", "am,bn,fa,gu,hi,kn,mr,zu; one: 0, 0.0, 0.00, 0.03, 0.1, 0.3, 0.30, 0.5, 1; other: 1.99, 2, 2.0, 2.00, 2.01, 2.1, 2.10, 3", "ff,fr,hy,kab; one: 0, 0.0, 0.00, 0.02, 0.1, 0.2, 0.20, 1, 1.99; other: 2, 2.0, 2.00, 2.01, 2.1, 2.10", -- 2.49.0