return null;
}
- affixPattern = ParsingUtils.maybeFold(affixPattern, parseFlags);
AffixPatternMatcher series = new AffixPatternMatcher(affixPattern);
series.factory = factory;
series.ignorables = (0 != (parseFlags & ParsingUtils.PARSE_FLAG_EXACT_AFFIX)) ? null
@Override
public boolean match(StringSegment segment, ParsedNumber result) {
- if (segment.getCodePoint() == cp) {
- segment.adjustOffset(Character.charCount(cp));
+ if (segment.matches(cp)) {
+ segment.adjustOffsetByCodePoint();
result.setCharsConsumed(segment);
}
return false;
private final String currency1;
private final String currency2;
- public static CurrencyMatcher getInstance(Currency currency, ULocale loc, int setupFlags) {
+ public static CurrencyMatcher getInstance(Currency currency, ULocale loc) {
return new CurrencyMatcher(currency.getSubtype(),
- ParsingUtils.maybeFold(currency.getSymbol(loc), setupFlags),
- ParsingUtils.maybeFold(currency.getCurrencyCode(), setupFlags));
+ currency.getSymbol(loc),
+ currency.getCurrencyCode());
}
private CurrencyMatcher(String isoCode, String currency1, String currency2) {
DecimalFormatSymbols symbols;
IgnorablesMatcher ignorables;
ULocale locale;
- int parseFlags;
public MinusSignMatcher minusSign(boolean allowTrailing) {
return MinusSignMatcher.getInstance(symbols, allowTrailing);
public AnyMatcher currency() {
AnyMatcher any = new AnyMatcher();
- any.addMatcher(CurrencyMatcher.getInstance(currency, locale, parseFlags));
+ any.addMatcher(CurrencyMatcher.getInstance(currency, locale));
any.addMatcher(CurrencyTrieMatcher.getInstance(locale));
any.freeze();
return any;
// License & terms of use: http://www.unicode.org/copyright.html#License
package com.ibm.icu.impl.number.parse;
-import com.ibm.icu.lang.UCharacter;
import com.ibm.icu.text.DecimalFormatSymbols;
import com.ibm.icu.text.UnicodeSet;
public class NanMatcher extends SymbolMatcher {
private static final NanMatcher DEFAULT = new NanMatcher("NaN");
- private static final NanMatcher DEFAULT_FOLDED = new NanMatcher(UCharacter.foldCase("NaN", true));
public static NanMatcher getInstance(DecimalFormatSymbols symbols, int parseFlags) {
- String symbolString = ParsingUtils.maybeFold(symbols.getNaN(), parseFlags);
+ String symbolString = symbols.getNaN();
if (DEFAULT.string.equals(symbolString)) {
return DEFAULT;
- } else if (DEFAULT_FOLDED.string.equals(symbolString)) {
- return DEFAULT_FOLDED;
} else {
return new NanMatcher(symbolString);
}
*/
public class NumberParserImpl {
+ @Deprecated
+ public static NumberParserImpl removeMeWhenMerged(ULocale locale, String pattern, int parseFlags) {
+ NumberParserImpl parser = new NumberParserImpl(parseFlags);
+ DecimalFormatSymbols symbols = DecimalFormatSymbols.getInstance(locale);
+ IgnorablesMatcher ignorables = IgnorablesMatcher.DEFAULT;
+
+ MatcherFactory factory = new MatcherFactory();
+ factory.currency = Currency.getInstance("USD");
+ factory.symbols = symbols;
+ factory.ignorables = ignorables;
+ factory.locale = locale;
+
+ ParsedPatternInfo patternInfo = PatternStringParser.parseToPatternInfo(pattern);
+ AffixMatcher.newGenerate(patternInfo, parser, factory, ignorables, parseFlags);
+
+ Grouper grouper = Grouper.forStrategy(GroupingStrategy.AUTO).withLocaleData(locale, patternInfo);
+ parser.addMatcher(DecimalMatcher.getInstance(symbols, grouper, parseFlags));
+ parser.addMatcher(CurrencyTrieMatcher.getInstance(locale));
+ parser.addMatcher(NanMatcher.getInstance(symbols, parseFlags));
+
+ parser.freeze();
+ return parser;
+ }
+
// TODO: Find a better place for this enum.
/** Controls the set of rules for parsing a string. */
public static enum ParseMode {
// Temporary frontend for testing.
int parseFlags = ParsingUtils.PARSE_FLAG_IGNORE_CASE
- | ParsingUtils.PARSE_FLAG_INCLUDE_UNPAIRED_AFFIXES;
+ | ParsingUtils.PARSE_FLAG_INCLUDE_UNPAIRED_AFFIXES
+ | ParsingUtils.PARSE_FLAG_OPTIMIZE;
if (strictGrouping) {
parseFlags |= ParsingUtils.PARSE_FLAG_STRICT_GROUPING_SIZE;
}
- NumberParserImpl parser = new NumberParserImpl(parseFlags, true);
+ NumberParserImpl parser = new NumberParserImpl(parseFlags);
DecimalFormatSymbols symbols = DecimalFormatSymbols.getInstance(locale);
IgnorablesMatcher ignorables = IgnorablesMatcher.DEFAULT;
factory.symbols = symbols;
factory.ignorables = ignorables;
factory.locale = locale;
- factory.parseFlags = parseFlags;
ParsedPatternInfo patternInfo = PatternStringParser.parseToPatternInfo(pattern);
AffixMatcher.newGenerate(patternInfo, parser, factory, ignorables, parseFlags);
parser.addMatcher(DecimalMatcher.getInstance(symbols, grouper, parseFlags));
parser.addMatcher(MinusSignMatcher.getInstance(symbols, false));
parser.addMatcher(NanMatcher.getInstance(symbols, parseFlags));
- parser.addMatcher(ScientificMatcher.getInstance(symbols, grouper, parseFlags));
+ parser.addMatcher(ScientificMatcher.getInstance(symbols, grouper));
parser.addMatcher(CurrencyTrieMatcher.getInstance(locale));
parser.addMatcher(new RequireNumberMatcher());
if (parseCurrency || patternInfo.hasCurrencySign()) {
parseFlags |= ParsingUtils.PARSE_FLAG_MONETARY_SEPARATORS;
}
+ if (optimize) {
+ parseFlags |= ParsingUtils.PARSE_FLAG_OPTIMIZE;
+ }
IgnorablesMatcher ignorables = isStrict ? IgnorablesMatcher.STRICT : IgnorablesMatcher.DEFAULT;
- NumberParserImpl parser = new NumberParserImpl(parseFlags, optimize);
+ NumberParserImpl parser = new NumberParserImpl(parseFlags);
MatcherFactory factory = new MatcherFactory();
factory.currency = currency;
factory.symbols = symbols;
factory.ignorables = ignorables;
factory.locale = locale;
- factory.parseFlags = parseFlags;
//////////////////////
/// AFFIX MATCHERS ///
////////////////////////
if (parseCurrency || patternInfo.hasCurrencySign()) {
- parser.addMatcher(CurrencyMatcher.getInstance(currency, locale, parseFlags));
+ parser.addMatcher(CurrencyMatcher.getInstance(currency, locale));
parser.addMatcher(CurrencyTrieMatcher.getInstance(locale));
}
parser.addMatcher(ignorables);
parser.addMatcher(DecimalMatcher.getInstance(symbols, grouper, parseFlags));
if (!properties.getParseNoExponent()) {
- parser.addMatcher(ScientificMatcher.getInstance(symbols, grouper, parseFlags));
+ parser.addMatcher(ScientificMatcher.getInstance(symbols, grouper));
}
//////////////////
* or twice, set this to false; if it is going to be used hundreds of times, set it to
* true.
*/
- public NumberParserImpl(int parseFlags, boolean optimize) {
+ public NumberParserImpl(int parseFlags) {
matchers = new ArrayList<NumberParseMatcher>();
- if (optimize) {
+ if (0 != (parseFlags & ParsingUtils.PARSE_FLAG_OPTIMIZE)) {
leadCodePointses = new ArrayList<UnicodeSet>();
} else {
leadCodePointses = null;
assert !frozen;
this.matchers.add(matcher);
if (leadCodePointses != null) {
- UnicodeSet leadCodePoints = matcher.getLeadCodePoints();
- assert leadCodePoints.isFrozen();
- this.leadCodePointses.add(leadCodePoints);
+ addLeadCodePointsForMatcher(matcher);
}
}
this.matchers.addAll(matchers);
if (leadCodePointses != null) {
for (NumberParseMatcher matcher : matchers) {
- UnicodeSet leadCodePoints = matcher.getLeadCodePoints();
- assert leadCodePoints.isFrozen();
- this.leadCodePointses.add(leadCodePoints);
+ addLeadCodePointsForMatcher(matcher);
}
}
}
+ private void addLeadCodePointsForMatcher(NumberParseMatcher matcher) {
+ UnicodeSet leadCodePoints = matcher.getLeadCodePoints();
+ assert leadCodePoints.isFrozen();
+ // TODO: Avoid the clone operation here.
+ if (0 != (parseFlags & ParsingUtils.PARSE_FLAG_IGNORE_CASE)) {
+ leadCodePoints = leadCodePoints.cloneAsThawed().closeOver(UnicodeSet.ADD_CASE_MAPPINGS)
+ .freeze();
+ }
+ this.leadCodePointses.add(leadCodePoints);
+ }
+
public void setComparator(Comparator<ParsedNumber> comparator) {
assert !frozen;
this.comparator = comparator;
public void parse(String input, int start, boolean greedy, ParsedNumber result) {
assert frozen;
assert start >= 0 && start < input.length();
- StringSegment segment = new StringSegment(ParsingUtils.maybeFold(input, parseFlags));
+ StringSegment segment = new StringSegment(input, parseFlags);
segment.adjustOffset(start);
if (greedy) {
parseGreedyRecursive(segment, result);
// License & terms of use: http://www.unicode.org/copyright.html#License
package com.ibm.icu.impl.number.parse;
-import com.ibm.icu.lang.UCharacter;
import com.ibm.icu.text.UnicodeSet;
import com.ibm.icu.text.UnicodeSet.EntryRange;
public static final int PARSE_FLAG_EXACT_AFFIX = 0x0200;
public static final int PARSE_FLAG_PLUS_SIGN_ALLOWED = 0x0400;
public static final int PARSE_FLAG_FRACTION_GROUPING_DISABLED = 0x0800;
+ public static final int PARSE_FLAG_OPTIMIZE = 0x1000;
public static void putLeadCodePoints(UnicodeSet input, UnicodeSet output) {
for (EntryRange range : input.ranges()) {
}
}
- /**
- * Case-folds the string if IGNORE_CASE flag is set; otherwise, returns the same string.
- */
- public static String maybeFold(String input, int parseFlags) {
- UnicodeSet cwcf = UnicodeSetStaticCache.get(UnicodeSetStaticCache.Key.CWCF);
- if (0 != (parseFlags & PARSE_FLAG_IGNORE_CASE) && cwcf.containsSome(input)) {
- return UCharacter.foldCase(input, true);
- } else {
- return input;
- }
- }
-
}
private final String exponentSeparatorString;
private final DecimalMatcher exponentMatcher;
- public static ScientificMatcher getInstance(
- DecimalFormatSymbols symbols,
- Grouper grouper,
- int parseFlags) {
+ public static ScientificMatcher getInstance(DecimalFormatSymbols symbols, Grouper grouper) {
// TODO: Static-initialize most common instances?
- return new ScientificMatcher(symbols, grouper, parseFlags);
+ return new ScientificMatcher(symbols, grouper);
}
- private ScientificMatcher(DecimalFormatSymbols symbols, Grouper grouper, int parseFlags) {
- exponentSeparatorString = ParsingUtils.maybeFold(symbols.getExponentSeparator(), parseFlags);
+ private ScientificMatcher(DecimalFormatSymbols symbols, Grouper grouper) {
+ exponentSeparatorString = symbols.getExponentSeparator();
exponentMatcher = DecimalMatcher.getInstance(symbols,
grouper,
ParsingUtils.PARSE_FLAG_DECIMAL_SCIENTIFIC | ParsingUtils.PARSE_FLAG_INTEGER_ONLY);
if (segment.length() == 0) {
return true;
}
- int leadCp = segment.getCodePoint();
- if (leadCp == -1) {
- // Partial code point match
- return true;
- }
// Allow a sign, and then try to match digits.
boolean minusSign = false;
- if (UnicodeSetStaticCache.get(UnicodeSetStaticCache.Key.MINUS_SIGN).contains(leadCp)) {
+ if (segment.matches(UnicodeSetStaticCache.get(UnicodeSetStaticCache.Key.MINUS_SIGN))) {
minusSign = true;
- segment.adjustOffset(Character.charCount(leadCp));
- } else if (UnicodeSetStaticCache.get(UnicodeSetStaticCache.Key.PLUS_SIGN).contains(leadCp)) {
- segment.adjustOffset(Character.charCount(leadCp));
+ segment.adjustOffsetByCodePoint();
+ } else if (segment.matches(UnicodeSetStaticCache.get(UnicodeSetStaticCache.Key.PLUS_SIGN))) {
+ segment.adjustOffsetByCodePoint();
}
int digitsOffset = segment.getOffset();
// License & terms of use: http://www.unicode.org/copyright.html#License
package com.ibm.icu.impl.number.parse;
+import com.ibm.icu.lang.UCharacter;
+import com.ibm.icu.text.UnicodeSet;
+
/**
* A mutable class allowing for a String with a variable offset and length. The charAt, length, and
* subSequence methods all operate relative to the fixed offset into the String.
private final String str;
private int start;
private int end;
+ private boolean foldCase;
- public StringSegment(String str) {
+ public StringSegment(String str, int parseFlags) {
this.str = str;
this.start = 0;
this.end = str.length();
+ this.foldCase = 0 != (parseFlags & ParsingUtils.PARSE_FLAG_IGNORE_CASE);
}
public int getOffset() {
start += delta;
}
+ /**
+ * Adjusts the offset by the width of the current code point, either 1 or 2 chars.
+ */
+ public void adjustOffsetByCodePoint() {
+ start += Character.charCount(getCodePoint());
+ }
+
public void setLength(int length) {
assert length >= 0;
assert start + length <= str.length();
/**
* Returns the first code point in the string segment, or -1 if the string starts with an invalid
* code point.
+ *
+ * <p>
+ * <strong>Important:</strong> Most of the time, you should use {@link #matches}, which handles case
+ * folding logic, instead of this method.
*/
public int getCodePoint() {
assert start < end;
}
}
+ /**
+ * Returns true if the first code point of this StringSegment equals the given code point.
+ *
+ * <p>
+ * This method will perform case folding if case folding is enabled for the parser.
+ */
+ public boolean matches(int otherCp) {
+ return codePointsEqual(getCodePoint(), otherCp, foldCase);
+ }
+
+ /**
+ * Returns true if the first code point of this StringSegment is in the given UnicodeSet.
+ */
+ public boolean matches(UnicodeSet uniset) {
+ // TODO: Move UnicodeSet case-folding logic here.
+ // TODO: Handle string matches here instead of separately.
+ int cp = getCodePoint();
+ if (cp == -1) {
+ return false;
+ }
+ return uniset.contains(cp);
+ }
+
/**
* Returns the length of the prefix shared by this StringSegment and the given CharSequence. For
* example, if this string segment is "aab", and the char sequence is "aac", this method returns 2,
* since the first 2 characters are the same.
+ *
+ * <p>
+ * This method will perform case folding if case folding is enabled for the parser.
*/
public int getCommonPrefixLength(CharSequence other) {
+ return getPrefixLengthInternal(other, foldCase);
+ }
+
+ /**
+ * Like {@link #getCommonPrefixLength}, but never performs case folding, even if case folding is
+ * enabled for the parser.
+ */
+ public int getCaseSensitivePrefixLength(CharSequence other) {
+ return getPrefixLengthInternal(other, false);
+ }
+
+ private int getPrefixLengthInternal(CharSequence other, boolean foldCase) {
int offset = 0;
for (; offset < Math.min(length(), other.length());) {
- if (charAt(offset) != other.charAt(offset)) {
+ // TODO: case-fold code points, not chars
+ char c1 = charAt(offset);
+ char c2 = other.charAt(offset);
+ if (!codePointsEqual(c1, c2, foldCase)) {
break;
}
offset++;
return offset;
}
+ // /**
+ // * Case-folds the string if IGNORE_CASE flag is set; otherwise, returns the same string.
+ // */
+ // public static String maybeFold(String input, int parseFlags) {
+ // UnicodeSet cwcf = UnicodeSetStaticCache.get(UnicodeSetStaticCache.Key.CWCF);
+ // if (0 != (parseFlags & ParsingUtils.PARSE_FLAG_IGNORE_CASE) && cwcf.containsSome(input)) {
+ // return UCharacter.foldCase(input, true);
+ // } else {
+ // return input;
+ // }
+ // }
+
+ private static final boolean codePointsEqual(int cp1, int cp2, boolean foldCase) {
+ if (cp1 == cp2) {
+ return true;
+ }
+ if (!foldCase) {
+ return false;
+ }
+ cp1 = UCharacter.foldCase(cp1, true);
+ cp2 = UCharacter.foldCase(cp2, true);
+ return cp1 == cp2;
+ }
+
@Override
public String toString() {
return str.substring(0, start) + "[" + str.substring(start, end) + "]" + str.substring(end);
}
}
- int cp = segment.getCodePoint();
- if (cp != -1 && uniSet.contains(cp)) {
- segment.adjustOffset(Character.charCount(cp));
+ if (segment.matches(uniSet)) {
+ segment.adjustOffsetByCodePoint();
accept(segment, result);
return false;
}
DIGITS,
NAN_LEAD,
SCIENTIFIC_LEAD,
- CWCF,
+ CWCF, // TODO: Check if this is being used and remove it if not.
// Combined Separators with Digits (for lead code points)
DIGITS_OR_ALL_SEPARATORS,
import java.io.Serializable;
import java.lang.reflect.Field;
import java.text.FieldPosition;
+import java.text.ParseException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import com.ibm.icu.text.MeasureFormat.FormatWidth;
import com.ibm.icu.text.NumberFormat;
import com.ibm.icu.util.Currency;
+import com.ibm.icu.util.CurrencyAmount;
import com.ibm.icu.util.Measure;
import com.ibm.icu.util.MeasureUnit;
import com.ibm.icu.util.NoUnit;
assertEquals("getCurrencyFormat ULocale/Locale", mfu, mfj);
}
+ @Test
+ public void testCurrencyFormatParseIsoCode() throws ParseException {
+ MeasureFormat mf = MeasureFormat.getCurrencyFormat(ULocale.ENGLISH);
+ CurrencyAmount result = (CurrencyAmount) mf.parseObject("GTQ 34.56");
+ assertEquals("Parse should succeed", result.getNumber().doubleValue(), 34.56, 0.0);
+ assertEquals("Should parse ISO code GTQ even though the currency is USD",
+ "GTQ", result.getCurrency().getCurrencyCode());
+ }
+
@Test
public void testDoubleZero() {
ULocale en = new ULocale("en");
new ParseCurrencyItem( "en_GB", "euros4", "4,00\u00A0\u20AC", 6,400, "EUR" ),
new ParseCurrencyItem( "en_GB", "euros6", "6\u00A0\u20AC", 3, 6, "EUR" ),
new ParseCurrencyItem( "en_GB", "euros8", "\u20AC8", 2, 8, "EUR" ),
- new ParseCurrencyItem( "en_GB", "dollars4", "US$4", 0, 0, "USD" ),
+ new ParseCurrencyItem( "en_GB", "dollars4", "US$4", 4, 4, "USD" ),
new ParseCurrencyItem( "fr_FR", "euros4", "4,00\u00A0\u20AC", 6, 4, "EUR" ),
new ParseCurrencyItem( "fr_FR", "euros6", "6\u00A0\u20AC", 3, 6, "EUR" ),
};
@SuppressWarnings("resource") // InputStream is will be closed by the ResourceReader.
- @Ignore("TODO: http://bugs.icu-project.org/trac/ticket/13571")
@Test
public void TestCases() {
String caseFileName = "NumberFormatTestCases.txt";
import com.ibm.icu.impl.number.parse.MinusSignMatcher;
import com.ibm.icu.impl.number.parse.NumberParserImpl;
import com.ibm.icu.impl.number.parse.ParsedNumber;
+import com.ibm.icu.impl.number.parse.ParsingUtils;
import com.ibm.icu.impl.number.parse.PercentMatcher;
import com.ibm.icu.impl.number.parse.PlusSignMatcher;
import com.ibm.icu.impl.number.parse.SeriesMatcher;
int expectedOffset = (Integer) cas[1];
boolean expectedMaybeMore = (Boolean) cas[2];
- StringSegment segment = new StringSegment(input);
+ StringSegment segment = new StringSegment(input, 0);
ParsedNumber result = new ParsedNumber();
boolean actualMaybeMore = series.match(segment, result);
int actualOffset = segment.getOffset();
result.getNumber().doubleValue(),
0.0);
}
+
+ @Test
+ public void testCaseFolding() {
+ Object[][] cases = new Object[][] {
+ // pattern, input string, case sensitive chars, case insensitive chars
+ { "0", "JP¥3456", 7, 7 },
+ { "0", "jp¥3456", 0, 0 }, // not to be accepted, even in case insensitive mode
+ { "A0", "A5", 2, 2 },
+ { "A0", "a5", 0, 2 },
+ { "0", "NaN", 3, 3 },
+ { "0", "nan", 0, 3 } };
+ for (Object[] cas : cases) {
+ String patternString = (String) cas[0];
+ String inputString = (String) cas[1];
+ int expectedCaseSensitiveChars = (Integer) cas[2];
+ int expectedCaseFoldingChars = (Integer) cas[3];
+
+ NumberParserImpl caseSensitiveParser = NumberParserImpl
+ .removeMeWhenMerged(ULocale.ENGLISH, patternString, ParsingUtils.PARSE_FLAG_OPTIMIZE);
+ ParsedNumber result = new ParsedNumber();
+ caseSensitiveParser.parse(inputString, true, result);
+ assertEquals("Case-Sensitive: " + inputString + " on " + patternString,
+ expectedCaseSensitiveChars,
+ result.charEnd);
+
+ NumberParserImpl caseFoldingParser = NumberParserImpl.removeMeWhenMerged(ULocale.ENGLISH,
+ patternString,
+ ParsingUtils.PARSE_FLAG_IGNORE_CASE | ParsingUtils.PARSE_FLAG_OPTIMIZE);
+ result = new ParsedNumber();
+ caseFoldingParser.parse(inputString, true, result);
+ assertEquals("Folded: " + inputString + " on " + patternString,
+ expectedCaseFoldingChars,
+ result.charEnd);
+ }
+ }
}
@Test
public void testOffset() {
- StringSegment segment = new StringSegment(SAMPLE_STRING);
+ StringSegment segment = new StringSegment(SAMPLE_STRING, 0);
assertEquals(0, segment.getOffset());
segment.adjustOffset(3);
assertEquals(3, segment.getOffset());
@Test
public void testLength() {
- StringSegment segment = new StringSegment(SAMPLE_STRING);
+ StringSegment segment = new StringSegment(SAMPLE_STRING, 0);
assertEquals(11, segment.length());
segment.adjustOffset(3);
assertEquals(8, segment.length());
@Test
public void testCharAt() {
- StringSegment segment = new StringSegment(SAMPLE_STRING);
+ StringSegment segment = new StringSegment(SAMPLE_STRING, 0);
assertCharSequenceEquals(SAMPLE_STRING, segment);
segment.adjustOffset(3);
assertCharSequenceEquals("radio 📻", segment);
@Test
public void testGetCodePoint() {
- StringSegment segment = new StringSegment(SAMPLE_STRING);
+ StringSegment segment = new StringSegment(SAMPLE_STRING, 0);
assertEquals(0x1F4FB, segment.getCodePoint());
segment.setLength(1);
assertEquals(-1, segment.getCodePoint());
@Test
public void testCommonPrefixLength() {
- StringSegment segment = new StringSegment(SAMPLE_STRING);
+ StringSegment segment = new StringSegment(SAMPLE_STRING, 0);
assertEquals(11, segment.getCommonPrefixLength(SAMPLE_STRING));
assertEquals(4, segment.getCommonPrefixLength("📻 r"));
assertEquals(3, segment.getCommonPrefixLength("📻 x"));
projects / icu / commitdiff