import java.text.CharacterIterator;
import java.util.HashSet;
+import java.util.Locale;
import com.ibm.icu.impl.ICUResourceBundle.OpenType;
import com.ibm.icu.text.BreakIterator;
import com.ibm.icu.text.FilteredBreakIteratorBuilder;
import com.ibm.icu.text.UCharacterIterator;
+import com.ibm.icu.text.UnicodeSet;
import com.ibm.icu.util.BytesTrie;
import com.ibm.icu.util.CharsTrie;
import com.ibm.icu.util.CharsTrieBuilder;
private UCharacterIterator text; // TODO(Tom): suffice to move into the local scope in next() ?
private CharsTrie backwardsTrie; // i.e. ".srM" for Mrs.
private CharsTrie forwardsPartialTrie; // Has ".a" for "a.M."
+ private UnicodeSet glueSet = null;
/**
* @param adoptBreakIterator
* forward & partial char trie to adopt
* @param backwardsTrie
* backward trie to adopt
+ * @param glueSet the glue set to adopt
*/
public SimpleFilteredSentenceBreakIterator(BreakIterator adoptBreakIterator, CharsTrie forwardsPartialTrie,
- CharsTrie backwardsTrie) {
+ CharsTrie backwardsTrie, UnicodeSet glueSet) {
this.delegate = adoptBreakIterator;
this.forwardsPartialTrie = forwardsPartialTrie;
this.backwardsTrie = backwardsTrie;
+ if(!glueSet.isEmpty()) {
+ this.glueSet = new UnicodeSet(glueSet).freeze(); // copy
+ }
}
/**
* Is there an exception at this point?
*
- * @param n
+ * @param n the location of the possible break
* @return
*/
private final boolean breakExceptionAt(int n) {
backwardsTrie.reset();
int uch;
+
+
// Assume a space is following the '.' (so we handle the case: "Mr. /Brown")
if ((uch = text.previousCodePoint()) == ' ') { // TODO: skip a class of chars here??
// TODO only do this the 1st time?
uch = text.nextCodePoint();
}
+ // Check for a glue character
+ if(this.glueSet != null && text.getIndex()<text.getLength()) {
+ uch = text.nextCodePoint();
+ if(glueSet.contains(uch)) {
+ // Glued - suppress this break.
+ return false; // no break here.
+ } else {
+ uch = text.previousCodePoint(); // move back again.
+ }
+ }
+
BytesTrie.Result r = BytesTrie.Result.INTERMEDIATE_VALUE;
while ((uch = text.previousCodePoint()) != UCharacterIterator.DONE && // more to consume backwards and..
/**
* filter set to store all exceptions
*/
- private HashSet<String> filterSet = new HashSet<String>();
+ private HashSet<CharSequence> filterSet = new HashSet<CharSequence>();
static final int PARTIAL = (1 << 0); // < partial - need to run through forward trie
static final int MATCH = (1 << 1); // < exact match - skip this one.
static final int SuppressInReverse = (1 << 0);
static final int AddToForward = (1 << 1);
+ private UnicodeSet glueSet = new UnicodeSet();
+
+ public Builder(Locale loc) {
+ this(ULocale.forLocale(loc));
+ }
/**
* Create SimpleFilteredBreakIteratorBuilder using given locale
* @param loc the locale to get filtered iterators
* Create SimpleFilteredBreakIteratorBuilder with no exception
*/
public Builder() {
- filterSet = new HashSet<String>();
}
@Override
- public boolean suppressBreakAfter(String str) {
- if (filterSet == null) {
- filterSet = new HashSet<String>();
- }
+ public boolean suppressBreakAfter(CharSequence str) {
return filterSet.add(str);
}
@Override
- public boolean unsuppressBreakAfter(String str) {
- if (filterSet == null) {
- return false;
- } else {
- return filterSet.remove(str);
- }
+ public boolean unsuppressBreakAfter(CharSequence str) {
+ return filterSet.remove(str);
}
@Override
- public BreakIterator build(BreakIterator adoptBreakIterator) {
+ public BreakIterator wrapIteratorWithFilter(BreakIterator adoptBreakIterator) {
if( filterSet.isEmpty() ) {
// Short circuit - nothing to except.
return adoptBreakIterator;
int fwdCount = 0;
int subCount = filterSet.size();
- String[] ustrs = new String[subCount];
+ CharSequence[] ustrs = new CharSequence[subCount];
int[] partials = new int[subCount];
CharsTrie backwardsTrie = null; // i.e. ".srM" for Mrs.
CharsTrie forwardsPartialTrie = null; // Has ".a" for "a.M."
int i = 0;
- for (String s : filterSet) {
+ for (CharSequence s : filterSet) {
ustrs[i] = s; // copy by value?
partials[i] = 0; // default: no partial
i++;
}
for (i = 0; i < subCount; i++) {
- int nn = ustrs[i].indexOf('.'); // TODO: non-'.' abbreviations
- if (nn > -1 && (nn + 1) != ustrs[i].length()) {
+ String thisStr = ustrs[i].toString(); // TODO: don't cast to String?
+ int nn = thisStr.indexOf('.'); // TODO: non-'.' abbreviations
+ if (nn > -1 && (nn + 1) != thisStr.length()) {
// is partial.
// is it unique?
int sameAs = -1;
for (int j = 0; j < subCount; j++) {
if (j == i)
continue;
- if (ustrs[i].regionMatches(0, ustrs[j], 0, nn + 1)) {
+ if (thisStr.regionMatches(0, ustrs[j].toString() /* TODO */, 0, nn + 1)) {
if (partials[j] == 0) { // hasn't been processed yet
partials[j] = SuppressInReverse | AddToForward;
} else if ((partials[j] & SuppressInReverse) != 0) {
}
if ((sameAs == -1) && (partials[i] == 0)) {
- StringBuilder prefix = new StringBuilder(ustrs[i].substring(0, nn + 1));
+ StringBuilder prefix = new StringBuilder(thisStr.substring(0, nn + 1));
// first one - add the prefix to the reverse table.
prefix.reverse();
builder.add(prefix, PARTIAL);
}
for (i = 0; i < subCount; i++) {
+ final String thisStr = ustrs[i].toString(); // TODO
if (partials[i] == 0) {
- StringBuilder reversed = new StringBuilder(ustrs[i]).reverse();
+ StringBuilder reversed = new StringBuilder(thisStr).reverse();
builder.add(reversed, MATCH);
revCount++;
} else {
// forward,
// instead of "Ph.D." since we already know the "Ph." part is a match.
// would need the trie to be able to hold 0-length strings, though.
- builder2.add(ustrs[i], MATCH); // forward
+ builder2.add(thisStr, MATCH); // forward
fwdCount++;
}
}
if (fwdCount > 0) {
forwardsPartialTrie = builder2.build(StringTrieBuilder.Option.FAST);
}
- return new SimpleFilteredSentenceBreakIterator(adoptBreakIterator, forwardsPartialTrie, backwardsTrie);
+ return new SimpleFilteredSentenceBreakIterator(adoptBreakIterator, forwardsPartialTrie, backwardsTrie, glueSet);
+ }
+
+ /* (non-Javadoc)
+ * @see com.ibm.icu.text.FilteredBreakIteratorBuilder#setGlueCharacters(com.ibm.icu.text.UnicodeSet)
+ * @internal
+ */
+ public void setGlueCharacters(UnicodeSet set) {
+ if (set == null || set.isEmpty()) {
+ glueSet.clear();
+ } else {
+ glueSet.set(set);
+ }
}
}
}
final String ssKeyword = locale.getKeywordValue("ss");
if (ssKeyword != null && ssKeyword.equals("standard")) {
final ULocale base = new ULocale(locale.getBaseName());
- return FilteredBreakIteratorBuilder.createInstance(base).build(iter);
+ return FilteredBreakIteratorBuilder.getInstance(base).wrapIteratorWithFilter(iter);
}
}
*/
package com.ibm.icu.text;
+import java.util.Locale;
+
import com.ibm.icu.impl.SimpleFilteredSentenceBreakIterator;
import com.ibm.icu.util.ULocale;
* in the string "Mr. Smith" (resulting in two segments),
* but with "Mr." as an exception, a filtered break iterator
* would consider the string "Mr. Smith" to be a single segment.
- *
- * <p><b>Note:</b> An instance of {@link BreakIterator} returned by this builder
- * class currently does not support following operations in this technology preview
- * version:
- * <ul>
- * <li>{@link BreakIterator#next(int) next(int n)}</li>
- * <li>{@link BreakIterator#previous() previous()}</li>
- * <li>{@link BreakIterator#following(int) following(int offset)}</li>
- * <li>{@link BreakIterator#preceding(int) preceding(int offset)}</li>
- * </ul>
- * When one of above methods is called, {@link UnsupportedOperationException} will be
- * thrown.
*
- * @author tomzhang
- *
- * @internal ICU 54 technology preview
- * @deprecated This API might change or be removed in a future release.
+ * <p>This class is not intended for public subclassing.
+ *
+ * @draft ICU 60
+ * @provisional This API might change or be removed in a future release.
*/
-@Deprecated
public abstract class FilteredBreakIteratorBuilder {
/**
- * Construct a FilteredBreakIteratorBuilder based on rules in a locale.
+ * Construct a FilteredBreakIteratorBuilder based on sentence break exception rules in a locale.
* The rules are taken from CLDR exception data for the locale,
* see http://www.unicode.org/reports/tr35/tr35-general.html#Segmentation_Exceptions
* This is the equivalent of calling createInstance(UErrorCode&)
* of the CLDR exception data.
* @param where the locale.
* @return the new builder
- * @internal ICU 54 technology preview
- * @deprecated This API might change or be removed in a future release.
+ * @draft ICU 60
+ * @provisional This API might change or be removed in a future release.
*/
- @Deprecated
- public static FilteredBreakIteratorBuilder createInstance(ULocale where) {
- FilteredBreakIteratorBuilder ret = new SimpleFilteredSentenceBreakIterator.Builder(where);
- return ret;
+ public static final FilteredBreakIteratorBuilder getInstance(Locale where) {
+ return new SimpleFilteredSentenceBreakIterator.Builder(where);
+ }
+
+ /**
+ * Construct a FilteredBreakIteratorBuilder based on sentence break exception rules in a locale.
+ * The rules are taken from CLDR exception data for the locale,
+ * see http://www.unicode.org/reports/tr35/tr35-general.html#Segmentation_Exceptions
+ * This is the equivalent of calling createInstance(UErrorCode&)
+ * and then repeatedly calling addNoBreakAfter(...) with the contents
+ * of the CLDR exception data.
+ * @param where the locale.
+ * @return the new builder
+ * @draft ICU 60
+ * @provisional This API might change or be removed in a future release.
+ */
+ public static final FilteredBreakIteratorBuilder getInstance(ULocale where) {
+ return new SimpleFilteredSentenceBreakIterator.Builder(where);
}
/**
* Construct an empty FilteredBreakIteratorBuilder.
* In this state, it will not suppress any segment boundaries.
* @return the new builder
- * @internal ICU 54 technology preview
- * @deprecated This API might change or be removed in a future release.
+ * @draft ICU 60
+ * @provisional This API might change or be removed in a future release.
*/
- @Deprecated
- public static FilteredBreakIteratorBuilder createInstance() {
- FilteredBreakIteratorBuilder ret = new SimpleFilteredSentenceBreakIterator.Builder();
- return ret;
+ public static final FilteredBreakIteratorBuilder getEmptyInstance() {
+ return new SimpleFilteredSentenceBreakIterator.Builder();
}
/**
* For example, suppressing "Mr.", then segments ending in "Mr." will not be returned
* by the iterator.
* @param str the string to suppress, such as "Mr."
- * @return returns true if the string was not present and now added,
+ * @return true if the string was not present and now added,
* false if the call was a no-op because the string was already being suppressed.
- * @internal ICU 54 technology preview
- * @deprecated This API might change or be removed in a future release.
+ * @draft ICU 60
+ * @provisional This API might change or be removed in a future release.
*/
- @Deprecated
- public abstract boolean suppressBreakAfter(String str);
+ public abstract boolean suppressBreakAfter(CharSequence str);
/**
* Stop suppressing a certain string from being the end of the segment.
* the effect of earlier calls to suppressBreakAfter, or to un-do the effect of
* locale data which may be suppressing certain strings.
* @param str the str the string to unsuppress, such as "Mr."
- * @return returns true if the string was present and now removed,
+ * @return true if the string was present and now removed,
* false if the call was a no-op because the string was not being suppressed.
- * @internal ICU 54 technology preview
- * @deprecated This API might change or be removed in a future release.
+ * @draft ICU 60
+ * @provisional This API might change or be removed in a future release.
*/
- @Deprecated
- public abstract boolean unsuppressBreakAfter(String str);
+ public abstract boolean unsuppressBreakAfter(CharSequence str);
/**
* Wrap (adopt) an existing break iterator in a new filtered instance.
- * The resulting BreakIterator is owned by the caller.
- * The BreakIteratorFilter may be destroyed before the BreakIterator is destroyed.
- * Note that the adoptBreakIterator is adopted by the new BreakIterator
+ * Note that the wrappedBreakIterator is adopted by the new BreakIterator
* and should no longer be used by the caller.
* The FilteredBreakIteratorBuilder may be reused.
- * @param adoptBreakIterator the break iterator to adopt
- * @return the new BreakIterator, owned by the caller.
- * @internal ICU 54 technology preview
- * @deprecated This API might change or be removed in a future release.
+ * @param wrappedBreakIterator the break iterator to wrap
+ * @return the new BreakIterator
+ * @draft ICU 60
+ * @provisional This API might change or be removed in a future release.
*/
- @Deprecated
- public abstract BreakIterator build(BreakIterator adoptBreakIterator);
+ public abstract BreakIterator wrapIteratorWithFilter(BreakIterator wrappedBreakIterator);
/**
* For subclass use
- * @internal ICU 54 technology preview
- * @deprecated This API might change or be removed in a future release.
+ * @internal
+ * @deprecated internal to ICU
*/
@Deprecated
- protected FilteredBreakIteratorBuilder() {}
+ protected FilteredBreakIteratorBuilder() {
+ }
}
\ No newline at end of file
public void TestFilteredJapanese() {
ULocale loc = ULocale.JAPANESE;
BreakIterator brk = FilteredBreakIteratorBuilder
- .createInstance(loc)
- .build(BreakIterator.getSentenceInstance(loc));
+ .getInstance(loc)
+ .wrapIteratorWithFilter(BreakIterator.getSentenceInstance(loc));
brk.setText("ＯＫです。");
assertEquals("Starting point", 0, brk.current());
assertEquals("Next point", 5, brk.next());
{
logln("Constructing empty builder\n");
- builder = FilteredBreakIteratorBuilder.createInstance();
+ builder = FilteredBreakIteratorBuilder.getEmptyInstance();
logln("Constructing base BI\n");
baseBI = BreakIterator.getSentenceInstance(Locale.ENGLISH);
logln("Building new BI\n");
- filteredBI = builder.build(baseBI);
+ filteredBI = builder.wrapIteratorWithFilter(baseBI);
assertDefaultBreakBehavior(filteredBI, text);
}
{
logln("Constructing empty builder\n");
- builder = FilteredBreakIteratorBuilder.createInstance();
+ builder = FilteredBreakIteratorBuilder.getEmptyInstance();
logln("Adding Mr. as an exception\n");
baseBI = BreakIterator.getSentenceInstance(Locale.ENGLISH);
logln("Building new BI\n");
- filteredBI = builder.build(baseBI);
+ filteredBI = builder.wrapIteratorWithFilter(baseBI);
logln("Testing:");
filteredBI.setText(text);
{
logln("Constructing empty builder\n");
- builder = FilteredBreakIteratorBuilder.createInstance();
+ builder = FilteredBreakIteratorBuilder.getEmptyInstance();
logln("Adding Mr. and Capt as an exception\n");
assertEquals("3.1 suppressBreakAfter", true, builder.suppressBreakAfter(ABBR_MR));
baseBI = BreakIterator.getSentenceInstance(Locale.ENGLISH);
logln("Building new BI\n");
- filteredBI = builder.build(baseBI);
+ filteredBI = builder.wrapIteratorWithFilter(baseBI);
logln("Testing:");
filteredBI.setText(text);
{
logln("Constructing English builder\n");
- builder = FilteredBreakIteratorBuilder.createInstance(ULocale.ENGLISH);
+ builder = FilteredBreakIteratorBuilder.getInstance(ULocale.ENGLISH);
logln("Constructing base BI\n");
baseBI = BreakIterator.getSentenceInstance(Locale.ENGLISH);
assertEquals("1st suppressBreakAfter", true, builder.unsuppressBreakAfter(ABBR_CAPT));
logln("Building new BI\n");
- filteredBI = builder.build(baseBI);
+ filteredBI = builder.wrapIteratorWithFilter(baseBI);
if(filteredBI != null) {
logln("Testing:");
{
logln("Constructing English builder\n");
- builder = FilteredBreakIteratorBuilder.createInstance(ULocale.ENGLISH);
+ builder = FilteredBreakIteratorBuilder.getInstance(ULocale.ENGLISH);
logln("Constructing base BI\n");
baseBI = BreakIterator.getSentenceInstance(Locale.ENGLISH);
logln("Building new BI\n");
- filteredBI = builder.build(baseBI);
+ filteredBI = builder.wrapIteratorWithFilter(baseBI);
if(filteredBI != null) {
assertEnglishBreakBehavior(filteredBI, text);
{
logln("Constructing French builder");
- builder = FilteredBreakIteratorBuilder.createInstance(ULocale.FRENCH);
+ builder = FilteredBreakIteratorBuilder.getInstance(ULocale.FRENCH);
logln("Constructing base BI\n");
baseBI = BreakIterator.getSentenceInstance(Locale.FRENCH);
logln("Building new BI\n");
- filteredBI = builder.build(baseBI);
+ filteredBI = builder.wrapIteratorWithFilter(baseBI);
if(filteredBI != null) {
assertFrenchBreakBehavior(filteredBI, text);
projects / icu / commitdiff