uscript.o uscript_props.o usc_impl.o unames.o \
utrie.o utrie2.o utrie2_builder.o bmpset.o unisetspan.o uset_props.o uniset_props.o uniset_closure.o uset.o uniset.o usetiter.o ruleiter.o caniter.o unifilt.o unifunct.o \
uarrsort.o brkiter.o ubrk.o brkeng.o dictbe.o filteredbrk.o \
-rbbi.o rbbidata.o rbbinode.o rbbirb.o rbbiscan.o rbbisetb.o rbbistbl.o rbbitblb.o \
+rbbi.o rbbidata.o rbbinode.o rbbirb.o rbbiscan.o rbbisetb.o rbbistbl.o rbbitblb.o rbbi_cache.o \
serv.o servnotf.o servls.o servlk.o servlkf.o servrbf.o servslkf.o \
uidna.o usprep.o uts46.o punycode.o \
util.o util_props.o parsepos.o locbased.o cwchar.o wintz.o dtintrv.o ucnvsel.o propsvec.o \
#if !UCONFIG_NO_BREAK_ITERATION
-#include "brkeng.h"
-#include "cmemory.h"
-#include "dictbe.h"
#include "unicode/uchar.h"
#include "unicode/uniset.h"
#include "unicode/chariter.h"
#include "unicode/uscript.h"
#include "unicode/ucharstrie.h"
#include "unicode/bytestrie.h"
+
+#include "brkeng.h"
+#include "cmemory.h"
+#include "dictbe.h"
#include "charstr.h"
#include "dictionarydata.h"
#include "mutex.h"
int32_t
UnhandledEngine::findBreaks( UText *text,
- int32_t startPos,
- int32_t endPos,
- UBool reverse,
- int32_t breakType,
- UStack &/*foundBreaks*/ ) const {
+ int32_t /* startPos */,
+ int32_t endPos,
+ int32_t breakType,
+ UVector32 &/*foundBreaks*/ ) const {
if (breakType >= 0 && breakType < UPRV_LENGTHOF(fHandled)) {
UChar32 c = utext_current32(text);
- if (reverse) {
- while((int32_t)utext_getNativeIndex(text) > startPos && fHandled[breakType]->contains(c)) {
- c = utext_previous32(text);
- }
- }
- else {
- while((int32_t)utext_getNativeIndex(text) < endPos && fHandled[breakType]->contains(c)) {
- utext_next32(text); // TODO: recast loop to work with post-increment operations.
- c = utext_current32(text);
- }
+ while((int32_t)utext_getNativeIndex(text) < endPos && fHandled[breakType]->contains(c)) {
+ utext_next32(text); // TODO: recast loop to work with post-increment operations.
+ c = utext_current32(text);
}
}
return 0;
class UnicodeSet;
class UStack;
+class UVector32;
class DictionaryMatcher;
/*******************************************************************
* is capable of handling.
* @param startPos The start of the run within the supplied text.
* @param endPos The end of the run within the supplied text.
- * @param reverse Whether the caller is looking for breaks in a reverse
- * direction.
* @param breakType The type of break desired, or -1.
- * @param foundBreaks An allocated C array of the breaks found, if any
+ * @param foundBreaks A Vector of int32_t to receive the breaks.
* @return The number of breaks found.
*/
virtual int32_t findBreaks( UText *text,
int32_t startPos,
int32_t endPos,
- UBool reverse,
int32_t breakType,
- UStack &foundBreaks ) const = 0;
+ UVector32 &foundBreaks ) const = 0;
};
* is capable of handling.
* @param startPos The start of the run within the supplied text.
* @param endPos The end of the run within the supplied text.
- * @param reverse Whether the caller is looking for breaks in a reverse
- * direction.
* @param breakType The type of break desired, or -1.
* @param foundBreaks An allocated C array of the breaks found, if any
* @return The number of breaks found.
virtual int32_t findBreaks( UText *text,
int32_t startPos,
int32_t endPos,
- UBool reverse,
int32_t breakType,
- UStack &foundBreaks ) const;
+ UVector32 &foundBreaks ) const;
/**
* <p>Tell the engine to handle a particular character and break type.</p>
</ClCompile>
<ClCompile Include="rbbitblb.cpp">
</ClCompile>
+ <ClCompile Include="rbbi_cache.cpp">
+ </ClCompile>
<ClCompile Include="dictionarydata.cpp" />
<ClCompile Include="ubrk.cpp" />
<ClCompile Include="ucol_swp.cpp">
<ClInclude Include="rbbiscan.h" />
<ClInclude Include="rbbisetb.h" />
<ClInclude Include="rbbitblb.h" />
+ <ClInclude Include="rbbi_cache.h" />
<ClInclude Include="dictionarydata.h" />
<CustomBuild Include="unicode\ubrk.h">
<Command Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">copy "%(FullPath)" ..\..\include\unicode
<ClCompile Include="rbbitblb.cpp">
<Filter>break iteration</Filter>
</ClCompile>
+ <ClCompile Include="rbbi_cache.cpp">
+ <Filter>break iteration</Filter>
+ </ClCompile>
<ClCompile Include="ubrk.cpp">
<Filter>break iteration</Filter>
</ClCompile>
<ClInclude Include="rbbitblb.h">
<Filter>break iteration</Filter>
</ClInclude>
+ <ClInclude Include="rbbi_cache.h">
+ <Filter>break iteration</Filter>
+ </ClInclude>
<ClInclude Include="ubrkimpl.h">
<Filter>break iteration</Filter>
</ClInclude>
DictionaryBreakEngine::findBreaks( UText *text,
int32_t startPos,
int32_t endPos,
- UBool reverse,
int32_t breakType,
- UStack &foundBreaks ) const {
+ UVector32 &foundBreaks ) const {
+ (void)startPos; // TODO: remove this param?
int32_t result = 0;
// Find the span of characters included in the set.
int32_t rangeStart;
int32_t rangeEnd;
UChar32 c = utext_current32(text);
- if (reverse) {
- UBool isDict = fSet.contains(c);
- while((current = (int32_t)utext_getNativeIndex(text)) > startPos && isDict) {
- c = utext_previous32(text);
- isDict = fSet.contains(c);
- }
- if (current < startPos) {
- rangeStart = startPos;
- } else {
- rangeStart = current;
- if (!isDict) {
- utext_next32(text);
- rangeStart = (int32_t)utext_getNativeIndex(text);
- }
- }
- // rangeEnd = start + 1;
- utext_setNativeIndex(text, start);
- utext_next32(text);
- rangeEnd = (int32_t)utext_getNativeIndex(text);
- }
- else {
- while((current = (int32_t)utext_getNativeIndex(text)) < endPos && fSet.contains(c)) {
- utext_next32(text); // TODO: recast loop for postincrement
- c = utext_current32(text);
- }
- rangeStart = start;
- rangeEnd = current;
+ while((current = (int32_t)utext_getNativeIndex(text)) < endPos && fSet.contains(c)) {
+ utext_next32(text); // TODO: recast loop for postincrement
+ c = utext_current32(text);
}
+ rangeStart = start;
+ rangeEnd = current;
if (breakType >= 0 && breakType < 32 && (((uint32_t)1 << breakType) & fTypes)) {
result = divideUpDictionaryRange(text, rangeStart, rangeEnd, foundBreaks);
utext_setNativeIndex(text, current);
ThaiBreakEngine::divideUpDictionaryRange( UText *text,
int32_t rangeStart,
int32_t rangeEnd,
- UStack &foundBreaks ) const {
+ UVector32 &foundBreaks ) const {
utext_setNativeIndex(text, rangeStart);
utext_moveIndex32(text, THAI_MIN_WORD_SPAN);
if (utext_getNativeIndex(text) >= rangeEnd) {
LaoBreakEngine::divideUpDictionaryRange( UText *text,
int32_t rangeStart,
int32_t rangeEnd,
- UStack &foundBreaks ) const {
+ UVector32 &foundBreaks ) const {
if ((rangeEnd - rangeStart) < LAO_MIN_WORD_SPAN) {
return 0; // Not enough characters for two words
}
BurmeseBreakEngine::divideUpDictionaryRange( UText *text,
int32_t rangeStart,
int32_t rangeEnd,
- UStack &foundBreaks ) const {
+ UVector32 &foundBreaks ) const {
if ((rangeEnd - rangeStart) < BURMESE_MIN_WORD_SPAN) {
return 0; // Not enough characters for two words
}
KhmerBreakEngine::divideUpDictionaryRange( UText *text,
int32_t rangeStart,
int32_t rangeEnd,
- UStack &foundBreaks ) const {
+ UVector32 &foundBreaks ) const {
if ((rangeEnd - rangeStart) < KHMER_MIN_WORD_SPAN) {
return 0; // Not enough characters for two words
}
* @param text A UText representing the text
* @param rangeStart The start of the range of dictionary characters
* @param rangeEnd The end of the range of dictionary characters
- * @param foundBreaks Output of C array of int32_t break positions, or 0
+ * @param foundBreaks vector<int32> to receive the break positions
* @return The number of breaks found
*/
int32_t
CjkBreakEngine::divideUpDictionaryRange( UText *inText,
int32_t rangeStart,
int32_t rangeEnd,
- UStack &foundBreaks ) const {
+ UVector32 &foundBreaks ) const {
if (rangeStart >= rangeEnd) {
return 0;
}
#include "unicode/utext.h"
#include "brkeng.h"
+#include "uvectr32.h"
U_NAMESPACE_BEGIN
*
* @param text A UText representing the text. The iterator is left at
* the end of the run of characters which the engine is capable of handling
- * that starts from the first (or last) character in the range.
+ * that starts from the first character in the range.
* @param startPos The start of the run within the supplied text.
* @param endPos The end of the run within the supplied text.
- * @param reverse Whether the caller is looking for breaks in a reverse
- * direction.
* @param breakType The type of break desired, or -1.
- * @param foundBreaks An allocated C array of the breaks found, if any
+ * @param foundBreaks vector of int32_t to receive the break positions
* @return The number of breaks found.
*/
virtual int32_t findBreaks( UText *text,
int32_t startPos,
int32_t endPos,
- UBool reverse,
int32_t breakType,
- UStack &foundBreaks ) const;
+ UVector32 &foundBreaks ) const;
protected:
virtual int32_t divideUpDictionaryRange( UText *text,
int32_t rangeStart,
int32_t rangeEnd,
- UStack &foundBreaks ) const = 0;
+ UVector32 &foundBreaks ) const = 0;
};
virtual int32_t divideUpDictionaryRange( UText *text,
int32_t rangeStart,
int32_t rangeEnd,
- UStack &foundBreaks ) const;
+ UVector32 &foundBreaks ) const;
};
virtual int32_t divideUpDictionaryRange( UText *text,
int32_t rangeStart,
int32_t rangeEnd,
- UStack &foundBreaks ) const;
+ UVector32 &foundBreaks ) const;
};
virtual int32_t divideUpDictionaryRange( UText *text,
int32_t rangeStart,
int32_t rangeEnd,
- UStack &foundBreaks ) const;
+ UVector32 &foundBreaks ) const;
};
virtual int32_t divideUpDictionaryRange( UText *text,
int32_t rangeStart,
int32_t rangeEnd,
- UStack &foundBreaks ) const;
+ UVector32 &foundBreaks ) const;
};
virtual int32_t divideUpDictionaryRange( UText *text,
int32_t rangeStart,
int32_t rangeEnd,
- UStack &foundBreaks ) const;
+ UVector32 &foundBreaks ) const;
};
***************************************************************************
*/
//
-// file: rbbi.c Contains the implementation of the rule based break iterator
+// file: rbbi.cpp Contains the implementation of the rule based break iterator
// runtime engine and the API implementation for
// class RuleBasedBreakIterator
//
#include "unicode/uchriter.h"
#include "unicode/uclean.h"
#include "unicode/udata.h"
+
#include "brkeng.h"
+#include "ucln_cmn.h"
#include "cmemory.h"
#include "cstring.h"
#include "rbbidata.h"
+#include "rbbi_cache.h"
#include "rbbirb.h"
#include "uassert.h"
-#include "ucln_cmn.h"
#include "umutex.h"
-#include "uvector.h"
+#include "uvectr32.h"
// if U_LOCAL_SERVICE_HOOK is defined, then localsvc.cpp is expected to be included.
#if U_LOCAL_SERVICE_HOOK
#endif
#ifdef RBBI_DEBUG
-static UBool fTrace = FALSE;
+static UBool gTrace = FALSE;
#endif
U_NAMESPACE_BEGIN
// The state number of the starting state
-#define START_STATE 1
+constexpr int32_t START_STATE = 1;
// The state-transition value indicating "stop"
-#define STOP_STATE 0
+constexpr int32_t STOP_STATE = 0;
UOBJECT_DEFINE_RTTI_IMPLEMENTATION(RuleBasedBreakIterator)
* Constructs a RuleBasedBreakIterator that uses the already-created
* tables object that is passed in as a parameter.
*/
-RuleBasedBreakIterator::RuleBasedBreakIterator(RBBIDataHeader* data, UErrorCode &status)
-{
- init();
+RuleBasedBreakIterator::RuleBasedBreakIterator(RBBIDataHeader* data, UErrorCode &status) {
+ init(status);
fData = new RBBIDataWrapper(data, status); // status checked in constructor
if (U_FAILURE(status)) {return;}
if(fData == 0) {
RuleBasedBreakIterator::RuleBasedBreakIterator(const uint8_t *compiledRules,
uint32_t ruleLength,
UErrorCode &status) {
- init();
+ init(status);
if (U_FAILURE(status)) {
return;
}
//-------------------------------------------------------------------------------
RuleBasedBreakIterator::RuleBasedBreakIterator(UDataMemory* udm, UErrorCode &status)
{
- init();
+ init(status);
fData = new RBBIDataWrapper(udm, status); // status checked in constructor
if (U_FAILURE(status)) {return;}
if(fData == 0) {
UParseError &parseError,
UErrorCode &status)
{
- init();
+ init(status);
if (U_FAILURE(status)) {return;}
RuleBasedBreakIterator *bi = (RuleBasedBreakIterator *)
RBBIRuleBuilder::createRuleBasedBreakIterator(rules, &parseError, status);
// of rules.
//-------------------------------------------------------------------------------
RuleBasedBreakIterator::RuleBasedBreakIterator() {
- init();
+ UErrorCode status = U_ZERO_ERROR;
+ init(status);
}
RuleBasedBreakIterator::RuleBasedBreakIterator(const RuleBasedBreakIterator& other)
: BreakIterator(other)
{
- this->init();
+ UErrorCode status = U_ZERO_ERROR;
+ this->init(status);
*this = other;
}
}
fCharIter = NULL;
delete fSCharIter;
- fCharIter = NULL;
+ fSCharIter = NULL;
delete fDCharIter;
fDCharIter = NULL;
fData->removeReference();
fData = NULL;
}
- if (fCachedBreakPositions) {
- uprv_free(fCachedBreakPositions);
- fCachedBreakPositions = NULL;
- }
- if (fLanguageBreakEngines) {
- delete fLanguageBreakEngines;
- fLanguageBreakEngines = NULL;
- }
- if (fUnhandledBreakEngine) {
- delete fUnhandledBreakEngine;
- fUnhandledBreakEngine = NULL;
- }
+ delete fBreakCache;
+ fBreakCache = NULL;
+
+ delete fDictionaryCache;
+ fDictionaryCache = NULL;
+
+ delete fLanguageBreakEngines;
+ fLanguageBreakEngines = NULL;
+
+ delete fUnhandledBreakEngine;
+ fUnhandledBreakEngine = NULL;
}
/**
}
BreakIterator::operator=(that);
- reset(); // Delete break cache information
fBreakType = that.fBreakType;
if (fLanguageBreakEngines != NULL) {
delete fLanguageBreakEngines;
fData = that.fData->addReference();
}
+ fPosition = that.fPosition;
+ fRuleStatusIndex = that.fRuleStatusIndex;
+ fDone = that.fDone;
+
+ // TODO: both the dictionary and the main cache need to be copied.
+ // Current position could be within a dictionary range. Trying to continue
+ // the iteration without the caches present would go to the rules, with
+ // the assumption that the current position is on a rule boundary.
+ fBreakCache->reset(fPosition, fRuleStatusIndex);
+ fDictionaryCache->reset();
+
return *this;
}
// Initializes all fields, leaving the object in a consistent state.
//
//-----------------------------------------------------------------------------
-void RuleBasedBreakIterator::init() {
- UErrorCode status = U_ZERO_ERROR;
- fText = utext_openUChars(NULL, NULL, 0, &status);
+void RuleBasedBreakIterator::init(UErrorCode &status) {
+ fText = NULL;
fCharIter = NULL;
fSCharIter = NULL;
fDCharIter = NULL;
fData = NULL;
- fLastRuleStatusIndex = 0;
- fLastStatusIndexValid = TRUE;
+ fPosition = 0;
+ fRuleStatusIndex = 0;
+ fDone = false;
fDictionaryCharCount = 0;
fBreakType = UBRK_WORD; // Defaulting BreakType to word gives reasonable
// dictionary behavior for Break Iterators that are
// built from rules. Even better would be the ability to
// declare the type in the rules.
- fCachedBreakPositions = NULL;
- fLanguageBreakEngines = NULL;
- fUnhandledBreakEngine = NULL;
- fNumCachedBreakPositions = 0;
- fPositionInCache = 0;
+ fLanguageBreakEngines = NULL;
+ fUnhandledBreakEngine = NULL;
+ fBreakCache = NULL;
+ fDictionaryCache = NULL;
+
+ if (U_FAILURE(status)) {
+ return;
+ }
+
+ fText = utext_openUChars(NULL, NULL, 0, &status);
+ fDictionaryCache = new DictionaryCache(this, status);
+ fBreakCache = new BreakCache(this, status);
+ if (U_SUCCESS(status) && (fText == NULL || fDictionaryCache == NULL || fBreakCache == NULL)) {
+ status = U_MEMORY_ALLOCATION_ERROR;
+ }
#ifdef RBBI_DEBUG
static UBool debugInitDone = FALSE;
if (debugInitDone == FALSE) {
char *debugEnv = getenv("U_RBBIDEBUG");
if (debugEnv && uprv_strstr(debugEnv, "trace")) {
- fTrace = TRUE;
+ gTrace = TRUE;
}
debugInitDone = TRUE;
}
if (typeid(*this) != typeid(that)) {
return FALSE;
}
+ if (this == &that) {
+ return TRUE;
+ }
// The base class BreakIterator carries no state that participates in equality,
// and does not implement an equality function that would otherwise be
return FALSE;
};
+ if (!(fPosition == that2.fPosition &&
+ fRuleStatusIndex == that2.fRuleStatusIndex &&
+ fDone == that2.fDone)) {
+ return FALSE;
+ }
+
if (that2.fData == fData ||
(fData != NULL && that2.fData != NULL && *that2.fData == *fData)) {
// The two break iterators are using the same rules.
if (U_FAILURE(status)) {
return;
}
- reset();
+ fBreakCache->reset();
+ fDictionaryCache->reset();
fText = utext_clone(fText, ut, FALSE, TRUE, &status);
// Set up a dummy CharacterIterator to be returned if anyone
fCharIter = newText;
UErrorCode status = U_ZERO_ERROR;
- reset();
+ fBreakCache->reset();
+ fDictionaryCache->reset();
if (newText==NULL || newText->startIndex() != 0) {
// startIndex !=0 wants to be an error, but there's no way to report it.
// Make the iterator text be an empty string.
void
RuleBasedBreakIterator::setText(const UnicodeString& newText) {
UErrorCode status = U_ZERO_ERROR;
- reset();
+ fBreakCache->reset();
+ fDictionaryCache->reset();
fText = utext_openConstUnicodeString(fText, &newText, &status);
// Set up a character iterator on the string.
* @return The new iterator position, which is zero.
*/
int32_t RuleBasedBreakIterator::first(void) {
- reset();
- fLastRuleStatusIndex = 0;
- fLastStatusIndexValid = TRUE;
- //if (fText == NULL)
- // return BreakIterator::DONE;
-
- utext_setNativeIndex(fText, 0);
+ UErrorCode status = U_ZERO_ERROR;
+ if (!fBreakCache->seek(0)) {
+ fBreakCache->populateNear(0, status);
+ }
+ fBreakCache->current();
+ U_ASSERT(fPosition == 0);
return 0;
}
* @return The text's past-the-end offset.
*/
int32_t RuleBasedBreakIterator::last(void) {
- reset();
- if (fText == NULL) {
- fLastRuleStatusIndex = 0;
- fLastStatusIndexValid = TRUE;
- return BreakIterator::DONE;
- }
-
- fLastStatusIndexValid = FALSE;
- int32_t pos = (int32_t)utext_nativeLength(fText);
- utext_setNativeIndex(fText, pos);
- return pos;
+ int32_t endPos = (int32_t)utext_nativeLength(fText);
+ UBool endShouldBeBoundary = isBoundary(endPos); // Has side effect of setting iterator position.
+ (void)endShouldBeBoundary;
+ U_ASSERT(endShouldBeBoundary);
+ U_ASSERT(fPosition == endPos);
+ return endPos;
}
/**
* the current one.
*/
int32_t RuleBasedBreakIterator::next(int32_t n) {
- int32_t result = current();
- while (n > 0) {
- result = next();
- --n;
- }
- while (n < 0) {
- result = previous();
- ++n;
+ int32_t result = 0;
+ if (n > 0) {
+ for (; n > 0 && result != UBRK_DONE; --n) {
+ result = next();
+ }
+ } else if (n < 0) {
+ for (; n < 0 && result != UBRK_DONE; ++n) {
+ result = previous();
+ }
+ } else {
+ result = current();
}
return result;
}
* @return The position of the first boundary after this one.
*/
int32_t RuleBasedBreakIterator::next(void) {
- // if we have cached break positions and we're still in the range
- // covered by them, just move one step forward in the cache
- if (fCachedBreakPositions != NULL) {
- if (fPositionInCache < fNumCachedBreakPositions - 1) {
- ++fPositionInCache;
- int32_t pos = fCachedBreakPositions[fPositionInCache];
- utext_setNativeIndex(fText, pos);
- return pos;
- }
- else {
- reset();
- }
- }
-
- int32_t startPos = current();
- fDictionaryCharCount = 0;
- int32_t result = handleNext(fData->fForwardTable);
- if (fDictionaryCharCount > 0) {
- result = checkDictionary(startPos, result, FALSE);
- }
- return result;
+ fBreakCache->next();
+ return fDone ? UBRK_DONE : fPosition;
}
/**
- * Advances the iterator backwards, to the last boundary preceding this one.
- * @return The position of the last boundary position preceding this one.
+ * Move the iterator backwards, to the boundary preceding the current one.
+ *
+ * Starts from the current position within fText.
+ * Starting position need not be on a boundary.
+ *
+ * @return The position of the boundary position immediately preceding the starting position.
*/
int32_t RuleBasedBreakIterator::previous(void) {
- int32_t result;
- int32_t startPos;
-
- // if we have cached break positions and we're still in the range
- // covered by them, just move one step backward in the cache
- if (fCachedBreakPositions != NULL) {
- if (fPositionInCache > 0) {
- --fPositionInCache;
- // If we're at the beginning of the cache, need to reevaluate the
- // rule status
- if (fPositionInCache <= 0) {
- fLastStatusIndexValid = FALSE;
- }
- int32_t pos = fCachedBreakPositions[fPositionInCache];
- utext_setNativeIndex(fText, pos);
- return pos;
- }
- else {
- reset();
- }
- }
-
- // if we're already sitting at the beginning of the text, return DONE
- if (fText == NULL || (startPos = current()) == 0) {
- fLastRuleStatusIndex = 0;
- fLastStatusIndexValid = TRUE;
- return BreakIterator::DONE;
- }
-
- if (fData->fSafeRevTable != NULL || fData->fSafeFwdTable != NULL) {
- result = handlePrevious(fData->fReverseTable);
- if (fDictionaryCharCount > 0) {
- result = checkDictionary(result, startPos, TRUE);
- }
- return result;
- }
-
- // old rule syntax
- // set things up. handlePrevious() will back us up to some valid
- // break position before the current position (we back our internal
- // iterator up one step to prevent handlePrevious() from returning
- // the current position), but not necessarily the last one before
- // where we started
-
- int32_t start = current();
-
- (void)UTEXT_PREVIOUS32(fText);
- int32_t lastResult = handlePrevious(fData->fReverseTable);
- if (lastResult == UBRK_DONE) {
- lastResult = 0;
- utext_setNativeIndex(fText, 0);
- }
- result = lastResult;
- int32_t lastTag = 0;
- UBool breakTagValid = FALSE;
-
- // iterate forward from the known break position until we pass our
- // starting point. The last break position before the starting
- // point is our return value
-
- for (;;) {
- result = next();
- if (result == BreakIterator::DONE || result >= start) {
- break;
- }
- lastResult = result;
- lastTag = fLastRuleStatusIndex;
- breakTagValid = TRUE;
- }
-
- // fLastBreakTag wants to have the value for section of text preceding
- // the result position that we are to return (in lastResult.) If
- // the backwards rules overshot and the above loop had to do two or more
- // next()s to move up to the desired return position, we will have a valid
- // tag value. But, if handlePrevious() took us to exactly the correct result position,
- // we wont have a tag value for that position, which is only set by handleNext().
-
- // Set the current iteration position to be the last break position
- // before where we started, and then return that value.
- utext_setNativeIndex(fText, lastResult);
- fLastRuleStatusIndex = lastTag; // for use by getRuleStatus()
- fLastStatusIndexValid = breakTagValid;
-
- // No need to check the dictionary; it will have been handled by
- // next()
-
- return lastResult;
+ UErrorCode status = U_ZERO_ERROR;
+ fBreakCache->previous(status);
+ return fDone ? UBRK_DONE : fPosition;
}
/**
* Sets the iterator to refer to the first boundary position following
* the specified position.
- * @offset The position from which to begin searching for a break position.
+ * @param startPos The position from which to begin searching for a break position.
* @return The position of the first break after the current position.
*/
-int32_t RuleBasedBreakIterator::following(int32_t offset) {
- // if the offset passed in is already past the end of the text,
- // just return DONE; if it's before the beginning, return the
+int32_t RuleBasedBreakIterator::following(int32_t startPos) {
+ // if the supplied position is before the beginning, return the
// text's starting offset
- if (fText == NULL || offset >= utext_nativeLength(fText)) {
- last();
- return next();
- }
- else if (offset < 0) {
+ if (startPos < 0) {
return first();
}
// Move requested offset to a code point start. It might be on a trail surrogate,
- // or on a trail byte if the input is UTF-8.
- utext_setNativeIndex(fText, offset);
- offset = (int32_t)utext_getNativeIndex(fText);
-
- // if we have cached break positions and offset is in the range
- // covered by them, use them
- // TODO: could use binary search
- // TODO: what if offset is outside range, but break is not?
- if (fCachedBreakPositions != NULL) {
- if (offset >= fCachedBreakPositions[0]
- && offset < fCachedBreakPositions[fNumCachedBreakPositions - 1]) {
- fPositionInCache = 0;
- // We are guaranteed not to leave the array due to range test above
- while (offset >= fCachedBreakPositions[fPositionInCache]) {
- ++fPositionInCache;
- }
- int32_t pos = fCachedBreakPositions[fPositionInCache];
- utext_setNativeIndex(fText, pos);
- return pos;
- }
- else {
- reset();
- }
- }
+ // or on a trail byte if the input is UTF-8. Or it may be beyond the end of the text.
+ utext_setNativeIndex(fText, startPos);
+ startPos = (int32_t)utext_getNativeIndex(fText);
- // Set our internal iteration position (temporarily)
- // to the position passed in. If this is the _beginning_ position,
- // then we can just use next() to get our return value
-
- int32_t result = 0;
-
- if (fData->fSafeRevTable != NULL) {
- // new rule syntax
- utext_setNativeIndex(fText, offset);
- // move forward one codepoint to prepare for moving back to a
- // safe point.
- // this handles offset being between a supplementary character
- // TODO: is this still needed, with move to code point boundary handled above?
- (void)UTEXT_NEXT32(fText);
- // handlePrevious will move most of the time to < 1 boundary away
- handlePrevious(fData->fSafeRevTable);
- int32_t result = next();
- while (result <= offset) {
- result = next();
- }
- return result;
- }
- if (fData->fSafeFwdTable != NULL) {
- // backup plan if forward safe table is not available
- utext_setNativeIndex(fText, offset);
- (void)UTEXT_PREVIOUS32(fText);
- // handle next will give result >= offset
- handleNext(fData->fSafeFwdTable);
- // previous will give result 0 or 1 boundary away from offset,
- // most of the time
- // we have to
- int32_t oldresult = previous();
- while (oldresult > offset) {
- int32_t result = previous();
- if (result <= offset) {
- return oldresult;
- }
- oldresult = result;
- }
- int32_t result = next();
- if (result <= offset) {
- return next();
- }
- return result;
- }
- // otherwise, we have to sync up first. Use handlePrevious() to back
- // up to a known break position before the specified position (if
- // we can determine that the specified position is a break position,
- // we don't back up at all). This may or may not be the last break
- // position at or before our starting position. Advance forward
- // from here until we've passed the starting position. The position
- // we stop on will be the first break position after the specified one.
- // old rule syntax
-
- utext_setNativeIndex(fText, offset);
- if (offset==0 ||
- (offset==1 && utext_getNativeIndex(fText)==0)) {
- return next();
- }
- result = previous();
-
- while (result != BreakIterator::DONE && result <= offset) {
- result = next();
- }
-
- return result;
+ UErrorCode status = U_ZERO_ERROR;
+ fBreakCache->following(startPos, status);
+ return fDone ? UBRK_DONE : fPosition;
}
/**
* Sets the iterator to refer to the last boundary position before the
* specified position.
- * @offset The position to begin searching for a break from.
+ * @param offset The position to begin searching for a break from.
* @return The position of the last boundary before the starting position.
*/
int32_t RuleBasedBreakIterator::preceding(int32_t offset) {
- // if the offset passed in is already past the end of the text,
- // just return DONE; if it's before the beginning, return the
- // text's starting offset
if (fText == NULL || offset > utext_nativeLength(fText)) {
return last();
}
- else if (offset < 0) {
- return first();
- }
// Move requested offset to a code point start. It might be on a trail surrogate,
// or on a trail byte if the input is UTF-8.
- utext_setNativeIndex(fText, offset);
- offset = (int32_t)utext_getNativeIndex(fText);
-
- // if we have cached break positions and offset is in the range
- // covered by them, use them
- if (fCachedBreakPositions != NULL) {
- // TODO: binary search?
- // TODO: What if offset is outside range, but break is not?
- if (offset > fCachedBreakPositions[0]
- && offset <= fCachedBreakPositions[fNumCachedBreakPositions - 1]) {
- fPositionInCache = 0;
- while (fPositionInCache < fNumCachedBreakPositions
- && offset > fCachedBreakPositions[fPositionInCache])
- ++fPositionInCache;
- --fPositionInCache;
- // If we're at the beginning of the cache, need to reevaluate the
- // rule status
- if (fPositionInCache <= 0) {
- fLastStatusIndexValid = FALSE;
- }
- utext_setNativeIndex(fText, fCachedBreakPositions[fPositionInCache]);
- return fCachedBreakPositions[fPositionInCache];
- }
- else {
- reset();
- }
- }
-
- // if we start by updating the current iteration position to the
- // position specified by the caller, we can just use previous()
- // to carry out this operation
-
- if (fData->fSafeFwdTable != NULL) {
- // new rule syntax
- utext_setNativeIndex(fText, offset);
- int32_t newOffset = (int32_t)UTEXT_GETNATIVEINDEX(fText);
- if (newOffset != offset) {
- // Will come here if specified offset was not a code point boundary AND
- // the underlying implmentation is using UText, which snaps any non-code-point-boundary
- // indices to the containing code point.
- // For breakitereator::preceding only, these non-code-point indices need to be moved
- // up to refer to the following codepoint.
- (void)UTEXT_NEXT32(fText);
- offset = (int32_t)UTEXT_GETNATIVEINDEX(fText);
- }
- // TODO: (synwee) would it be better to just check for being in the middle of a surrogate pair,
- // rather than adjusting the position unconditionally?
- // (Change would interact with safe rules.)
- // TODO: change RBBI behavior for off-boundary indices to match that of UText?
- // affects only preceding(), seems cleaner, but is slightly different.
- (void)UTEXT_PREVIOUS32(fText);
- handleNext(fData->fSafeFwdTable);
- int32_t result = (int32_t)UTEXT_GETNATIVEINDEX(fText);
- while (result >= offset) {
- result = previous();
- }
- return result;
- }
- if (fData->fSafeRevTable != NULL) {
- // backup plan if forward safe table is not available
- // TODO: check whether this path can be discarded
- // It's probably OK to say that rules must supply both safe tables
- // if they use safe tables at all. We have certainly never described
- // to anyone how to work with just one safe table.
- utext_setNativeIndex(fText, offset);
- (void)UTEXT_NEXT32(fText);
-
- // handle previous will give result <= offset
- handlePrevious(fData->fSafeRevTable);
-
- // next will give result 0 or 1 boundary away from offset,
- // most of the time
- // we have to
- int32_t oldresult = next();
- while (oldresult < offset) {
- int32_t result = next();
- if (result >= offset) {
- return oldresult;
- }
- oldresult = result;
- }
- int32_t result = previous();
- if (result >= offset) {
- return previous();
- }
- return result;
- }
-
- // old rule syntax
utext_setNativeIndex(fText, offset);
- return previous();
+ int32_t adjustedOffset = utext_getNativeIndex(fText);
+
+ UErrorCode status = U_ZERO_ERROR;
+ fBreakCache->preceding(adjustedOffset, status);
+ return fDone ? UBRK_DONE : fPosition;
}
/**
* Returns true if the specfied position is a boundary position. As a side
* effect, leaves the iterator pointing to the first boundary position at
* or after "offset".
+ *
* @param offset the offset to check.
* @return True if "offset" is a boundary position.
*/
UBool RuleBasedBreakIterator::isBoundary(int32_t offset) {
- // the beginning index of the iterator is always a boundary position by definition
- if (offset == 0) {
- first(); // For side effects on current position, tag values.
- return TRUE;
- }
-
- if (offset == (int32_t)utext_nativeLength(fText)) {
- last(); // For side effects on current position, tag values.
- return TRUE;
- }
-
// out-of-range indexes are never boundary positions
if (offset < 0) {
first(); // For side effects on current position, tag values.
return FALSE;
}
- if (offset > utext_nativeLength(fText)) {
- last(); // For side effects on current position, tag values.
- return FALSE;
+ // Adjust offset to be on a code point boundary and not beyond the end of the text.
+ // Note that isBoundary() is always be false for offsets that are not on code point boundaries.
+ // But we still need the side effect of leaving iteration at the following boundary.
+
+ utext_setNativeIndex(fText, offset);
+ int32_t adjustedOffset = utext_getNativeIndex(fText);
+
+ bool result = false;
+ UErrorCode status = U_ZERO_ERROR;
+ if (fBreakCache->seek(adjustedOffset) || fBreakCache->populateNear(adjustedOffset, status)) {
+ result = (fBreakCache->current() == offset);
}
- // otherwise, we can use following() on the position before the specified
- // one and return true if the position we get back is the one the user
- // specified
- utext_previous32From(fText, offset);
- int32_t backOne = (int32_t)UTEXT_GETNATIVEINDEX(fText);
- UBool result = following(backOne) == offset;
+ if (result && adjustedOffset < offset && utext_char32At(fText, offset) == U_SENTINEL) {
+ // Original offset is beyond the end of the text. Return FALSE, it's not a boundary,
+ // but the iteration position remains set to the end of the text, which is a boundary.
+ return FALSE;
+ }
+ if (!result) {
+ // Not on a boundary. isBoundary() must leave iterator on the following boundary.
+ // Cache->seek(), above, left us on the preceding boundary, so advance one.
+ next();
+ }
return result;
}
+
/**
* Returns the current iteration position.
* @return The current iteration position.
*/
int32_t RuleBasedBreakIterator::current(void) const {
- int32_t pos = (int32_t)UTEXT_GETNATIVEINDEX(fText);
- return pos;
+ return fPosition;
}
+
//=======================================================================
// implementation
//=======================================================================
//-----------------------------------------------------------------------------------
//
-// handleNext(stateTable)
-// This method is the actual implementation of the rbbi next() method.
-// This method initializes the state machine to state 1
-// and advances through the text character by character until we reach the end
-// of the text or the state machine transitions to state 0. We update our return
-// value every time the state machine passes through an accepting state.
+// handleNext()
+// Run the state machine to find a boundary
//
//-----------------------------------------------------------------------------------
-int32_t RuleBasedBreakIterator::handleNext(const RBBIStateTable *statetable) {
+int32_t RuleBasedBreakIterator::handleNext() {
int32_t state;
uint16_t category = 0;
RBBIRunMode mode;
LookAheadResults lookAheadMatches;
int32_t result = 0;
int32_t initialPosition = 0;
+ const RBBIStateTable *statetable = fData->fForwardTable;
const char *tableData = statetable->fTableData;
uint32_t tableRowLen = statetable->fRowLen;
-
#ifdef RBBI_DEBUG
- if (fTrace) {
+ if (gTrace) {
RBBIDebugPuts("Handle Next pos char state category");
}
#endif
- // No matter what, handleNext alway correctly sets the break tag value.
- fLastStatusIndexValid = TRUE;
- fLastRuleStatusIndex = 0;
+ // handleNext alway sets the break tag value.
+ // Set the default for it.
+ fRuleStatusIndex = 0;
+
+ fDictionaryCharCount = 0;
// if we're already at the end of the text, return DONE.
- initialPosition = (int32_t)UTEXT_GETNATIVEINDEX(fText);
+ initialPosition = fPosition;
+ UTEXT_SETNATIVEINDEX(fText, initialPosition);
result = initialPosition;
c = UTEXT_NEXT32(fText);
if (fData == NULL || c==U_SENTINEL) {
- return BreakIterator::DONE;
+ fDone = TRUE;
+ return UBRK_DONE;
}
// Set the initial state for the state machine
category = UTRIE2_GET16(fData->fTrie, c);
// Check the dictionary bit in the character's category.
- // Counter is only used by dictionary based iterators (subclasses).
+ // Counter is only used by dictionary based iteration.
// Chars that need to be handled by a dictionary have a flag bit set
// in their category values.
//
}
#ifdef RBBI_DEBUG
- if (fTrace) {
+ if (gTrace) {
RBBIDebugPrintf(" %4ld ", utext_getNativeIndex(fText));
if (0x20<=c && c<0x7f) {
RBBIDebugPrintf("\"%c\" ", c);
if (mode != RBBI_START) {
result = (int32_t)UTEXT_GETNATIVEINDEX(fText);
}
- fLastRuleStatusIndex = row->fTagIdx; // Remember the break status (tag) values.
+ fRuleStatusIndex = row->fTagIdx; // Remember the break status (tag) values.
}
int16_t completedRule = row->fAccepting;
// Lookahead match is completed.
int32_t lookaheadResult = lookAheadMatches.getPosition(completedRule);
if (lookaheadResult >= 0) {
- fLastRuleStatusIndex = row->fTagIdx;
- UTEXT_SETNATIVEINDEX(fText, lookaheadResult);
+ fRuleStatusIndex = row->fTagIdx;
+ fPosition = lookaheadResult;
return lookaheadResult;
}
}
mode = RBBI_RUN;
}
}
-
-
}
// The state machine is done. Check whether it found a match...
// (This really indicates a defect in the break rules. They should always match
// at least one character.)
if (result == initialPosition) {
- UTEXT_SETNATIVEINDEX(fText, initialPosition);
- UTEXT_NEXT32(fText);
- result = (int32_t)UTEXT_GETNATIVEINDEX(fText);
+ utext_setNativeIndex(fText, initialPosition);
+ utext_next32(fText);
+ result = (int32_t)utext_getNativeIndex(fText);
+ fRuleStatusIndex = 0;
}
// Leave the iterator at our result position.
- UTEXT_SETNATIVEINDEX(fText, result);
+ fPosition = result;
#ifdef RBBI_DEBUG
- if (fTrace) {
+ if (gTrace) {
RBBIDebugPrintf("result = %d\n\n", result);
}
#endif
//
// handlePrevious()
//
-// Iterate backwards, according to the logic of the reverse rules.
-// This version handles the exact style backwards rules.
-//
+// Iterate backwards using the safe reverse rules.
// The logic of this function is very similar to handleNext(), above.
//
//-----------------------------------------------------------------------------------
-int32_t RuleBasedBreakIterator::handlePrevious(const RBBIStateTable *statetable) {
+int32_t RuleBasedBreakIterator::handlePrevious(int32_t fromPosition) {
int32_t state;
uint16_t category = 0;
RBBIRunMode mode;
int32_t result = 0;
int32_t initialPosition = 0;
+ const RBBIStateTable *stateTable = fData->fSafeRevTable;
+ UTEXT_SETNATIVEINDEX(fText, fromPosition);
#ifdef RBBI_DEBUG
- if (fTrace) {
+ if (gTrace) {
RBBIDebugPuts("Handle Previous pos char state category");
}
#endif
- // handlePrevious() never gets the rule status.
- // Flag the status as invalid; if the user ever asks for status, we will need
- // to back up, then re-find the break position using handleNext(), which does
- // get the status value.
- fLastStatusIndexValid = FALSE;
- fLastRuleStatusIndex = 0;
-
// if we're already at the start of the text, return DONE.
if (fText == NULL || fData == NULL || UTEXT_GETNATIVEINDEX(fText)==0) {
return BreakIterator::DONE;
// Set the initial state for the state machine
state = START_STATE;
row = (RBBIStateTableRow *)
- (statetable->fTableData + (statetable->fRowLen * state));
+ (stateTable->fTableData + (stateTable->fRowLen * state));
category = 3;
mode = RBBI_RUN;
- if (statetable->fFlags & RBBI_BOF_REQUIRED) {
+ if (stateTable->fFlags & RBBI_BOF_REQUIRED) {
category = 2;
mode = RBBI_START;
}
// We have already run the loop one last time with the
// character set to the psueudo {eof} value. Now it is time
// to unconditionally bail out.
- if (result == initialPosition) {
- // Ran off start, no match found.
- // move one index one (towards the start, since we are doing a previous())
- UTEXT_SETNATIVEINDEX(fText, initialPosition);
- (void)UTEXT_PREVIOUS32(fText); // TODO: shouldn't be necessary. We're already at beginning. Check.
- }
break;
}
// Run the loop one last time with the fake end-of-input character category.
// Note: the 16 in UTRIE_GET16 refers to the size of the data being returned,
// not the size of the character going in, which is a UChar32.
//
+ // And off the dictionary flag bit. For reverse iteration it is not used.
category = UTRIE2_GET16(fData->fTrie, c);
-
- // Check the dictionary bit in the character's category.
- // Counter is only used by dictionary based iterators (subclasses).
- // Chars that need to be handled by a dictionary have a flag bit set
- // in their category values.
- //
- if ((category & 0x4000) != 0) {
- fDictionaryCharCount++;
- // And off the dictionary flag bit.
- category &= ~0x4000;
- }
+ category &= ~0x4000;
}
#ifdef RBBI_DEBUG
- if (fTrace) {
+ if (gTrace) {
RBBIDebugPrintf(" %4d ", (int32_t)utext_getNativeIndex(fText));
if (0x20<=c && c<0x7f) {
RBBIDebugPrintf("\"%c\" ", c);
U_ASSERT(category<fData->fHeader->fCatCount);
state = row->fNextState[category]; /*Not accessing beyond memory*/
row = (RBBIStateTableRow *)
- (statetable->fTableData + (statetable->fRowLen * state));
+ (stateTable->fTableData + (stateTable->fRowLen * state));
if (row->fAccepting == -1) {
// Match found, common case.
result = (int32_t)UTEXT_GETNATIVEINDEX(fText);
}
- // Leave the iterator at our result position.
- UTEXT_SETNATIVEINDEX(fText, result);
#ifdef RBBI_DEBUG
- if (fTrace) {
+ if (gTrace) {
RBBIDebugPrintf("result = %d\n\n", result);
}
#endif
}
-void
-RuleBasedBreakIterator::reset()
-{
- if (fCachedBreakPositions) {
- uprv_free(fCachedBreakPositions);
- }
- fCachedBreakPositions = NULL;
- fNumCachedBreakPositions = 0;
- fDictionaryCharCount = 0;
- fPositionInCache = 0;
-}
-
-
-
//-------------------------------------------------------------------------------
//
// getRuleStatus() Return the break rule tag associated with the current
// position by iterating forwards, the value will have been
// cached by the handleNext() function.
//
-// If no cached status value is available, the status is
-// found by doing a previous() followed by a next(), which
-// leaves the iterator where it started, and computes the
-// status while doing the next().
-//
//-------------------------------------------------------------------------------
-void RuleBasedBreakIterator::makeRuleStatusValid() {
- if (fLastStatusIndexValid == FALSE) {
- // No cached status is available.
- if (fText == NULL || current() == 0) {
- // At start of text, or there is no text. Status is always zero.
- fLastRuleStatusIndex = 0;
- fLastStatusIndexValid = TRUE;
- } else {
- // Not at start of text. Find status the tedious way.
- int32_t pa = current();
- previous();
- if (fNumCachedBreakPositions > 0) {
- reset(); // Blow off the dictionary cache
- }
- int32_t pb = next();
- if (pa != pb) {
- // note: the if (pa != pb) test is here only to eliminate warnings for
- // unused local variables on gcc. Logically, it isn't needed.
- U_ASSERT(pa == pb);
- }
- }
- }
- U_ASSERT(fLastRuleStatusIndex >= 0 && fLastRuleStatusIndex < fData->fStatusMaxIdx);
-}
-
int32_t RuleBasedBreakIterator::getRuleStatus() const {
- RuleBasedBreakIterator *nonConstThis = (RuleBasedBreakIterator *)this;
- nonConstThis->makeRuleStatusValid();
// fLastRuleStatusIndex indexes to the start of the appropriate status record
// (the number of status values.)
// This function returns the last (largest) of the array of status values.
- int32_t idx = fLastRuleStatusIndex + fData->fRuleStatusTable[fLastRuleStatusIndex];
+ int32_t idx = fRuleStatusIndex + fData->fRuleStatusTable[fRuleStatusIndex];
int32_t tagVal = fData->fRuleStatusTable[idx];
return tagVal;
}
-
-
int32_t RuleBasedBreakIterator::getRuleStatusVec(
- int32_t *fillInVec, int32_t capacity, UErrorCode &status)
-{
+ int32_t *fillInVec, int32_t capacity, UErrorCode &status) {
if (U_FAILURE(status)) {
return 0;
}
- RuleBasedBreakIterator *nonConstThis = (RuleBasedBreakIterator *)this;
- nonConstThis->makeRuleStatusValid();
- int32_t numVals = fData->fRuleStatusTable[fLastRuleStatusIndex];
+ int32_t numVals = fData->fRuleStatusTable[fRuleStatusIndex];
int32_t numValsToCopy = numVals;
if (numVals > capacity) {
status = U_BUFFER_OVERFLOW_ERROR;
}
int i;
for (i=0; i<numValsToCopy; i++) {
- fillInVec[i] = fData->fRuleStatusTable[fLastRuleStatusIndex + i + 1];
+ fillInVec[i] = fData->fRuleStatusTable[fRuleStatusIndex + i + 1];
}
return numVals;
}
return (RuleBasedBreakIterator *)clonedBI;
}
-
-//-------------------------------------------------------------------------------
-//
-// checkDictionary This function handles all processing of characters in
-// the "dictionary" set. It will determine the appropriate
-// course of action, and possibly set up a cache in the
-// process.
-//
-//-------------------------------------------------------------------------------
-int32_t RuleBasedBreakIterator::checkDictionary(int32_t startPos,
- int32_t endPos,
- UBool reverse) {
- // Reset the old break cache first.
- reset();
-
- // note: code segment below assumes that dictionary chars are in the
- // startPos-endPos range
- // value returned should be next character in sequence
- if ((endPos - startPos) <= 1) {
- return (reverse ? startPos : endPos);
- }
-
- // Starting from the starting point, scan towards the proposed result,
- // looking for the first dictionary character (which may be the one
- // we're on, if we're starting in the middle of a range).
- utext_setNativeIndex(fText, reverse ? endPos : startPos);
- if (reverse) {
- UTEXT_PREVIOUS32(fText);
- }
-
- int32_t rangeStart = startPos;
- int32_t rangeEnd = endPos;
-
- uint16_t category;
- int32_t current;
- UErrorCode status = U_ZERO_ERROR;
- UStack breaks(status);
- int32_t foundBreakCount = 0;
- UChar32 c = utext_current32(fText);
-
- category = UTRIE2_GET16(fData->fTrie, c);
-
- // Is the character we're starting on a dictionary character? If so, we
- // need to back up to include the entire run; otherwise the results of
- // the break algorithm will differ depending on where we start. Since
- // the result is cached and there is typically a non-dictionary break
- // within a small number of words, there should be little performance impact.
- if (category & 0x4000) {
- if (reverse) {
- do {
- utext_next32(fText); // TODO: recast to work directly with postincrement.
- c = utext_current32(fText);
- category = UTRIE2_GET16(fData->fTrie, c);
- } while (c != U_SENTINEL && (category & 0x4000));
- // Back up to the last dictionary character
- rangeEnd = (int32_t)UTEXT_GETNATIVEINDEX(fText);
- if (c == U_SENTINEL) {
- // c = fText->last32();
- // TODO: why was this if needed?
- c = UTEXT_PREVIOUS32(fText);
- }
- else {
- c = UTEXT_PREVIOUS32(fText);
- }
- }
- else {
- do {
- c = UTEXT_PREVIOUS32(fText);
- category = UTRIE2_GET16(fData->fTrie, c);
- }
- while (c != U_SENTINEL && (category & 0x4000));
- // Back up to the last dictionary character
- if (c == U_SENTINEL) {
- // c = fText->first32();
- c = utext_current32(fText);
- }
- else {
- utext_next32(fText);
- c = utext_current32(fText);
- }
- rangeStart = (int32_t)UTEXT_GETNATIVEINDEX(fText);;
- }
- category = UTRIE2_GET16(fData->fTrie, c);
- }
-
- // Loop through the text, looking for ranges of dictionary characters.
- // For each span, find the appropriate break engine, and ask it to find
- // any breaks within the span.
- // Note: we always do this in the forward direction, so that the break
- // cache is built in the right order.
- if (reverse) {
- utext_setNativeIndex(fText, rangeStart);
- c = utext_current32(fText);
- category = UTRIE2_GET16(fData->fTrie, c);
- }
- while(U_SUCCESS(status)) {
- while((current = (int32_t)UTEXT_GETNATIVEINDEX(fText)) < rangeEnd && (category & 0x4000) == 0) {
- utext_next32(fText); // TODO: tweak for post-increment operation
- c = utext_current32(fText);
- category = UTRIE2_GET16(fData->fTrie, c);
- }
- if (current >= rangeEnd) {
- break;
- }
-
- // We now have a dictionary character. Get the appropriate language object
- // to deal with it.
- const LanguageBreakEngine *lbe = getLanguageBreakEngine(c);
-
- // Ask the language object if there are any breaks. It will leave the text
- // pointer on the other side of its range, ready to search for the next one.
- if (lbe != NULL) {
- foundBreakCount += lbe->findBreaks(fText, rangeStart, rangeEnd, FALSE, fBreakType, breaks);
- }
-
- // Reload the loop variables for the next go-round
- c = utext_current32(fText);
- category = UTRIE2_GET16(fData->fTrie, c);
- }
-
- // If we found breaks, build a new break cache. The first and last entries must
- // be the original starting and ending position.
- if (foundBreakCount > 0) {
- U_ASSERT(foundBreakCount == breaks.size());
- int32_t totalBreaks = foundBreakCount;
- if (startPos < breaks.elementAti(0)) {
- totalBreaks += 1;
- }
- if (endPos > breaks.peeki()) {
- totalBreaks += 1;
- }
- fCachedBreakPositions = (int32_t *)uprv_malloc(totalBreaks * sizeof(int32_t));
- if (fCachedBreakPositions != NULL) {
- int32_t out = 0;
- fNumCachedBreakPositions = totalBreaks;
- if (startPos < breaks.elementAti(0)) {
- fCachedBreakPositions[out++] = startPos;
- }
- for (int32_t i = 0; i < foundBreakCount; ++i) {
- fCachedBreakPositions[out++] = breaks.elementAti(i);
- }
- if (endPos > fCachedBreakPositions[out-1]) {
- fCachedBreakPositions[out] = endPos;
- }
- // If there are breaks, then by definition, we are replacing the original
- // proposed break by one of the breaks we found. Use following() and
- // preceding() to do the work. They should never recurse in this case.
- if (reverse) {
- return preceding(endPos);
- }
- else {
- return following(startPos);
- }
- }
- // If the allocation failed, just fall through to the "no breaks found" case.
- }
-
- // If we get here, there were no language-based breaks. Set the text pointer
- // to the original proposed break.
- utext_setNativeIndex(fText, reverse ? startPos : endPos);
- return (reverse ? startPos : endPos);
-}
-
U_NAMESPACE_END
void RuleBasedBreakIterator::setBreakType(int32_t type) {
fBreakType = type;
- reset();
}
+void RuleBasedBreakIterator::dumpCache() {
+ fBreakCache->dumpCache();
+}
/**
* Returns the description used to create this iterator
--- /dev/null
+// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// License & terms of use: http://www.unicode.org/copyright.html
+
+// file: rbbi_cache.cpp
+
+#include "unicode/utypes.h"
+#include "unicode/ubrk.h"
+#include "unicode/rbbi.h"
+
+#include "rbbi_cache.h"
+
+#include "brkeng.h"
+#include "cmemory.h"
+#include "rbbidata.h"
+#include "uassert.h"
+#include "uvectr32.h"
+
+U_NAMESPACE_BEGIN
+
+/*
+ * DictionaryCache implementation
+ */
+
+RuleBasedBreakIterator::DictionaryCache::DictionaryCache(RuleBasedBreakIterator *bi, UErrorCode &status) :
+ fBI(bi), fBreaks(NULL), fPositionInCache(-1),
+ fStart(0), fLimit(0), fFirstRuleStatusIndex(0), fOtherRuleStatusIndex(0) {
+ fBreaks = new UVector32(status);
+}
+
+RuleBasedBreakIterator::DictionaryCache::~DictionaryCache() {
+ delete fBreaks;
+ fBreaks = NULL;
+}
+
+void RuleBasedBreakIterator::DictionaryCache::reset() {
+ fPositionInCache = -1;
+ fStart = 0;
+ fLimit = 0;
+ fFirstRuleStatusIndex = 0;
+ fOtherRuleStatusIndex = 0;
+ fBreaks->removeAllElements();
+}
+
+UBool RuleBasedBreakIterator::DictionaryCache::following(int32_t fromPos, int32_t *result, int32_t *statusIndex) {
+ if (fromPos >= fLimit || fromPos < fStart) {
+ fPositionInCache = -1;
+ return FALSE;
+ }
+
+ // Sequential iteration, move from previous boundary to the following
+
+ int32_t r = 0;
+ if (fPositionInCache >= 0 && fPositionInCache < fBreaks->size() && fBreaks->elementAti(fPositionInCache) == fromPos) {
+ ++fPositionInCache;
+ if (fPositionInCache >= fBreaks->size()) {
+ fPositionInCache = -1;
+ return FALSE;
+ }
+ r = fBreaks->elementAti(fPositionInCache);
+ U_ASSERT(r > fromPos);
+ *result = r;
+ *statusIndex = fOtherRuleStatusIndex;
+ return TRUE;
+ }
+
+ // Random indexing. Linear search for the boundary following the given position.
+
+ for (fPositionInCache = 0; fPositionInCache < fBreaks->size(); ++fPositionInCache) {
+ r= fBreaks->elementAti(fPositionInCache);
+ if (r > fromPos) {
+ *result = r;
+ *statusIndex = fOtherRuleStatusIndex;
+ return TRUE;
+ }
+ }
+ U_ASSERT(FALSE);
+ fPositionInCache = -1;
+ return FALSE;
+}
+
+
+UBool RuleBasedBreakIterator::DictionaryCache::preceding(int32_t fromPos, int32_t *result, int32_t *statusIndex) {
+ if (fromPos <= fStart || fromPos > fLimit) {
+ fPositionInCache = -1;
+ return FALSE;
+ }
+
+ if (fromPos == fLimit) {
+ fPositionInCache = fBreaks->size() - 1;
+ if (fPositionInCache >= 0) {
+ U_ASSERT(fBreaks->elementAti(fPositionInCache) == fromPos);
+ }
+ }
+
+ int32_t r;
+ if (fPositionInCache > 0 && fPositionInCache < fBreaks->size() && fBreaks->elementAti(fPositionInCache) == fromPos) {
+ --fPositionInCache;
+ r = fBreaks->elementAti(fPositionInCache);
+ U_ASSERT(r < fromPos);
+ *result = r;
+ *statusIndex = ( r== fStart) ? fFirstRuleStatusIndex : fOtherRuleStatusIndex;
+ return TRUE;
+ }
+
+ if (fPositionInCache == 0) {
+ fPositionInCache = -1;
+ return FALSE;
+ }
+
+ for (fPositionInCache = fBreaks->size()-1; fPositionInCache >= 0; --fPositionInCache) {
+ r = fBreaks->elementAti(fPositionInCache);
+ if (r < fromPos) {
+ *result = r;
+ *statusIndex = ( r == fStart) ? fFirstRuleStatusIndex : fOtherRuleStatusIndex;
+ return TRUE;
+ }
+ }
+ U_ASSERT(FALSE);
+ fPositionInCache = -1;
+ return FALSE;
+}
+
+void RuleBasedBreakIterator::DictionaryCache::populateDictionary(int32_t startPos, int32_t endPos,
+ int32_t firstRuleStatus, int32_t otherRuleStatus) {
+ if ((endPos - startPos) <= 1) {
+ return;
+ }
+
+ reset();
+ fFirstRuleStatusIndex = firstRuleStatus;
+ fOtherRuleStatusIndex = otherRuleStatus;
+
+ int32_t rangeStart = startPos;
+ int32_t rangeEnd = endPos;
+
+ uint16_t category;
+ int32_t current;
+ UErrorCode status = U_ZERO_ERROR;
+ int32_t foundBreakCount = 0;
+ UText *text = fBI->fText;
+
+ // Loop through the text, looking for ranges of dictionary characters.
+ // For each span, find the appropriate break engine, and ask it to find
+ // any breaks within the span.
+
+ utext_setNativeIndex(text, rangeStart);
+ UChar32 c = utext_current32(text);
+ category = UTRIE2_GET16(fBI->fData->fTrie, c);
+
+ while(U_SUCCESS(status)) {
+ while((current = (int32_t)UTEXT_GETNATIVEINDEX(text)) < rangeEnd && (category & 0x4000) == 0) {
+ utext_next32(text); // TODO: cleaner loop structure.
+ c = utext_current32(text);
+ category = UTRIE2_GET16(fBI->fData->fTrie, c);
+ }
+ if (current >= rangeEnd) {
+ break;
+ }
+
+ // We now have a dictionary character. Get the appropriate language object
+ // to deal with it.
+ const LanguageBreakEngine *lbe = fBI->getLanguageBreakEngine(c);
+
+ // Ask the language object if there are any breaks. It will add them to the cache and
+ // leave the text pointer on the other side of its range, ready to search for the next one.
+ if (lbe != NULL) {
+ foundBreakCount += lbe->findBreaks(text, rangeStart, rangeEnd, fBI->fBreakType, *fBreaks);
+ }
+
+ // Reload the loop variables for the next go-round
+ c = utext_current32(text);
+ category = UTRIE2_GET16(fBI->fData->fTrie, c);
+ }
+
+ // If we found breaks, ensure that the first and last entries are
+ // the original starting and ending position. And initialize the
+ // cache iteration position to the first entry.
+
+ // printf("foundBreakCount = %d\n", foundBreakCount);
+ if (foundBreakCount > 0) {
+ U_ASSERT(foundBreakCount == fBreaks->size());
+ if (startPos < fBreaks->elementAti(0)) {
+ // The dictionary did not place a boundary at the start of the segment of text.
+ // Add one now. This should not commonly happen, but it would be easy for interactions
+ // of the rules for dictionary segments and the break engine implementations to
+ // inadvertently cause it. Cover it here, just in case.
+ fBreaks->insertElementAt(startPos, 0, status);
+ }
+ if (endPos > fBreaks->peeki()) {
+ fBreaks->push(endPos, status);
+ }
+ fPositionInCache = 0;
+ // Note: Dictionary matching may extend beyond the original limit.
+ fStart = fBreaks->elementAti(0);
+ fLimit = fBreaks->peeki();
+ } else {
+ // there were no language-based breaks, even though the segment contained
+ // dictionary characters. Subsequent attempts to fetch boundaries from the dictionary cache
+ // for this range will fail, and the calling code will fall back to the rule based boundaries.
+ }
+}
+
+
+/*
+ * BreakCache implemetation
+ */
+
+RuleBasedBreakIterator::BreakCache::BreakCache(RuleBasedBreakIterator *bi, UErrorCode &status) :
+ fBI(bi), fSideBuffer(status) {
+ reset();
+}
+
+
+RuleBasedBreakIterator::BreakCache::~BreakCache() {
+}
+
+
+void RuleBasedBreakIterator::BreakCache::reset(int32_t pos, int32_t ruleStatus) {
+ fStartBufIdx = 0;
+ fEndBufIdx = 0;
+ fTextIdx = pos;
+ fBufIdx = 0;
+ fBoundaries[0] = pos;
+ fStatuses[0] = (uint16_t)ruleStatus;
+}
+
+
+int32_t RuleBasedBreakIterator::BreakCache::current() {
+ fBI->fPosition = fTextIdx;
+ fBI->fRuleStatusIndex = fStatuses[fBufIdx];
+ fBI->fDone = FALSE;
+ return fTextIdx;
+}
+
+
+void RuleBasedBreakIterator::BreakCache::following(int32_t startPos, UErrorCode &status) {
+ if (U_FAILURE(status)) {
+ return;
+ }
+ if (startPos == fTextIdx || seek(startPos) || populateNear(startPos, status)) {
+ // startPos is in the cache. Do a next() from that position.
+ // TODO: an awkward set of interactions with bi->fDone
+ // seek() does not clear it; it can't because of interactions with populateNear().
+ // next() does not clear it in the fast-path case, where everything matters. Maybe it should.
+ // So clear it here, for the case where seek() succeeded on an iterator that had previously run off the end.
+ fBI->fDone = false;
+ next();
+ }
+ return;
+}
+
+
+void RuleBasedBreakIterator::BreakCache::preceding(int32_t startPos, UErrorCode &status) {
+ if (U_FAILURE(status)) {
+ return;
+ }
+ if (startPos == fTextIdx || seek(startPos) || populateNear(startPos, status)) {
+ if (startPos == fTextIdx) {
+ previous(status);
+ } else {
+ // seek() leaves the BreakCache positioned at the preceding boundary
+ // if the requested position is between two bounaries.
+ // current() pushes the BreakCache position out to the BreakIterator itself.
+ U_ASSERT(startPos > fTextIdx);
+ current();
+ }
+ }
+ return;
+}
+
+
+/*
+ * Out-of-line code for BreakCache::next().
+ * Cache does not already contain the boundary
+ */
+void RuleBasedBreakIterator::BreakCache::nextOL() {
+ fBI->fDone = !populateFollowing();
+ fBI->fPosition = fTextIdx;
+ fBI->fRuleStatusIndex = fStatuses[fBufIdx];
+ return;
+}
+
+
+void RuleBasedBreakIterator::BreakCache::previous(UErrorCode &status) {
+ if (U_FAILURE(status)) {
+ return;
+ }
+ int32_t initialBufIdx = fBufIdx;
+ if (fBufIdx == fStartBufIdx) {
+ // At start of cache. Prepend to it.
+ populatePreceding(status);
+ } else {
+ // Cache already holds the next boundary
+ fBufIdx = modChunkSize(fBufIdx - 1);
+ fTextIdx = fBoundaries[fBufIdx];
+ }
+ fBI->fDone = (fBufIdx == initialBufIdx);
+ fBI->fPosition = fTextIdx;
+ fBI->fRuleStatusIndex = fStatuses[fBufIdx];
+ return;
+}
+
+
+UBool RuleBasedBreakIterator::BreakCache::seek(int32_t pos) {
+ if (pos < fBoundaries[fStartBufIdx] || pos > fBoundaries[fEndBufIdx]) {
+ return FALSE;
+ }
+ if (pos == fBoundaries[fStartBufIdx]) {
+ // Common case: seek(0), from BreakIterator::first()
+ fBufIdx = fStartBufIdx;
+ fTextIdx = fBoundaries[fBufIdx];
+ return TRUE;
+ }
+ if (pos == fBoundaries[fEndBufIdx]) {
+ fBufIdx = fEndBufIdx;
+ fTextIdx = fBoundaries[fBufIdx];
+ return TRUE;
+ }
+
+ int32_t min = fStartBufIdx;
+ int32_t max = fEndBufIdx;
+ while (min != max) {
+ int32_t probe = (min + max + (min>max ? CACHE_SIZE : 0)) / 2;
+ probe = modChunkSize(probe);
+ if (fBoundaries[probe] > pos) {
+ max = probe;
+ } else {
+ min = modChunkSize(probe + 1);
+ }
+ }
+ U_ASSERT(fBoundaries[max] > pos);
+ fBufIdx = modChunkSize(max - 1);
+ fTextIdx = fBoundaries[fBufIdx];
+ U_ASSERT(fTextIdx <= pos);
+ return TRUE;
+}
+
+
+UBool RuleBasedBreakIterator::BreakCache::populateNear(int32_t position, UErrorCode &status) {
+ if (U_FAILURE(status)) {
+ return FALSE;
+ }
+ U_ASSERT(position < fBoundaries[fStartBufIdx] || position > fBoundaries[fEndBufIdx]);
+
+ // Find a boundary somewhere in the vicinity of the requested position.
+ // Depending on the safe rules and the text data, it could be either before, at, or after
+ // the requested position.
+
+
+ // If the requested position is not near already cached positions, clear the existing cache,
+ // find a near-by boundary and begin new cache contents there.
+
+ if ((position < fBoundaries[fStartBufIdx] - 15) || position > (fBoundaries[fEndBufIdx] + 15)) {
+ int32_t aBoundary = 0;
+ int32_t ruleStatusIndex = 0;
+ // TODO: check for position == length of text. Although may still need to back up to get rule status.
+ if (position > 20) {
+ int32_t backupPos = fBI->handlePrevious(position);
+ fBI->fPosition = backupPos;
+ aBoundary = fBI->handleNext(); // Ignore dictionary, just finding a rule based boundary.
+ ruleStatusIndex = fBI->fRuleStatusIndex;
+ }
+ reset(aBoundary, ruleStatusIndex); // Reset cache to hold aBoundary as a single starting point.
+ }
+
+ // Fill in boundaries between existing cache content and the new requested position.
+
+ if (fBoundaries[fEndBufIdx] < position) {
+ // The last position in the cache precedes the requested position.
+ // Add following position(s) to the cache.
+ while (fBoundaries[fEndBufIdx] < position) {
+ if (!populateFollowing()) {
+ U_ASSERT(false);
+ return false;
+ }
+ }
+ fBufIdx = fEndBufIdx; // Set iterator position to the end of the buffer.
+ fTextIdx = fBoundaries[fBufIdx]; // Required because populateFollowing may add extra boundaries.
+ while (fTextIdx > position) { // Move backwards to a position at or preceding the requested pos.
+ previous(status);
+ }
+ return true;
+ }
+
+ if (fBoundaries[fStartBufIdx] > position) {
+ // The first position in the cache is beyond the requested position.
+ // back up more until we get a boundary <= the requested position.
+ while (fBoundaries[fStartBufIdx] > position) {
+ populatePreceding(status);
+ }
+ fBufIdx = fStartBufIdx; // Set iterator position to the start of the buffer.
+ fTextIdx = fBoundaries[fBufIdx]; // Required because populatePreceding may add extra boundaries.
+ while (fTextIdx < position) { // Move forwards to a position at or following the requested pos.
+ next();
+ }
+ if (fTextIdx > position) {
+ // If position is not itself a boundary, the next() loop above will overshoot.
+ // Back up one, leaving cache position at the boundary preceding the requested position.
+ previous(status);
+ }
+ return true;
+ }
+
+ U_ASSERT(fTextIdx == position);
+ return true;
+}
+
+
+
+UBool RuleBasedBreakIterator::BreakCache::populateFollowing() {
+ int32_t fromPosition = fBoundaries[fEndBufIdx];
+ int32_t fromRuleStatusIdx = fStatuses[fEndBufIdx];
+ int32_t pos = 0;
+ int32_t ruleStatusIdx = 0;
+
+ if (fBI->fDictionaryCache->following(fromPosition, &pos, &ruleStatusIdx)) {
+ addFollowing(pos, ruleStatusIdx, UpdateCachePosition);
+ return TRUE;
+ }
+
+ fBI->fPosition = fromPosition;
+ pos = fBI->handleNext();
+ if (pos == UBRK_DONE) {
+ return FALSE;
+ }
+
+ ruleStatusIdx = fBI->fRuleStatusIndex;
+ if (fBI->fDictionaryCharCount > 0) {
+ // The text segment obtained from the rules includes dictionary characters.
+ // Subdivide it, with subdivided results going into the dictionary cache.
+ fBI->fDictionaryCache->populateDictionary(fromPosition, pos, fromRuleStatusIdx, ruleStatusIdx);
+ if (fBI->fDictionaryCache->following(fromPosition, &pos, &ruleStatusIdx)) {
+ addFollowing(pos, ruleStatusIdx, UpdateCachePosition);
+ return TRUE;
+ // TODO: may want to move a sizable chunk of dictionary cache to break cache at this point.
+ // But be careful with interactions with populateNear().
+ }
+ }
+
+ // Rule based segment did not include dictionary characters.
+ // Or, it did contain dictionary chars, but the dictionary segmenter didn't handle them,
+ // meaning that we didn't take the return, above.
+ // Add its end point to the cache.
+ addFollowing(pos, ruleStatusIdx, UpdateCachePosition);
+
+ // Add several non-dictionary boundaries at this point, to optimize straight forward iteration.
+ // (subsequent calls to BreakIterator::next() will take the fast path, getting cached results.
+ //
+ for (int count=0; count<6; ++count) {
+ pos = fBI->handleNext();
+ if (pos == UBRK_DONE || fBI->fDictionaryCharCount > 0) {
+ break;
+ }
+ addFollowing(pos, fBI->fRuleStatusIndex, RetainCachePosition);
+ }
+
+ return TRUE;
+}
+
+
+UBool RuleBasedBreakIterator::BreakCache::populatePreceding(UErrorCode &status) {
+ if (U_FAILURE(status)) {
+ return FALSE;
+ }
+
+ int32_t fromPosition = fBoundaries[fStartBufIdx];
+ if (fromPosition == 0) {
+ return FALSE;
+ }
+
+ int32_t position = 0;
+ int32_t positionStatusIdx = 0;
+
+ if (fBI->fDictionaryCache->preceding(fromPosition, &position, &positionStatusIdx)) {
+ addPreceding(position, positionStatusIdx, UpdateCachePosition);
+ return TRUE;
+ }
+
+ int32_t backupPosition = fromPosition;
+
+ // Find a boundary somewhere preceding the first already-cached boundary
+ do {
+ backupPosition = backupPosition - 30;
+ if (backupPosition <= 0) {
+ backupPosition = 0;
+ } else {
+ backupPosition = fBI->handlePrevious(backupPosition);
+ }
+ if (backupPosition == UBRK_DONE || backupPosition == 0) {
+ position = 0;
+ positionStatusIdx = 0;
+ } else {
+ fBI->fPosition = backupPosition; // TODO: pass starting position in a clearer way.
+ position = fBI->handleNext();
+ positionStatusIdx = fBI->fRuleStatusIndex;
+
+ }
+ } while (position >= fromPosition);
+
+ // Find boundaries between the one we just located and the first already-cached boundary
+ // Put them in a side buffer, because we don't yet know where they will fall in the circular cache buffer..
+
+ fSideBuffer.removeAllElements();
+ fSideBuffer.addElement(position, status);
+ fSideBuffer.addElement(positionStatusIdx, status);
+
+ do {
+ int32_t prevPosition = fBI->fPosition = position;
+ int32_t prevStatusIdx = positionStatusIdx;
+ position = fBI->handleNext();
+ positionStatusIdx = fBI->fRuleStatusIndex;
+ if (position == UBRK_DONE) {
+ break;
+ }
+
+ UBool segmentHandledByDictionary = FALSE;
+ if (fBI->fDictionaryCharCount != 0) {
+ // Segment from the rules includes dictionary characters.
+ // Subdivide it, with subdivided results going into the dictionary cache.
+ int32_t dictSegEndPosition = position;
+ fBI->fDictionaryCache->populateDictionary(prevPosition, dictSegEndPosition, prevStatusIdx, positionStatusIdx);
+ while (fBI->fDictionaryCache->following(prevPosition, &position, &positionStatusIdx)) {
+ segmentHandledByDictionary = true;
+ U_ASSERT(position > prevPosition);
+ if (position >= fromPosition) {
+ break;
+ }
+ U_ASSERT(position <= dictSegEndPosition);
+ fSideBuffer.addElement(position, status);
+ fSideBuffer.addElement(positionStatusIdx, status);
+ prevPosition = position;
+ }
+ U_ASSERT(position==dictSegEndPosition || position>=fromPosition);
+ }
+
+ if (!segmentHandledByDictionary && position < fromPosition) {
+ fSideBuffer.addElement(position, status);
+ fSideBuffer.addElement(positionStatusIdx, status);
+ }
+ } while (position < fromPosition);
+
+ // Move boundaries from the side buffer to the main circular buffer.
+ UBool success = FALSE;
+ if (!fSideBuffer.isEmpty()) {
+ positionStatusIdx = fSideBuffer.popi();
+ position = fSideBuffer.popi();
+ addPreceding(position, positionStatusIdx, UpdateCachePosition);
+ success = TRUE;
+ }
+
+ while (!fSideBuffer.isEmpty()) {
+ positionStatusIdx = fSideBuffer.popi();
+ position = fSideBuffer.popi();
+ if (!addPreceding(position, positionStatusIdx, RetainCachePosition)) {
+ // No space in circular buffer to hold a new preceding result while
+ // also retaining the current cache (iteration) position.
+ // Bailing out is safe; the cache will refill again if needed.
+ break;
+ }
+ }
+
+ return success;
+}
+
+
+void RuleBasedBreakIterator::BreakCache::addFollowing(int32_t position, int32_t ruleStatusIdx, UpdatePositionValues update) {
+ U_ASSERT(position > fBoundaries[fEndBufIdx]);
+ U_ASSERT(ruleStatusIdx <= UINT16_MAX);
+ int32_t nextIdx = modChunkSize(fEndBufIdx + 1);
+ if (nextIdx == fStartBufIdx) {
+ fStartBufIdx = modChunkSize(fStartBufIdx + 6); // TODO: experiment. Probably revert to 1.
+ }
+ fBoundaries[nextIdx] = position;
+ fStatuses[nextIdx] = ruleStatusIdx;
+ fEndBufIdx = nextIdx;
+ if (update == UpdateCachePosition) {
+ // Set current position to the newly added boundary.
+ fBufIdx = nextIdx;
+ fTextIdx = position;
+ } else {
+ // Retaining the original cache position.
+ // Check if the added boundary wraps around the buffer, and would over-write the original position.
+ // It's the responsibility of callers of this function to not add too many.
+ U_ASSERT(nextIdx != fBufIdx);
+ }
+}
+
+bool RuleBasedBreakIterator::BreakCache::addPreceding(int32_t position, int32_t ruleStatusIdx, UpdatePositionValues update) {
+ U_ASSERT(position < fBoundaries[fStartBufIdx]);
+ U_ASSERT(ruleStatusIdx <= UINT16_MAX);
+ int32_t nextIdx = modChunkSize(fStartBufIdx - 1);
+ if (nextIdx == fEndBufIdx) {
+ if (fBufIdx == fEndBufIdx && update == RetainCachePosition) {
+ // Failure. The insertion of the new boundary would claim the buffer position that is the
+ // current iteration position. And we also want to retain the current iteration position.
+ // (The buffer is already completely full of entries that precede the iteration position.)
+ return false;
+ }
+ fEndBufIdx = modChunkSize(fEndBufIdx - 1);
+ }
+ fBoundaries[nextIdx] = position;
+ fStatuses[nextIdx] = ruleStatusIdx;
+ fStartBufIdx = nextIdx;
+ if (update == UpdateCachePosition) {
+ fBufIdx = nextIdx;
+ fTextIdx = position;
+ }
+ return true;
+}
+
+
+void RuleBasedBreakIterator::BreakCache::dumpCache() {
+ printf("fTextIdx:%d fBufIdx:%d\n", fTextIdx, fBufIdx);
+ for (int32_t i=fStartBufIdx; ; i=modChunkSize(i+1)) {
+ printf("%d %d\n", i, fBoundaries[i]);
+ if (i == fEndBufIdx) {
+ break;
+ }
+ }
+}
+
+U_NAMESPACE_END
--- /dev/null
+// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// License & terms of use: http://www.unicode.org/copyright.html
+
+// file: rbbi_cache.h
+//
+#ifndef RBBI_CACHE_H
+#define RBBI_CACHE_H
+
+#include "unicode/utypes.h"
+
+#include "unicode/rbbi.h"
+#include "unicode/uobject.h"
+
+#include "uvectr32.h"
+
+U_NAMESPACE_BEGIN
+
+/* DictionaryCache stores the boundaries obtained from a run of dictionary characters.
+ * Dictionary boundaries are moved first to this cache, then from here
+ * to the main BreakCache, where they may inter-leave with non-dictionary
+ * boundaries. The public BreakIterator API always fetches directly
+ * from the main BreakCache, not from here.
+ *
+ * In common situations, the number of boundaries in a single dictionary run
+ * should be quite small, it will be terminated by punctuation, spaces,
+ * or any other non-dictionary characters. The main BreakCache may end
+ * up with boundaries from multiple dictionary based runs.
+ *
+ * The boundaries are stored in a simple ArrayList (vector), with the
+ * assumption that they will be accessed sequentially.
+ */
+class RuleBasedBreakIterator::DictionaryCache: public UMemory {
+ public:
+ DictionaryCache(RuleBasedBreakIterator *bi, UErrorCode &status);
+ ~DictionaryCache();
+
+ void reset();
+
+ UBool following(int32_t fromPos, int32_t *pos, int32_t *statusIndex);
+ UBool preceding(int32_t fromPos, int32_t *pos, int32_t *statusIndex);
+
+ /**
+ * Populate the cache with the dictionary based boundaries within a region of text.
+ * @param startPos The start position of a range of text
+ * @param endPos The end position of a range of text
+ * @param firstRuleStatus The rule status index that applies to the break at startPos
+ * @param otherRuleStatus The rule status index that applies to boundaries other than startPos
+ * @internal
+ */
+ void populateDictionary(int32_t startPos, int32_t endPos,
+ int32_t firstRuleStatus, int32_t otherRuleStatus);
+
+
+
+ RuleBasedBreakIterator *fBI;
+
+ UVector32 *fBreaks; // A vector containing the boundaries.
+ int32_t fPositionInCache; // Index in fBreaks of last boundary returned by following()
+ // or preceding(). Optimizes sequential access.
+ int32_t fStart; // Text position of first boundary in cache.
+ int32_t fLimit; // Last boundary in cache. Which is the limit of the
+ // text segment being handled by the dictionary.
+ int32_t fFirstRuleStatusIndex; // Rule status info for first boundary.
+ int32_t fOtherRuleStatusIndex; // Rule status info for 2nd through last boundaries.
+};
+
+
+/*
+ * class BreakCache
+ *
+ * Cache of break boundary positions and rule status values.
+ * Break iterator API functions, next(), previous(), etc., will use cached results
+ * when possible, and otherwise cache new results as they are obtained.
+ *
+ * Uniformly caches both dictionary and rule based (non-dictionary) boundaries.
+ *
+ * The cache is implemented as a single circular buffer.
+ */
+
+/*
+ * size of the circular cache buffer.
+ */
+
+class RuleBasedBreakIterator::BreakCache: public UMemory {
+ public:
+ BreakCache(RuleBasedBreakIterator *bi, UErrorCode &status);
+ virtual ~BreakCache();
+ void reset(int32_t pos = 0, int32_t ruleStatus = 0);
+ void next() { if (fBufIdx == fEndBufIdx) {
+ nextOL();
+ } else {
+ fBufIdx = modChunkSize(fBufIdx + 1);
+ fTextIdx = fBI->fPosition = fBoundaries[fBufIdx];
+ fBI->fRuleStatusIndex = fStatuses[fBufIdx];
+ }
+ };
+
+
+ void nextOL();
+ void previous(UErrorCode &status);
+
+ // Move the iteration state to the position following the startPosition.
+ // Input position must be pinned to the input length.
+ void following(int32_t startPosition, UErrorCode &status);
+
+ void preceding(int32_t startPosition, UErrorCode &status);
+
+ /*
+ * Update the state of the public BreakIterator (fBI) to reflect the
+ * current state of the break iterator cache (this).
+ */
+ int32_t current();
+
+ /**
+ * Add boundaries to the cache near the specified position.
+ * The given position need not be a boundary itself.
+ * The input position must be within the range of the text, and
+ * on a code point boundary.
+ * If the requested position is a break boundary, leave the iteration
+ * position on it.
+ * If the requested position is not a boundary, leave the iteration
+ * position on the preceding boundary and include both the the
+ * preceding and following boundaries in the cache.
+ * Additional boundaries, either preceding or following, may be added
+ * to the cache as a side effect.
+ *
+ * Return FALSE if the operation failed.
+ */
+ UBool populateNear(int32_t position, UErrorCode &status);
+
+ /**
+ * Add boundary(s) to the cache following the current last boundary.
+ * Return FALSE if at the end of the text, and no more boundaries can be added.
+ * Leave iteration position at the first newly added boundary, or unchanged if no boundary was added.
+ */
+ UBool populateFollowing();
+
+ /**
+ * Add one or more boundaries to the cache preceding the first currently cached boundary.
+ * Leave the iteration position on the first added boundary.
+ * Return false if no boundaries could be added (if at the start of the text.)
+ */
+ UBool populatePreceding(UErrorCode &status);
+
+ enum UpdatePositionValues {
+ RetainCachePosition = 0,
+ UpdateCachePosition = 1
+ };
+
+ /*
+ * Add the boundary following the current position.
+ * The current position can be left as it was, or changed to the newly added boundary,
+ * as specified by the update parameter.
+ */
+ void addFollowing(int32_t position, int32_t ruleStatusIdx, UpdatePositionValues update);
+
+
+ /*
+ * Add the boundary preceding the current position.
+ * The current position can be left as it was, or changed to the newly added boundary,
+ * as specified by the update parameter.
+ */
+ bool addPreceding(int32_t position, int32_t ruleStatusIdx, UpdatePositionValues update);
+
+ /**
+ * Set the cache position to the specified position, or, if the position
+ * falls between to cached boundaries, to the preceding boundary.
+ * Fails if the requested position is outside of the range of boundaries currently held by the cache.
+ * The startPosition must be on a code point boundary.
+ *
+ * Return TRUE if successful, FALSE if the specified position is after
+ * the last cached boundary or before the first.
+ */
+ UBool seek(int32_t startPosition);
+
+ void dumpCache();
+
+ private:
+ static inline int32_t modChunkSize(int index) { return index & (CACHE_SIZE - 1); };
+
+ static constexpr int32_t CACHE_SIZE = 128;
+ static_assert((CACHE_SIZE & (CACHE_SIZE-1)) == 0, "CACHE_SIZE must be power of two.");
+
+ RuleBasedBreakIterator *fBI;
+ int32_t fStartBufIdx;
+ int32_t fEndBufIdx; // inclusive
+
+ int32_t fTextIdx;
+ int32_t fBufIdx;
+
+ int32_t fBoundaries[CACHE_SIZE];
+ uint16_t fStatuses[CACHE_SIZE];
+
+ UVector32 fSideBuffer;
+};
+
+U_NAMESPACE_END
+
+#endif // RBBI_CACHE_H
#include "unicode/utypes.h"
#include "rbbidata.h"
#include "rbbirb.h"
-#include "utrie.h"
+#include "utrie2.h"
#include "udatamem.h"
#include "cmemory.h"
#include "cstring.h"
fReverseTable = NULL;
fSafeFwdTable = NULL;
fSafeRevTable = NULL;
- fRuleSource = NULL;
+ fRuleSource = NULL;
fRuleStatusTable = NULL;
- fTrie = NULL;
- fUDataMem = NULL;
- fRefCount = 0;
+ fTrie = NULL;
+ fUDataMem = NULL;
+ fRefCount = 0;
fDontFreeData = TRUE;
}
fSafeRevTable = (RBBIStateTable *)((char *)data + fHeader->fSRTable);
}
+ // Rule Compatibility Hacks
+ // If a rule set includes reverse rules but does not explicitly include safe reverse rules,
+ // the reverse rules are to be treated as safe reverse rules.
+
+ if (fSafeRevTable == NULL && fReverseTable != NULL) {
+ fSafeRevTable = fReverseTable;
+ fReverseTable = NULL;
+ }
fTrie = utrie2_openFromSerialized(UTRIE2_16_VALUE_BITS,
(uint8_t *)data + fHeader->fTrie,
/* number of int32_t values in the rule status table. Used to sanity check indexing */
int32_t fStatusMaxIdx;
- UTrie2 *fTrie;
+ UTrie2 *fTrie;
private:
u_atomic_int32_t fRefCount;
- UDataMemory *fUDataMem;
+ UDataMemory *fUDataMem;
UnicodeString fRuleString;
UBool fDontFreeData;
#include "unicode/uchriter.h"
#include "unicode/parsepos.h"
#include "unicode/parseerr.h"
+
#include "cmemory.h"
#include "cstring.h"
-
#include "rbbirb.h"
#include "rbbinode.h"
-
#include "rbbiscan.h"
#include "rbbisetb.h"
#include "rbbitblb.h"
#include "rbbidata.h"
+#include "uassert.h"
U_NAMESPACE_BEGIN
int32_t statusTableSize = align8(fRuleStatusVals->size() * sizeof(int32_t));
int32_t rulesSize = align8((strippedRules.length()+1) * sizeof(UChar));
- int32_t totalSize = headerSize + forwardTableSize + reverseTableSize
- + safeFwdTableSize + safeRevTableSize
+ (void)safeFwdTableSize;
+
+ int32_t totalSize = headerSize
+ + forwardTableSize
+ + /* reverseTableSize */ 0
+ + /* safeFwdTableSize */ 0
+ + (safeRevTableSize ? safeRevTableSize : reverseTableSize)
+ statusTableSize + trieSize + rulesSize;
RBBIDataHeader *data = (RBBIDataHeader *)uprv_malloc(totalSize);
data->fLength = totalSize;
data->fCatCount = fSetBuilder->getNumCharCategories();
+ // Only save the forward table and the safe reverse table,
+ // because these are the only ones used at run-time.
+ //
+ // For the moment, we still build the other tables if they are present in the rule source files,
+ // for backwards compatibility. Old rule files need to work, and this is the simplest approach.
+ //
+ // Additional backwards compatibility consideration: if no safe rules are provided, consider the
+ // reverse rules to actually be the safe reverse rules.
+
data->fFTable = headerSize;
data->fFTableLen = forwardTableSize;
+
+ // Do not save Reverse Table.
data->fRTable = data->fFTable + forwardTableSize;
- data->fRTableLen = reverseTableSize;
- data->fSFTable = data->fRTable + reverseTableSize;
- data->fSFTableLen = safeFwdTableSize;
- data->fSRTable = data->fSFTable + safeFwdTableSize;
- data->fSRTableLen = safeRevTableSize;
+ data->fRTableLen = 0;
+
+ // Do not save the Safe Forward table.
+ data->fSFTable = data->fRTable + 0;
+ data->fSFTableLen = 0;
+
+ data->fSRTable = data->fSFTable + 0;
+ if (safeRevTableSize > 0) {
+ data->fSRTableLen = safeRevTableSize;
+ } else if (reverseTableSize > 0) {
+ data->fSRTableLen = reverseTableSize;
+ } else {
+ U_ASSERT(FALSE); // Rule build should have failed for lack of a reverse table
+ // before reaching this point.
+ }
+
- data->fTrie = data->fSRTable + safeRevTableSize;
+ data->fTrie = data->fSRTable + data->fSRTableLen;
data->fTrieLen = fSetBuilder->getTrieSize();
data->fStatusTable = data->fTrie + trieSize;
data->fStatusTableLen= statusTableSize;
uprv_memset(data->fReserved, 0, sizeof(data->fReserved));
fForwardTables->exportTable((uint8_t *)data + data->fFTable);
- fReverseTables->exportTable((uint8_t *)data + data->fRTable);
- fSafeFwdTables->exportTable((uint8_t *)data + data->fSFTable);
- fSafeRevTables->exportTable((uint8_t *)data + data->fSRTable);
+ // fReverseTables->exportTable((uint8_t *)data + data->fRTable);
+ // fSafeFwdTables->exportTable((uint8_t *)data + data->fSFTable);
+ if (safeRevTableSize > 0) {
+ fSafeRevTables->exportTable((uint8_t *)data + data->fSRTable);
+ } else {
+ fReverseTables->exportTable((uint8_t *)data + data->fSRTable);
+ }
+
fSetBuilder->serializeTrie ((uint8_t *)data + data->fTrie);
int32_t *ruleStatusTable = (int32_t *)((uint8_t *)data + data->fStatusTable);
//
//------------------------------------------------------------------------------
static const UChar gRuleSet_rule_char_pattern[] = {
+ // Characters that may appear as literals in patterns without escaping or quoting.
// [ ^ [ \ p { Z } \ u 0 0 2 0
0x5b, 0x5e, 0x5b, 0x5c, 0x70, 0x7b, 0x5a, 0x7d, 0x5c, 0x75, 0x30, 0x30, 0x32, 0x30,
// - \ u 0 0 7 f ] - [ \ p
fRB->fDefaultTree = &fRB->fSafeRevTree;
} else if (opt == UNICODE_STRING("lookAheadHardBreak", 18)) {
fRB->fLookAheadHardBreak = TRUE;
+ } else if (opt == UNICODE_STRING("quoted_literals_only", 20)) {
+ fRuleSets[kRuleSet_rule_char-128].clear();
+ } else if (opt == UNICODE_STRING("unquoted_literals", 17)) {
+ fRuleSets[kRuleSet_rule_char-128].applyPattern(UnicodeString(gRuleSet_rule_char_pattern), *fRB->fStatus);
} else {
error(U_BRK_UNRECOGNIZED_OPTION);
}
// Build the Trie table for mapping UChar32 values to the corresponding
// range group number
//
- fTrie = utrie2_open(0, // Initial value for all code points
- 0, // errorValue
+ fTrie = utrie2_open(0, // Initial value for all code points.
+ 0, // Error value for out-of-range input.
fStatus);
- for (rlRange = fRangeList; rlRange!=0; rlRange=rlRange->fNext) {
- utrie2_setRange32(fTrie, rlRange->fStartChar, rlRange->fEndChar, rlRange->fNum, TRUE, fStatus);
+ for (rlRange = fRangeList; rlRange!=0 && U_SUCCESS(*fStatus); rlRange=rlRange->fNext) {
+ utrie2_setRange32(fTrie,
+ rlRange->fStartChar, // Range start
+ rlRange->fEndChar, // Range end (inclusive)
+ rlRange->fNum, // value for range
+ TRUE, // Overwrite previously written values
+ fStatus);
}
}
// getTrieSize() Return the size that will be required to serialize the Trie.
//
//-----------------------------------------------------------------------------------
-int32_t RBBISetBuilder::getTrieSize() /*const*/ {
+int32_t RBBISetBuilder::getTrieSize() {
+ if (U_FAILURE(*fStatus)) {
+ return 0;
+ }
utrie2_freeze(fTrie, UTRIE2_16_VALUE_BITS, fStatus);
fTrieSize = utrie2_serialize(fTrie,
NULL, // Buffer
RangeDescriptor *fRangeList; // Head of the linked list of RangeDescriptors
- UTrie2 *fTrie; // The mapping TRIE that is the end result of processing
- uint32_t fTrieSize; // the Unicode Sets.
+ UTrie2 *fTrie; // The mapping TRIE that is the end result of processing
+ uint32_t fTrieSize; // the Unicode Sets.
// Groups correspond to character categories -
// groups of ranges that are in the same original UnicodeSets.
#include "unicode/schriter.h"
#include "unicode/uchriter.h"
-
U_NAMESPACE_BEGIN
/** @internal */
+class LanguageBreakEngine;
struct RBBIDataHeader;
-class RuleBasedBreakIteratorTables;
-class BreakIterator;
class RBBIDataWrapper;
-class UStack;
-class LanguageBreakEngine;
class UnhandledEngine;
-struct RBBIStateTable;
-
-
-
+class UStack;
/**
*
*/
RBBIDataWrapper *fData;
- /** Index of the Rule {tag} values for the most recent match.
+ /**
+ * The iteration state - current position, rule status for the current position,
+ * and whether the iterator ran off the end, yielding UBRK_DONE.
+ * Current position is pinned to be 0 < position <= text.length.
+ * Current position is always set to a boundary.
* @internal
*/
- int32_t fLastRuleStatusIndex;
+ /**
+ * The current position of the iterator. Pinned, 0 < fPosition <= text.length.
+ * Never has the value UBRK_DONE (-1).
+ */
+ int32_t fPosition;
/**
- * Rule tag value valid flag.
- * Some iterator operations don't intrinsically set the correct tag value.
- * This flag lets us lazily compute the value if we are ever asked for it.
- * @internal
- */
- UBool fLastStatusIndexValid;
+ * TODO:
+ */
+ int32_t fRuleStatusIndex;
/**
- * Counter for the number of characters encountered with the "dictionary"
- * flag set.
- * @internal
- */
- uint32_t fDictionaryCharCount;
+ * True when iteration has run off the end, and iterator functions should return UBRK_DONE.
+ */
+ UBool fDone;
/**
- * When a range of characters is divided up using the dictionary, the break
- * positions that are discovered are stored here, preventing us from having
- * to use either the dictionary or the state table again until the iterator
- * leaves this range of text. Has the most impact for line breaking.
- * @internal
+ * Cache of previously determined boundary positions.
*/
- int32_t* fCachedBreakPositions;
-
+ public: // TODO: debug, return to private.
+ class BreakCache;
+ BreakCache *fBreakCache;
+ private:
/**
- * The number of elements in fCachedBreakPositions
+ * Counter for the number of characters encountered with the "dictionary"
+ * flag set.
* @internal
*/
- int32_t fNumCachedBreakPositions;
+ uint32_t fDictionaryCharCount;
/**
- * if fCachedBreakPositions is not null, this indicates which item in the
- * cache the current iteration position refers to
- * @internal
+ * Cache of boundary positions within a region of text that has been
+ * sub-divided by dictionary based breaking.
*/
- int32_t fPositionInCache;
+ class DictionaryCache;
+ DictionaryCache *fDictionaryCache;
/**
*
*/
RuleBasedBreakIterator(RBBIDataHeader* data, UErrorCode &status);
-
+ /** @internal */
friend class RBBIRuleBuilder;
/** @internal */
friend class BreakIterator;
-
-
public:
/** Default constructor. Creates an empty shell of an iterator, with no
virtual UBool isBoundary(int32_t offset);
/**
- * Returns the current iteration position.
+ * Returns the current iteration position. Note that UBRK_DONE is never
+ * returned from this function; if iteration has run to the end of a
+ * string, current() will return the length of the string while
+ * next() will return UBRK_DONE).
* @return The current iteration position.
* @stable ICU 2.0
*/
* Note: this function is not thread safe. It should not have been
* declared const, and the const remains only for compatibility
* reasons. (The function is logically const, but not bit-wise const).
+ * TODO: check this. Probably thread safe now.
* <p>
* @return the status from the break rule that determined the most recently
* returned break position.
* Common initialization function, used by constructors and bufferClone.
* @internal
*/
- void init();
+ void init(UErrorCode &status);
/**
- * This method backs the iterator back up to a "safe position" in the text.
- * This is a position that we know, without any context, must be a break position.
- * The various calling methods then iterate forward from this safe position to
- * the appropriate position to return. (For more information, see the description
- * of buildBackwardsStateTable() in RuleBasedBreakIterator.Builder.)
- * @param statetable state table used of moving backwards
+ * Iterate backwards from an arbitrary position in the input text using the Safe Reverse rules.
+ * This locates a "Safe Position" from which the forward break rules
+ * will operate correctly. A Safe Position is not necessarily a boundary itself.
+ *
+ * @param fromPosition the position in the input text to begin the iteration.
* @internal
*/
- int32_t handlePrevious(const RBBIStateTable *statetable);
+ int32_t handlePrevious(int32_t fromPosition);
/**
- * This method is the actual implementation of the next() method. All iteration
- * vectors through here. This method initializes the state machine to state 1
- * and advances through the text character by character until we reach the end
- * of the text or the state machine transitions to state 0. We update our return
- * value every time the state machine passes through a possible end state.
- * @param statetable state table used of moving forwards
- * @internal
- */
- int32_t handleNext(const RBBIStateTable *statetable);
-
-
- /**
- * This is the function that actually implements dictionary-based
- * breaking. Covering at least the range from startPos to endPos,
- * it checks for dictionary characters, and if it finds them determines
- * the appropriate object to deal with them. It may cache found breaks in
- * fCachedBreakPositions as it goes. It may well also look at text outside
- * the range startPos to endPos.
- * If going forward, endPos is the normal Unicode break result, and
- * if goind in reverse, startPos is the normal Unicode break result
- * @param startPos The start position of a range of text
- * @param endPos The end position of a range of text
- * @param reverse The call is for the reverse direction
+ * Find a rule-based boundary by running the state machine.
+ * Input
+ * fPosition, the position in the text to begin from.
+ * Output
+ * fPosition: the boundary following the starting position.
+ * fDictionaryCharCount the number of dictionary characters encountered.
+ * If > 0, the segment will be further subdivided
+ * fRuleStatusIndex Info from the state table indicating which rules caused the boundary.
+ *
* @internal
*/
- int32_t checkDictionary(int32_t startPos, int32_t endPos, UBool reverse);
+ int32_t handleNext();
/**
*/
const LanguageBreakEngine *getLanguageBreakEngine(UChar32 c);
+ public:
/**
- * @internal
+ * Debugging function only.
+ * @internal
*/
- void makeRuleStatusValid();
-
+ void dumpCache();
};
//------------------------------------------------------------------------------
-#
+#
# Copyright (C) 2016 and later: Unicode, Inc. and others.
# License & terms of use: http://www.unicode.org/copyright.html
# Copyright (C) 2002-2016, International Business Machines Corporation and others.
# Plus revisions to rule GB 11 from http://unicode.org/cldr/trac/ticket/10088
# Plus additional characters introduces with Emoji 5, http://www.unicode.org/reports/tr51/proposed.html
+!!quoted_literals_only;
+
#
# Character Class Definitions.
#
## -------------------------------------------------
-!!reverse;
-$LF $CR;
-($L | $V | $LV | $LVT) $L;
-($V | $T) ($LV | $V);
-$T ($LVT | $T);
-
-# GB 9
-($Extend | $ZWJ) [^$Control $CR $LF]; #note that this will chain into Regional_Indicator when needed.
-
-# GB 9a
-$SpacingMark [^$Control $CR $LF];
-
-# GB 9b
-[^$Control $CR $LF] $Prepend;
-
-# GB 10
-$E_Modifier $Extend* ($E_Base | $E_Base_GAZ);
-
-# GB 11 Don't break between ZWJ and Glue_After_ZWJ
-($Extended_Pict | $EmojiNRK) $ZWJ $Extend* ($Extended_Pict | $EmojiNRK);
-
-# GB 12-13. Going backwards, we must scan through any number of regional indicators as pairs.
-#
-[{bof} $Extend $ZWJ $SpacingMark] $Regional_Indicator $Regional_Indicator / ($Regional_Indicator $Regional_Indicator)+ [{eof}[^$Regional_Indicator]];
-[{bof} $Extend $ZWJ $SpacingMark] $Regional_Indicator / ($Regional_Indicator $Regional_Indicator)+ [{eof}[^$Regional_Indicator]];
-$Regional_Indicator $Regional_Indicator;
-$Regional_Indicator $Prepend;
-
-## -------------------------------------------------
-
!!safe_reverse;
$Regional_Indicator $Regional_Indicator;
($Extend | $ZWJ | $EmojiNRK | $Extended_Pict)+ .;
-
-## -------------------------------------------------
-
-!!safe_forward;
-$Regional_Indicator $Regional_Indicator;
-($Extend | $ZWJ | $EmojiNRK | $Extended_Pict)+ .;
-# Copyright (C) 2016 and later: Unicode, Inc. and others.
+# Copyright (C) 2016 and later: Unicode, Inc. and others.
# License & terms of use: http://www.unicode.org/copyright.html
# Copyright (c) 2002-2016 International Business Machines Corporation and
# others. All Rights Reserved.
#
!!chain;
+!!quoted_literals_only;
$AI = [:LineBreak = Ambiguous:];
$AL = [:LineBreak = Alphabetic:];
# LB 30b Do not break between an Emoji Base and an Emoji Modifier
$EB $CM* $EM;
-#
-# Reverse Rules.
-#
-## -------------------------------------------------
-
-!!reverse;
-
-# LB 9 Combining Marks.
-# Stick together any combining sequences that don't match other rules.
-
-^$CM+ $CAN_CM?;
-
-#
-# Sequences of the form (shown forwards)
-# [CANT_CM] <break> [CM] [whatever]
-# The CM needs to behave as an AL
-#
-$AL_FOLLOW $CM+ / (
- [$BK $CR $LF $NL $ZW {eof}] |
- $SP+ $CM+ $SP |
- $SP+ $CM* ([^$OP $CM $SP] | [$AL {eof}])); # if LB 14 will match, need to surpress this break.
- # LB14 says OP SP* x .
- # becomes OP SP* x AL
- # becomes OP SP* x CM+ AL_FOLLOW
- #
- # Further note: the $AL in [$AL {eof}] is only to work around
- # a rule compiler bug which complains about
- # empty sets otherwise.
-
-
-# LB 4, 5, 6
-
-$LB4Breaks [$LB4NonBreaks-$CM];
-$LB4Breaks $CM+ $CAN_CM;
-$LF $CR;
-
-
-# LB 7 x SP
-# x ZW
-[$SP $ZW] [$LB4NonBreaks-$CM];
-[$SP $ZW] $CM+ $CAN_CM;
-
-# LB 8 ZW SP* <break>
-# TODO: to implement this, we need more than one look-ahead hard break in play at a time.
-# Requires an engine enhancement.
-# / $SP* $ZW
-
-# LB 8a ZWJ x (ID | Extended_Pict | EmojiNRK)
-#
-($ID | $Extended_Pict | $EmojiNRK) $ZWJ $CM* $CAN_CM?;
-
-
-# LB 9,10 Combining marks.
-# X $CM needs to behave like X, where X is not $SP or controls.
-# $CM not covered by the above needs to behave like $AL
-# Stick together any combining sequences that don't match other rules.
-^$CM+ $CAN_CM;
-
-
-# LB 11
-#
-$WJ $CM* $CAN_CM;
-$WJ [$LB8NonBreaks-$CM];
-
- $CANT_CM $CM* $WJ;
-$CAN_CM $CM* $WJ;
-
-# LB 12a
-# [^SP BA HY] x GL
-#
-$GL $CM* [$LB8NonBreaks-[$CM $SP $BA $HY]];
-
-# LB 12
-# GL x
-#
-$CANT_CM $CM* $GL;
-$CAN_CM $CM* $GL;
-
-
-# LB 13
-$CL $CM+ $CAN_CM;
-$CP $CM+ $CAN_CM;
-$EX $CM+ $CAN_CM;
-$IS $CM+ $CAN_CM;
-$SY $CM+ $CAN_CM;
-
-$CL [$LB8NonBreaks-$CM];
-$CP [$LB8NonBreaks-$CM];
-$EX [$LB8NonBreaks-$CM];
-$IS [$LB8NonBreaks-$CM];
-$SY [$LB8NonBreaks-$CM];
-
-
-# LB 14 OP SP* x
-#
-. $SP* $CM* $OP;
-$AL_FOLLOW? $CM+ $SP+ $CM* $OP; # by LB 10, behaves like $AL_FOLLOW? $AL $SP+ $CM* $OP
-
-
-# LB 15
-$OP $SP* $CM* $QU;
-
-# LB 16
-$NS $SP* $CM* ($CL | $CP);
-
-# LB 17
-$B2 $SP* $CM* $B2;
-
-# LB 18 break after spaces
-# Nothing explicit needed here.
-
-
-#
-# LB 19
-#
-$QU $CM* $CAN_CM; # . x QU
-$QU $LB18NonBreaks;
-
-
-$CAN_CM $CM* $QU; # QU x .
- $CANT_CM $CM* $QU;
-
-#
-# LB 20 Break before and after CB.
-# nothing needed here.
-#
-
-# LB 21
-($BA | $HY | $NS) $CM* [$LB20NonBreaks-$CM]; # . x (BA | HY | NS)
-
-[$LB20NonBreaks-$CM] $CM* $BB; # BB x .
-[^$CB] $CM* $BB; #
-
-# LB21a
-[^$CB] $CM* ($HY | $BA) $CM* $HL;
-
-# LB21b (reverse)
-$HL $CM* $SY;
-
-# LB 22
-$IN $CM* ($ALPlus | $HL);
-$IN $CM* $EX;
-$IN $CM* ($ID | $EB | $EM);
-$IN $CM* $IN;
-$IN $CM* $NU;
-
-# LB 23
-$NU $CM* ($ALPlus | $HL);
-($ALPlus | $HL) $CM* $NU;
-
-# LB23a
-($ID | $EB | $EM) $CM* $PR;
-$PO $CM* ($ID | $EB | $EM);
-
-# LB 24
-($ALPlus | $HL) $CM* ($PR | $PO);
-($PR | $PO) $CM* ($ALPlus | $HL);
-
-
-# LB 25
-($CM* ($PR | $PO))? ($CM* ($CL | $CP))? ($CM* ($NU | $IS | $SY))* $CM* $NU ($CM* ($OP | $HY))? ($CM* ($PR | $PO))?;
-
-# LB 26
-($H3 | $H2 | $JV | $JL) $CM* $JL;
-($JT | $JV) $CM* ($H2 | $JV);
-$JT $CM* ($H3 | $JT);
-
-# LB 27
-$IN $CM* ($H3 | $H2 | $JT | $JV | $JL);
-$PO $CM* ($H3 | $H2 | $JT | $JV | $JL);
- ($H3 | $H2 | $JT | $JV | $JL) $CM* $PR;
-
-# LB 28
-($ALPlus | $HL) $CM* ($ALPlus | $HL);
-
-
-# LB 29
-($ALPlus | $HL) $CM* $IS;
-
-# LB 30
-$OP $CM* ($ALPlus | $HL | $NU);
-($ALPlus | $HL | $NU) $CM* $CP;
-
-# LB 30a
-# Pairs of Regional Indicators.
-# The following two rules are nearly identical. The first matches only sequences with an odd number of adjacent RIs,
-# the second with an even number. Stripping away the cruft they look like
-# [^RI] RI / (RI RI)+ ^RI;
-# [^RI] RI RI / (RI RI)+ ^RI;
-#
-[{bof} $NS $HY $BA $QU $CL $CP $EX $IS $SY $WJ $GL $ZW $SP $BK $CR $LF $NL $ZWJ] $CM* $RI / ($CM* $RI $CM* $RI)+ $CM* [{eof}[^$RI $CM]];
-[{bof} $NS $HY $BA $QU $CL $CP $EX $IS $SY $WJ $GL $ZW $SP $BK $CR $LF $NL $ZWJ] $CM* $RI $CM* $RI / ($CM* $RI $CM* $RI)+ $CM* [{eof}[^$RI $CM]];
-
-# In general, adjacent RIs stay together. The hard-break rules, above, overide this, forcing in the boundaries between pairs.
-$RI $CM* $RI;
-
-# WJ, GL, QU, etc. are classes with rules like "WJ x " which includes "WJ x RI".
-$RI $CM* ([$WJ $GL $QU $BB] | (($HY | $BA)$CM* $HL));
-
-
-# LB 30b Do not break between an Emoji Base and an Emoji Modifier
-$EM $CM* $EB;
-
## -------------------------------------------------
# LB 9
^$CM+ [^$CM $BK $CR $LF $NL $ZW $SP];
-^$CM+ $SP / .;
# LB 14
$SP+ $CM* $OP;
# For dictionary-based break
$dictionary $dictionary;
-## -------------------------------------------------
-
-!!safe_forward;
-
-# Skip forward over all character classes that are involved in
-# rules containing patterns with possibly more than one char
-# of context.
-#
-# It might be slightly more efficient to have specific rules
-# instead of one generic one, but only if we could
-# turn off rule chaining. We don't want to move more
-# than necessary.
-#
-^[$CM $OP $QU $CL $CP $B2 $PR $HY $BA $SP $RI $ZWJ $dictionary]+ [^$CM $OP $QU $CL $CP $B2 $PR $HY $BA $RI $ZWJ $dictionary];
-$dictionary $dictionary;
-
-# Copyright (C) 2016 and later: Unicode, Inc. and others.
+# Copyright (C) 2016 and later: Unicode, Inc. and others.
# License & terms of use: http://www.unicode.org/copyright.html
# Copyright (c) 2002-2016 International Business Machines Corporation and
# others. All Rights Reserved.
#
!!chain;
+!!quoted_literals_only;
$AI = [:LineBreak = Ambiguous:];
$AL = [:LineBreak = Alphabetic:];
# LB 30b Do not break between an Emoji Base and an Emoji Modifier
$EB $CM* $EM;
-#
-# Reverse Rules.
-#
-## -------------------------------------------------
-
-!!reverse;
-
-# LB 9 Combining Marks.
-# Stick together any combining sequences that don't match other rules.
-
-^$CM+ $CAN_CM?;
-
-#
-# Sequences of the form (shown forwards)
-# [CANT_CM] <break> [CM] [whatever]
-# The CM needs to behave as an AL
-#
-$AL_FOLLOW $CM+ / (
- [$BK $CR $LF $NL $ZW {eof}] |
- $SP+ $CM+ $SP |
- $SP+ $CM* ([^$OP $CM $SP] | [$AL {eof}])); # if LB 14 will match, need to surpress this break.
- # LB14 says OP SP* x .
- # becomes OP SP* x AL
- # becomes OP SP* x CM+ AL_FOLLOW
- #
- # Further note: the $AL in [$AL {eof}] is only to work around
- # a rule compiler bug which complains about
- # empty sets otherwise.
-
-
-# LB 4, 5, 6
-
-$LB4Breaks [$LB4NonBreaks-$CM];
-$LB4Breaks $CM+ $CAN_CM;
-$LF $CR;
-
-
-# LB 7 x SP
-# x ZW
-[$SP $ZW] [$LB4NonBreaks-$CM];
-[$SP $ZW] $CM+ $CAN_CM;
-
-# LB 8 ZW SP* <break>
-# TODO: to implement this, we need more than one look-ahead hard break in play at a time.
-# Requires an engine enhancement.
-# / $SP* $ZW
-
-# LB 8a ZWJ x (ID | Extended_Pict | EmojiNRK)
-#
-($ID | $Extended_Pict | $EmojiNRK) $ZWJ $CM* $CAN_CM?;
-
-
-# LB 9,10 Combining marks.
-# X $CM needs to behave like X, where X is not $SP or controls.
-# $CM not covered by the above needs to behave like $AL
-# Stick together any combining sequences that don't match other rules.
-^$CM+ $CAN_CM;
-
-
-# LB 11
-#
-$WJ $CM* $CAN_CM;
-$WJ [$LB8NonBreaks-$CM];
-
- $CANT_CM $CM* $WJ;
-$CAN_CM $CM* $WJ;
-
-# LB 12a
-# [^SP BA HY] x GL
-#
-$GL $CM* [$LB8NonBreaks-[$CM $SP $BA $HH $HY]];
-
-# LB 12
-# GL x
-#
-$CANT_CM $CM* $GL;
-$CAN_CM $CM* $GL;
-
-
-# LB 13
-$CL $CM+ $CAN_CM;
-$CP $CM+ $CAN_CM;
-$EX $CM+ $CAN_CM;
-$IS $CM+ $CAN_CM;
-$SY $CM+ $CAN_CM;
-
-$CL [$LB8NonBreaks-$CM];
-$CP [$LB8NonBreaks-$CM];
-$EX [$LB8NonBreaks-$CM];
-$IS [$LB8NonBreaks-$CM];
-$SY [$LB8NonBreaks-$CM];
-
-
-# LB 14 OP SP* x
-#
-. $SP* $CM* $OP;
-$AL_FOLLOW? $CM+ $SP+ $CM* $OP; # by LB 10, behaves like $AL_FOLLOW? $AL $SP+ $CM* $OP
-
-
-# LB 15
-$OP $SP* $CM* $QU;
-
-# LB 16
-$NS $SP* $CM* ($CL | $CP);
-
-# LB 17
-$B2 $SP* $CM* $B2;
-
-# LB 18 break after spaces
-# Nothing explicit needed here.
-
-
-#
-# LB 19
-#
-$QU $CM* $CAN_CM; # . x QU
-$QU $LB18NonBreaks;
-
-
-$CAN_CM $CM* $QU; # QU x .
- $CANT_CM $CM* $QU;
-
-#
-# LB 20 Break before and after CB.
-# nothing needed here.
-#
-
-# LB 20.09 added rule for Finnish tailoring
-$AL ($HY | $HH) / $SP;
-
-# LB 21
-($BA | $HH | $HY | $NS) $CM* [$LB20NonBreaks-$CM]; # . x (BA | HY | NS)
-
-[$LB20NonBreaks-$CM] $CM* $BB; # BB x .
-[^$CB] $CM* $BB; #
-
-# LB21a
-[^$CB] $CM* ($HY | $BA | $HH) $CM* $HL;
-
-# LB21b (reverse)
-$HL $CM* $SY;
-
-# LB 22
-$IN $CM* ($ALPlus | $HL);
-$IN $CM* $EX;
-$IN $CM* ($ID | $EB | $EM);
-$IN $CM* $IN;
-$IN $CM* $NU;
-
-# LB 23
-$NU $CM* ($ALPlus | $HL);
-($ALPlus | $HL) $CM* $NU;
-
-# LB23a
-($ID | $EB | $EM) $CM* $PR;
-$PO $CM* ($ID | $EB | $EM);
-
-# LB 24
-($ALPlus | $HL) $CM* ($PR | $PO);
-($PR | $PO) $CM* ($ALPlus | $HL);
-
-
-# LB 25
-($CM* ($PR | $PO))? ($CM* ($CL | $CP))? ($CM* ($NU | $IS | $SY))* $CM* $NU ($CM* ($OP | $HY))? ($CM* ($PR | $PO))?;
-
-# LB 26
-($H3 | $H2 | $JV | $JL) $CM* $JL;
-($JT | $JV) $CM* ($H2 | $JV);
-$JT $CM* ($H3 | $JT);
-
-# LB 27
-$IN $CM* ($H3 | $H2 | $JT | $JV | $JL);
-$PO $CM* ($H3 | $H2 | $JT | $JV | $JL);
- ($H3 | $H2 | $JT | $JV | $JL) $CM* $PR;
-
-# LB 28
-($ALPlus | $HL) $CM* ($ALPlus | $HL);
-
-
-# LB 29
-($ALPlus | $HL) $CM* $IS;
-
-# LB 30
-$OP $CM* ($ALPlus | $HL | $NU);
-($ALPlus | $HL | $NU) $CM* $CP;
-
-# LB 30a
-# Pairs of Regional Indicators.
-# The following two rules are nearly identical. The first matches only sequences with an odd number of adjacent RIs,
-# the second with an even number. Stripping away the cruft they look like
-# [^RI] RI / (RI RI)+ ^RI;
-# [^RI] RI RI / (RI RI)+ ^RI;
-#
-[{bof} $NS $HY $BA $QU $CL $CP $EX $IS $SY $WJ $GL $ZW $SP $BK $CR $LF $NL $ZWJ] $CM* $RI / ($CM* $RI $CM* $RI)+ $CM* [{eof}[^$RI $CM]];
-[{bof} $NS $HY $BA $QU $CL $CP $EX $IS $SY $WJ $GL $ZW $SP $BK $CR $LF $NL $ZWJ] $CM* $RI $CM* $RI / ($CM* $RI $CM* $RI)+ $CM* [{eof}[^$RI $CM]];
-
-# In general, adjacent RIs stay together. The hard-break rules, above, overide this, forcing in the boundaries between pairs.
-$RI $CM* $RI;
-
-# WJ, GL, QU, etc. are classes with rules like "WJ x " which includes "WJ x RI".
-$RI $CM* ([$WJ $GL $QU $BB] | (($HY | $BA)$CM* $HL));
-
-
-# LB 30b Do not break between an Emoji Base and an Emoji Modifier
-$EM $CM* $EB;
-
-
## -------------------------------------------------
!!safe_reverse;
# LB 9
^$CM+ [^$CM $BK $CR $LF $NL $ZW $SP];
-^$CM+ $SP / .;
# LB 14
$SP+ $CM* $OP;
# For dictionary-based break
$dictionary $dictionary;
-
-## -------------------------------------------------
-
-!!safe_forward;
-
-# Skip forward over all character classes that are involved in
-# rules containing patterns with possibly more than one char
-# of context.
-#
-# It might be slightly more efficient to have specific rules
-# instead of one generic one, but only if we could
-# turn off rule chaining. We don't want to move more
-# than necessary.
-#
-^[$CM $OP $QU $CL $CP $B2 $PR $HY $BA $SP $RI $ZWJ $dictionary]+ [^$CM $OP $QU $CL $CP $B2 $PR $HY $BA $RI $ZWJ $dictionary];
-$dictionary $dictionary;
-
-# Copyright (C) 2016 and later: Unicode, Inc. and others.
+# Copyright (C) 2016 and later: Unicode, Inc. and others.
# License & terms of use: http://www.unicode.org/copyright.html
+#
# Copyright (c) 2002-2016 International Business Machines Corporation and
# others. All Rights Reserved.
#
#
!!chain;
+!!quoted_literals_only;
$AI = [:LineBreak = Ambiguous:];
$AL = [:LineBreak = Alphabetic:];
# LB 30b Do not break between an Emoji Base and an Emoji Modifier
$EB $CM* $EM;
-#
-# Reverse Rules.
-#
-## -------------------------------------------------
-
-!!reverse;
-
-# LB 9 Combining Marks.
-# Stick together any combining sequences that don't match other rules.
-
-^$CM+ $CAN_CM?;
-
-#
-# Sequences of the form (shown forwards)
-# [CANT_CM] <break> [CM] [whatever]
-# The CM needs to behave as an AL
-#
-$AL_FOLLOW $CM+ / (
- [$BK $CR $LF $NL $ZW {eof}] |
- $SP+ $CM+ $SP |
- $SP+ $CM* ([^$OP $CM $SP] | [$AL {eof}])); # if LB 14 will match, need to surpress this break.
- # LB14 says OP SP* x .
- # becomes OP SP* x AL
- # becomes OP SP* x CM+ AL_FOLLOW
- #
- # Further note: the $AL in [$AL {eof}] is only to work around
- # a rule compiler bug which complains about
- # empty sets otherwise.
-
-
-# LB 4, 5, 6
-
-$LB4Breaks [$LB4NonBreaks-$CM];
-$LB4Breaks $CM+ $CAN_CM;
-$LF $CR;
-
-
-# LB 7 x SP
-# x ZW
-[$SP $ZW] [$LB4NonBreaks-$CM];
-[$SP $ZW] $CM+ $CAN_CM;
-
-# LB 8 ZW SP* <break>
-# TODO: to implement this, we need more than one look-ahead hard break in play at a time.
-# Requires an engine enhancement.
-# / $SP* $ZW
-
-# LB 8a ZWJ x (ID | Extended_Pict | EmojiNRK)
-#
-($ID | $Extended_Pict | $EmojiNRK) $ZWJ $CM* $CAN_CM?;
-
-
-# LB 9,10 Combining marks.
-# X $CM needs to behave like X, where X is not $SP or controls.
-# $CM not covered by the above needs to behave like $AL
-# Stick together any combining sequences that don't match other rules.
-^$CM+ $CAN_CM;
-
-
-# LB 11
-#
-$WJ $CM* $CAN_CM;
-$WJ [$LB8NonBreaks-$CM];
-
- $CANT_CM $CM* $WJ;
-$CAN_CM $CM* $WJ;
-
-# LB 12a
-# [^SP BA HY] x GL
-#
-$GL $CM* [$LB8NonBreaks-[$CM $SP $BA $HY]];
-
-# LB 12
-# GL x
-#
-$CANT_CM $CM* $GL;
-$CAN_CM $CM* $GL;
-
-
-# LB 13
-$CL $CM+ $CAN_CM;
-$CP $CM+ $CAN_CM;
-$EX $CM+ $CAN_CM;
-$IS $CM+ $CAN_CM;
-$SY $CM+ $CAN_CM;
-
-$CL [$LB8NonBreaks-$CM];
-$CP [$LB8NonBreaks-$CM];
-$EX [$LB8NonBreaks-$CM];
-$IS [$LB8NonBreaks-$CM];
-$SY [$LB8NonBreaks-$CM];
-
-
-# LB 14 OP SP* x
-#
-. $SP* $CM* $OP;
-$AL_FOLLOW? $CM+ $SP+ $CM* $OP; # by LB 10, behaves like $AL_FOLLOW? $AL $SP+ $CM* $OP
-
-
-# LB 15
-$OP $SP* $CM* $QU;
-
-# LB 16
-# Don't include $NSX here
-$NS $SP* $CM* ($CL | $CP);
-
-# LB 17
-$B2 $SP* $CM* $B2;
-
-# LB 18 break after spaces
-# Nothing explicit needed here.
-
-
-#
-# LB 19
-#
-$QU $CM* $CAN_CM; # . x QU
-$QU $LB18NonBreaks;
-
-
-$CAN_CM $CM* $QU; # QU x .
- $CANT_CM $CM* $QU;
-
-#
-# LB 20 Break before and after CB.
-# nothing needed here.
-#
-
-# LB 21
-# Don't include $NSX here
-($BA | $HY | $NS) $CM* [$LB20NonBreaks-$CM]; # . x (BA | HY | NS)
-
-[$LB20NonBreaks-$CM] $CM* $BB; # BB x .
-[^$CB] $CM* $BB; #
-
-# LB21a
-[^$CB] $CM* ($HY | $BA) $CM* $HL;
-
-# LB21b (reverse)
-$HL $CM* $SY;
-
-# LB 22
-$IN $CM* ($ALPlus | $HL);
-$IN $CM* $EX;
-$IN $CM* ($ID | $EB | $EM);
-# $IN $CM* $IN; # delete this rule for CSS loose
-$IN $CM* $NU;
-
-# LB 23
-$NU $CM* ($ALPlus | $HL);
-($ALPlus | $HL) $CM* $NU;
-
-# LB23a
-($ID | $EB | $EM) $CM* $PR;
-$PO $CM* ($ID | $EB | $EM);
-
-# LB 24
-($ALPlus | $HL) $CM* ($PR | $PO);
-($PR | $PO) $CM* ($ALPlus | $HL);
-
-
-# LB 25
-($CM* ($PR | $PO))? ($CM* ($CL | $CP))? ($CM* ($NU | $IS | $SY))* $CM* $NU ($CM* ($OP | $HY))? ($CM* ($PR | $PO))?;
-
-# LB 26
-($H3 | $H2 | $JV | $JL) $CM* $JL;
-($JT | $JV) $CM* ($H2 | $JV);
-$JT $CM* ($H3 | $JT);
-
-# LB 27
-$IN $CM* ($H3 | $H2 | $JT | $JV | $JL);
-$PO $CM* ($H3 | $H2 | $JT | $JV | $JL);
- ($H3 | $H2 | $JT | $JV | $JL) $CM* $PR;
-
-# LB 28
-($ALPlus | $HL) $CM* ($ALPlus | $HL);
-
-
-# LB 29
-($ALPlus | $HL) $CM* $IS;
-
-# LB 30
-$OP $CM* ($ALPlus | $HL | $NU);
-($ALPlus | $HL | $NU) $CM* $CP;
-
-# LB 30a
-# Pairs of Regional Indicators.
-# The following two rules are nearly identical. The first matches only sequences with an odd number of adjacent RIs,
-# the second with an even number. Stripping away the cruft they look like
-# [^RI] RI / (RI RI)+ ^RI;
-# [^RI] RI RI / (RI RI)+ ^RI;
-#
-# Line Loose tailoring: Don't include NSX here.
-[{bof} $NS $HY $BA $QU $CL $CP $EX $IS $SY $WJ $GL $ZW $SP $BK $CR $LF $NL $ZWJ] $CM* $RI / ($CM* $RI $CM* $RI)+ $CM* [{eof}[^$RI $CM]];
-[{bof} $NS $HY $BA $QU $CL $CP $EX $IS $SY $WJ $GL $ZW $SP $BK $CR $LF $NL $ZWJ] $CM* $RI $CM* $RI / ($CM* $RI $CM* $RI)+ $CM* [{eof}[^$RI $CM]];
-
-# In general, adjacent RIs stay together. The hard-break rules, above, overide this, forcing in the boundaries between pairs.
-$RI $CM* $RI;
-
-# WJ, GL, QU, etc. are classes with rules like "WJ x " which includes "WJ x RI".
-$RI $CM* ([$WJ $GL $QU $BB] | (($HY | $BA)$CM* $HL));
-
-
-# LB 30b Do not break between an Emoji Base and an Emoji Modifier
-$EM $CM* $EB;
-
## -------------------------------------------------
# LB 9
^$CM+ [^$CM $BK $CR $LF $NL $ZW $SP];
-^$CM+ $SP / .;
# LB 14
$SP+ $CM* $OP;
# For dictionary-based break
$dictionary $dictionary;
-
-## -------------------------------------------------
-
-!!safe_forward;
-
-# Skip forward over all character classes that are involved in
-# rules containing patterns with possibly more than one char
-# of context.
-#
-# It might be slightly more efficient to have specific rules
-# instead of one generic one, but only if we could
-# turn off rule chaining. We don't want to move more
-# than necessary.
-#
-^[$CM $OP $QU $CL $CP $B2 $PR $HY $BA $SP $RI $ZWJ $dictionary]+ [^$CM $OP $QU $CL $CP $B2 $PR $HY $BA $RI $ZWJ $dictionary];
-$dictionary $dictionary;
-
-# Copyright (C) 2016 and later: Unicode, Inc. and others.
+# Copyright (C) 2016 and later: Unicode, Inc. and others.
# License & terms of use: http://www.unicode.org/copyright.html
# Copyright (c) 2002-2016 International Business Machines Corporation and
# others. All Rights Reserved.
#
!!chain;
+!!quoted_literals_only;
$AI = [:LineBreak = Ambiguous:];
$AL = [:LineBreak = Alphabetic:];
# LB 30b Do not break between an Emoji Base and an Emoji Modifier
$EB $CM* $EM;
-#
-# Reverse Rules.
-#
-## -------------------------------------------------
-
-!!reverse;
-
-# LB 9 Combining Marks.
-# Stick together any combining sequences that don't match other rules.
-
-^$CM+ $CAN_CM?;
-
-#
-# Sequences of the form (shown forwards)
-# [CANT_CM] <break> [CM] [whatever]
-# The CM needs to behave as an AL
-#
-$AL_FOLLOW $CM+ / (
- [$BK $CR $LF $NL $ZW {eof}] |
- $SP+ $CM+ $SP |
- $SP+ $CM* ([^$OP $CM $SP] | [$AL {eof}])); # if LB 14 will match, need to surpress this break.
- # LB14 says OP SP* x .
- # becomes OP SP* x AL
- # becomes OP SP* x CM+ AL_FOLLOW
- #
- # Further note: the $AL in [$AL {eof}] is only to work around
- # a rule compiler bug which complains about
- # empty sets otherwise.
-
-
-# LB 4, 5, 6
-
-$LB4Breaks [$LB4NonBreaks-$CM];
-$LB4Breaks $CM+ $CAN_CM;
-$LF $CR;
-
-
-# LB 7 x SP
-# x ZW
-[$SP $ZW] [$LB4NonBreaks-$CM];
-[$SP $ZW] $CM+ $CAN_CM;
-
-# LB 8 ZW SP* <break>
-# TODO: to implement this, we need more than one look-ahead hard break in play at a time.
-# Requires an engine enhancement.
-# / $SP* $ZW
-
-# LB 8a ZWJ x (ID | Extended_Pict | EmojiNRK)
-#
-($ID | $Extended_Pict | $EmojiNRK) $ZWJ $CM* $CAN_CM?;
-
-
-# LB 9,10 Combining marks.
-# X $CM needs to behave like X, where X is not $SP or controls.
-# $CM not covered by the above needs to behave like $AL
-# Stick together any combining sequences that don't match other rules.
-^$CM+ $CAN_CM;
-
-
-# LB 11
-#
-$WJ $CM* $CAN_CM;
-$WJ [$LB8NonBreaks-$CM];
-
- $CANT_CM $CM* $WJ;
-$CAN_CM $CM* $WJ;
-
-# LB 12a
-# [^SP BA HY] x GL
-#
-$GL $CM* [$LB8NonBreaks-[$CM $SP $BA $BAX $HY]];
-
-# LB 12
-# GL x
-#
-$CANT_CM $CM* $GL;
-$CAN_CM $CM* $GL;
-
-
-# LB 13
-# Do not include $EXX here
-$CL $CM+ $CAN_CM;
-$CP $CM+ $CAN_CM;
-$EX $CM+ $CAN_CM;
-$IS $CM+ $CAN_CM;
-$SY $CM+ $CAN_CM;
-
-$CL [$LB8NonBreaks-$CM];
-$CP [$LB8NonBreaks-$CM];
-$EX [$LB8NonBreaks-$CM];
-$IS [$LB8NonBreaks-$CM];
-$SY [$LB8NonBreaks-$CM];
-
-
-# LB 14 OP SP* x
-#
-. $SP* $CM* $OP;
-$AL_FOLLOW? $CM+ $SP+ $CM* $OP; # by LB 10, behaves like $AL_FOLLOW? $AL $SP+ $CM* $OP
-
-
-# LB 15
-$OP $SP* $CM* $QU;
-
-# LB 16
-# Don't include $NSX here
-$NS $SP* $CM* ($CL | $CP);
-
-# LB 17
-$B2 $SP* $CM* $B2;
-
-# LB 18 break after spaces
-# Nothing explicit needed here.
-
-
-#
-# LB 19
-#
-$QU $CM* $CAN_CM; # . x QU
-$QU $LB18NonBreaks;
-
-
-$CAN_CM $CM* $QU; # QU x .
- $CANT_CM $CM* $QU;
-
-#
-# LB 20 Break before and after CB.
-# nothing needed here.
-#
-
-# LB 21
-# Don't include $BAX or $NSX here
-($BA | $HY | $NS) $CM* [$LB20NonBreaks-$CM]; # . x (BA | HY | NS)
-
-[$LB20NonBreaks-$CM] $CM* $BB; # BB x .
-[^$CB] $CM* $BB; #
-
-# LB21a Don't break after Hebrew + Hyphen.
-([^$CB] $CM*)? ($HY | $BA | $BAX) $CM* $HL;
-
-# LB21b (reverse)
-$HL $CM* $SY;
-
-# LB 22
-$IN $CM* ($ALPlus | $HL);
-$IN $CM* $EX;
-$IN $CM* ($ID | $EB | $EM);
-# $IN $CM* $IN; # delete this rule for CSS loose
-$IN $CM* $NU;
-
-# LB 23
-# Do not include $POX here
-$NU $CM* ($ALPlus | $HL);
-($ALPlus | $HL) $CM* $NU;
-
-# LB23a
-# Do not include $PRX here
-($ID | $EB | $EM) $CM* $PR;
-$PO $CM* ($ID | $EB | $EM);
-
-# LB 24
-# Do not include $PRX here
-($ALPlus | $HL) $CM* ($PR | $PO | $POX);
-($PR | $PO | $POX) $CM* ($ALPlus | $HL);
-
-
-# LB 25
-# Here do not include $POX at the beginning or $PRX at the end
-($CM* ($PR | $PRX | $PO))? ($CM* ($CL | $CP))? ($CM* ($NU | $IS | $SY))* $CM* $NU ($CM* ($OP | $HY))? ($CM* ($PR | $PO | $POX))?;
-
-# LB 26
-($H3 | $H2 | $JV | $JL) $CM* $JL;
-($JT | $JV) $CM* ($H2 | $JV);
-$JT $CM* ($H3 | $JT);
-
-# LB 27
-# Do not include $POX or $PRX here
-$IN $CM* ($H3 | $H2 | $JT | $JV | $JL);
-$PO $CM* ($H3 | $H2 | $JT | $JV | $JL);
- ($H3 | $H2 | $JT | $JV | $JL) $CM* $PR;
-
-# LB 28
-($ALPlus | $HL) $CM* ($ALPlus | $HL);
-
-
-# LB 29
-($ALPlus | $HL) $CM* $IS;
-
-# LB 30
-$OP $CM* ($ALPlus | $HL | $NU);
-($ALPlus | $HL | $NU) $CM* $CP;
-
-# LB 30a
-# Pairs of Regional Indicators.
-# The following two rules are nearly identical. The first matches only sequences with an odd number of adjacent RIs,
-# the second with an even number. Stripping away the cruft they look like
-# [^RI] RI / (RI RI)+ ^RI;
-# [^RI] RI RI / (RI RI)+ ^RI;
-#
-# Line Loose tailoring: Don't include NSX here.
-[{bof} $NS $HY $BA $QU $CL $CP $EX $IS $SY $WJ $GL $ZW $SP $BK $CR $LF $NL $ZWJ] $CM* $RI / ($CM* $RI $CM* $RI)+ $CM* [{eof}[^$RI $CM]];
-[{bof} $NS $HY $BA $QU $CL $CP $EX $IS $SY $WJ $GL $ZW $SP $BK $CR $LF $NL $ZWJ] $CM* $RI $CM* $RI / ($CM* $RI $CM* $RI)+ $CM* [{eof}[^$RI $CM]];
-
-# In general, adjacent RIs stay together. The hard-break rules, above, overide this, forcing in the boundaries between pairs.
-$RI $CM* $RI;
-
-# WJ, GL, QU, etc. are classes with rules like "WJ x " which includes "WJ x RI".
-$RI $CM* ([$WJ $GL $QU $BB] | (($HY | $BA)$CM* $HL));
-
-
-# LB 30b Do not break between an Emoji Base and an Emoji Modifier
-$EM $CM* $EB;
-
-
## -------------------------------------------------
!!safe_reverse;
# LB 9
^$CM+ [^$CM $BK $CR $LF $NL $ZW $SP];
-^$CM+ $SP / .;
# LB 14
$SP+ $CM* $OP;
# For dictionary-based break
$dictionary $dictionary;
-
-## -------------------------------------------------
-
-!!safe_forward;
-
-# Skip forward over all character classes that are involved in
-# rules containing patterns with possibly more than one char
-# of context.
-#
-# It might be slightly more efficient to have specific rules
-# instead of one generic one, but only if we could
-# turn off rule chaining. We don't want to move more
-# than necessary.
-#
-^[$CM $OP $QU $CL $CP $B2 $PR $PRX $HY $BA $BAX $SP $RI $ZWJ $dictionary]+ [^$CM $OP $QU $CL $CP $B2 $PR $PRX $HY $BA $BAX $RI $ZWJ $dictionary];
-$dictionary $dictionary;
-
-# Copyright (C) 2016 and later: Unicode, Inc. and others.
+# Copyright (C) 2016 and later: Unicode, Inc. and others.
# License & terms of use: http://www.unicode.org/copyright.html
# Copyright (c) 2002-2016 International Business Machines Corporation and
# others. All Rights Reserved.
#
!!chain;
+!!quoted_literals_only;
$AI = [:LineBreak = Ambiguous:];
$AL = [:LineBreak = Alphabetic:];
# LB 30b Do not break between an Emoji Base and an Emoji Modifier
$EB $CM* $EM;
-#
-# Reverse Rules.
-#
-## -------------------------------------------------
-
-!!reverse;
-
-# LB 9 Combining Marks.
-# Stick together any combining sequences that don't match other rules.
-
-^$CM+ $CAN_CM?;
-
-#
-# Sequences of the form (shown forwards)
-# [CANT_CM] <break> [CM] [whatever]
-# The CM needs to behave as an AL
-#
-$AL_FOLLOW $CM+ / (
- [$BK $CR $LF $NL $ZW {eof}] |
- $SP+ $CM+ $SP |
- $SP+ $CM* ([^$OP $CM $SP] | [$AL {eof}])); # if LB 14 will match, need to surpress this break.
- # LB14 says OP SP* x .
- # becomes OP SP* x AL
- # becomes OP SP* x CM+ AL_FOLLOW
- #
- # Further note: the $AL in [$AL {eof}] is only to work around
- # a rule compiler bug which complains about
- # empty sets otherwise.
-
-
-# LB 4, 5, 6
-
-$LB4Breaks [$LB4NonBreaks-$CM];
-$LB4Breaks $CM+ $CAN_CM;
-$LF $CR;
-
-
-# LB 7 x SP
-# x ZW
-[$SP $ZW] [$LB4NonBreaks-$CM];
-[$SP $ZW] $CM+ $CAN_CM;
-
-# LB 8 ZW SP* <break>
-# TODO: to implement this, we need more than one look-ahead hard break in play at a time.
-# Requires an engine enhancement.
-# / $SP* $ZW
-
-# LB 8a ZWJ x (ID | Extended_Pict | EmojiNRK)
-#
-($ID | $Extended_Pict | $EmojiNRK) $ZWJ $CM* $CAN_CM?;
-
-
-# LB 9,10 Combining marks.
-# X $CM needs to behave like X, where X is not $SP or controls.
-# $CM not covered by the above needs to behave like $AL
-# Stick together any combining sequences that don't match other rules.
-^$CM+ $CAN_CM;
-
-
-# LB 11
-#
-$WJ $CM* $CAN_CM;
-$WJ [$LB8NonBreaks-$CM];
-
- $CANT_CM $CM* $WJ;
-$CAN_CM $CM* $WJ;
-
-# LB 12a
-# [^SP BA HY] x GL
-#
-$GL $CM* [$LB8NonBreaks-[$CM $SP $BA $HH $HY]];
-
-# LB 12
-# GL x
-#
-$CANT_CM $CM* $GL;
-$CAN_CM $CM* $GL;
-
-
-# LB 13
-$CL $CM+ $CAN_CM;
-$CP $CM+ $CAN_CM;
-$EX $CM+ $CAN_CM;
-$IS $CM+ $CAN_CM;
-$SY $CM+ $CAN_CM;
-
-$CL [$LB8NonBreaks-$CM];
-$CP [$LB8NonBreaks-$CM];
-$EX [$LB8NonBreaks-$CM];
-$IS [$LB8NonBreaks-$CM];
-$SY [$LB8NonBreaks-$CM];
-
-
-# LB 14 OP SP* x
-#
-. $SP* $CM* $OP;
-$AL_FOLLOW? $CM+ $SP+ $CM* $OP; # by LB 10, behaves like $AL_FOLLOW? $AL $SP+ $CM* $OP
-
-
-# LB 15
-$OP $SP* $CM* $QU;
-
-# LB 16
-# Don't include $NSX here
-$NS $SP* $CM* ($CL | $CP);
-
-# LB 17
-$B2 $SP* $CM* $B2;
-
-# LB 18 break after spaces
-# Nothing explicit needed here.
-
-
-#
-# LB 19
-#
-$QU $CM* $CAN_CM; # . x QU
-$QU $LB18NonBreaks;
-
-
-$CAN_CM $CM* $QU; # QU x .
- $CANT_CM $CM* $QU;
-
-#
-# LB 20 Break before and after CB.
-# nothing needed here.
-#
-
-# LB 20.09 added rule for Finnish tailoring
-$AL ($HY | $HH) / $SP;
-
-# LB 21
-# Don't include $NSX here
-($BA | $HH | $HY | $NS) $CM* [$LB20NonBreaks-$CM]; # . x (BA | HY | NS)
-
-[$LB20NonBreaks-$CM] $CM* $BB; # BB x .
-[^$CB] $CM* $BB; #
-
-# LB21a
-[^$CB] $CM* ($HY | $BA | $HH) $CM* $HL;
-
-# LB21b (reverse)
-$HL $CM* $SY;
-
-# LB 22
-$IN $CM* ($ALPlus | $HL);
-$IN $CM* $EX;
-$IN $CM* ($ID | $EB | $EM);
-# $IN $CM* $IN; # delete this rule for CSS loose
-$IN $CM* $NU;
-
-# LB 23
-$NU $CM* ($ALPlus | $HL);
-($ALPlus | $HL) $CM* $NU;
-
-# LB23a
-($ID | $EB | $EM) $CM* $PR;
-$PO $CM* ($ID | $EB | $EM);
-
-# LB 24
-($ALPlus | $HL) $CM* ($PR | $PO);
-($PR | $PO) $CM* ($ALPlus | $HL);
-
-
-# LB 25
-($CM* ($PR | $PO))? ($CM* ($CL | $CP))? ($CM* ($NU | $IS | $SY))* $CM* $NU ($CM* ($OP | $HY))? ($CM* ($PR | $PO))?;
-
-# LB 26
-($H3 | $H2 | $JV | $JL) $CM* $JL;
-($JT | $JV) $CM* ($H2 | $JV);
-$JT $CM* ($H3 | $JT);
-
-# LB 27
-$IN $CM* ($H3 | $H2 | $JT | $JV | $JL);
-$PO $CM* ($H3 | $H2 | $JT | $JV | $JL);
- ($H3 | $H2 | $JT | $JV | $JL) $CM* $PR;
-
-# LB 28
-($ALPlus | $HL) $CM* ($ALPlus | $HL);
-
-
-# LB 29
-($ALPlus | $HL) $CM* $IS;
-
-# LB 30
-$OP $CM* ($ALPlus | $HL | $NU);
-($ALPlus | $HL | $NU) $CM* $CP;
-
-# LB 30a
-# Pairs of Regional Indicators.
-# The following two rules are nearly identical. The first matches only sequences with an odd number of adjacent RIs,
-# the second with an even number. Stripping away the cruft they look like
-# [^RI] RI / (RI RI)+ ^RI;
-# [^RI] RI RI / (RI RI)+ ^RI;
-#
-# Line Loose tailoring: Don't include NSX here.
-[{bof} $NS $HY $BA $QU $CL $CP $EX $IS $SY $WJ $GL $ZW $SP $BK $CR $LF $NL $ZWJ] $CM* $RI / ($CM* $RI $CM* $RI)+ $CM* [{eof}[^$RI $CM]];
-[{bof} $NS $HY $BA $QU $CL $CP $EX $IS $SY $WJ $GL $ZW $SP $BK $CR $LF $NL $ZWJ] $CM* $RI $CM* $RI / ($CM* $RI $CM* $RI)+ $CM* [{eof}[^$RI $CM]];
-
-# In general, adjacent RIs stay together. The hard-break rules, above, overide this, forcing in the boundaries between pairs.
-$RI $CM* $RI;
-
-# WJ, GL, QU, etc. are classes with rules like "WJ x " which includes "WJ x RI".
-$RI $CM* ([$WJ $GL $QU $BB] | (($HY | $BA)$CM* $HL));
-
-
-# LB 30b Do not break between an Emoji Base and an Emoji Modifier
-$EM $CM* $EB;
-
## -------------------------------------------------
# LB 9
^$CM+ [^$CM $BK $CR $LF $NL $ZW $SP];
-^$CM+ $SP / .;
# LB 14
$SP+ $CM* $OP;
# For dictionary-based break
$dictionary $dictionary;
-
-## -------------------------------------------------
-
-!!safe_forward;
-
-# Skip forward over all character classes that are involved in
-# rules containing patterns with possibly more than one char
-# of context.
-#
-# It might be slightly more efficient to have specific rules
-# instead of one generic one, but only if we could
-# turn off rule chaining. We don't want to move more
-# than necessary.
-#
-^[$CM $OP $QU $CL $CP $B2 $PR $HY $BA $SP $RI $ZWJ $dictionary]+ [^$CM $OP $QU $CL $CP $B2 $PR $HY $BA $RI $ZWJ $dictionary];
-$dictionary $dictionary;
-
-# Copyright (C) 2016 and later: Unicode, Inc. and others.
+# Copyright (C) 2016 and later: Unicode, Inc. and others.
# License & terms of use: http://www.unicode.org/copyright.html
# Copyright (c) 2002-2016 International Business Machines Corporation and
# others. All Rights Reserved.
#
!!chain;
+!!quoted_literals_only;
$AI = [:LineBreak = Ambiguous:];
$AL = [:LineBreak = Alphabetic:];
# LB 30b Do not break between an Emoji Base and an Emoji Modifier
$EB $CM* $EM;
-#
-# Reverse Rules.
-#
-## -------------------------------------------------
-
-!!reverse;
-
-# LB 9 Combining Marks.
-# Stick together any combining sequences that don't match other rules.
-
-^$CM+ $CAN_CM?;
-
-#
-# Sequences of the form (shown forwards)
-# [CANT_CM] <break> [CM] [whatever]
-# The CM needs to behave as an AL
-#
-$AL_FOLLOW $CM+ / (
- [$BK $CR $LF $NL $ZW {eof}] |
- $SP+ $CM+ $SP |
- $SP+ $CM* ([^$OP $CM $SP] | [$AL {eof}])); # if LB 14 will match, need to surpress this break.
- # LB14 says OP SP* x .
- # becomes OP SP* x AL
- # becomes OP SP* x CM+ AL_FOLLOW
- #
- # Further note: the $AL in [$AL {eof}] is only to work around
- # a rule compiler bug which complains about
- # empty sets otherwise.
-
-
-# LB 4, 5, 6
-
-$LB4Breaks [$LB4NonBreaks-$CM];
-$LB4Breaks $CM+ $CAN_CM;
-$LF $CR;
-
-
-# LB 7 x SP
-# x ZW
-[$SP $ZW] [$LB4NonBreaks-$CM];
-[$SP $ZW] $CM+ $CAN_CM;
-
-# LB 8 ZW SP* <break>
-# TODO: to implement this, we need more than one look-ahead hard break in play at a time.
-# Requires an engine enhancement.
-# / $SP* $ZW
-
-# LB 8a ZWJ x (ID | Extended_Pict | EmojiNRK)
-#
-($ID | $Extended_Pict | $EmojiNRK) $ZWJ $CM* $CAN_CM?;
-
-
-# LB 9,10 Combining marks.
-# X $CM needs to behave like X, where X is not $SP or controls.
-# $CM not covered by the above needs to behave like $AL
-# Stick together any combining sequences that don't match other rules.
-^$CM+ $CAN_CM;
-
-
-# LB 11
-#
-$WJ $CM* $CAN_CM;
-$WJ [$LB8NonBreaks-$CM];
-
- $CANT_CM $CM* $WJ;
-$CAN_CM $CM* $WJ;
-
-# LB 12a
-# [^SP BA HY] x GL
-#
-$GL $CM* [$LB8NonBreaks-[$CM $SP $BA $HY]];
-
-# LB 12
-# GL x
-#
-$CANT_CM $CM* $GL;
-$CAN_CM $CM* $GL;
-
-
-# LB 13
-$CL $CM+ $CAN_CM;
-$CP $CM+ $CAN_CM;
-$EX $CM+ $CAN_CM;
-$IS $CM+ $CAN_CM;
-$SY $CM+ $CAN_CM;
-
-$CL [$LB8NonBreaks-$CM];
-$CP [$LB8NonBreaks-$CM];
-$EX [$LB8NonBreaks-$CM];
-$IS [$LB8NonBreaks-$CM];
-$SY [$LB8NonBreaks-$CM];
-
-
-# LB 14 OP SP* x
-#
-. $SP* $CM* $OP;
-$AL_FOLLOW? $CM+ $SP+ $CM* $OP; # by LB 10, behaves like $AL_FOLLOW? $AL $SP+ $CM* $OP
-
-
-# LB 15
-$OP $SP* $CM* $QU;
-
-# LB 16
-$NS $SP* $CM* ($CL | $CP);
-
-# LB 17
-$B2 $SP* $CM* $B2;
-
-# LB 18 break after spaces
-# Nothing explicit needed here.
-
-
-#
-# LB 19
-#
-$QU $CM* $CAN_CM; # . x QU
-$QU $LB18NonBreaks;
-
-
-$CAN_CM $CM* $QU; # QU x .
- $CANT_CM $CM* $QU;
-
-#
-# LB 20 Break before and after CB.
-# nothing needed here.
-#
-
-# LB 21
-($BA | $HY | $NS) $CM* [$LB20NonBreaks-$CM]; # . x (BA | HY | NS)
-
-[$LB20NonBreaks-$CM] $CM* $BB; # BB x .
-[^$CB] $CM* $BB; #
-
-# LB21a
-[^$CB] $CM* ($HY | $BA) $CM* $HL;
-
-# LB21b (reverse)
-$HL $CM* $SY;
-
-# LB 22
-$IN $CM* ($ALPlus | $HL);
-$IN $CM* $EX;
-$IN $CM* ($ID | $EB | $EM);
-$IN $CM* $IN;
-$IN $CM* $NU;
-
-# LB 23
-$NU $CM* ($ALPlus | $HL);
-($ALPlus | $HL) $CM* $NU;
-
-# LB23a
-($ID | $EB | $EM) $CM* $PR;
-$PO $CM* ($ID | $EB | $EM);
-
-# LB 24
-($ALPlus | $HL) $CM* ($PR | $PO);
-($PR | $PO) $CM* ($ALPlus | $HL);
-
-
-# LB 25
-($CM* ($PR | $PO))? ($CM* ($CL | $CP))? ($CM* ($NU | $IS | $SY))* $CM* $NU ($CM* ($OP | $HY))? ($CM* ($PR | $PO))?;
-
-# LB 26
-($H3 | $H2 | $JV | $JL) $CM* $JL;
-($JT | $JV) $CM* ($H2 | $JV);
-$JT $CM* ($H3 | $JT);
-
-# LB 27
-$IN $CM* ($H3 | $H2 | $JT | $JV | $JL);
-$PO $CM* ($H3 | $H2 | $JT | $JV | $JL);
- ($H3 | $H2 | $JT | $JV | $JL) $CM* $PR;
-
-# LB 28
-($ALPlus | $HL) $CM* ($ALPlus | $HL);
-
-
-# LB 29
-($ALPlus | $HL) $CM* $IS;
-
-# LB 30
-$OP $CM* ($ALPlus | $HL | $NU);
-($ALPlus | $HL | $NU) $CM* $CP;
-
-# LB 30a
-# Pairs of Regional Indicators.
-# The following two rules are nearly identical. The first matches only sequences with an odd number of adjacent RIs,
-# the second with an even number. Stripping away the cruft they look like
-# [^RI] RI / (RI RI)+ ^RI;
-# [^RI] RI RI / (RI RI)+ ^RI;
-#
-[{bof} $NS $HY $BA $QU $CL $CP $EX $IS $SY $WJ $GL $ZW $SP $BK $CR $LF $NL $ZWJ] $CM* $RI / ($CM* $RI $CM* $RI)+ $CM* [{eof}[^$RI $CM]];
-[{bof} $NS $HY $BA $QU $CL $CP $EX $IS $SY $WJ $GL $ZW $SP $BK $CR $LF $NL $ZWJ] $CM* $RI $CM* $RI / ($CM* $RI $CM* $RI)+ $CM* [{eof}[^$RI $CM]];
-
-# In general, adjacent RIs stay together. The hard-break rules, above, overide this, forcing in the boundaries between pairs.
-$RI $CM* $RI;
-
-# WJ, GL, QU, etc. are classes with rules like "WJ x " which includes "WJ x RI".
-$RI $CM* ([$WJ $GL $QU $BB] | (($HY | $BA)$CM* $HL));
-
-
-# LB 30b Do not break between an Emoji Base and an Emoji Modifier
-$EM $CM* $EB;
-
-
## -------------------------------------------------
!!safe_reverse;
# LB 9
^$CM+ [^$CM $BK $CR $LF $NL $ZW $SP];
-^$CM+ $SP / .;
# LB 14
$SP+ $CM* $OP;
# For dictionary-based break
$dictionary $dictionary;
-
-## -------------------------------------------------
-
-!!safe_forward;
-
-# Skip forward over all character classes that are involved in
-# rules containing patterns with possibly more than one char
-# of context.
-#
-# It might be slightly more efficient to have specific rules
-# instead of one generic one, but only if we could
-# turn off rule chaining. We don't want to move more
-# than necessary.
-#
-^[$CM $OP $QU $CL $CP $B2 $PR $HY $BA $SP $RI $ZWJ $dictionary]+ [^$CM $OP $QU $CL $CP $B2 $PR $HY $BA $RI $ZWJ $dictionary];
-$dictionary $dictionary;
-
-# Copyright (C) 2016 and later: Unicode, Inc. and others.
+# Copyright (C) 2016 and later: Unicode, Inc. and others.
# License & terms of use: http://www.unicode.org/copyright.html
# Copyright (c) 2002-2016 International Business Machines Corporation and
# others. All Rights Reserved.
#
!!chain;
+!!quoted_literals_only;
$AI = [:LineBreak = Ambiguous:];
$AL = [:LineBreak = Alphabetic:];
# LB 30b Do not break between an Emoji Base and an Emoji Modifier
$EB $CM* $EM;
-#
-# Reverse Rules.
-#
-## -------------------------------------------------
-
-!!reverse;
-
-# LB 9 Combining Marks.
-# Stick together any combining sequences that don't match other rules.
-
-^$CM+ $CAN_CM?;
-
-#
-# Sequences of the form (shown forwards)
-# [CANT_CM] <break> [CM] [whatever]
-# The CM needs to behave as an AL
-#
-$AL_FOLLOW $CM+ / (
- [$BK $CR $LF $NL $ZW {eof}] |
- $SP+ $CM+ $SP |
- $SP+ $CM* ([^$OP $CM $SP] | [$AL {eof}])); # if LB 14 will match, need to surpress this break.
- # LB14 says OP SP* x .
- # becomes OP SP* x AL
- # becomes OP SP* x CM+ AL_FOLLOW
- #
- # Further note: the $AL in [$AL {eof}] is only to work around
- # a rule compiler bug which complains about
- # empty sets otherwise.
-
-
-# LB 4, 5, 6
-
-$LB4Breaks [$LB4NonBreaks-$CM];
-$LB4Breaks $CM+ $CAN_CM;
-$LF $CR;
-
-
-# LB 7 x SP
-# x ZW
-[$SP $ZW] [$LB4NonBreaks-$CM];
-[$SP $ZW] $CM+ $CAN_CM;
-
-# LB 8 ZW SP* <break>
-# TODO: to implement this, we need more than one look-ahead hard break in play at a time.
-# Requires an engine enhancement.
-# / $SP* $ZW
-
-# LB 8a ZWJ x (ID | Extended_Pict | EmojiNRK)
-#
-($ID | $Extended_Pict | $EmojiNRK) $ZWJ $CM* $CAN_CM?;
-
-
-# LB 9,10 Combining marks.
-# X $CM needs to behave like X, where X is not $SP or controls.
-# $CM not covered by the above needs to behave like $AL
-# Stick together any combining sequences that don't match other rules.
-^$CM+ $CAN_CM;
-
-
-# LB 11
-#
-$WJ $CM* $CAN_CM;
-$WJ [$LB8NonBreaks-$CM];
-
- $CANT_CM $CM* $WJ;
-$CAN_CM $CM* $WJ;
-
-# LB 12a
-# [^SP BA HY] x GL
-#
-$GL $CM* [$LB8NonBreaks-[$CM $SP $BA $BAX $HY]];
-
-# LB 12
-# GL x
-#
-$CANT_CM $CM* $GL;
-$CAN_CM $CM* $GL;
-
-
-# LB 13
-$CL $CM+ $CAN_CM;
-$CP $CM+ $CAN_CM;
-$EX $CM+ $CAN_CM;
-$IS $CM+ $CAN_CM;
-$SY $CM+ $CAN_CM;
-
-$CL [$LB8NonBreaks-$CM];
-$CP [$LB8NonBreaks-$CM];
-$EX [$LB8NonBreaks-$CM];
-$IS [$LB8NonBreaks-$CM];
-$SY [$LB8NonBreaks-$CM];
-
-
-# LB 14 OP SP* x
-#
-. $SP* $CM* $OP;
-$AL_FOLLOW? $CM+ $SP+ $CM* $OP; # by LB 10, behaves like $AL_FOLLOW? $AL $SP+ $CM* $OP
-
-
-# LB 15
-$OP $SP* $CM* $QU;
-
-# LB 16
-# Don't include $NSX here
-$NS $SP* $CM* ($CL | $CP);
-
-# LB 17
-$B2 $SP* $CM* $B2;
-
-# LB 18 break after spaces
-# Nothing explicit needed here.
-
-
-#
-# LB 19
-#
-$QU $CM* $CAN_CM; # . x QU
-$QU $LB18NonBreaks;
-
-
-$CAN_CM $CM* $QU; # QU x .
- $CANT_CM $CM* $QU;
-
-#
-# LB 20 Break before and after CB.
-# nothing needed here.
-#
-
-# LB 21
-# Don't include $BAX or $NSX here
-($BA | $HY | $NS) $CM* [$LB20NonBreaks-$CM]; # . x (BA | HY | NS)
-
-[$LB20NonBreaks-$CM] $CM* $BB; # BB x .
-[^$CB] $CM* $BB; #
-
-# LB21a Don't break after Hebrew + Hyphen.
-([^$CB] $CM*)? ($HY | $BA | $BAX) $CM* $HL;
-
-# LB21b (reverse)
-$HL $CM* $SY;
-
-# LB 22
-$IN $CM* ($ALPlus | $HL);
-$IN $CM* $EX;
-$IN $CM* ($ID | $EB | $EM);
-$IN $CM* $IN;
-$IN $CM* $NU;
-
-# LB 23
-$NU $CM* ($ALPlus | $HL);
-($ALPlus | $HL) $CM* $NU;
-
-# LB23a
-($ID | $EB | $EM) $CM* $PR;
-$PO $CM* ($ID | $EB | $EM);
-
-# LB 24
-($ALPlus | $HL) $CM* ($PR | $PO);
-($PR | $PO) $CM* ($ALPlus | $HL);
-
-
-# LB 25
-($CM* ($PR | $PO))? ($CM* ($CL | $CP))? ($CM* ($NU | $IS | $SY))* $CM* $NU ($CM* ($OP | $HY))? ($CM* ($PR | $PO))?;
-
-# LB 26
-($H3 | $H2 | $JV | $JL) $CM* $JL;
-($JT | $JV) $CM* ($H2 | $JV);
-$JT $CM* ($H3 | $JT);
-
-# LB 27
-$IN $CM* ($H3 | $H2 | $JT | $JV | $JL);
-$PO $CM* ($H3 | $H2 | $JT | $JV | $JL);
- ($H3 | $H2 | $JT | $JV | $JL) $CM* $PR;
-
-# LB 28
-($ALPlus | $HL) $CM* ($ALPlus | $HL);
-
-
-# LB 29
-($ALPlus | $HL) $CM* $IS;
-
-# LB 30
-$OP $CM* ($ALPlus | $HL | $NU);
-($ALPlus | $HL | $NU) $CM* $CP;
-
-# LB 30a
-# Pairs of Regional Indicators.
-# The following two rules are nearly identical. The first matches only sequences with an odd number of adjacent RIs,
-# the second with an even number. Stripping away the cruft they look like
-# [^RI] RI / (RI RI)+ ^RI;
-# [^RI] RI RI / (RI RI)+ ^RI;
-#
-[{bof} $NS $HY $BA $QU $CL $CP $EX $IS $SY $WJ $GL $ZW $SP $BK $CR $LF $NL $ZWJ] $CM* $RI / ($CM* $RI $CM* $RI)+ $CM* [{eof}[^$RI $CM]];
-[{bof} $NS $HY $BA $QU $CL $CP $EX $IS $SY $WJ $GL $ZW $SP $BK $CR $LF $NL $ZWJ] $CM* $RI $CM* $RI / ($CM* $RI $CM* $RI)+ $CM* [{eof}[^$RI $CM]];
-
-# In general, adjacent RIs stay together. The hard-break rules, above, overide this, forcing in the boundaries between pairs.
-$RI $CM* $RI;
-
-# WJ, GL, QU, etc. are classes with rules like "WJ x " which includes "WJ x RI".
-$RI $CM* ([$WJ $GL $QU $BB] | (($HY | $BA)$CM* $HL));
-
-
-# LB 30b Do not break between an Emoji Base and an Emoji Modifier
-$EM $CM* $EB;
-
-
## -------------------------------------------------
!!safe_reverse;
# LB 9
^$CM+ [^$CM $BK $CR $LF $NL $ZW $SP];
-^$CM+ $SP / .;
# LB 14
$SP+ $CM* $OP;
# For dictionary-based break
$dictionary $dictionary;
-
-## -------------------------------------------------
-
-!!safe_forward;
-
-# Skip forward over all character classes that are involved in
-# rules containing patterns with possibly more than one char
-# of context.
-#
-# It might be slightly more efficient to have specific rules
-# instead of one generic one, but only if we could
-# turn off rule chaining. We don't want to move more
-# than necessary.
-#
-^[$CM $OP $QU $CL $CP $B2 $PR $HY $BA $BAX $SP $RI $ZWJ $dictionary]+ [^$CM $OP $QU $CL $CP $B2 $PR $HY $BA $BAX $RI $ZWJ $dictionary];
-$dictionary $dictionary;
-
-# Copyright (C) 2016 and later: Unicode, Inc. and others.
+# Copyright (C) 2016 and later: Unicode, Inc. and others.
# License & terms of use: http://www.unicode.org/copyright.html
# Copyright (c) 2002-2016 International Business Machines Corporation and
# others. All Rights Reserved.
#
!!chain;
+!!quoted_literals_only;
$AI = [:LineBreak = Ambiguous:];
$AL = [:LineBreak = Alphabetic:];
# LB 30b Do not break between an Emoji Base and an Emoji Modifier
$EB $CM* $EM;
-#
-# Reverse Rules.
-#
-## -------------------------------------------------
-
-!!reverse;
-
-# LB 9 Combining Marks.
-# Stick together any combining sequences that don't match other rules.
-
-^$CM+ $CAN_CM?;
-
-#
-# Sequences of the form (shown forwards)
-# [CANT_CM] <break> [CM] [whatever]
-# The CM needs to behave as an AL
-#
-$AL_FOLLOW $CM+ / (
- [$BK $CR $LF $NL $ZW {eof}] |
- $SP+ $CM+ $SP |
- $SP+ $CM* ([^$OP $CM $SP] | [$AL {eof}])); # if LB 14 will match, need to surpress this break.
- # LB14 says OP SP* x .
- # becomes OP SP* x AL
- # becomes OP SP* x CM+ AL_FOLLOW
- #
- # Further note: the $AL in [$AL {eof}] is only to work around
- # a rule compiler bug which complains about
- # empty sets otherwise.
-
-
-# LB 4, 5, 6
-
-$LB4Breaks [$LB4NonBreaks-$CM];
-$LB4Breaks $CM+ $CAN_CM;
-$LF $CR;
-
-
-# LB 7 x SP
-# x ZW
-[$SP $ZW] [$LB4NonBreaks-$CM];
-[$SP $ZW] $CM+ $CAN_CM;
-
-# LB 8 ZW SP* <break>
-# TODO: to implement this, we need more than one look-ahead hard break in play at a time.
-# Requires an engine enhancement.
-# / $SP* $ZW
-
-# LB 8a ZWJ x (ID | Extended_Pict | EmojiNRK)
-#
-($ID | $Extended_Pict | $EmojiNRK) $ZWJ $CM* $CAN_CM?;
-
-
-# LB 9,10 Combining marks.
-# X $CM needs to behave like X, where X is not $SP or controls.
-# $CM not covered by the above needs to behave like $AL
-# Stick together any combining sequences that don't match other rules.
-^$CM+ $CAN_CM;
-
-
-# LB 11
-#
-$WJ $CM* $CAN_CM;
-$WJ [$LB8NonBreaks-$CM];
-
- $CANT_CM $CM* $WJ;
-$CAN_CM $CM* $WJ;
-
-# LB 12a
-# [^SP BA HY] x GL
-#
-$GL $CM* [$LB8NonBreaks-[$CM $SP $BA $HH $HY]];
-
-# LB 12
-# GL x
-#
-$CANT_CM $CM* $GL;
-$CAN_CM $CM* $GL;
-
-
-# LB 13
-$CL $CM+ $CAN_CM;
-$CP $CM+ $CAN_CM;
-$EX $CM+ $CAN_CM;
-$IS $CM+ $CAN_CM;
-$SY $CM+ $CAN_CM;
-
-$CL [$LB8NonBreaks-$CM];
-$CP [$LB8NonBreaks-$CM];
-$EX [$LB8NonBreaks-$CM];
-$IS [$LB8NonBreaks-$CM];
-$SY [$LB8NonBreaks-$CM];
-
-
-# LB 14 OP SP* x
-#
-. $SP* $CM* $OP;
-$AL_FOLLOW? $CM+ $SP+ $CM* $OP; # by LB 10, behaves like $AL_FOLLOW? $AL $SP+ $CM* $OP
-
-
-# LB 15
-$OP $SP* $CM* $QU;
-
-# LB 16
-$NS $SP* $CM* ($CL | $CP);
-
-# LB 17
-$B2 $SP* $CM* $B2;
-
-# LB 18 break after spaces
-# Nothing explicit needed here.
-
-
-#
-# LB 19
-#
-$QU $CM* $CAN_CM; # . x QU
-$QU $LB18NonBreaks;
-
-
-$CAN_CM $CM* $QU; # QU x .
- $CANT_CM $CM* $QU;
-
-#
-# LB 20 Break before and after CB.
-# nothing needed here.
-#
-
-# LB 20.09 added rule for Finnish tailoring
-$AL ($HY | $HH) / $SP;
-
-# LB 21
-($BA | $HH | $HY | $NS) $CM* [$LB20NonBreaks-$CM]; # . x (BA | HY | NS)
-
-[$LB20NonBreaks-$CM] $CM* $BB; # BB x .
-[^$CB] $CM* $BB; #
-
-# LB21a
-[^$CB] $CM* ($HY | $BA | $HH) $CM* $HL;
-
-# LB21b (reverse)
-$HL $CM* $SY;
-
-# LB 22
-$IN $CM* ($ALPlus | $HL);
-$IN $CM* $EX;
-$IN $CM* ($ID | $EB | $EM);
-$IN $CM* $IN;
-$IN $CM* $NU;
-
-# LB 23
-$NU $CM* ($ALPlus | $HL);
-($ALPlus | $HL) $CM* $NU;
-
-# LB23a
-($ID | $EB | $EM) $CM* $PR;
-$PO $CM* ($ID | $EB | $EM);
-
-# LB 24
-($ALPlus | $HL) $CM* ($PR | $PO);
-($PR | $PO) $CM* ($ALPlus | $HL);
-
-
-# LB 25
-($CM* ($PR | $PO))? ($CM* ($CL | $CP))? ($CM* ($NU | $IS | $SY))* $CM* $NU ($CM* ($OP | $HY))? ($CM* ($PR | $PO))?;
-
-# LB 26
-($H3 | $H2 | $JV | $JL) $CM* $JL;
-($JT | $JV) $CM* ($H2 | $JV);
-$JT $CM* ($H3 | $JT);
-
-# LB 27
-$IN $CM* ($H3 | $H2 | $JT | $JV | $JL);
-$PO $CM* ($H3 | $H2 | $JT | $JV | $JL);
- ($H3 | $H2 | $JT | $JV | $JL) $CM* $PR;
-
-# LB 28
-($ALPlus | $HL) $CM* ($ALPlus | $HL);
-
-
-# LB 29
-($ALPlus | $HL) $CM* $IS;
-
-# LB 30
-$OP $CM* ($ALPlus | $HL | $NU);
-($ALPlus | $HL | $NU) $CM* $CP;
-
-# LB 30a
-# Pairs of Regional Indicators.
-# The following two rules are nearly identical. The first matches only sequences with an odd number of adjacent RIs,
-# the second with an even number. Stripping away the cruft they look like
-# [^RI] RI / (RI RI)+ ^RI;
-# [^RI] RI RI / (RI RI)+ ^RI;
-#
-[{bof} $NS $HY $BA $QU $CL $CP $EX $IS $SY $WJ $GL $ZW $SP $BK $CR $LF $NL $ZWJ] $CM* $RI / ($CM* $RI $CM* $RI)+ $CM* [{eof}[^$RI $CM]];
-[{bof} $NS $HY $BA $QU $CL $CP $EX $IS $SY $WJ $GL $ZW $SP $BK $CR $LF $NL $ZWJ] $CM* $RI $CM* $RI / ($CM* $RI $CM* $RI)+ $CM* [{eof}[^$RI $CM]];
-
-# In general, adjacent RIs stay together. The hard-break rules, above, overide this, forcing in the boundaries between pairs.
-$RI $CM* $RI;
-
-# WJ, GL, QU, etc. are classes with rules like "WJ x " which includes "WJ x RI".
-$RI $CM* ([$WJ $GL $QU $BB] | (($HY | $BA)$CM* $HL));
-
-
-# LB 30b Do not break between an Emoji Base and an Emoji Modifier
-$EM $CM* $EB;
-
-
## -------------------------------------------------
!!safe_reverse;
# For dictionary-based break
$dictionary $dictionary;
-
-## -------------------------------------------------
-
-!!safe_forward;
-
-# Skip forward over all character classes that are involved in
-# rules containing patterns with possibly more than one char
-# of context.
-#
-# It might be slightly more efficient to have specific rules
-# instead of one generic one, but only if we could
-# turn off rule chaining. We don't want to move more
-# than necessary.
-#
-^[$CM $OP $QU $CL $CP $B2 $PR $HY $BA $SP $RI $ZWJ $dictionary]+ [^$CM $OP $QU $CL $CP $B2 $PR $HY $BA $RI $ZWJ $dictionary];
-$dictionary $dictionary;
-
-#
-# Copyright (C) 2016 and later: Unicode, Inc. and others.
-# License & terms of use: http://www.unicode.org/copyright.html#License
+# Copyright (C) 2016 and later: Unicode, Inc. and others.
+# License & terms of use: http://www.unicode.org/copyright.html
#
# Copyright (C) 2002-2015, International Business Machines Corporation and others.
# All Rights Reserved.
# These rules are based on UAX #29 Revision 26 for Unicode Version 8.0
#
+!!quoted_literals_only;
#
# Character categories as defined in TR 29
## -------------------------------------------------
-!!reverse;
+!!safe_reverse;
$SpEx_R = ($Extend | $Format)* $Sp;
$ATermEx_R = ($Extend | $Format)* $ATerm;
$STermEx_R = ($Extend | $Format)* $STerm;
$CloseEx_R = ($Extend | $Format)* $Close;
-#
-# Reverse rules.
-# For now, use the old style inexact reverse rules, which are easier
-# to write, but less efficient.
-# TODO: exact reverse rules. It appears that exact reverse rules
-# may require improving support for look-ahead breaks in the
-# builder. Needs more investigation.
-#
-
[{bof}] (.? | $LF $CR) [^$Sep $CR $LF]* [$Sep $CR $LF {eof}] ($SpEx_R* $CloseEx_R* ($STermEx_R | $ATermEx_R))*;
#.*;
# The preceding $Sep, which will be the second one that the rule matches.
# Any immediately preceding STerm or ATerm sequences. We need to see these
# to get the correct rule status when moving forwards again.
-#
+#
# [{bof}] inhibit rule chaining. Without this, rule would loop on itself and match
-# the entire string.
+# the entire string. TODO: can bof be replaced with ^
#
# (.? | $LF $CR) Match one $Sep instance. Use .? rather than $Sep because position might be
# at the beginning of the string at this point, and we don't want to fail.
+# Copyright (C) 2016 and later: Unicode, Inc. and others.
+# License & terms of use: http://www.unicode.org/copyright.html
#
-# Copyright (C) 2016 and later: Unicode, Inc. and others.
-# License & terms of use: http://www.unicode.org/copyright.html#License
#
# Copyright (C) 2002-2015, International Business Machines Corporation and others.
# All Rights Reserved.
# These rules are based on UAX #29 Revision 26 for Unicode Version 8.0
#
+!!quoted_literals_only;
#
# Character categories as defined in TR 29
## -------------------------------------------------
-!!reverse;
+!!safe_reverse;
$SpEx_R = ($Extend | $Format)* $Sp;
$ATermEx_R = ($Extend | $Format)* $ATerm;
#
[{bof}] (.? | $LF $CR) [^$Sep $CR $LF]* [$Sep $CR $LF {eof}] ($SpEx_R* $CloseEx_R* ($STermEx_R | $ATermEx_R))*;
-#.*;
# Explanation for this rule:
#
# The preceding $Sep, which will be the second one that the rule matches.
# Any immediately preceding STerm or ATerm sequences. We need to see these
# to get the correct rule status when moving forwards again.
-#
+#
# [{bof}] inhibit rule chaining. Without this, rule would loop on itself and match
# the entire string.
#
# Copyright (C) 2016 and later: Unicode, Inc. and others.
-# License & terms of use: http://www.unicode.org/copyright.html#License
+# License & terms of use: http://www.unicode.org/copyright.html
#
# Copyright (c) 2002-2015, International Business Machines Corporation and
# others. All Rights Reserved.
# Title Casing Break Rules
#
+!!quoted_literals_only;
$CaseIgnorable = [[:Mn:][:Me:][:Cf:][:Lm:][:Sk:] \u0027 \u00AD \u2019];
$Cased = [[:Upper_Case:][:Lower_Case:][:Lt:] - $CaseIgnorable];
$Cased ($Cased | $CaseIgnorable)* ($NotCased | $CaseIgnorable)*;
-# Reverse Rules
-!!reverse;
-
-# Normal Rule, will work nearly universally, so long as there is a
-# start-of-word preceding the current iteration position.
-
-($NotCased | $CaseIgnorable)* ($Cased | $CaseIgnorable)* $Cased;
-
-# Short rule, will be effective only when moving to the start of text,
-# with no word (cased character) preceding the current iteration position.
-
-($NotCased | $CaseIgnorable)*;
-
!!safe_reverse;
# Safe Reverse: the exact forward rule must not start in the middle
# leaving it just before the start of a word.
($Cased | $CaseIgnorable)*;
-
-!!safe_forward;
-
-# Safe Forward, nothing needs to be done, the exact Reverse rules will
-# always find valid boundaries from any starting position.
-# Still, some rule is needed, so '.', a one character movement.
-.;
-#
+#
# Copyright (C) 2016 and later: Unicode, Inc. and others.
# License & terms of use: http://www.unicode.org/copyright.html
-# Copyright (C) 2002-2016, International Business Machines Corporation
+# Copyright (C) 2002-2016, International Business Machines Corporation
# and others. All Rights Reserved.
#
# file: word.txt
##############################################################################
!!chain;
+!!quoted_literals_only;
#
$KanaKanji $KanaKanji {400}; # different rule status if both kana and kanji found
-## -------------------------------------------------
-
-!!reverse;
-
-$BackHebrew_LetterEx = ($Format | $Extend | $ZWJ)* $Hebrew_Letter;
-$BackALetterEx = ($Format | $Extend | $ZWJ)* $ALetterPlus;
-$BackSingle_QuoteEx = ($Format | $Extend | $ZWJ)* $Single_Quote;
-$BackDouble_QuoteEx = ($Format | $Extend | $ZWJ)* $Double_Quote;
-$BackMidNumLetEx = ($Format | $Extend | $ZWJ)* $MidNumLet;
-$BackNumericEx = ($Format | $Extend | $ZWJ)* $Numeric;
-$BackMidNumEx = ($Format | $Extend | $ZWJ)* $MidNum;
-$BackMidLetterEx = ($Format | $Extend | $ZWJ)* $MidLetter;
-$BackKatakanaEx = ($Format | $Extend | $ZWJ)* $Katakana;
-$BackHiraganaEx = ($Format | $Extend | $ZWJ)* $Hiragana;
-$BackExtendNumLetEx = ($Format | $Extend | $ZWJ)* $ExtendNumLet;
-$BackRegional_IndicatorEx = ($Format | $Extend | $ZWJ)* $Regional_Indicator;
-
-# rule 3
-$LF $CR;
-
-# Rule 3c ZWJ x (Extended_Pict | EmojiNRK). Precedes WB4, so no intervening Extend chars allowed.
-#
-($Extended_Pict | $EmojiNRK) $ZWJ;
-
-# rule 4
-($Format | $Extend | $ZWJ)* [^$CR $LF $Newline]?;
-
-# rule 5
-
-($BackALetterEx | $BackHebrew_LetterEx) ($BackALetterEx | $BackHebrew_LetterEx);
-
-# rule 6 and 7
-
-($BackALetterEx | $BackHebrew_LetterEx) ($BackMidLetterEx | $BackMidNumLetEx | $BackSingle_QuoteEx) ($BackALetterEx | $BackHebrew_LetterEx);
-
-# rule 7a
-$BackSingle_QuoteEx $BackHebrew_LetterEx;
-
-# Rule 7b and 7c
-$BackHebrew_LetterEx $BackDouble_QuoteEx $BackHebrew_LetterEx;
-
-# rule 8
-
-$BackNumericEx $BackNumericEx;
-
-# rule 9
-
-$BackNumericEx ($BackALetterEx | $BackHebrew_LetterEx);
-
-# rule 10
-
-($BackALetterEx | $BackHebrew_LetterEx) $BackNumericEx;
-
-# rule 11 and 12
-
-$BackNumericEx ($BackMidNumEx | $BackMidNumLetEx | $BackSingle_QuoteEx) $BackNumericEx;
-
-# rule 13
-
-$BackKatakanaEx $BackKatakanaEx;
-
-# rules 13 a/b
-#
-$BackExtendNumLetEx ($BackALetterEx | $BackHebrew_LetterEx | $BackNumericEx | $BackKatakanaEx | $BackExtendNumLetEx);
-($BackALetterEx | $BackHebrew_LetterEx | $BackNumericEx | $BackKatakanaEx) $BackExtendNumLetEx;
-
-# special handling for CJK characters: chain for later dictionary segmentation
-$HangulSyllable $HangulSyllable;
-$KanaKanji $KanaKanji; #different rule status if both kanji and kana found
-
-# rule 14
-
-$E_Modifier ($Format | $Extend | $ZWJ)* ($E_Base | $EBG);
-
-# rule 15 - 17
-# Pairs of Regional Indicators stay together.
-
-^$BackRegional_IndicatorEx / ($BackRegional_IndicatorEx $BackRegional_IndicatorEx)*
- ($Format | $Extend | $ZWJ)* [[^$Regional_Indicator $Format $Extend $ZWJ] {eof}];
-^$BackRegional_IndicatorEx $BackRegional_IndicatorEx / ($BackRegional_IndicatorEx $BackRegional_IndicatorEx)*
- ($Format | $Extend | $ZWJ)* [[^$Regional_Indicator $Format $Extend $ZWJ] {eof}];
-
-($Extended_Pict | $EmojiNRK) $ZWJ $BackRegional_IndicatorEx / ($BackRegional_IndicatorEx $BackRegional_IndicatorEx)*
- ($Format | $Extend | $ZWJ)* [[^$Regional_Indicator $Format $Extend $ZWJ] {eof}];
-($Extended_Pict | $EmojiNRK) $ZWJ $BackRegional_IndicatorEx $BackRegional_IndicatorEx / ($BackRegional_IndicatorEx $BackRegional_IndicatorEx)*
- ($Format | $Extend | $ZWJ)* [[^$Regional_Indicator $Format $Extend $ZWJ] {eof}];
-
-
-
## -------------------------------------------------
!!safe_reverse;
($Extend | $Format | $ZWJ)+ .?;
# rule 6
-($MidLetter | $MidNumLet | $Single_Quote) ($BackALetterEx | $BackHebrew_LetterEx);
+($MidLetter | $MidNumLet | $Single_Quote) ($Format | $Extend | $ZWJ)* ($Hebrew_Letter | $ALetterPlus);
# rule 7b
-$Double_Quote $BackHebrew_LetterEx;
-
+$Double_Quote ($Format | $Extend | $ZWJ)* $Hebrew_Letter;
-# rule 11
-($MidNum | $MidNumLet | $Single_Quote) $BackNumericEx;
-
-# rule 13c
-$BackRegional_IndicatorEx*;
-
-# For dictionary-based break
-$dictionary $dictionary;
-
-## -------------------------------------------------
-
-!!safe_forward;
-
-# rule 4
-($Extend | $Format | $ZWJ)+ .?;
-
-# rule 6
-($MidLetterEx | $MidNumLetEx | $Single_QuoteEx) ($ALetterEx | $Hebrew_LetterEx);
-
-# rule 7b
-$Double_QuoteEx $Hebrew_LetterEx;
# rule 11
-($MidNumEx | $MidNumLetEx | $Single_QuoteEx) $NumericEx;
+($MidNum | $MidNumLet | $Single_Quote) ($Format | $Extend | $ZWJ)* $Numeric;
# rule 13c
-$Regional_IndicatorEx*;
+$Regional_Indicator ($Format | $Extend | $ZWJ)* $Regional_Indicator;
# For dictionary-based break
$dictionary $dictionary;
-#
+#
# Copyright (C) 2016 and later: Unicode, Inc. and others.
# License & terms of use: http://www.unicode.org/copyright.html
-# Copyright (C) 2002-2016, International Business Machines Corporation
+# Copyright (C) 2002-2016, International Business Machines Corporation
# and others. All Rights Reserved.
#
# file: word_POSIX.txt
##############################################################################
!!chain;
+!!quoted_literals_only;
#
# 5.0 or later as the definition of Complex_Context was corrected to include all
# characters requiring dictionary break.
-$Control = [\p{Grapheme_Cluster_Break = Control}];
+$Control = [\p{Grapheme_Cluster_Break = Control}];
$HangulSyllable = [\uac00-\ud7a3];
$ComplexContext = [:LineBreak = Complex_Context:];
$KanaKanji = [$Han $Hiragana $Katakana];
#
-# Rules 4 Ignore Format and Extend characters,
+# Rules 4 Ignore Format and Extend characters,
# except when they appear at the beginning of a region of text.
#
# TODO: check if handling of katakana in dictionary makes rules incorrect/void
$NumericEx ($ALetterEx | $Hebrew_LetterEx) {200};
-# rule 11 and 12
+# rule 11 and 12
$NumericEx ($MidNumEx | $MidNumLetEx | $Single_QuoteEx) $NumericEx {100};
# special handling for CJK characters: chain for later dictionary segmentation
$HangulSyllable $HangulSyllable {200};
-$KanaKanji $KanaKanji {400}; # different rule status if both kana and kanji found
-
-
-## -------------------------------------------------
-
-!!reverse;
-
-$BackHebrew_LetterEx = ($Format | $Extend | $ZWJ)* $Hebrew_Letter;
-$BackALetterEx = ($Format | $Extend | $ZWJ)* $ALetterPlus;
-$BackSingle_QuoteEx = ($Format | $Extend | $ZWJ)* $Single_Quote;
-$BackDouble_QuoteEx = ($Format | $Extend | $ZWJ)* $Double_Quote;
-$BackMidNumLetEx = ($Format | $Extend | $ZWJ)* $MidNumLet;
-$BackNumericEx = ($Format | $Extend | $ZWJ)* $Numeric;
-$BackMidNumEx = ($Format | $Extend | $ZWJ)* $MidNum;
-$BackMidLetterEx = ($Format | $Extend | $ZWJ)* $MidLetter;
-$BackKatakanaEx = ($Format | $Extend | $ZWJ)* $Katakana;
-$BackHiraganaEx = ($Format | $Extend | $ZWJ)* $Hiragana;
-$BackExtendNumLetEx = ($Format | $Extend | $ZWJ)* $ExtendNumLet;
-$BackRegional_IndicatorEx = ($Format | $Extend | $ZWJ)* $Regional_Indicator;
-
-# rule 3
-$LF $CR;
-
-# Rule 3c ZWJ x (Extended_Pict | EmojiNRK). Precedes WB4, so no intervening Extend chars allowed.
-#
-($Extended_Pict | $EmojiNRK) $ZWJ;
-
-# rule 4
-($Format | $Extend | $ZWJ)* [^$CR $LF $Newline]?;
-
-# rule 5
-
-($BackALetterEx | $BackHebrew_LetterEx) ($BackALetterEx | $BackHebrew_LetterEx);
-
-# rule 6 and 7
-
-($BackALetterEx | $BackHebrew_LetterEx) ($BackMidLetterEx | $BackMidNumLetEx | $BackSingle_QuoteEx) ($BackALetterEx | $BackHebrew_LetterEx);
-
-# rule 7a
-$BackSingle_QuoteEx $BackHebrew_LetterEx;
-
-# Rule 7b and 7c
-$BackHebrew_LetterEx $BackDouble_QuoteEx $BackHebrew_LetterEx;
-
-# rule 8
-
-$BackNumericEx $BackNumericEx;
-
-# rule 9
-
-$BackNumericEx ($BackALetterEx | $BackHebrew_LetterEx);
-
-# rule 10
-
-($BackALetterEx | $BackHebrew_LetterEx) $BackNumericEx;
-
-# rule 11 and 12
-
-$BackNumericEx ($BackMidNumEx | $BackMidNumLetEx | $BackSingle_QuoteEx) $BackNumericEx;
-
-# rule 13
-
-$BackKatakanaEx $BackKatakanaEx;
-
-# rules 13 a/b
-#
-$BackExtendNumLetEx ($BackALetterEx | $BackHebrew_LetterEx | $BackNumericEx | $BackKatakanaEx | $BackExtendNumLetEx);
-($BackALetterEx | $BackHebrew_LetterEx | $BackNumericEx | $BackKatakanaEx) $BackExtendNumLetEx;
-
-# special handling for CJK characters: chain for later dictionary segmentation
-$HangulSyllable $HangulSyllable;
-$KanaKanji $KanaKanji; #different rule status if both kanji and kana found
-
-# rule 14
-
-$E_Modifier ($Format | $Extend | $ZWJ)* ($E_Base | $EBG);
-
-# rule 15 - 17
-# Pairs of Regional Indicators stay together.
-
-^$BackRegional_IndicatorEx / ($BackRegional_IndicatorEx $BackRegional_IndicatorEx)*
- ($Format | $Extend | $ZWJ)* [[^$Regional_Indicator $Format $Extend $ZWJ] {eof}];
-^$BackRegional_IndicatorEx $BackRegional_IndicatorEx / ($BackRegional_IndicatorEx $BackRegional_IndicatorEx)*
- ($Format | $Extend | $ZWJ)* [[^$Regional_Indicator $Format $Extend $ZWJ] {eof}];
-
-($Extended_Pict | $EmojiNRK) $ZWJ $BackRegional_IndicatorEx / ($BackRegional_IndicatorEx $BackRegional_IndicatorEx)*
- ($Format | $Extend | $ZWJ)* [[^$Regional_Indicator $Format $Extend $ZWJ] {eof}];
-($Extended_Pict | $EmojiNRK) $ZWJ $BackRegional_IndicatorEx $BackRegional_IndicatorEx / ($BackRegional_IndicatorEx $BackRegional_IndicatorEx)*
- ($Format | $Extend | $ZWJ)* [[^$Regional_Indicator $Format $Extend $ZWJ] {eof}];
-
+$KanaKanji $KanaKanji {400}; # different rule status if both kana and kanji found
## -------------------------------------------------
($Extend | $Format | $ZWJ)+ .?;
# rule 6
-($MidLetter | $MidNumLet | $Single_Quote) ($BackALetterEx | $BackHebrew_LetterEx);
+($MidLetter | $MidNumLet | $Single_Quote) ($Format | $Extend | $ZWJ)* ($Hebrew_Letter | $ALetterPlus);
# rule 7b
-$Double_Quote $BackHebrew_LetterEx;
+$Double_Quote ($Format | $Extend | $ZWJ)* $Hebrew_Letter;
# rule 11
-($MidNum | $MidNumLet | $Single_Quote) $BackNumericEx;
-
-# rule 13c
-$BackRegional_IndicatorEx*;
-
-# For dictionary-based break
-$dictionary $dictionary;
-
-## -------------------------------------------------
-
-!!safe_forward;
-
-# rule 4
-($Extend | $Format | $ZWJ)+ .?;
-
-# rule 6
-($MidLetterEx | $MidNumLetEx | $Single_QuoteEx) ($ALetterEx | $Hebrew_LetterEx);
-
-# rule 7b
-$Double_QuoteEx $Hebrew_LetterEx;
-
-# rule 11
-($MidNumEx | $MidNumLetEx | $Single_QuoteEx) $NumericEx;
+($MidNum | $MidNumLet | $Single_Quote) ($Format | $Extend | $ZWJ)* $Numeric;
# rule 13c
-$Regional_IndicatorEx*;
+$Regional_Indicator ($Format | $Extend | $ZWJ)* $Regional_Indicator;
# For dictionary-based break
$dictionary $dictionary;
bi->first();
i = bi->current();
if (i != 0) {
- errln("%s:%d Incorrect value from bi->previous(). Expected 0, got %d", __FILE__, __LINE__, i);
+ errln("%s:%d Incorrect value from bi->current(). Expected 0, got %d", __FILE__, __LINE__, i);
}
bi->next();
i = bi->current();
if (i != 1) {
- errln("%s:%d Incorrect value from bi->previous(). Expected 1, got %d", __FILE__, __LINE__, i);
+ errln("%s:%d Incorrect value from bi->current(). Expected 1, got %d", __FILE__, __LINE__, i);
}
bi->last();
bi->next();
i = bi->current();
if (i != 10) {
- errln("%s:%d Incorrect value from bi->previous(). Expected 10, got %d", __FILE__, __LINE__, i);
+ errln("%s:%d Incorrect value from bi->current(). Expected 10, got %d", __FILE__, __LINE__, i);
}
bi->first();
bi->previous();
i = bi->current();
if (i != 0) {
- errln("%s:%d Incorrect value from bi->previous(). Expected 0, got %d", __FILE__, __LINE__, i);
+ errln("%s:%d Incorrect value from bi->current(). Expected 0, got %d", __FILE__, __LINE__, i);
}
testFollowing(status);
testPreceding(status);
testIsBoundary(status);
+ testIsBoundaryRandom(status);
if (fLoopCount < 0 && loopCount % 100 == 0) {
fprintf(stderr, ".");
checkResults("testForwards", FORWARD, status);
}
+void RBBIMonkeyImpl::testIsBoundaryRandom(UErrorCode &status) {
+ if (U_FAILURE(status)) {
+ return;
+ }
+ fBI->setText(fTestData->fString);
+
+ int stringLen = fTestData->fString.length();
+ for (int i=stringLen; i>=0; --i) {
+ int strIdx = fRandomGenerator() % stringLen;
+ if (fTestData->fExpectedBreaks.charAt(strIdx) != fBI->isBoundary(strIdx)) {
+ IntlTest::gTest->errln("%s:%d testIsBoundaryRandom failure at index %d. Parameters to reproduce: @rules=%s,seed=%u,loop=1,verbose ",
+ __FILE__, __LINE__, strIdx, fRuleFileName, fTestData->fRandomSeed);
+ if (fVerbose) {
+ fTestData->dump(i);
+ }
+ status = U_INVALID_STATE_ERROR;
+ break;
+ }
+ }
+}
+
+
+
void RBBIMonkeyImpl::checkResults(const char *msg, CheckDirection direction, UErrorCode &status) {
if (U_FAILURE(status)) {
return;
void testFollowing(UErrorCode &status);
void testPreceding(UErrorCode &status);
void testIsBoundary(UErrorCode &status);
+ void testIsBoundaryRandom(UErrorCode &status);
void checkResults(const char *msg, CheckDirection dir, UErrorCode &status);
class RBBIMonkeyThread: public SimpleThread {
// Run the iterator backwards, verify that the same breaks are found.
//
prevBP = utext_nativeLength(t->textToBreak)+2; // start with a phony value for the last break pos seen.
- for (bp = t->bi->last(); bp != BreakIterator::DONE; bp = t->bi->previous()) {
+ bp = t->bi->last();
+ while (bp != BreakIterator::DONE) {
if (prevBP == bp) {
// Fail for lack of progress.
errln("Reverse Iteration, no progress. Break Pos=%4d File line,col=%4d,%4d",
}
prevBP = bp;
+ bp = t->bi->previous();
}
// Verify that there were no missed breaks prior to the last one found
// Reached end of test file. Raise an error if parseState indicates that we are
// within a block that should have been terminated.
+
if (parseState == PARSE_RULES) {
errln("rbbitst.txt:%d <rules> block beginning at line %d is not closed.",
lineNum, rulesFirstLine);
</badrules>
<rules>
-#TODO: uncomment this line when quoted_literals_only is implemented.
-#!!quoted_literals_only;
+!!quoted_literals_only;
!!forward;
'Hello World';
!!reverse;
</rules>
<data>•Hello World•</data>
+
+# Test for circular buffer overflow during reverse iteration with inefficient reverse rules,
+# Too many boundaries between safe back up position and current position.
+
+<rules>
+!!forward;
+.;
+!!reverse;
+.*;
+</rules>
+<data>•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•</data>
+
+#
+# Dictionary regression check
+# Intended to find unexpected behavior changes when changing dictionary implementation code,
+# but may also be fragile, failing with intended improvements to dictionary breaking.
+#
+<locale en>
+<word>
+<data>•Unicode<200> •คือ<200>อะไร<200>?•
+•Unicode<200> •กำหนด<200>หมายเลข<200>เฉพาะ<200>สำหรับ<200>ทุก<200>อักขระ<200>
+•โดย<200>ไม่<200>สนใจ<200>ว่า<200>เป็น<200>แพ<200>ล็ต<200>ฟอร์ม<200>ใด<200>
+•ไม่<200>ขึ้น<200>กับ<200>ว่า<200>จะ<200>เป็น<200>โปรแกรม<200>ใด<200>
+•และ<200>ไม่<200>ว่า<200>จะ<200>เป็น<200>ภาษา<200>ใด<200>
+•
+•โดย<200>พื้น<200>ฐาน<200>แล้ว<200>,• •คอมพิวเตอร์<200>จะ<200>เกี่ยวข้อง<200>กับ<200>เรื่อง<200>ของ<200>ตัวเลข<200>.• •คอมพิวเตอร์<200>จัด<200>เก็บ<200>ตัว<200>อักษร<200>และ<200>อักข<200>ระ<200>อื่นๆ<200> •โดย<200>การ<200>กำหนด<200>หมายเลข<200>ให้<200>สำหรับ<200>แต่ละ<200>ตัว<200>.• •ก่อน<200>หน้า<200>ที่๊<200> •Unicode<200> •จะ<200>ถูก<200>สร้าง<200>ขึ้น<200>,• •ได้<200>มี<200>ระบบ<200> •encoding<200> •อยู่<200>หลาย<200>ร้อย<200>ระบบ<200>สำหรับ<200>การ<200>กำหนด<200>หมายเลข<200>เหล่า<200>นี้<200>.• •ไม่มี<200> •encoding<200> •ใด<200>ที่<200>มี<200>จำนวน<200>ตัว<200>อักขระ<200>มาก<200>เพียง<200>พอ<200>:• •ยก<200>ตัวอย่าง<200>เช่น<200>,• •เฉพาะ<200>ใน<200>กลุ่ม<200>สหภาพ<200>ยุโรป<200>เพียง<200>แห่ง<200>เดียว<200> •ก็<200>ต้องการ<200>หลาย<200> •encoding<200> •ใน<200>การ<200>ครอบคลุม<200>ทุก<200>ภาษา<200>ใน<200>กลุ่ม<200>.• •หรือ<200>แม้แต่<200>ใน<200>ภาษา<200>เดี่ยว<200> •เช่น<200> •ภาษา<200>อังกฤษ<200> •ก็<200>ไม่มี<200> •encoding<200> •ใด<200>ที่<200>เพียง<200>พอ<200>สำหรับ<200>ทุก<200>ตัว<200>อักษร<200>,• •เครื่องหมาย<200>วรรค<200>ตอน<200> •และ<200>สัญลักษณ์<200>ทาง<200>เทคนิค<200>ที่<200>ใช้<200>กัน<200>อยู่<200>ทั่วไป<200>.•
+•
+•ระบบ<200> •encoding<200> •เหล่า<200>นี้<200>ยัง<200>ขัด<200>แย้ง<200>ซึ่ง<200>กัน<200>และ<200>กัน<200>.• •นั่น<200>ก็<200>คือ<200>,• •ใน<200>สอง<200> •encoding<200> •สามารถ<200>ใช้<200>หมายเลข<200>เดียวกัน<200>สำหรับ<200>ตัว<200>อักขระ<200>สอง<200>ตัว<200>ที่<200>แตก<200>ต่าง<200>กัน<200>,•หรือ<200>ใช้<200>หมายเลข<200>ต่าง<200>กัน<200>สำหรับ<200>อักขระ<200>ตัว<200>เดียวกัน<200>.• •ใน<200>ระบบ<200>คอมพิวเตอร์<200> •(•โดย<200>เฉพาะ<200>เซิร์ฟเวอร์<200>)• •ต้อง<200>มี<200>การ<200>สนับสนุน<200>หลาย<200> •encoding<200>;• •และ<200>เมื่อ<200>ข้อมูล<200>ที่<200>ผ่าน<200>ไป<200>มาระ<200>หว่าง<200>การ<200>เข้า<200>รหัส<200>หรือ<200>แพ<200>ล็ต<200>ฟอร์ม<200>ที่<200>ต่าง<200>กัน<200>,• •ข้อมูล<200>นั้น<200>จะ<200>เสี่ยง<200>ต่อ<200>การ<200>ผิด<200>พลาด<200>เสีย<200>หาย<200>.•
+•
+•Unicode<200> •จะ<200>เปลี่ยนแปลง<200>สิ่ง<200>เหล่า<200>นั้น<200>ทั้งหมด<200>!•
+•
+•Unicode<200> •กำหนด<200>หมายเลข<200>เฉพาะ<200>สำหรับ<200>แต่ละ<200>อักขระ<200>,• •โดย<200>ไม่<200>สนใจ<200>ว่า<200>เป็น<200>แพ<200>ล็ต<200>ฟอร์ม<200>ใด<200>,• •ไม่<200>ขึ้น<200>กับ<200>ว่า<200>จะ<200>เป็น<200>โปรแกรม<200>ใด<200>และ<200>ไม่<200>ว่า<200>จะ<200>เป็น<200>ภาษา<200>ใด<200>.• •มาตรฐาน<200> •Unicode<200> •ได้<200>ถูก<200>นำ<200>ไป<200>ใช้<200>โดย<200>ผู้นำ<200>ใน<200>อุตสาหกรรม<200> •เช่น<200> •Apple<200>,• •HP<200>,• •IBM<200>,• •JustSystem<200>,• •Microsoft<200>,• •Oracle<200>,• •SAP<200>,• •Sun<200>,• •Sybase<200>,• •Unisys<200> •และ<200>อื่นๆ<200> •อีก<200>มาก<200>.• •Unicode<200> •เป็น<200>สิ่ง<200>ที่<200>จำเป็น<200>สำหรับ<200>มาตร<200>ฐาน<200>ใหม่ๆ<200> •เช่น<200> •XML<200>,• •Java<200>,• •ECMAScript<200> •(•JavaScript<200>)•,• •LDAP<200>,• •CORBA<200> •3.0<100>,• •WML<200> •ฯลฯ<200>.•,• •และ<200>เป็น<200>แนวทาง<200>อย่าง<200>เป็น<200>ทางการ<200>ใน<200>การ<200>ทำ<200> •ISO<200>/•IEC<200> •10646<100>.• •Unicode<200> •ได้<200>รับ<200>การ<200>สนับสนุน<200>ใน<200>ระบบ<200>ปฏิบัติ<200>การ<200>จำนวน<200>มาก<200>,• •บราวเซอร์<200>ใหม่ๆ<200> •ทก<200>ตัว<200>,• •และ<200>ผลิต<200>ภัณฑ์<200>อื่นๆ<200> •อีก<200>มาก<200>.• •การ<200>เกิด<200>ขึ้น<200>ของ<200> •Unicode<200> •Standard<200> •และ<200>ทูล<200>ส์<200>ต่างๆ<200> •ที่<200>มี<200>ใน<200>การ<200>สนับสนุน<200> •Unicode<200>,• •เป็น<200>หนึ่ง<200>ใน<200>แนว<200>โน้ม<200>ทาง<200>เทคโนโลยี<200>ซอฟต์แวร์<200>ระดับ<200>โลก<200>ที่<200>มี<200>ความ<200>สำคัญ<200>ที่สุด<200>.•
+•
+•การ<200>รวม<200> •Unicode<200> •เข้าไป<200>ใน<200>ระบบ<200>ไคลเอ็นต์<200>-•เซิร์ฟเวอร์<200> •หรือ<200>แอ็พ<200>พลิ<200>เค<200>ชัน<200>แบบ<200> •multi<200>-•tiered<200> •และ<200>เว็บไซต์<200> •จะ<200>ทำให้<200>เกิด<200>การ<200>ประหยัด<200>ค่า<200>ใช้<200>จ่าย<200>มากกว่า<200>การ<200>ใช้<200>ชุด<200>อักขระ<200>แบบ<200>เดิม<200>.• •Unicode<200> •ทำให้<200>ผลิตภัณฑ์<200>ซอฟต์แวร์<200>หนึ่ง<200>เดียว<200> •หรือ<200>เว็บไซต์<200>แห่ง<200>เดียว<200> •รองรับ<200>ได้<200>หลาย<200>แพ<200>ล็ต<200>ฟอร์ม<200>,• •หลาย<200>ภาษา<200>และ<200>หลาย<200>ประเทศ<200>โดย<200>ไม่<200>ต้อง<200>ทำการ<200>รื้อ<200>ปรับ<200>ระบบ<200>.• •Unicode<200> •ยัง<200>ทำให้<200>ข้อมูล<200>สามารถ<200>เคลื่อน<200>ย้าย<200>ไป<200>มา<200>ใน<200>หลายๆ<200> •ระบบ<200>โดย<200>ไม่<200>เกิด<200>ความ<200>ผิด<200>พลาด<200>เสีย<200>หาย<200>.•
+•
+•เกี่ยว<200>กับ<200> •Unicode<200> •Consortium<200>
+•
+•Unicode<200> •Consortium<200> •เป็น<200>องค์กร<200>ไม่<200>แสวงหา<200>กำไร<200>ที่<200>ก่อ<200>ตั้ง<200>ขึ้น<200>เพื่อ<200>พัฒนา<200>,• •ขยาย<200>และ<200>ส่ง<200>เสริม<200>การ<200>ใช้<200> •Unicode<200> •Standard<200>,• •ซึ่ง<200>กำหนด<200>รูป<200>แบบ<200>การ<200>แทน<200>ค่า<200>ของ<200>ข้อความ<200>ใน<200>ผลิตภัณฑ์<200>ซอฟต์แวร์<200>และ<200>มาตร<200>ฐาน<200>ใหม่ๆ<200>.• •สมาชิก<200>ของ<200>สมาคม<200>เป็น<200>ตัวแทน<200>จาก<200>บริษัท<200>และ<200>องค์กร<200>ใน<200>อุตสาหกรรม<200>คอมพิวเตอร์<200>และ<200>การ<200>ประมวล<200>ผล<200>สารสนเทศ<200>.• •สมาคม<200>ได้<200>รับ<200>การ<200>สนับสนุน<200>ทางการ<200>เงิน<200>ผ่าน<200>ทาง<200>ค่า<200>ธรรมเนียม<200>ของ<200>การ<200>เป็น<200>สมาชิก<200>เท่านั้น<200>.• •สมาชิก<200>ภาพ<200>ของ<200> •Unicode<200> •Consortium<200> •เปิด<200>กว้าง<200>สำหรับ<200>องค์กร<200>หรือ<200>บุคคล<200>ใดๆ<200> •ใน<200>โลก<200>ที่<200>ต้องการ<200>สนับสนุน<200> •Unicode<200> •Standard<200> •และ<200>ช่วย<200>เหลือ<200>การ<200>ขยาย<200>ตัว<200>และ<200>การนำ<200> •Unicode<200> •ไป<200>ใช้<200>งาน<200>.•
+•
+•สำหรับ<200>ข้อมูล<200>เพิ่ม<200>เติม<200>,• •ให้<200>ดู<200>ที่<200> •Glossary<200>,• •Sample<200> •Unicode<200>-•Enabled<200> •Products<200>,• •Technical<200> •Introduction<200> •และ<200> •Useful<200> •Resources<200>.•</data>
+
+<word>
+# Burmese
+<data>•အ<200>လော<200>င္<200>မ<200>င္<200>တရား<200>
+• • • • • •မဟာ<200>ဓမ္မရာဇာ<200>မိ<200>ပတိ<200>လ<200>က္<200>ထ<200>က္<200>တ္<200>ဝ<200>င္<200> •အ<200>င္<200>ဝ<200>နေ<200>ပ္<200>ရ<200>ည္<200>တော္<200>က္<200>ရီး<200>သ<200>ည္<200> •မ<200>င္<200>ရိ<200>မ္<200>မ<200>သ<200>က္<200>ဖ္<200>ရ<200>စ္<200>နေ<200>သ<200>ည္<200>။• •မဏိ<200>ပူ<200>ရ<200> •က<200>သ<200>ည္<200>မ္<200>ယား<200>က<200> •အ<200>င္<200>ဝ<200>နေ<200>ပ္<200>ရ<200>ည္<200>တော္<200>၏• •မ္<200>ရော<200>က္<200>ဘ<200>က္<200>တ<200>လ္<200>ဝ္<200>ဟား<200>ကုိ<200> •တုိ<200>က္<200>ခုိ<200>က္<200>ဖ္<200>ယ<200>က္<200>ဆီး<200>သ<200>ည္<200>။• •အော<200>က္<200>မ္<200>ရ<200>န္<200>မာ<200>နုိ<200>င္<200>ငံ<200> •ဟံ<200>သာ<200>ဝ<200>တီ<200>သား<200>တုိ့<200>က<200>လ<200>ည္<200> •ပု<200>န္<200>က<200>န္<200>သ<200>ည္<200>။• •မတ္တ<200>ရာ<200>အု<200>တ္<200>ဖုိ<200>ရ္<200>ဟိ<200> •က္<200>ဝေ့<200>ရ္<200>ဟ<200>မ္<200>မ္<200>ယား<200>က<200>လ<200>ည္<200> •ထ<200>က္<200>ရ္<200>ဝ<200>သ<200>ည္<200>။•
+•
+• • • • •ထုိ<200>အ<200>ခ္<200>ယိ<200>န္<200>တ္<200>ဝ<200>င္<200> •မု<200>ဆုိး<200>ဖုိ<200>ရ္<200>ဝာ<200>သူ<200>က္<200>ရီး<200> •အော<200>င္<200>ဇေ<200>ယ္<200>ယ<200>သ<200>ည္<200> •မိမိ<200>၏•ရ္<200>ဝာ<200>ကုိ<200> •လုံ<200>ခ္<200>ရုံ<200>အော<200>င္<200>ထ<200>န္<200>လုံး<200>တ<200>ပ္<200>မ္<200>ယား<200>ကာ<200>ရ<200>သ<200>ည္<200>။• •အနီး<200>အ<200>ပား<200> •က္<200>ယေး<200>ရ္<200>ဝာ<200> •လေး<200>ဆ<200>ယ့္<200>ခ္<200>ရော<200>က္<200>ရ္<200>ဝာ<200>ကုိ<200> •သိ<200>မ္း<200>သ္<200>ဝ<200>င္<200>ထား<200>သ<200>ည္<200>။• •မ<200>က္<200>ရာ<200>မီ<200>ပ<200>င္<200> •အ<200>င္<200>ဝ<200>နေ<200>ပ္<200>ရ<200>ည္<200>တော္<200>က္<200>ရီး<200>သ<200>ည္<200> •ဟံ<200>သာ<200>ဝ<200>တီ<200>တ<200>ပ္<200>မ္<200>ယား<200> •လ<200>က္<200>တ္<200>ဝ<200>င္<200>သ<200>က္<200>ဆ<200>င္<200>ရ<200>တော့<200>သ<200>ည္<200>။•
+•
+• • • • •အ<200>င္<200>ဝ<200>ကုိ<200> •သိ<200>မ္<200>ပုိ<200>က္<200>ပ္<200>ရီး<200>သော<200> •ဟံ<200>သာ<200>ဝ<200>တီ<200>တ<200>ပ္<200>မ္<200>ယား<200>သ<200>ည္<200> •မ္<200>ရော<200>က္<200>ဘ<200>က္<200>တ<200>လ္<200>ဝ္<200>ဟား<200>က္<200>ယေး<200>ရ္<200>ဝာ<200>မ္<200>ယား<200>ကုိ<200> •သစ္စာ<200>ခံ<200>ခုိ<200>င္<200>ရ<200>န္<200> •လာ<200>က္<200>ရ<200>ရာ<200> •မု<200>ဆုိး<200>ဖုိ<200>ရ္<200>ဝာ<200>သုိ့<200> •ရော<200>က္<200>ရ္<200>ဟိ<200>လာ<200>သ<200>ည္<200>။• •တ<200>ခ္<200>ယိ<200>န္<200>တ<200>ည္<200>မ္<200>ဟာ<200>ပ<200>င္<200> •က္<200>ဝေ့<200>ရ္<200>ဟ<200>မ္<200>မ္<200>ယား<200>က<200>လ<200>ည္<200> •သစ္စာ<200>ခံ<200>ခုိ<200>င္<200>ရ<200>န္<200> •ရော<200>က္<200>ရ္<200>ဟိ<200>လာ<200>သ<200>ည္<200>။• •ဦး<200>အော<200>င္<200>ဇေ<200>ယ္<200>ယ<200>သ<200>ည္<200> •အ<200>ဖ္<200>ဝဲ့<200>န္<200>ဟ<200>စ္<200>ဖ္<200>ဝဲ့<200>ကုိ<200> •ခ္<200>ရေ<200>ငံ<200>စ္<200>ဝာ<200> •ဆ<200>က္<200>ဆံ<200>သ<200>ည္<200>။• •မ<200>ည္<200>သူ့<200>သ<200>စ္<200>စာ<200>ကုိ<200>မ္<200>ယ္<200>ဟ<200> •ခံ<200>ယူ<200>ခ္<200>ရ<200>င္<200>မ<200>ပ္<200>ရု<200>ပေ<200>။• •ဟံ<200>သာ<200>ဝ<200>တီ<200>တ<200>ပ္<200>ဖ္<200>ဝဲ့<200>ကုိ<200> •အ<200>ပ္<200>ရ<200>န္<200>ခ<200>ရီး<200>တ္<200>ဝ<200>င္<200> •လ<200>မ္<200>မ္<200>ဟ<200>ဖ္<200>ရ<200>တ္<200>၍• •တုိ<200>က္<200>ခုိ<200>က္<200>သ<200>ည္<200>။• •ဟံ<200>သာ<200>ဝ<200>တီ<200>တ<200>ပ္<200>ဖ္<200>ဝဲ့<200>မ္<200>ယား<200> •အထိ<200>အ<200>ခုိ<200>က္<200>အ<200>က္<200>ယ<200>အ<200>ဆုံး<200>မ္<200>ယား<200>စ္<200>ဝာ<200>ဖ္<200>ရ<200>င္<200> •ပ္<200>ရ<200>န္<200>ရ<200>သ<200>ည္<200>။•
+•
+• • • • •ဟံ<200>သာ<200>ဝ<200>တီ<200>တ<200>ပ္<200>မ္<200>ယား<200>သ<200>ည္<200> •မု<200>ဆုိး<200>ဖုိ<200>ရ္<200>ဝာ<200>ကုိ<200> •လာ<200>ရော<200>က္<200>တုိ<200>က္<200>ခုိ<200>က္<200>က္<200>ရ<200>ပ္<200>ရ<200>န္<200>သ<200>ည္<200>။• •ဦး<200>အော<200>င္<200>ဇေ<200>ယ္<200>ယ<200>သ<200>ည္<200> •သ္<200>ဝေး<200>သော<200>က္<200>ရဲ<200>ဘော္<200> •ခ္<200>ရော<200>က္<200>က္<200>ယိ<200>ပ္<200>ရ္<200>ဟ<200>စ္<200>ယော<200>က္<200>န္<200>ဟ<200>င္<200>အတူ<200> •ဦးစီး<200>ကာ<200>အော<200>င္<200>မ္<200>ရ<200>င္<200>စ္<200>ဝာ<200>ခု<200>ခံ<200>တ္<200>ဝ<200>န္<200>လ္<200>ဟ<200>န္<200>နုိ<200>င္<200>ခဲ့<200>သ<200>ည္<200>။• •ထုိ့<200>နော<200>က္<200> •ဦး<200>အော<200>င္<200>ဇေ<200>ယ္<200>ယ<200>သ<200>ည္<200> •မ္<200>ရော<200>က္<200>ဘ<200>က္<200>တ<200>လ္<200>ဝ္<200>ဟား<200>ရ္<200>ဟိ<200> •ရ္<200>ဟ<200>မ္<200>မ္<200>ယား<200>န္<200>ဟ<200>င္<200> •မ္<200>ရ<200>န္<200>မာ<200>မ္<200>ယား<200>ကုိ<200>လ<200>ည္<200> •ဆ<200>က္<200>သ္<200>ဝ<200>ယ္<200>စ<200>ည္<200>ရုံး<200>နုိ<200>င္<200>ခဲ့<200>သ<200>ည္<200>။• •ဤ<200>သုိ့<200>ဖ္<200>ရ<200>င္<200> •ဦး<200>အော<200>င္<200>ဇေ<200>ယ္<200>ယ<200>၏• •အ<200>ရ္<200>ဟိ<200>န္<200>အ<200>ဝာ<200> •မ္<200>ရ<200>င္<200>မား<200>လာ<200>လေ<200>သ<200>ည္<200>။•
+•
+• • • • •ဦး<200>အော<200>င္<200>ဇေ<200>ယ္<200>ယ<200>သ<200>ည္<200> •အ<200>လော<200>င္<200>မ<200>င္<200>တ<200>ရား<200>ဘ္<200>ဝဲ့<200>ကုိ<200> •ခံယူ<200>ကာ<200> •ကု<200>န္<200>ဘော<200>င္<200>မ<200>င္<200>ဆ<200>က္<200>ကုိ<200>စ<200>တ<200>င္<200>တ<200>ည္<200>ထော<200>င္<200>သ<200>ည္<200>။• •မု<200>ဆုိး<200>ဖုိ<200>ရ္<200>ဝာ<200>ကုိ<200> •ရ္<200>ဝ္<200>ဟ<200>ဝေ<200>ဘုိ<200>ဟု<200> •သ<200>မု<200>တ္<200>ကာ<200> •မ္<200>ရုိ့<200>န<200>န္<200>တ<200>ည္<200>သ<200>ည္<200>။• •န<200>န္<200>တ<200>ည္<200>သ<200>က္<200>က<200>ရာ<200>ဇ္<200>ဖ္<200>ရ<200>စ္<200>သော<200> •၁၁၁၅<100> •ခု<200>ကုိ<200> •ဥ<200>ဩ<200>အော္<200>မ္<200>ရ<200>ည္<200> •ကု<200>န္<200>ဘော<200>င္<200>တ<200>ည္<200>ဟု<200> •အ<200>မ္<200>ဟ<200>တ္<200>အ<200>သား<200>ပ္<200>ရု<200>က္<200>ရ<200>သ<200>ည္<200>။•
+•
+• • • • •အ<200>လော<200>င္<200>မ<200>င္<200>တရား<200>သ<200>ည္<200> •ဧရာ<200>ဝ<200>တီ<200>န္<200>ဟ<200>င္<200>ခ္<200>ယ<200>င္<200>တ္<200>ဝ<200>င္<200> •မ္<200>ရ<200>စ္<200>န္<200>ဟ<200>စ္<200>သ္<200>ဝ<200>ယ္<200>အ<200>က္<200>ရား<200> •ဒေ<200>သ<200>မ္<200>ယား<200>ကုိ<200>အ<200>ခုိ<200>င္<200>အ<200>မာ<200> •စု<200>စ<200>ည္<200>ပ္<200>ရီး<200>နော<200>က္<200> •အ<200>င္<200>ဝ<200>ကုိ<200> •တုိ<200>က္<200>ခုိ<200>က္<200>အော<200>င္<200>မ္<200>ရ<200>င္<200>သ<200>ည္<200>။• •ထုိ<200>နော<200>က္<200>တ္<200>ဝ<200>င္<200>ပ္<200>ရ<200>ည္<200>၊• •လ္<200>ဝ<200>န္<200>ဆေး<200>၊• •ဒ<200>ဂုံ<200>မ္<200>ရုိ့<200>မ္<200>ယား<200>ကုိ<200> •သိ<200>မ္<200>ပုိ<200>က္<200>သ<200>ည္<200>။• •လ္<200>ဝ<200>န္<200>ဆေး<200> •ကုိ<200>မ္<200>ရ<200>န္<200>အော<200>င္<200>ဟူ<200>၍• •သ<200>မု<200>တ္<200>သ<200>ည္<200>။• •ဒ<200>ဂုံ<200>ကုိ<200>ရ<200>န္<200>ကု<200>န္<200>ဟူ<200>၍• •သ<200>မု<200>တ္<200>ထ<200>သ<200>ည္<200>။•</data>
+
+<word>
+# japanese
+<data>•ユニ<400>コード<400>と<400>は<400>何<400>か<400>？•
+•ユニ<400>コード<400>は<400>、•すべて<400>の<400>文字<400>に<400>固有<400>の<400>番号<400>を<400>付与<400>し<400>ます<400>
+•プラットフォーム<400>に<400>は<400>依存<400>しま<400>せん<400>
+•プログラム<400>に<400>も<400>依存<400>しま<400>せん<400>
+•言語<400>に<400>も<400>依存<400>しま<400>せん<400>
+•
+•コンピューター<400>は<400>、•本質<400>的<400>に<400>は<400>数字<400>しか<400>扱う<400>こと<400>が<400>でき<400>ま<400>せん<400>。•コンピューター<400>は<400>、•文字<400>や<400>記号<400>など<400>の<400>それぞれに<400>番号<400>を<400>割り振る<400>こと<400>によって<400>扱える<400>よう<400>にし<400>ます<400>。•ユニ<400>コード<400>が<400>出来る<400>まで<400>は<400>、•これらの<400>番号<400>を<400>割り振る<400>仕組み<400>が<400>何<400>百<400>種類<400>も<400>存在<400>しま<400>した<400>。•どの<400>一つ<400>を<400>とっても<400>、•十分<400>な<400>文字<400>を<400>含<400>んで<400>は<400>いま<400>せん<400>で<400>した<400>。•例えば<400>、•欧州<400>連合<400>一つ<400>を<400>見<400>て<400>も<400>、•その<400>すべて<400>の<400>言語<400>を<400>カバー<400>する<400>ため<400>に<400>は<400>、•いくつか<400>の<400>異なる<400>符号<400>化<400>の<400>仕組み<400>が<400>必要<400>で<400>した<400>。•英語<400>の<400>よう<400>な<400>一つ<400>の<400>言語<400>に<400>限<400>って<400>も<400>、•一つ<400>だけ<400>の<400>符号<400>化<400>の<400>仕組み<400>では<400>、•一般<400>的<400>に<400>使<400>われる<400>すべて<400>の<400>文字<400>、•句読点<400>、•技術<400>的<400>な<400>記号<400>など<400>を<400>扱う<400>に<400>は<400>不十分<400>で<400>した<400>。•
+•
+•これらの<400>符号<400>化<400>の<400>仕組み<400>は<400>、•相互<400>に<400>矛盾<400>する<400>もの<400>でも<400>ありま<400>した<400>。•二つ<400>の<400>異なる<400>符号<400>化<400>の<400>仕組み<400>が<400>、•二つ<400>の<400>異なる<400>文字<400>に<400>同一<400>の<400>番号<400>を<400>付ける<400>こと<400>も<400>できる<400>し<400>、•同じ<400>文字<400>に<400>異なる<400>番号<400>を<400>付ける<400>こと<400>も<400>できる<400>の<400>です<400>。•どの<400>よう<400>な<400>コンピューター<400>も<400>（•特に<400>サーバー<400>は<400>）•多く<400>の<400>異<400>な<400>っ<400>た<400>符号<400>化<400>の<400>仕組み<400>を<400>サポート<400>する<400>必要<400>が<400>あり<400>ます<400>。•たとえ<400>データ<400>が<400>異なる<400>符号<400>化<400>の<400>仕組み<400>や<400>プラットフォーム<400>を<400>通過<400>し<400>て<400>も<400>、•いつ<400>どこ<400>で<400>データ<400>が<400>乱れる<400>か<400>分<400>から<400>ない<400>危険<400>を<400>冒す<400>こと<400>の<400>なる<400>の<400>です<400>。•
+•
+•ユニ<400>コード<400>は<400>すべて<400>を<400>変<400>え<400>ます<400>
+•
+•ユニ<400>コード<400>は<400>、•プラットフォーム<400>に<400>係<400>わら<400>ず<400>、•プログラム<400>に<400>係<400>わら<400>ず<400>、•言語<400>に<400>係<400>わら<400>ず<400>、•すべて<400>の<400>文字<400>に<400>独立<400>した<400>番号<400>を<400>与<400>え<400>ます<400>。•ユニ<400>コード<400>標準<400>は<400>、•アップル<400>、•ヒュー<400>レット<400>パッ<400>カード<400>、•IBM<200>、•ジャスト<400>システム<400>、•マイクロ<400>ソフト<400>、•オラクル<400>、•SAＰ<200>、•サン<400>、•サイ<400>ベース<400>など<400>の<400>産業<400>界<400>の<400>主導<400>的<400>企業<400>と<400>他の<400>多く<400>の<400>企業<400>に<400>採用<400>さ<400>れ<400>てい<400>ます<400>。•ユニ<400>コード<400>は<400>、•XML<200>、•Java<200>、•ECMAScript<200>(•JavaScript<200>)•、•LDAP<200>、•CORBA<200> •3.0<100>など<400>の<400>最先端<400>の<400>標準<400>の<400>前提<400>と<400>な<400>って<400>おり<400>、•ユニ<400>コード<400>を<400>実装<400>す<400>れ<400>ば<400>、•ISO<200>/•IEC<200> •10646<100>に<400>適合<400>する<400>ことに<400>なり<400>ます<400>。•ユニ<400>コード<400>は<400>、•多く<400>の<400>オペレーティングシステム<400>と<400>すべて<400>の<400>最新<400>の<400>ブラウザー<400>と<400>他の<400>多く<400>の<400>製品<400>で<400>サポート<400>さ<400>れ<400>てい<400>ます<400>。•ユニ<400>コード<400>標準<400>の<400>出現<400>と<400>ユニ<400>コード<400>を<400>サポート<400>する<400>ツール<400>類<400>は<400>、•昨今<400>顕著<400>に<400>な<400>って<400>いる<400>ソフトウエア<400>技術<400>の<400>グローバル<400>化<400>の<400>流れ<400>に対して<400>、•特に<400>役<400>に<400>立<400>って<400>い<400>ます<400>。•
+•
+•ユニ<400>コード<400>を<400>ク<400>ライアン<400>ト<400>サーバー<400>型<400>の<400>アプリケーション<400>や<400>、•多層<400>構造<400>を<400>持つ<400>アプリケーション<400>、•ウェブサイト<400>など<400>に<400>組み込む<400>こと<400>で<400>、•従来<400>の<400>文字<400>コードセット<400>を<400>用いる<400>より<400>も<400>明らか<400>な<400>コスト<400>削減<400>が<400>可能<400>です<400>。•ユニ<400>コード<400>は<400>、•単一<400>の<400>ソフトウエア<400>製品<400>、•単一<400>の<400>ウェブサイト<400>に<400>、•何ら<400>手<400>を<400>加える<400>こと<400>なく<400>、•複数<400>の<400>プラットフォーム<400>、•複数<400>の<400>言語<400>、•複数<400>の<400>国<400>を<400>カバー<400>する<400>こと<400>が<400>出来る<400>の<400>です<400>。•ユニ<400>コード<400>は<400>、•データ<400>が<400>多く<400>の<400>異なる<400>システム<400>の<400>間<400>を<400>、•何<400>の<400>乱れ<400>も<400>なし<400>に<400>転送<400>する<400>こと<400>を<400>可能<400>と<400>する<400>の<400>です<400>。•
+•
+•ユニ<400>コード<400>コンソーシアム<400>について<400>
+•
+•ユニ<400>コード<400>コンソーシアム<400>は<400>、•最新<400>の<400>ソフトウエア<400>製品<400>と<400>標準<400>において<400>テキスト<400>を<400>表現<400>する<400>こと<400>を<400>意味<400>する<400>“•ユニ<400>コード<400>標準<400>”•の<400>構築<400>、•発展<400>、•普及<400>、•利用<400>促進<400>を<400>目的<400>として<400>設立<400>さ<400>れ<400>た<400>非<400>営利<400>組織<400>です<400>。•同<400>コンソーシアム<400>の<400>会員<400>は<400>、•コンピューター<400>と<400>情報処理<400>に<400>係わる<400>広汎<400>な<400>企業<400>や<400>組織<400>から<400>構成<400>さ<400>れ<400>てい<400>ます<400>。•同<400>コンソーシアム<400>は<400>、•財政<400>的<400>に<400>は<400>、•純粋<400>に<400>会費<400>のみ<400>によって<400>運営<400>さ<400>れ<400>てい<400>ます<400>。•ユニ<400>コード<400>標準<400>を<400>支持<400>し<400>、•その<400>拡張<400>と<400>実装<400>を<400>支援<400>する<400>世界中<400>の<400>組織<400>や<400>個人<400>は<400>、•だれ<400>も<400>が<400>ユニ<400>コード<400>コンソーシアム<400>の<400>会員<400>なる<400>こと<400>が<400>でき<400>ます<400>。•
+•
+•より<400>詳しい<400>こと<400>を<400>お<400>知<400>り<400>に<400>なり<400>たい<400>方<400>は<400>、•Glossary<200>,• •Technical<200> •Introduction<200> •および<400> •Useful<200> •Resources<200>を<400>ご<400>参照<400>くだ<400>さい<400>。•
+•</data>
* For internal use only.
* @internal
*/
- static class DequeI {
+ static class DequeI implements Cloneable {
private int[] data = new int[50];
private int lastIdx = 4; // or base of stack. Index of element.
private int firstIdx = 4; // or Top of Stack. Index of element + 1.
+ @Override
+ public Object clone() throws CloneNotSupportedException {
+ DequeI result = (DequeI)super.clone();
+ data = data.clone();
+ return result;
+ }
+
int size() {
return firstIdx - lastIdx;
}
}
return false;
}
+
+ int elementAt(int i) {
+ assert i < size();
+ return data[lastIdx + i];
+ }
+
+ void removeAllElements() {
+ lastIdx = firstIdx = 4;
+ }
}
UnicodeSet fSet = new UnicodeSet();
@Override
public int findBreaks(CharacterIterator text, int startPos, int endPos,
- boolean reverse, int breakType, DequeI foundBreaks) {
- int result = 0;
+ int breakType, DequeI foundBreaks) {
+ int result = 0;
// Find the span of characters included in the set.
// The span to break begins at the current position int the text, and
int rangeStart;
int rangeEnd;
int c = CharacterIteration.current32(text);
- if (reverse) {
- boolean isDict = fSet.contains(c);
- while ((current = text.getIndex()) > startPos && isDict) {
- c = CharacterIteration.previous32(text);
- isDict = fSet.contains(c);
- }
- rangeStart = (current < startPos) ? startPos :
- current + (isDict ? 0 : 1);
- rangeEnd = start + 1;
- } else {
- while ((current = text.getIndex()) < endPos && fSet.contains(c)) {
- CharacterIteration.next32(text);
- c = CharacterIteration.current32(text);
- }
- rangeStart = start;
- rangeEnd = current;
+ while ((current = text.getIndex()) < endPos && fSet.contains(c)) {
+ CharacterIteration.next32(text);
+ c = CharacterIteration.current32(text);
}
+ rangeStart = start;
+ rangeEnd = current;
+ // if (breakType >= 0 && breakType < 32 && (((uint32_t)1 << breakType) & fTypes)) {
+ // TODO: Why does icu4c have this?
result = divideUpDictionaryRange(text, rangeStart, rangeEnd, foundBreaks);
text.setIndex(current);
import java.text.CharacterIterator;
/**
- * The LanguageBreakEngine interface is to be used to implement any
+ * The LanguageBreakEngine interface is to be used to implement any
* language-specific logic for break iteration.
*/
interface LanguageBreakEngine {
boolean handles(int c, int breakType);
/**
- * Implements the actual breaking logic.
- * @param text The text to break over
- * @param startPos The index of the beginning of our range
+ * Implements the actual breaking logic. Find any breaks within a run in the supplied text.
+ * @param text The text to break over. The iterator is left at
+ * the end of the run of characters which the engine has handled.
+ * @param startPos The index of the beginning of the range
* @param endPos The index of the possible end of our range. It is possible,
- * however, that our range ends earlier
- * @param reverse true iff we are iterating backwards (in a call to
- * previous(), for example)
+ * however, that the range ends earlier
* @param breakType The kind of break iterator that is wanting to make use
* of this engine - character, word, line, sentence
- * @param foundBreaks A Stack that the breaks found will be added to
- * @return the number of words found
+ * @param foundBreaks A data structure to receive the break positions.
+ * @return the number of breaks found
*/
int findBreaks(CharacterIterator text, int startPos, int endPos,
- boolean reverse, int breakType, DictionaryBreakEngine.DequeI foundBreaks);
+ int breakType, DictionaryBreakEngine.DequeI foundBreaks);
}
-
-
-
+
+
+
pos += This.fHeader.fSRTableLen;
}
+ // Rule Compatibility Hacks
+ // If a rule set includes reverse rules but does not explicitly include safe reverse rules,
+ // the reverse rules are to be treated as safe reverse rules.
+
+ if (This.fSRTable == null && This.fRTable != null) {
+ This.fSRTable = This.fRTable;
+ This.fRTable = null;
+ }
+
//
// Unserialize the Character categories TRIE
// Because we can't be absolutely certain where the Trie deserialize will
///CLOVER:OFF
/** Dump a state table. (A full set of RBBI rules has 4 state tables.) */
private void dumpTable(java.io.PrintStream out, short table[]) {
- if (table == null) {
+ if (table == null || table.length == 0) {
out.println(" -- null -- ");
} else {
int n;
int headerSize = 24 * 4; // align8(sizeof(RBBIDataHeader));
int forwardTableSize = align8(fForwardTables.getTableSize());
int reverseTableSize = align8(fReverseTables.getTableSize());
- int safeFwdTableSize = align8(fSafeFwdTables.getTableSize());
+ // int safeFwdTableSize = align8(fSafeFwdTables.getTableSize());
int safeRevTableSize = align8(fSafeRevTables.getTableSize());
int trieSize = align8(fSetBuilder.getTrieSize());
int statusTableSize = align8(fRuleStatusVals.size() * 4);
int rulesSize = align8((strippedRules.length()) * 2);
- int totalSize = headerSize + forwardTableSize + reverseTableSize
- + safeFwdTableSize + safeRevTableSize
- + statusTableSize + trieSize + rulesSize;
+
+ int totalSize = headerSize
+ + forwardTableSize
+ + /* reverseTableSize */ 0
+ + /* safeFwdTableSize */ 0
+ + (safeRevTableSize > 0 ? safeRevTableSize : reverseTableSize)
+ + statusTableSize + trieSize + rulesSize;
int outputPos = 0; // Track stream position, starting from RBBIDataHeader.
//
header[RBBIDataWrapper.DH_FORMATVERSION] = RBBIDataWrapper.FORMAT_VERSION;
header[RBBIDataWrapper.DH_LENGTH] = totalSize; // fLength, the total size of all rule sections.
header[RBBIDataWrapper.DH_CATCOUNT] = fSetBuilder.getNumCharCategories(); // fCatCount.
+
+ // Only save the forward table and the safe reverse table,
+ // because these are the only ones used at run-time.
+ //
+ // For the moment, we still build the other tables if they are present in the rule source files,
+ // for backwards compatibility. Old rule files need to work, and this is the simplest approach.
+ //
+ // Additional backwards compatibility consideration: if no safe rules are provided, consider the
+ // reverse rules to actually be the safe reverse rules.
+
header[RBBIDataWrapper.DH_FTABLE] = headerSize; // fFTable
header[RBBIDataWrapper.DH_FTABLELEN] = forwardTableSize; // fTableLen
+
+ // Do not save Reverse Table.
header[RBBIDataWrapper.DH_RTABLE] = header[RBBIDataWrapper.DH_FTABLE] + forwardTableSize; // fRTable
- header[RBBIDataWrapper.DH_RTABLELEN] = reverseTableSize; // fRTableLen
+ header[RBBIDataWrapper.DH_RTABLELEN] = 0; // fRTableLen
+
+ // Do not save the Safe Forward table.
header[RBBIDataWrapper.DH_SFTABLE] = header[RBBIDataWrapper.DH_RTABLE]
- + reverseTableSize; // fSTable
- header[RBBIDataWrapper.DH_SFTABLELEN] = safeFwdTableSize; // fSTableLen
+ + 0; // fSTable
+ header[RBBIDataWrapper.DH_SFTABLELEN] = 0; // fSTableLen
+
+ // Safe reverse table. Use if present, otherwise save regular reverse table as the safe reverse.
header[RBBIDataWrapper.DH_SRTABLE] = header[RBBIDataWrapper.DH_SFTABLE]
- + safeFwdTableSize; // fSRTable
- header[RBBIDataWrapper.DH_SRTABLELEN] = safeRevTableSize; // fSRTableLen
+ + 0; // fSRTable
+ if (safeRevTableSize > 0) {
+ header[RBBIDataWrapper.DH_SRTABLELEN] = safeRevTableSize;
+ } else {
+ assert reverseTableSize > 0;
+ header[RBBIDataWrapper.DH_SRTABLELEN] = reverseTableSize;
+ }
+
header[RBBIDataWrapper.DH_TRIE] = header[RBBIDataWrapper.DH_SRTABLE]
- + safeRevTableSize; // fTrie
+ + header[RBBIDataWrapper.DH_SRTABLELEN]; // fTrie
header[RBBIDataWrapper.DH_TRIELEN] = fSetBuilder.getTrieSize(); // fTrieLen
header[RBBIDataWrapper.DH_STATUSTABLE] = header[RBBIDataWrapper.DH_TRIE]
+ header[RBBIDataWrapper.DH_TRIELEN];
outputPos += 2;
}
+ /* do not write the reverse table
tableData = fReverseTables.exportTable();
Assert.assrt(outputPos == header[6]);
for (i = 0; i < tableData.length; i++) {
dos.writeShort(tableData[i]);
outputPos += 2;
}
+ */
+ /* do not write safe forwards table
Assert.assrt(outputPos == header[8]);
tableData = fSafeFwdTables.exportTable();
for (i = 0; i < tableData.length; i++) {
dos.writeShort(tableData[i]);
outputPos += 2;
}
+ */
+ // Write the safe reverse table.
+ // If not present, write the plain reverse table (old style rule compatibility)
Assert.assrt(outputPos == header[10]);
- tableData = fSafeRevTables.exportTable();
+ if (safeRevTableSize > 0) {
+ tableData = fSafeRevTables.exportTable();
+ } else {
+ tableData = fReverseTables.exportTable();
+ }
for (i = 0; i < tableData.length; i++) {
dos.writeShort(tableData[i]);
outputPos += 2;
UnicodeSet fRuleSets[] = new UnicodeSet[10]; // Unicode Sets that are needed during
// the scanning of RBBI rules. The
- // indicies for these are assigned by the
+ // Indices for these are assigned by the
// perl script that builds the state tables.
// See rbbirpt.h.
// keyword, while being scanned.
-
+ // gRuleSet_rule_char_pattern is characters that may appear as literals in patterns without escaping or quoting.
static private String gRuleSet_rule_char_pattern = "[^[\\p{Z}\\u0020-\\u007f]-[\\p{L}]-[\\p{N}]]";
static private String gRuleSet_name_char_pattern = "[_\\p{L}\\p{N}]";
static private String gRuleSet_digit_char_pattern = "[0-9]";
fRB.fDefaultTree = RBBIRuleBuilder.fSafeRevTree;
} else if (opt.equals("lookAheadHardBreak")) {
fRB.fLookAheadHardBreak = true;
+ } else if (opt.equals("quoted_literals_only")) {
+ fRuleSets[RBBIRuleParseTable.kRuleSet_rule_char - 128].clear();
+ } else if (opt.equals("unquoted_literals")) {
+ fRuleSets[RBBIRuleParseTable.kRuleSet_rule_char - 128].applyPattern(gRuleSet_rule_char_pattern);
} else {
error(RBBIRuleBuilder.U_BRK_UNRECOGNIZED_OPTION);
}
}
}
-
+
// If there are no forward rules throw an error.
//
if (fRB.fTreeRoots[RBBIRuleBuilder.fForwardTree] == null) {
if (fRB.fDebugEnv!=null && fRB.fDebugEnv.indexOf("rgroup")>=0) {printRangeGroups();}
if (fRB.fDebugEnv!=null && fRB.fDebugEnv.indexOf("esets")>=0) {printSets();}
- fTrie = new Trie2Writable(0, // Initial value for all code points
- 0); // Error value.
+ fTrie = new Trie2Writable(0, // Initial value for all code points.
+ 0); // Error value for out-of-range input.
for (rlRange = fRangeList; rlRange!=null; rlRange=rlRange.fNext) {
- fTrie.setRange(rlRange.fStartChar, rlRange.fEndChar, rlRange.fNum, true);
+ fTrie.setRange(
+ rlRange.fStartChar, // Range start
+ rlRange.fEndChar, // Range end (inclusive)
+ rlRange.fNum, // value for range
+ true // Overwrite previously written values
+ );
}
}
import java.util.ArrayList;
import java.util.List;
-import com.ibm.icu.impl.Assert;
import com.ibm.icu.impl.CharacterIteration;
import com.ibm.icu.impl.ICUBinary;
import com.ibm.icu.impl.ICUDebug;
* private constructor
*/
private RuleBasedBreakIterator() {
- fLastStatusIndexValid = true;
fDictionaryCharCount = 0;
synchronized(gAllBreakEngines) {
fBreakEngines = new ArrayList<LanguageBreakEngine>(gAllBreakEngines);
* @stable ICU 2.0
*/
@Override
- public Object clone()
- {
- RuleBasedBreakIterator result = (RuleBasedBreakIterator)super.clone();
+ public Object clone() {
+ RuleBasedBreakIterator result;
+ result = (RuleBasedBreakIterator)super.clone();
if (fText != null) {
result.fText = (CharacterIterator)(fText.clone());
}
result.fBreakEngines = new ArrayList<LanguageBreakEngine>(gAllBreakEngines);
}
result.fLookAheadMatches = new LookAheadResults();
- if (fCachedBreakPositions != null) {
- result.fCachedBreakPositions = fCachedBreakPositions.clone();
- }
+ result.fBreakCache = result.new BreakCache(fBreakCache);
+ result.fDictionaryCache = result.new DictionaryCache(fDictionaryCache);
return result;
}
+
/**
* Returns true if both BreakIterators are of the same class, have the same
* rules, and iterate over the same text.
if (fText == null && other.fText == null) {
return true;
}
- if (fText == null || other.fText == null) {
+ if (fText == null || other.fText == null || !fText.equals(other.fText)) {
return false;
}
- return fText.equals(other.fText);
+ return fPosition == other.fPosition;
}
catch(ClassCastException e) {
return false;
*/
RBBIDataWrapper fRData;
- /*
+ /**
+ * The iteration state - current position, rule status for the current position,
+ * and whether the iterator ran off the end, yielding UBRK_DONE.
+ * Current position is pinned to be 0 < position <= text.length.
+ * Current position is always set to a boundary.
+ *
+ * The current position of the iterator. Pinned, 0 < fPosition <= text.length.
+ * Never has the value UBRK_DONE (-1).
+ */
+ private int fPosition;
+
+ /**
* Index of the Rule {tag} values for the most recent match.
*/
- private int fLastRuleStatusIndex;
+ private int fRuleStatusIndex;
- /*
- * Rule tag value valid flag.
- * Some iterator operations don't intrinsically set the correct tag value.
- * This flag lets us lazily compute the value if we are ever asked for it.
+ /**
+ * True when iteration has run off the end, and iterator functions should return UBRK_DONE.
+ */
+ private boolean fDone;
+
+ /**
+ * Cache of previously determined boundary positions.
*/
- private boolean fLastStatusIndexValid;
+ private BreakCache fBreakCache = new BreakCache();
+
/**
* Counter for the number of characters encountered with the "dictionary"
*/
private int fDictionaryCharCount;
+ private DictionaryCache fDictionaryCache = new DictionaryCache();
+
/*
* ICU debug argument name for RBBI
*/
&& ICUDebug.value(RBBI_DEBUG_ARG).indexOf("trace") >= 0;
/**
- * What kind of break iterator this is. Set to KIND_LINE by default,
- * since this produces sensible output.
+ * What kind of break iterator this is.
+ * Defaulting BreakType to word gives reasonable dictionary behavior for
+ * Break Iterators that are built from rules.
*/
- private int fBreakType = KIND_LINE;
+ private int fBreakType = KIND_WORD;
/**
* The "default" break engine - just skips over ranges of dictionary words,
*/
private List<LanguageBreakEngine> fBreakEngines;
- /**
- * when a range of characters is divided up using the dictionary, the break
- * positions that are discovered are stored here, preventing us from having
- * to use either the dictionary or the state table again until the iterator
- * leaves this range of text
- */
- private int[] fCachedBreakPositions;
-
- /**
- * if fCachedBreakPositions is not null, this indicates which item in the
- * cache the current iteration position refers to
- */
- private int fPositionInCache;
-
- /**
- * Dumps caches and performs other actions associated with a complete change
- * in text or iteration position.
- */
- private void reset() {
- fCachedBreakPositions = null;
- // fNumCachedBreakPositions = 0;
- fDictionaryCharCount = 0;
- fPositionInCache = 0;
-
- }
/**
* Dump the contents of the state table and character classes for this break iterator.
* For debugging only.
*/
@Override
public int first() {
- fCachedBreakPositions = null;
- fDictionaryCharCount = 0;
- fPositionInCache = 0;
- fLastRuleStatusIndex = 0;
- fLastStatusIndexValid = true;
if (fText == null) {
return BreakIterator.DONE;
}
fText.first();
- return fText.getIndex();
+ int start = fText.getIndex();
+ if (!fBreakCache.seek(start)) {
+ fBreakCache.populateNear(start);
+ }
+ fBreakCache.current();
+ assert(fPosition == start);
+ return fPosition;
}
/**
*/
@Override
public int last() {
- fCachedBreakPositions = null;
- fDictionaryCharCount = 0;
- fPositionInCache = 0;
-
if (fText == null) {
- fLastRuleStatusIndex = 0;
- fLastStatusIndexValid = true;
return BreakIterator.DONE;
}
-
- // t.last() returns the offset of the last character,
- // rather than the past-the-end offset
- // so a loop like for(p=it.last(); p!=DONE; p=it.previous()) ...
- // will work correctly.
- fLastStatusIndexValid = false;
- int pos = fText.getEndIndex();
- fText.setIndex(pos);
- return pos;
+ int endPos = fText.getEndIndex();
+ boolean endShouldBeBoundary = isBoundary(endPos); // Has side effect of setting iterator position.
+ assert(endShouldBeBoundary);
+ if (fPosition != endPos) {
+ assert(fPosition == endPos);
+ }
+ return endPos;
}
/**
*/
@Override
public int next(int n) {
- int result = current();
- while (n > 0) {
- result = next();
- --n;
- }
- while (n < 0) {
- result = previous();
- ++n;
+ int result = 0;
+ if (n > 0) {
+ for (; n > 0 && result != DONE; --n) {
+ result = next();
+ }
+ } else if (n < 0) {
+ for (; n < 0 && result != DONE; ++n) {
+ result = previous();
+ }
+ } else {
+ result = current();
}
return result;
}
*/
@Override
public int next() {
- // if we have cached break positions and we're still in the range
- // covered by them, just move one step forward in the cache
- if (fCachedBreakPositions != null) {
- if (fPositionInCache < fCachedBreakPositions.length - 1) {
- ++fPositionInCache;
- int pos = fCachedBreakPositions[fPositionInCache];
- fText.setIndex(pos);
- return pos;
- }
- else {
- reset();
- }
- }
-
- int startPos = current();
- fDictionaryCharCount = 0;
- int result = handleNext(fRData.fFTable);
- if (fDictionaryCharCount > 0) {
- result = checkDictionary(startPos, result, false);
- }
- return result;
+ fBreakCache.next();
+ return fDone ? DONE : fPosition;
}
/**
- * checkDictionary This function handles all processing of characters in
- * the "dictionary" set. It will determine the appropriate
- * course of action, and possibly set up a cache in the
- * process.
- */
- private int checkDictionary(int startPos, int endPos, boolean reverse) {
-
- // Reset the old break cache first.
- reset();
-
- // note: code segment below assumes that dictionary chars are in the
- // startPos-endPos range
- // value returned should be next character in sequence
- if ((endPos - startPos) <= 1) {
- return (reverse ? startPos : endPos);
- }
-
- // Starting from the starting point, scan towards the proposed result,
- // looking for the first dictionary character (which may be the one
- // we're on, if we're starting in the middle of a range).
- fText.setIndex(reverse ? endPos : startPos);
- if (reverse) {
- CharacterIteration.previous32(fText);
- }
-
- int rangeStart = startPos;
- int rangeEnd = endPos;
-
- int category;
- int current;
- DictionaryBreakEngine.DequeI breaks = new DictionaryBreakEngine.DequeI();
- int foundBreakCount = 0;
- int c = CharacterIteration.current32(fText);
- category = (short)fRData.fTrie.get(c);
-
- // Is the character we're starting on a dictionary character? If so, we
- // need to back up to include the entire run; otherwise the results of
- // the break algorithm will differ depending on where we start. Since
- // the result is cached and there is typically a non-dictionary break
- // within a small number of words, there should be little performance impact.
- if ((category & 0x4000) != 0) {
- if (reverse) {
- do {
- CharacterIteration.next32(fText);
- c = CharacterIteration.current32(fText);
- category = (short)fRData.fTrie.get(c);
- } while (c != CharacterIteration.DONE32 && ((category & 0x4000)) != 0);
-
- // Back up to the last dictionary character
- rangeEnd = fText.getIndex();
- if (c == CharacterIteration.DONE32) {
- // c = fText->last32();
- // TODO: why was this if needed?
- c = CharacterIteration.previous32(fText);
- }
- else {
- c = CharacterIteration.previous32(fText);
- }
- }
- else {
- do {
- c = CharacterIteration.previous32(fText);
- category = (short)fRData.fTrie.get(c);
- }
- while (c != CharacterIteration.DONE32 && ((category & 0x4000) != 0));
- // Back up to the last dictionary character
- if (c == CharacterIteration.DONE32) {
- // c = fText->first32();
- c = CharacterIteration.current32(fText);
- }
- else {
- CharacterIteration.next32(fText);
- c = CharacterIteration.current32(fText);
- }
- rangeStart = fText.getIndex();
- }
- category = (short)fRData.fTrie.get(c);
- }
-
-
- // Loop through the text, looking for ranges of dictionary characters.
- // For each span, find the appropriate break engine, and ask it to find
- // any breaks within the span.
- // Note: we always do this in the forward direction, so that the break
- // cache is built in the right order.
- if (reverse) {
- fText.setIndex(rangeStart);
- c = CharacterIteration.current32(fText);
- category = (short)fRData.fTrie.get(c);
- }
- LanguageBreakEngine lbe = null;
- while(true) {
- while((current = fText.getIndex()) < rangeEnd && (category & 0x4000) == 0) {
- CharacterIteration.next32(fText);
- c = CharacterIteration.current32(fText);
- category = (short)fRData.fTrie.get(c);
- }
- if (current >= rangeEnd) {
- break;
- }
-
- // We now have a dictionary character. Get the appropriate language object
- // to deal with it.
- lbe = getLanguageBreakEngine(c);
-
- // Ask the language object if there are any breaks. It will leave the text
- // pointer on the other side of its range, ready to search for the next one.
- if (lbe != null) {
- int startingIdx = fText.getIndex();
- foundBreakCount += lbe.findBreaks(fText, rangeStart, rangeEnd, false, fBreakType, breaks);
- assert fText.getIndex() > startingIdx;
- }
-
- // Reload the loop variables for the next go-round
- c = CharacterIteration.current32(fText);
- category = (short)fRData.fTrie.get(c);
- }
-
- // If we found breaks, build a new break cache. The first and last entries must
- // be the original starting and ending position.
- if (foundBreakCount > 0) {
- if (foundBreakCount != breaks.size()) {
- System.out.println("oops, foundBreakCount != breaks.size(). LBE = " + lbe.getClass());
- }
- assert foundBreakCount == breaks.size();
- if (startPos < breaks.peekLast()) {
- breaks.offer(startPos);
- }
- if (endPos > breaks.peek()) {
- breaks.push(endPos);
- }
-
- // TODO: get rid of this array, use results from the deque directly
- fCachedBreakPositions = new int[breaks.size()];
-
- int i = 0;
- while (breaks.size() > 0) {
- fCachedBreakPositions[i++] = breaks.pollLast();
- }
-
- // If there are breaks, then by definition, we are replacing the original
- // proposed break by one of the breaks we found. Use following() and
- // preceding() to do the work. They should never recurse in this case.
- if (reverse) {
- return preceding(endPos);
- }
- else {
- return following(startPos);
- }
- }
-
- // If we get here, there were no language-based breaks. Set the text pointer
- // to the original proposed break.
- fText.setIndex(reverse ? startPos : endPos);
- return (reverse ? startPos : endPos);
-
- }
-
-
- /**
- * Moves the iterator backwards, to the last boundary preceding this one.
- * @return The position of the last boundary position preceding this one.
+ * Moves the iterator backwards, to the boundary preceding the current one.
+ * @return The position of the boundary position immediately preceding the starting position.
* @stable ICU 2.0
*/
@Override
public int previous() {
- int result;
- int startPos;
-
- CharacterIterator text = getText();
-
- fLastStatusIndexValid = false;
-
- // if we have cached break positions and we're still in the range
- // covered by them, just move one step backward in the cache
- if (fCachedBreakPositions != null) {
- if (fPositionInCache > 0) {
- --fPositionInCache;
- // If we're at the beginning of the cache, need to reevaluate the
- // rule status
- if (fPositionInCache <= 0) {
- fLastStatusIndexValid = false;
- }
- int pos = fCachedBreakPositions[fPositionInCache];
- text.setIndex(pos);
- return pos;
- } else {
- reset();
- }
- }
-
- // if we're already sitting at the beginning of the text, return DONE
- startPos = current();
- if (fText == null || startPos == fText.getBeginIndex()) {
- fLastRuleStatusIndex = 0;
- fLastStatusIndexValid = true;
- return BreakIterator.DONE;
- }
-
- // Rules with an exact reverse table are handled here.
- if (fRData.fSRTable != null || fRData.fSFTable != null) {
- result = handlePrevious(fRData.fRTable);
- if (fDictionaryCharCount > 0) {
- result = checkDictionary(result, startPos, true);
- }
- return result;
- }
-
- // old rule syntax
- // set things up. handlePrevious() will back us up to some valid
- // break position before the current position (we back our internal
- // iterator up one step to prevent handlePrevious() from returning
- // the current position), but not necessarily the last one before
- // where we started
-
- int start = current();
-
- previous32(fText);
- int lastResult = handlePrevious(fRData.fRTable);
- if (lastResult == BreakIterator.DONE) {
- lastResult = fText.getBeginIndex();
- fText.setIndex(lastResult);
- }
- result = lastResult;
- int lastTag = 0;
- boolean breakTagValid = false;
-
- // iterate forward from the known break position until we pass our
- // starting point. The last break position before the starting
- // point is our return value
-
- for (;;) {
- result = next();
- if (result == BreakIterator.DONE || result >= start) {
- break;
- }
- lastResult = result;
- lastTag = fLastRuleStatusIndex;
- breakTagValid = true;
- }
-
- // fLastBreakTag wants to have the value for section of text preceding
- // the result position that we are to return (in lastResult.) If
- // the backwards rules overshot and the above loop had to do two or more
- // handleNext()s to move up to the desired return position, we will have a valid
- // tag value. But, if handlePrevious() took us to exactly the correct result position,
- // we wont have a tag value for that position, which is only set by handleNext().
-
- // Set the current iteration position to be the last break position
- // before where we started, and then return that value.
- fText.setIndex(lastResult);
- fLastRuleStatusIndex = lastTag; // for use by getRuleStatus()
- fLastStatusIndexValid = breakTagValid;
- return lastResult;
+ fBreakCache.previous();
+ return fDone ? DONE : fPosition;
}
/**
* Sets the iterator to refer to the first boundary position following
* the specified position.
- * @param offset The position from which to begin searching for a break position.
+ * @param startPos The position from which to begin searching for a break position.
* @return The position of the first break after the current position.
* @stable ICU 2.0
*/
@Override
- public int following(int offset) {
- CharacterIterator text = getText();
-
- // if we have no cached break positions, or if "offset" is outside the
- // range covered by the cache, then dump the cache and call our
- // inherited following() method. This will call other methods in this
- // class that may refresh the cache.
- if (fCachedBreakPositions == null || offset < fCachedBreakPositions[0] ||
- offset >= fCachedBreakPositions[fCachedBreakPositions.length - 1]) {
- fCachedBreakPositions = null;
- return rulesFollowing(offset);
- }
-
- // on the other hand, if "offset" is within the range covered by the
- // cache, then just search the cache for the first break position
- // after "offset"
- else {
- fPositionInCache = 0;
- while (fPositionInCache < fCachedBreakPositions.length
- && offset >= fCachedBreakPositions[fPositionInCache])
- ++fPositionInCache;
- text.setIndex(fCachedBreakPositions[fPositionInCache]);
- return text.getIndex();
- }
- }
-
- private int rulesFollowing(int offset) {
- // if the offset passed in is already past the end of the text,
- // just return DONE; if it's before the beginning, return the
+ public int following(int startPos) {
+ // if the supplied position is before the beginning, return the
// text's starting offset
- fLastRuleStatusIndex = 0;
- fLastStatusIndexValid = true;
- if (fText == null || offset >= fText.getEndIndex()) {
- last();
- return next();
- }
- else if (offset < fText.getBeginIndex()) {
+ if (startPos < fText.getBeginIndex()) {
return first();
}
- // otherwise, set our internal iteration position (temporarily)
- // to the position passed in. If this is the _beginning_ position,
- // then we can just use next() to get our return value
-
- int result = 0;
-
- if (fRData.fSRTable != null) {
- // Safe Point Reverse rules exist.
- // This allows us to use the optimum algorithm.
- fText.setIndex(offset);
- // move forward one codepoint to prepare for moving back to a
- // safe point.
- // this handles offset being between a supplementary character
- next32(fText);
- // handlePrevious will move most of the time to < 1 boundary away
- handlePrevious(fRData.fSRTable);
- result = next();
- while (result <= offset) {
- result = next();
- }
- return result;
- }
- if (fRData.fSFTable != null) {
- // No Safe point reverse table, but there is a safe pt forward table.
- //
- fText.setIndex(offset);
- previous32(fText);
- // handle next will give result >= offset
- handleNext(fRData.fSFTable);
- // previous will give result 0 or 1 boundary away from offset,
- // most of the time
- // we have to
- int oldresult = previous();
- while (oldresult > offset) {
- result = previous();
- if (result <= offset) {
- return oldresult;
- }
- oldresult = result;
- }
- result = next();
- if (result <= offset) {
- return next();
- }
- return result;
- }
- // otherwise, we have to sync up first. Use handlePrevious() to back
- // us up to a known break position before the specified position (if
- // we can determine that the specified position is a break position,
- // we don't back up at all). This may or may not be the last break
- // position at or before our starting position. Advance forward
- // from here until we've passed the starting position. The position
- // we stop on will be the first break position after the specified one.
- // old rule syntax
-
- fText.setIndex(offset);
- if (offset == fText.getBeginIndex()) {
- return next();
- }
- result = previous();
+ // Move requested offset to a code point start. It might be on a trail surrogate.
+ // Or it may be beyond the end of the text.
+ startPos = CISetIndex32(fText, startPos);
+ fBreakCache.following(startPos);
+ return fDone ? DONE : fPosition;
+ }
- while (result != BreakIterator.DONE && result <= offset) {
- result = next();
- }
- return result;
- }
/**
* Sets the iterator to refer to the last boundary position before the
* specified position.
*/
@Override
public int preceding(int offset) {
- CharacterIterator text = getText();
-
- // if we have no cached break positions, or "offset" is outside the
- // range covered by the cache, we can just call the inherited routine
- // (which will eventually call other routines in this class that may
- // refresh the cache)
- if (fCachedBreakPositions == null || offset <= fCachedBreakPositions[0] ||
- offset > fCachedBreakPositions[fCachedBreakPositions.length - 1]) {
- fCachedBreakPositions = null;
- return rulesPreceding(offset);
- }
-
- // on the other hand, if "offset" is within the range covered by the cache,
- // then all we have to do is search the cache for the last break position
- // before "offset"
- else {
- fPositionInCache = 0;
- while (fPositionInCache < fCachedBreakPositions.length
- && offset > fCachedBreakPositions[fPositionInCache])
- ++fPositionInCache;
- --fPositionInCache;
- text.setIndex(fCachedBreakPositions[fPositionInCache]);
- return text.getIndex();
- }
- }
-
- private int rulesPreceding(int offset) {
- // if the offset passed in is already past the end of the text,
- // just return DONE; if it's before the beginning, return the
-
- // text's starting offset
if (fText == null || offset > fText.getEndIndex()) {
- // return BreakIterator::DONE;
return last();
- }
- else if (offset < fText.getBeginIndex()) {
+ } else if (offset < fText.getBeginIndex()) {
return first();
}
- // if we start by updating the current iteration position to the
- // position specified by the caller, we can just use previous()
- // to carry out this operation
-
- int result;
- if (fRData.fSFTable != null) {
- /// todo synwee
- // new rule syntax
- fText.setIndex(offset);
- // move backwards one codepoint to prepare for moving forwards to a
- // safe point.
- // this handles offset being between a supplementary character
- previous32(fText);
- handleNext(fRData.fSFTable);
- result = previous();
- while (result >= offset) {
- result = previous();
- }
- return result;
- }
- if (fRData.fSRTable != null) {
- // backup plan if forward safe table is not available
- fText.setIndex(offset);
- next32(fText);
- // handle previous will give result <= offset
- handlePrevious(fRData.fSRTable);
-
- // next will give result 0 or 1 boundary away from offset,
- // most of the time
- // we have to
- int oldresult = next();
- while (oldresult < offset) {
- result = next();
- if (result >= offset) {
- return oldresult;
- }
- oldresult = result;
- }
- result = previous();
- if (result >= offset) {
- return previous();
- }
- return result;
- }
+ // Move requested offset to a code point start. It might be on a trail surrogate.
+ // int adjustedOffset = CISetIndex32(fText, offset); // TODO: restore to match ICU4C behavior.
+ int adjustedOffset = offset;
+ fBreakCache.preceding(adjustedOffset);
+ return fDone ? DONE : fPosition;
- // old rule syntax
- fText.setIndex(offset);
- return previous();
}
+
/**
* Throw IllegalArgumentException unless begin <= offset < end.
* @stable ICU 2.0
*/
@Override
public boolean isBoundary(int offset) {
+ // TODO: behavior difference with ICU4C, which considers out-of-range offsets
+ // to not be boundaries, and to not be errors.
checkOffset(offset, fText);
- // the beginning index of the iterator is always a boundary position by definition
- if (offset == fText.getBeginIndex()) {
- first(); // For side effects on current position, tag values.
- return true;
- }
+ // Adjust offset to be on a code point boundary and not beyond the end of the text.
+ // Note that isBoundary() is always be false for offsets that are not on code point boundaries.
+ // But we still need the side effect of leaving iteration at the following boundary.
+ int adjustedOffset = CISetIndex32(fText, offset);
- if (offset == fText.getEndIndex()) {
- last(); // For side effects on current position, tag values.
- return true;
+ boolean result = false;
+ if (fBreakCache.seek(adjustedOffset) || fBreakCache.populateNear(adjustedOffset)) {
+ result = (fBreakCache.current() == offset);
}
- // otherwise, we can use following() on the position before the specified
- // one and return true if the position we get back is the one the user
- // specified
-
- // return following(offset - 1) == offset;
- // TODO: check whether it is safe to revert to the simpler offset-1 code
- // The safe rules may take care of unpaired surrogates ok.
- fText.setIndex(offset);
- previous32(fText);
- int pos = fText.getIndex();
- boolean result = following(pos) == offset;
+ if (!result) {
+ // Not on a boundary. isBoundary() must leave iterator on the following boundary.
+ // fBreakCache.seek(), above, left us on the preceding boundary, so advance one.
+ next();
+ }
return result;
+
}
/**
- * Returns the current iteration position.
+ * Returns the current iteration position. Note that UBRK_DONE is never
+ * returned from this function; if iteration has run to the end of a
+ * string, current() will return the length of the string while
+ * next() will return BreakIterator.DONE).
* @return The current iteration position.
* @stable ICU 2.0
*/
@Override
public int current() {
- return (fText != null) ? fText.getIndex() : BreakIterator.DONE;
+ return (fText != null) ? fPosition : BreakIterator.DONE;
}
- private void makeRuleStatusValid() {
- if (fLastStatusIndexValid == false) {
- // No cached status is available.
- int curr = current();
- if (curr == BreakIterator.DONE || curr == fText.getBeginIndex()) {
- // At start of text, or there is no text. Status is always zero.
- fLastRuleStatusIndex = 0;
- fLastStatusIndexValid = true;
- } else {
- // Not at start of text. Find status the tedious way.
- int pa = fText.getIndex();
- first();
- int pb = current();
- while (fText.getIndex() < pa) {
- pb = next();
- }
- Assert.assrt(pa == pb);
- }
- Assert.assrt(fLastStatusIndexValid == true);
- Assert.assrt(fLastRuleStatusIndex >= 0 && fLastRuleStatusIndex < fRData.fStatusTable.length);
- }
- }
/**
* Return the status tag from the break rule that determined the most recently
* status, a default value of 0 is returned. If more than one rule applies,
* the numerically largest of the possible status values is returned.
* <p>
- * Of the standard types of ICU break iterators, only the word break
+ * Of the standard types of ICU break iterators, only the word and line break
* iterator provides status values. The values are defined in
* class RuleBasedBreakIterator, and allow distinguishing between words
* that contain alphabetic letters, "words" that appear to be numbers,
* @return the status from the break rule that determined the most recently
* returned break position.
*
- * @draft ICU 3.0 (retain)
- * @provisional This is a draft API and might change in a future release of ICU.
+ * @stable ICU 60
*/
@Override
public int getRuleStatus() {
- makeRuleStatusValid();
// Status records have this form:
// Count N <-- fLastRuleStatusIndex points here.
// Status val 0
// Status val N-1 <-- the value we need to return
// The status values are sorted in ascending order.
// This function returns the last (largest) of the array of status values.
- int idx = fLastRuleStatusIndex + fRData.fStatusTable[fLastRuleStatusIndex];
+ int idx = fRuleStatusIndex + fRData.fStatusTable[fRuleStatusIndex];
int tagVal = fRData.fStatusTable[idx];
return tagVal;
}
* In the event that the array is too small, the return value
* is the total number of status values that were available,
* not the reduced number that were actually returned.
- * @draft ICU 3.0 (retain)
- * @provisional This is a draft API and might change in a future release of ICU.
+ * @stable ICU 60
*/
@Override
public int getRuleStatusVec(int[] fillInArray) {
- makeRuleStatusValid();
- int numStatusVals = fRData.fStatusTable[fLastRuleStatusIndex];
+ int numStatusVals = fRData.fStatusTable[fRuleStatusIndex];
if (fillInArray != null) {
int numToCopy = Math.min(numStatusVals, fillInArray.length);
for (int i=0; i<numToCopy; i++) {
- fillInArray[i] = fRData.fStatusTable[fLastRuleStatusIndex + i + 1];
+ fillInArray[i] = fRData.fStatusTable[fRuleStatusIndex + i + 1];
}
}
return numStatusVals;
*/
@Override
public void setText(CharacterIterator newText) {
+ if (newText != null) {
+ fBreakCache.reset(newText.getBeginIndex(), 0);
+ } else {
+ fBreakCache.reset();
+ }
+ fDictionaryCache.reset();
fText = newText;
- // first() resets the caches
this.first();
}
* The State Machine Engine for moving forward is here.
* This function is the heart of the RBBI run time engine.
*
- * @param stateTable
+ * Input
+ * fPosition, the position in the text to begin from.
+ * Output
+ * fPosition: the boundary following the starting position.
+ * fDictionaryCharCount the number of dictionary characters encountered.
+ * If > 0, the segment will be further subdivided
+ * fRuleStatusIndex Info from the state table indicating which rules caused the boundary.
+ *
* @return the new iterator position
*
* A note on supplementary characters and the position of underlying
* This is different from everywhere else, where an iterator always
* points at the lead surrogate of a supplementary.
*/
- private int handleNext(short stateTable[]) {
+ private int handleNext() {
if (TRACE) {
System.out.println("Handle Next pos char state category");
}
- // No matter what, handleNext alway correctly sets the break tag value.
- fLastStatusIndexValid = true;
- fLastRuleStatusIndex = 0;
+ // handleNext always sets the break tag value.
+ // Set the default for it.
+ fRuleStatusIndex = 0;
+ fDictionaryCharCount = 0;
// caches for quicker access
CharacterIterator text = fText;
Trie2 trie = fRData.fTrie;
+ short[] stateTable = fRData.fFTable;
+ int initialPosition = fPosition;
+ text.setIndex(initialPosition);
+ int result = initialPosition;
+
// Set up the starting char
- int c = text.current();
+ int c = text.current();
if (c >= UTF16.LEAD_SURROGATE_MIN_VALUE) {
c = nextTrail32(text, c);
if (c == DONE32) {
+ fDone = true;
return BreakIterator.DONE;
}
}
- int initialPosition = text.getIndex();
- int result = initialPosition;
// Set the initial state for the state machine
int state = START_STATE;
}
// Remember the break status (tag) values.
- fLastRuleStatusIndex = stateTable[row + RBBIDataWrapper.TAGIDX];
+ fRuleStatusIndex = stateTable[row + RBBIDataWrapper.TAGIDX];
}
int completedRule = stateTable[row + RBBIDataWrapper.ACCEPTING];
// Lookahead match is completed
int lookaheadResult = fLookAheadMatches.getPosition(completedRule);
if (lookaheadResult >= 0) {
- fLastRuleStatusIndex = stateTable[row + RBBIDataWrapper.TAGIDX];
- text.setIndex(lookaheadResult);
+ fRuleStatusIndex = stateTable[row + RBBIDataWrapper.TAGIDX];
+ fPosition = lookaheadResult;
return lookaheadResult;
}
}
text.setIndex(initialPosition);
next32(text);
result = text.getIndex();
+ fRuleStatusIndex = 0;
}
- else {
- // Leave the iterator at our result position.
- // (we may have advanced beyond the last accepting position chasing after
- // longer matches that never completed.)
- text.setIndex(result);
- }
+
+ // Leave the iterator at our result position.
+ // (we may have advanced beyond the last accepting position chasing after
+ // longer matches that never completed.)
+ fPosition = result;
+
if (TRACE) {
System.out.println("result = " + result);
}
return result;
}
- private int handlePrevious(short stateTable[]) {
- if (fText == null || stateTable == null) {
+ /**
+ * Iterate backwards from an arbitrary position in the input text using the Safe Reverse rules.
+ * This locates a "Safe Position" from which the forward break rules
+ * will operate correctly. A Safe Position is not necessarily a boundary itself.
+ *
+ * The logic of this function is very similar to handleNext(), above.
+ *
+ * @param fromPosition the position in the input text to begin the iteration.
+ * @internal
+ */
+ private int handlePrevious(int fromPosition) {
+ if (fText == null) {
return 0;
}
int row;
int c;
int result = 0;
- int initialPosition = 0;
+ int initialPosition = fromPosition;
fLookAheadMatches.reset();
-
- // handlePrevious() never gets the rule status.
- // Flag the status as invalid; if the user ever asks for status, we will need
- // to back up, then re-find the break position using handleNext(), which does
- // get the status value.
- fLastStatusIndexValid = false;
- fLastRuleStatusIndex = 0;
+ short[] stateTable = fRData.fSRTable;
+ CISetIndex32(fText, fromPosition);
+ if (fromPosition == fText.getBeginIndex()) {
+ return BreakIterator.DONE;
+ }
// set up the starting char
- initialPosition = fText.getIndex();
result = initialPosition;
c = previous32(fText);
if (mode == RBBI_END) {
// We have already done the {eof} iteration. Now is the time
// to unconditionally bail out.
- if (result == initialPosition) {
- // Ran off start, no match found.
- // Move one position (towards the start, since we are doing previous.)
- fText.setIndex(initialPosition);
- previous32(fText);
- }
break mainLoop;
}
mode = RBBI_END;
// look up the current character's category, which tells us
// which column in the state table to look at.
//
+ // And off the dictionary flag bit. For reverse iteration it is not used.
category = (short) fRData.fTrie.get(c);
-
- // Check the dictionary bit in the character's category.
- // Counter is only used by dictionary based iterators (subclasses).
- // Chars that need to be handled by a dictionary have a flag bit set
- // in their category values.
- //
- if ((category & 0x4000) != 0) {
- fDictionaryCharCount++;
- // And off the dictionary flag bit.
- category &= ~0x4000;
- }
+ category &= ~0x4000;
}
-
if (TRACE) {
System.out.print(" " + fText.getIndex() + " ");
if (0x20 <= c && c < 0x7f) {
// The state machine is done. Check whether it found a match...
//
- // If the iterator failed to advance in the match engine, force it ahead by one.
+ // If the iterator failed to move in the match engine, force it back by one code point.
// (This really indicates a defect in the break rules. They should always match
// at least one character.)
if (result == initialPosition) {
- result = fText.setIndex(initialPosition);
+ CISetIndex32(fText, initialPosition);
previous32(fText);
result = fText.getIndex();
}
- fText.setIndex(result);
if (TRACE) {
System.out.println("Result = " + result);
}
return result;
}
+
+ /**
+ * Set the index of a CharacterIterator.
+ * Pin the index to the valid range range of BeginIndex <= index <= EndIndex.
+ * If the index points to a trail surrogate of a supplementary character, adjust it
+ * to the start (lead surrogate) index.
+ *
+ * @param ci A CharacterIterator to set
+ * @param index the index to set
+ * @return the resulting index, possibly pinned or adjusted.
+ */
+ private static int CISetIndex32(CharacterIterator ci, int index) {
+ if (index <= ci.getBeginIndex()) {
+ ci.first();
+ } else if (index >= ci.getEndIndex()) {
+ ci.setIndex(ci.getEndIndex());
+ } else if (Character.isLowSurrogate(ci.setIndex(index))) {
+ if (!Character.isHighSurrogate(ci.previous())) {
+ ci.next();
+ }
+ }
+ return ci.getIndex();
+ }
+
+ /* DictionaryCache stores the boundaries obtained from a run of dictionary characters.
+ * Dictionary boundaries are moved first to this cache, then from here
+ * to the main BreakCache, where they may inter-leave with non-dictionary
+ * boundaries. The public BreakIterator API always fetches directly
+ * from the main BreakCache, not from here.
+ *
+ * In common situations, the number of boundaries in a single dictionary run
+ * should be quite small, it will be terminated by punctuation, spaces,
+ * or any other non-dictionary characters. The main BreakCache may end
+ * up with boundaries from multiple dictionary based runs.
+ *
+ * The boundaries are stored in a simple ArrayList (vector), with the
+ * assumption that they will be accessed sequentially.
+ */
+ class DictionaryCache {
+
+ void reset() {
+ fPositionInCache = -1;
+ fStart = 0;
+ fLimit = 0;
+ fFirstRuleStatusIndex = 0;
+ fOtherRuleStatusIndex = 0;
+ fBreaks.removeAllElements();
+ };
+
+ boolean following(int fromPos) {
+ if (fromPos >= fLimit || fromPos < fStart) {
+ fPositionInCache = -1;
+ return false;
+ }
+
+ // Sequential iteration, move from previous boundary to the following
+
+ int r = 0;
+ if (fPositionInCache >= 0 && fPositionInCache < fBreaks.size() && fBreaks.elementAt(fPositionInCache) == fromPos) {
+ ++fPositionInCache;
+ if (fPositionInCache >= fBreaks.size()) {
+ fPositionInCache = -1;
+ return false;
+ }
+ r = fBreaks.elementAt(fPositionInCache);
+ assert(r > fromPos);
+ fBoundary = r;
+ fStatusIndex = fOtherRuleStatusIndex;
+ return true;
+ }
+
+ // Random indexing. Linear search for the boundary following the given position.
+
+ for (fPositionInCache = 0; fPositionInCache < fBreaks.size(); ++fPositionInCache) {
+ r= fBreaks.elementAt(fPositionInCache);
+ if (r > fromPos) {
+ fBoundary = r;
+ fStatusIndex = fOtherRuleStatusIndex;
+ return true;
+ }
+ }
+
+ // Internal error. fStart <= fromPos < fLimit, but no cached boundary.
+ assert(false);
+ fPositionInCache = -1;
+ return false;
+ };
+
+ boolean preceding(int fromPos) {
+ if (fromPos <= fStart || fromPos > fLimit) {
+ fPositionInCache = -1;
+ return false;
+ }
+
+ if (fromPos == fLimit) {
+ fPositionInCache = fBreaks.size() - 1;
+ if (fPositionInCache >= 0) {
+ assert(fBreaks.elementAt(fPositionInCache) == fromPos);
+ }
+ }
+
+ int r;
+ if (fPositionInCache > 0 && fPositionInCache < fBreaks.size() && fBreaks.elementAt(fPositionInCache) == fromPos) {
+ --fPositionInCache;
+ r = fBreaks.elementAt(fPositionInCache);
+ assert(r < fromPos);
+ fBoundary = r;
+ fStatusIndex = ( r== fStart) ? fFirstRuleStatusIndex : fOtherRuleStatusIndex;
+ return true;
+ }
+
+ if (fPositionInCache == 0) {
+ fPositionInCache = -1;
+ return false;
+ }
+
+ for (fPositionInCache = fBreaks.size()-1; fPositionInCache >= 0; --fPositionInCache) {
+ r = fBreaks.elementAt(fPositionInCache);
+ if (r < fromPos) {
+ fBoundary = r;
+ fStatusIndex = ( r == fStart) ? fFirstRuleStatusIndex : fOtherRuleStatusIndex;
+ return true;
+ }
+ }
+ assert(false);
+ fPositionInCache = -1;
+ return false;
+ };
+
+ /**
+ * Populate the cache with the dictionary based boundaries within a region of text.
+ * @param startPos The start position of a range of text
+ * @param endPos The end position of a range of text
+ * @param firstRuleStatus The rule status index that applies to the break at startPos
+ * @param otherRuleStatus The rule status index that applies to boundaries other than startPos
+ * @internal
+ */
+ void populateDictionary(int startPos, int endPos,
+ int firstRuleStatus, int otherRuleStatus) {
+ if ((endPos - startPos) <= 1) {
+ return;
+ }
+
+ reset();
+ fFirstRuleStatusIndex = firstRuleStatus;
+ fOtherRuleStatusIndex = otherRuleStatus;
+
+ int rangeStart = startPos;
+ int rangeEnd = endPos;
+
+ int category;
+ int current;
+ int foundBreakCount = 0;
+
+ // Loop through the text, looking for ranges of dictionary characters.
+ // For each span, find the appropriate break engine, and ask it to find
+ // any breaks within the span.
+
+ fText.setIndex(rangeStart);
+ int c = CharacterIteration.current32(fText);
+ category = (short)fRData.fTrie.get(c);
+
+ while(true) {
+ while((current = fText.getIndex()) < rangeEnd && (category & 0x4000) == 0) {
+ c = CharacterIteration.next32(fText); // pre-increment
+ category = (short)fRData.fTrie.get(c);
+ }
+ if (current >= rangeEnd) {
+ break;
+ }
+
+ // We now have a dictionary character. Get the appropriate language object
+ // to deal with it.
+ LanguageBreakEngine lbe = getLanguageBreakEngine(c);
+
+ // Ask the language object if there are any breaks. It will add them to the cache and
+ // leave the text pointer on the other side of its range, ready to search for the next one.
+ if (lbe != null) {
+ foundBreakCount += lbe.findBreaks(fText, rangeStart, rangeEnd, fBreakType, fBreaks);
+ }
+
+ // Reload the loop variables for the next go-round
+ c = CharacterIteration.current32(fText);
+ category = (short)fRData.fTrie.get(c);
+ }
+
+ // If we found breaks, ensure that the first and last entries are
+ // the original starting and ending position. And initialize the
+ // cache iteration position to the first entry.
+
+ // System.out.printf("foundBreakCount = %d\n", foundBreakCount);
+ if (foundBreakCount > 0) {
+ assert(foundBreakCount == fBreaks.size());
+ if (startPos < fBreaks.elementAt(0)) {
+ // The dictionary did not place a boundary at the start of the segment of text.
+ // Add one now. This should not commonly happen, but it would be easy for interactions
+ // of the rules for dictionary segments and the break engine implementations to
+ // inadvertently cause it. Cover it here, just in case.
+ fBreaks.offer(startPos);
+ }
+ if (endPos > fBreaks.peek()) {
+ fBreaks.push(endPos);
+ }
+ fPositionInCache = 0;
+ // Note: Dictionary matching may extend beyond the original limit.
+ fStart = fBreaks.elementAt(0);
+ fLimit = fBreaks.peek();
+ } else {
+ // there were no language-based breaks, even though the segment contained
+ // dictionary characters. Subsequent attempts to fetch boundaries from the dictionary cache
+ // for this range will fail, and the calling code will fall back to the rule based boundaries.
+ }
+
+ };
+
+
+ DictionaryCache() {
+ fPositionInCache = -1;
+ fBreaks = new DictionaryBreakEngine.DequeI();
+ }
+
+ /**
+ * copy constructor. Used by RuleBasedBreakIterator.clone().
+ *
+ * @param src the source object to be copied.
+ */
+ DictionaryCache(DictionaryCache src) {
+ try {
+ fBreaks = (DictionaryBreakEngine.DequeI)src.fBreaks.clone();
+ }
+ catch (CloneNotSupportedException e) {
+ throw new RuntimeException(e);
+ }
+ fPositionInCache = src.fPositionInCache;
+ fStart = src.fStart;
+ fLimit = src.fLimit;
+ fFirstRuleStatusIndex = src.fFirstRuleStatusIndex;
+ fOtherRuleStatusIndex = src.fOtherRuleStatusIndex;
+ fBoundary = src.fBoundary;
+ fStatusIndex = src.fStatusIndex;
+ }
+
+ // A data structure containing the boundaries themselves. Essentially a vector of raw ints.
+ DictionaryBreakEngine.DequeI fBreaks;
+ int fPositionInCache; // Index in fBreaks of last boundary returned by following()
+ // // or preceding(). Optimizes sequential access.
+ int fStart; // Text position of first boundary in cache.
+ int fLimit; // Last boundary in cache. Which is the limit of the
+ // // text segment being handled by the dictionary.
+ int fFirstRuleStatusIndex; // Rule status info for first boundary.
+ int fOtherRuleStatusIndex; // Rule status info for 2nd through last boundaries.
+ int fBoundary; // Current boundary. Set by preceding(), following().
+ int fStatusIndex; // Current rule status index. Set by preceding, following().
+ };
+
+
+
+
+/*
+ * class BreakCache
+ *
+ * Cache of break boundary positions and rule status values.
+ * Break iterator API functions, next(), previous(), etc., will use cached results
+ * when possible, and otherwise cache new results as they are obtained.
+ *
+ * Uniformly caches both dictionary and rule based (non-dictionary) boundaries.
+ *
+ * The cache is implemented as a single circular buffer.
+ */
+
+/*
+ * size of the circular cache buffer.
+ */
+
+class BreakCache {
+
+ BreakCache() {
+ reset();
+ };
+
+ void reset(int pos, int ruleStatus) {
+ fStartBufIdx = 0;
+ fEndBufIdx = 0;
+ fTextIdx = pos;
+ fBufIdx = 0;
+ fBoundaries[0] = pos;
+ fStatuses[0] = (short)ruleStatus;
+ }
+
+ void reset() {reset(0, 0); };
+
+ void next() {
+ if (fBufIdx == fEndBufIdx) {
+ fDone = !populateFollowing();
+ fPosition = fTextIdx;
+ fRuleStatusIndex = fStatuses[fBufIdx];
+ } else {
+ fBufIdx = modChunkSize(fBufIdx + 1);
+ fTextIdx = fPosition = fBoundaries[fBufIdx];
+ fRuleStatusIndex = fStatuses[fBufIdx];
+ }
+ };
+
+ void previous() {
+ int initialBufIdx = fBufIdx;
+ if (fBufIdx == fStartBufIdx) {
+ // At start of cache. Prepend to it.
+ populatePreceding();
+ } else {
+ // Cache already holds the next boundary
+ fBufIdx = modChunkSize(fBufIdx - 1);
+ fTextIdx = fBoundaries[fBufIdx];
+ }
+ fDone = (fBufIdx == initialBufIdx);
+ fPosition = fTextIdx;
+ fRuleStatusIndex = fStatuses[fBufIdx];
+ return;
+ };
+
+ // Move the iteration state to the position following the startPosition.
+ // Input position must be pinned to the input length.
+ void following(int startPos) {
+ if (startPos == fTextIdx || seek(startPos) || populateNear(startPos)) {
+ // startPos is in the cache. Do a next() from that position.
+ // TODO: an awkward set of interactions with bi->fDone
+ // seek() does not clear it; it can't because of interactions with populateNear().
+ // next() does not clear it in the fast-path case, where everything matters. Maybe it should.
+ // So clear it here, for the case where seek() succeeded on an iterator that had previously run off the end.
+ fDone = false;
+ next();
+ }
+
+ };
+
+ void preceding(int startPos) {
+ if (startPos == fTextIdx || seek(startPos) || populateNear(startPos)) {
+ if (startPos == fTextIdx) {
+ previous();
+ } else {
+ // seek() leaves the BreakCache positioned at the preceding boundary
+ // if the requested position is between two bounaries.
+ // current() pushes the BreakCache position out to the BreakIterator itself.
+ assert(startPos > fTextIdx);
+ current();
+ }
+ }
+ return;
+ };
+
+ /*
+ * Update the state of the public BreakIterator (fBI) to reflect the
+ * current state of the break iterator cache (this).
+ */
+ int current() {
+ fPosition = fTextIdx;
+ fRuleStatusIndex = fStatuses[fBufIdx];
+ fDone = false;
+ return fTextIdx;
+ };
+
+ /**
+ * Add boundaries to the cache near the specified position.
+ * The given position need not be a boundary itself.
+ * The input position must be within the range of the text, and
+ * on a code point boundary.
+ * If the requested position is a break boundary, leave the iteration
+ * position on it.
+ * If the requested position is not a boundary, leave the iteration
+ * position on the preceding boundary and include both the the
+ * preceding and following boundaries in the cache.
+ * Additional boundaries, either preceding or following, may be added
+ * to the cache as a side effect.
+ *
+ * Return false if the operation failed.
+ */
+ boolean populateNear(int position) {
+ assert(position < fBoundaries[fStartBufIdx] || position > fBoundaries[fEndBufIdx]);
+
+ // Find a boundary somewhere in the vicinity of the requested position.
+ // Depending on the safe rules and the text data, it could be either before, at, or after
+ // the requested position.
+
+
+ // If the requested position is not near already cached positions, clear the existing cache,
+ // find a near-by boundary and begin new cache contents there.
+
+ if ((position < fBoundaries[fStartBufIdx] - 15) || position > (fBoundaries[fEndBufIdx] + 15)) {
+ int aBoundary = fText.getBeginIndex();
+ int ruleStatusIndex = 0;
+ // TODO: check for position == length of text. Although may still need to back up to get rule status.
+ if (position > aBoundary + 20) {
+ int backupPos = handlePrevious(position);
+ fPosition = backupPos;
+ aBoundary = handleNext(); // Ignore dictionary, just finding a rule based boundary.
+ ruleStatusIndex = fRuleStatusIndex;
+ }
+ reset(aBoundary, ruleStatusIndex); // Reset cache to hold aBoundary as a single starting point.
+ }
+
+ // Fill in boundaries between existing cache content and the new requested position.
+
+ if (fBoundaries[fEndBufIdx] < position) {
+ // The last position in the cache precedes the requested position.
+ // Add following position(s) to the cache.
+ while (fBoundaries[fEndBufIdx] < position) {
+ if (!populateFollowing()) {
+ assert false;
+ return false;
+ }
+ }
+ fBufIdx = fEndBufIdx; // Set iterator position to the end of the buffer.
+ fTextIdx = fBoundaries[fBufIdx]; // Required because populateFollowing may add extra boundaries.
+ while (fTextIdx > position) { // Move backwards to a position at or preceding the requested pos.
+ previous();
+ }
+ return true;
+ }
+
+ if (fBoundaries[fStartBufIdx] > position) {
+ // The first position in the cache is beyond the requested position.
+ // back up more until we get a boundary <= the requested position.
+ while (fBoundaries[fStartBufIdx] > position) {
+ populatePreceding();
+ }
+ fBufIdx = fStartBufIdx; // Set iterator position to the start of the buffer.
+ fTextIdx = fBoundaries[fBufIdx]; // Required because populatePreceding may add extra boundaries.
+ while (fTextIdx < position) { // Move forwards to a position at or following the requested pos.
+ next();
+ }
+ if (fTextIdx > position) {
+ // If position is not itself a boundary, the next() loop above will overshoot.
+ // Back up one, leaving cache position at the boundary preceding the requested position.
+ previous();
+ }
+ return true;
+ }
+
+ assert fTextIdx == position;
+ return true;
+
+ };
+
+ /**
+ * Add boundary(s) to the cache following the current last boundary.
+ * Return false if at the end of the text, and no more boundaries can be added.
+ * Leave iteration position at the first newly added boundary, or unchanged if no boundary was added.
+ */
+ boolean populateFollowing() {
+ int fromPosition = fBoundaries[fEndBufIdx];
+ int fromRuleStatusIdx = fStatuses[fEndBufIdx];
+ int pos = 0;
+ int ruleStatusIdx = 0;
+
+ if (fDictionaryCache.following(fromPosition)) {
+ addFollowing(fDictionaryCache.fBoundary, fDictionaryCache.fStatusIndex, UpdateCachePosition);
+ return true;
+ }
+
+ fPosition = fromPosition;
+ pos = handleNext();
+ if (pos == BreakIterator.DONE) {
+ return false;
+ }
+
+ ruleStatusIdx = fRuleStatusIndex;
+ if (fDictionaryCharCount > 0) {
+ // The text segment obtained from the rules includes dictionary characters.
+ // Subdivide it, with subdivided results going into the dictionary cache.
+ fDictionaryCache.populateDictionary(fromPosition, pos, fromRuleStatusIdx, ruleStatusIdx);
+ if (fDictionaryCache.following(fromPosition)) {
+ addFollowing(fDictionaryCache.fBoundary, fDictionaryCache.fStatusIndex, UpdateCachePosition);
+ return true;
+ // TODO: may want to move a sizable chunk of the dictionary cache to the break cache at this point.
+ // But be careful with interactions with populateNear().
+ }
+ }
+
+ // Rule based segment did not include dictionary characters.
+ // Or, it did contain dictionary chars, but the dictionary segmenter didn't handle them,
+ // meaning that we didn't take the return, above.
+ // Add its end point to the cache.
+ addFollowing(pos, ruleStatusIdx, UpdateCachePosition);
+
+ // Add several non-dictionary boundaries at this point, to optimize straight forward iteration.
+ // (subsequent calls to BreakIterator::next() will take the fast path, getting cached results.
+ //
+ for (int count=0; count<6; ++count) {
+ pos = handleNext();
+ if (pos == BreakIterator.DONE || fDictionaryCharCount > 0) {
+ break;
+ }
+ addFollowing(pos, fRuleStatusIndex, RetainCachePosition);
+ }
+ return true;
+ };
+
+ /**
+ * Add one or more boundaries to the cache preceding the first currently cached boundary.
+ * Leave the iteration position on the first added boundary.
+ * Return false if no boundaries could be added (if at the start of the text.)
+ */
+ boolean populatePreceding() {
+ int textBegin = fText.getBeginIndex();
+ int fromPosition = fBoundaries[fStartBufIdx];
+ if (fromPosition == textBegin) {
+ return false;
+ }
+
+ int position = textBegin;
+ int positionStatusIdx = 0;
+
+ if (fDictionaryCache.preceding(fromPosition)) {
+ addPreceding(fDictionaryCache.fBoundary, fDictionaryCache.fStatusIndex, UpdateCachePosition);
+ return true;
+ }
+
+ int backupPosition = fromPosition;
+
+ // Find a boundary somewhere preceding the first already-cached boundary
+ do {
+ backupPosition = backupPosition - 30;
+ if (backupPosition <= textBegin) {
+ backupPosition = textBegin;
+ } else {
+ backupPosition = handlePrevious(backupPosition);
+ }
+ if (backupPosition == BreakIterator.DONE || backupPosition == textBegin) {
+ position = textBegin;
+ positionStatusIdx = 0;
+ } else {
+ fPosition = backupPosition; // TODO: pass starting position in a clearer way.
+ position = handleNext();
+ positionStatusIdx = fRuleStatusIndex;
+
+ }
+ } while (position >= fromPosition);
+
+ // Find boundaries between the one we just located and the first already-cached boundary
+ // Put them in a side buffer, because we don't yet know where they will fall in the circular cache buffer..
+
+ fSideBuffer.removeAllElements();
+ fSideBuffer.push(position);
+ fSideBuffer.push(positionStatusIdx);
+
+ do {
+ int prevPosition = fPosition = position;
+ int prevStatusIdx = positionStatusIdx;
+ position = handleNext();
+ positionStatusIdx = fRuleStatusIndex;
+ if (position == BreakIterator.DONE) {
+ break;
+ }
+
+ boolean segmentHandledByDictionary = false;
+ if (fDictionaryCharCount != 0) {
+ // Segment from the rules includes dictionary characters.
+ // Subdivide it, with subdivided results going into the dictionary cache.
+ int dictSegEndPosition = position;
+ fDictionaryCache.populateDictionary(prevPosition, dictSegEndPosition, prevStatusIdx, positionStatusIdx);
+ while (fDictionaryCache.following(prevPosition)) {
+ position = fDictionaryCache.fBoundary;
+ positionStatusIdx = fDictionaryCache.fStatusIndex;
+ segmentHandledByDictionary = true;
+ assert(position > prevPosition);
+ if (position >= fromPosition) {
+ break;
+ }
+ assert(position <= dictSegEndPosition);
+ fSideBuffer.push(position);
+ fSideBuffer.push(positionStatusIdx);
+ prevPosition = position;
+ }
+ assert(position==dictSegEndPosition || position>=fromPosition);
+ }
+
+ if (!segmentHandledByDictionary && position < fromPosition) {
+ fSideBuffer.push(position);
+ fSideBuffer.push(positionStatusIdx);
+ }
+ } while (position < fromPosition);
+
+ // Move boundaries from the side buffer to the main circular buffer.
+ boolean success = false;
+ if (!fSideBuffer.isEmpty()) {
+ positionStatusIdx = fSideBuffer.pop();
+ position = fSideBuffer.pop();
+ addPreceding(position, positionStatusIdx, UpdateCachePosition);
+ success = true;
+ }
+
+ while (!fSideBuffer.isEmpty()) {
+ positionStatusIdx = fSideBuffer.pop();
+ position = fSideBuffer.pop();
+ if (!addPreceding(position, positionStatusIdx, RetainCachePosition)) {
+ // No space in circular buffer to hold a new preceding result while
+ // also retaining the current cache (iteration) position.
+ // Bailing out is safe; the cache will refill again if needed.
+ break;
+ }
+ }
+ return success;
+ };
+
+
+ static final boolean RetainCachePosition = false;
+ static final boolean UpdateCachePosition = true;
+
+ /*
+ * Add the boundary following the current position.
+ * The current position can be left as it was, or changed to the newly added boundary,
+ * as specified by the update parameter.
+ */
+ void addFollowing(int position, int ruleStatusIdx, boolean update) {
+ assert(position > fBoundaries[fEndBufIdx]);
+ assert(ruleStatusIdx <= Short.MAX_VALUE);
+ int nextIdx = modChunkSize(fEndBufIdx + 1);
+ if (nextIdx == fStartBufIdx) {
+ fStartBufIdx = modChunkSize(fStartBufIdx + 6); // TODO: experiment. Probably revert to 1.
+ }
+ fBoundaries[nextIdx] = position;
+ fStatuses[nextIdx] = (short)ruleStatusIdx;
+ fEndBufIdx = nextIdx;
+ if (update == UpdateCachePosition) {
+ // Set current position to the newly added boundary.
+ fBufIdx = nextIdx;
+ fTextIdx = position;
+ } else {
+ // Retaining the original cache position.
+ // Check if the added boundary wraps around the buffer, and would over-write the original position.
+ // It's the responsibility of callers of this function to not add too many.
+ assert(nextIdx != fBufIdx);
+ }
+
+ };
+
+
+ /*
+ * Add the boundary preceding the current position.
+ * The current position can be left as it was, or changed to the newly added boundary,
+ * as specified by the update parameter.
+ */
+ boolean addPreceding(int position, int ruleStatusIdx, boolean update) {
+ assert(position < fBoundaries[fStartBufIdx]);
+ assert(ruleStatusIdx <= Short.MAX_VALUE);
+ int nextIdx = modChunkSize(fStartBufIdx - 1);
+ if (nextIdx == fEndBufIdx) {
+ if (fBufIdx == fEndBufIdx && update == RetainCachePosition) {
+ // Failure. The insertion of the new boundary would claim the buffer position that is the
+ // current iteration position. And we also want to retain the current iteration position.
+ // (The buffer is already completely full of entries that precede the iteration position.)
+ return false;
+ }
+ fEndBufIdx = modChunkSize(fEndBufIdx - 1);
+ }
+ fBoundaries[nextIdx] = position;
+ fStatuses[nextIdx] = (short)ruleStatusIdx;
+ fStartBufIdx = nextIdx;
+ if (update == UpdateCachePosition) {
+ fBufIdx = nextIdx;
+ fTextIdx = position;
+ }
+ return true;
+ };
+
+ /**
+ * Set the cache position to the specified position, or, if the position
+ * falls between to cached boundaries, to the preceding boundary.
+ * Fails if the requested position is outside of the range of boundaries currently held by the cache.
+ * The startPosition must be on a code point boundary.
+ *
+ * Return true if successful, false if the specified position is after
+ * the last cached boundary or before the first.
+ */
+ boolean seek(int pos) {
+ if (pos < fBoundaries[fStartBufIdx] || pos > fBoundaries[fEndBufIdx]) {
+ return false;
+ }
+ if (pos == fBoundaries[fStartBufIdx]) {
+ // Common case: seek(0), from BreakIterator::first()
+ fBufIdx = fStartBufIdx;
+ fTextIdx = fBoundaries[fBufIdx];
+ return true;
+ }
+ if (pos == fBoundaries[fEndBufIdx]) {
+ fBufIdx = fEndBufIdx;
+ fTextIdx = fBoundaries[fBufIdx];
+ return true;
+ }
+
+ int min = fStartBufIdx;
+ int max = fEndBufIdx;
+ while (min != max) {
+ int probe = (min + max + (min>max ? CACHE_SIZE : 0)) / 2;
+ probe = modChunkSize(probe);
+ if (fBoundaries[probe] > pos) {
+ max = probe;
+ } else {
+ min = modChunkSize(probe + 1);
+ }
+ }
+ assert(fBoundaries[max] > pos);
+ fBufIdx = modChunkSize(max - 1);
+ fTextIdx = fBoundaries[fBufIdx];
+ assert(fTextIdx <= pos);
+ return true;
+
+ };
+
+
+ /**
+ * copy constructor, used from RuleBasedBreakIterator.clone().
+ *
+ * @param src
+ */
+ BreakCache(BreakCache src) {
+ fStartBufIdx = src.fStartBufIdx;
+ fEndBufIdx = src.fEndBufIdx;
+ fTextIdx = src.fTextIdx;
+ fBufIdx = src.fBufIdx;
+ fBoundaries = src.fBoundaries.clone();
+ fStatuses = src.fStatuses.clone();
+ fSideBuffer = new DictionaryBreakEngine.DequeI(); // Transient, no need to clone contents.
+ }
+
+ void dumpCache() {
+ System.out.printf("fTextIdx:%d fBufIdx:%d\n", fTextIdx, fBufIdx);
+ for (int i=fStartBufIdx; ; i=modChunkSize(i+1)) {
+ System.out.printf("%d %d\n", i, fBoundaries[i]);
+ if (i == fEndBufIdx) {
+ break;
+ }
+ }
+ };
+
+ private final int modChunkSize(int index) { return index & (CACHE_SIZE - 1); };
+
+ static final int CACHE_SIZE = 128;
+ // static_assert((CACHE_SIZE & (CACHE_SIZE-1)) == 0, "CACHE_SIZE must be power of two.");
+
+ int fStartBufIdx;
+ int fEndBufIdx; // inclusive
+
+ int fTextIdx;
+ int fBufIdx;
+
+ int[] fBoundaries = new int[CACHE_SIZE];
+ short[] fStatuses = new short[CACHE_SIZE];
+
+ DictionaryBreakEngine.DequeI fSideBuffer = new DictionaryBreakEngine.DequeI();
+};
+
+
+
+
}
@Override
public int findBreaks(CharacterIterator text, int startPos, int endPos,
- boolean reverse, int breakType, DictionaryBreakEngine.DequeI foundBreaks) {
+ int breakType, DictionaryBreakEngine.DequeI foundBreaks) {
if (breakType >= 0 && breakType < fHandled.length()) {
UnicodeSet uniset = fHandled.get(breakType);
int c = CharacterIteration.current32(text);
- if (reverse) {
- while (text.getIndex() > startPos && uniset.contains(c)) {
- CharacterIteration.previous32(text);
- c = CharacterIteration.current32(text);
- }
- } else {
- while (text.getIndex() < endPos && uniset.contains(c)) {
- CharacterIteration.next32(text);
- c = CharacterIteration.current32(text);
- }
+ while (text.getIndex() < endPos && uniset.contains(c)) {
+ CharacterIteration.next32(text);
+ c = CharacterIteration.current32(text);
}
}
return 0;
version https://git-lfs.github.com/spec/v1
-oid sha256:e8f0af1c8a7b33d8ff22dea3143ecde79291218f0cea9b18181d3fdbd462c2fb
-size 12226267
+oid sha256:c147e785f0d4400d571af2fd76930e9a94b87d63b27dd5039e385e92969dd296
+size 12197459
version https://git-lfs.github.com/spec/v1
-oid sha256:acb00f0af330fbce209ffc15f9af2368edda40ced40f1b3367f040063f6e268a
+oid sha256:bf9291b5ec8a8bd2be603c4aaadcf9c5fbb3f59fd3b0498bbe357e0fbd5fc5e5
size 92486
}
/**
- * Testing the methods lastt(), previous(), and preceding() of RuleBasedBreakIterator
+ * Testing the methods last(), previous(), and preceding() of RuleBasedBreakIterator
**/
@Test
public void TestLastPreviousPreceding() {
String testString = "This is a word break. Isn't it? 2.25 dollars";
logln("Testing last(),previous(), preceding() with custom rules");
logln("testing word iteration for string \"" + testString + "\"");
- RuleBasedBreakIterator wordIter1 = (RuleBasedBreakIterator) BreakIterator.getWordInstance(Locale.getDefault());
+ RuleBasedBreakIterator wordIter1 = (RuleBasedBreakIterator) BreakIterator.getWordInstance(Locale.ENGLISH);
wordIter1.setText(testString);
p = wordIter1.last();
if (p != testString.length()) {
- errln("ERROR: first() returned" + p + "instead of" + testString.length());
+ errln("ERROR: last() returned" + p + "instead of" + testString.length());
}
q = wordIter1.previous();
doTest(testString, p, q, 37, "dollars");
@Test
public void TestIsBoundary() {
String testString1 = "Write here. \u092d\u0301\u0930\u0924 \u0938\u0941\u0902\u0926\u0930 a\u0301u";
- RuleBasedBreakIterator charIter1 = (RuleBasedBreakIterator) BreakIterator.getCharacterInstance(Locale.getDefault());
+ RuleBasedBreakIterator charIter1 = (RuleBasedBreakIterator) BreakIterator.getCharacterInstance(Locale.ENGLISH);
charIter1.setText(testString1);
int bounds1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 15, 16, 17, 20, 21, 22, 23, 25, 26};
doBoundaryTest(charIter1, testString1, bounds1);
- RuleBasedBreakIterator wordIter2 = (RuleBasedBreakIterator) BreakIterator.getWordInstance(Locale.getDefault());
+ RuleBasedBreakIterator wordIter2 = (RuleBasedBreakIterator) BreakIterator.getWordInstance(Locale.ENGLISH);
wordIter2.setText(testString1);
int bounds2[] = {0, 5, 6, 10, 11, 12, 16, 17, 22, 23, 26};
doBoundaryTest(wordIter2, testString1, bounds2);
static class TestParams {
BreakIterator bi;
StringBuilder dataToBreak = new StringBuilder();
- int[] expectedBreaks = new int[1000];
- int[] srcLine = new int[1000];
- int[] srcCol = new int[1000];
+ int[] expectedBreaks = new int[4000];
+ int[] srcLine = new int[4000];
+ int[] srcCol = new int[4000];
ULocale currentLocale = new ULocale("en_US");
}
</badrules>
<rules>
-#TODO: uncomment this line when quoted_literals_only is implemented.
-#!!quoted_literals_only;
+!!quoted_literals_only;
!!forward;
'Hello World';
!!reverse;
</rules>
<data>•Hello World•</data>
+
+# Test for circular buffer overflow during reverse iteration with inefficient reverse rules,
+# Too many boundaries between safe back up position and current position.
+
+<rules>
+!!forward;
+.;
+!!reverse;
+.*;
+</rules>
+<data>•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•a•</data>
+
+#
+# Dictionary regression check
+# Intended to find unexpected behavior changes when changing dictionary implementation code,
+# but may also be fragile, failing with intended improvements to dictionary breaking.
+#
+<locale en>
+<word>
+<data>•Unicode<200> •คือ<200>อะไร<200>?•
+•Unicode<200> •กำหนด<200>หมายเลข<200>เฉพาะ<200>สำหรับ<200>ทุก<200>อักขระ<200>
+•โดย<200>ไม่<200>สนใจ<200>ว่า<200>เป็น<200>แพ<200>ล็ต<200>ฟอร์ม<200>ใด<200>
+•ไม่<200>ขึ้น<200>กับ<200>ว่า<200>จะ<200>เป็น<200>โปรแกรม<200>ใด<200>
+•และ<200>ไม่<200>ว่า<200>จะ<200>เป็น<200>ภาษา<200>ใด<200>
+•
+•โดย<200>พื้น<200>ฐาน<200>แล้ว<200>,• •คอมพิวเตอร์<200>จะ<200>เกี่ยวข้อง<200>กับ<200>เรื่อง<200>ของ<200>ตัวเลข<200>.• •คอมพิวเตอร์<200>จัด<200>เก็บ<200>ตัว<200>อักษร<200>และ<200>อักข<200>ระ<200>อื่นๆ<200> •โดย<200>การ<200>กำหนด<200>หมายเลข<200>ให้<200>สำหรับ<200>แต่ละ<200>ตัว<200>.• •ก่อน<200>หน้า<200>ที่๊<200> •Unicode<200> •จะ<200>ถูก<200>สร้าง<200>ขึ้น<200>,• •ได้<200>มี<200>ระบบ<200> •encoding<200> •อยู่<200>หลาย<200>ร้อย<200>ระบบ<200>สำหรับ<200>การ<200>กำหนด<200>หมายเลข<200>เหล่า<200>นี้<200>.• •ไม่มี<200> •encoding<200> •ใด<200>ที่<200>มี<200>จำนวน<200>ตัว<200>อักขระ<200>มาก<200>เพียง<200>พอ<200>:• •ยก<200>ตัวอย่าง<200>เช่น<200>,• •เฉพาะ<200>ใน<200>กลุ่ม<200>สหภาพ<200>ยุโรป<200>เพียง<200>แห่ง<200>เดียว<200> •ก็<200>ต้องการ<200>หลาย<200> •encoding<200> •ใน<200>การ<200>ครอบคลุม<200>ทุก<200>ภาษา<200>ใน<200>กลุ่ม<200>.• •หรือ<200>แม้แต่<200>ใน<200>ภาษา<200>เดี่ยว<200> •เช่น<200> •ภาษา<200>อังกฤษ<200> •ก็<200>ไม่มี<200> •encoding<200> •ใด<200>ที่<200>เพียง<200>พอ<200>สำหรับ<200>ทุก<200>ตัว<200>อักษร<200>,• •เครื่องหมาย<200>วรรค<200>ตอน<200> •และ<200>สัญลักษณ์<200>ทาง<200>เทคนิค<200>ที่<200>ใช้<200>กัน<200>อยู่<200>ทั่วไป<200>.•
+•
+•ระบบ<200> •encoding<200> •เหล่า<200>นี้<200>ยัง<200>ขัด<200>แย้ง<200>ซึ่ง<200>กัน<200>และ<200>กัน<200>.• •นั่น<200>ก็<200>คือ<200>,• •ใน<200>สอง<200> •encoding<200> •สามารถ<200>ใช้<200>หมายเลข<200>เดียวกัน<200>สำหรับ<200>ตัว<200>อักขระ<200>สอง<200>ตัว<200>ที่<200>แตก<200>ต่าง<200>กัน<200>,•หรือ<200>ใช้<200>หมายเลข<200>ต่าง<200>กัน<200>สำหรับ<200>อักขระ<200>ตัว<200>เดียวกัน<200>.• •ใน<200>ระบบ<200>คอมพิวเตอร์<200> •(•โดย<200>เฉพาะ<200>เซิร์ฟเวอร์<200>)• •ต้อง<200>มี<200>การ<200>สนับสนุน<200>หลาย<200> •encoding<200>;• •และ<200>เมื่อ<200>ข้อมูล<200>ที่<200>ผ่าน<200>ไป<200>มาระ<200>หว่าง<200>การ<200>เข้า<200>รหัส<200>หรือ<200>แพ<200>ล็ต<200>ฟอร์ม<200>ที่<200>ต่าง<200>กัน<200>,• •ข้อมูล<200>นั้น<200>จะ<200>เสี่ยง<200>ต่อ<200>การ<200>ผิด<200>พลาด<200>เสีย<200>หาย<200>.•
+•
+•Unicode<200> •จะ<200>เปลี่ยนแปลง<200>สิ่ง<200>เหล่า<200>นั้น<200>ทั้งหมด<200>!•
+•
+•Unicode<200> •กำหนด<200>หมายเลข<200>เฉพาะ<200>สำหรับ<200>แต่ละ<200>อักขระ<200>,• •โดย<200>ไม่<200>สนใจ<200>ว่า<200>เป็น<200>แพ<200>ล็ต<200>ฟอร์ม<200>ใด<200>,• •ไม่<200>ขึ้น<200>กับ<200>ว่า<200>จะ<200>เป็น<200>โปรแกรม<200>ใด<200>และ<200>ไม่<200>ว่า<200>จะ<200>เป็น<200>ภาษา<200>ใด<200>.• •มาตรฐาน<200> •Unicode<200> •ได้<200>ถูก<200>นำ<200>ไป<200>ใช้<200>โดย<200>ผู้นำ<200>ใน<200>อุตสาหกรรม<200> •เช่น<200> •Apple<200>,• •HP<200>,• •IBM<200>,• •JustSystem<200>,• •Microsoft<200>,• •Oracle<200>,• •SAP<200>,• •Sun<200>,• •Sybase<200>,• •Unisys<200> •และ<200>อื่นๆ<200> •อีก<200>มาก<200>.• •Unicode<200> •เป็น<200>สิ่ง<200>ที่<200>จำเป็น<200>สำหรับ<200>มาตร<200>ฐาน<200>ใหม่ๆ<200> •เช่น<200> •XML<200>,• •Java<200>,• •ECMAScript<200> •(•JavaScript<200>)•,• •LDAP<200>,• •CORBA<200> •3.0<100>,• •WML<200> •ฯลฯ<200>.•,• •และ<200>เป็น<200>แนวทาง<200>อย่าง<200>เป็น<200>ทางการ<200>ใน<200>การ<200>ทำ<200> •ISO<200>/•IEC<200> •10646<100>.• •Unicode<200> •ได้<200>รับ<200>การ<200>สนับสนุน<200>ใน<200>ระบบ<200>ปฏิบัติ<200>การ<200>จำนวน<200>มาก<200>,• •บราวเซอร์<200>ใหม่ๆ<200> •ทก<200>ตัว<200>,• •และ<200>ผลิต<200>ภัณฑ์<200>อื่นๆ<200> •อีก<200>มาก<200>.• •การ<200>เกิด<200>ขึ้น<200>ของ<200> •Unicode<200> •Standard<200> •และ<200>ทูล<200>ส์<200>ต่างๆ<200> •ที่<200>มี<200>ใน<200>การ<200>สนับสนุน<200> •Unicode<200>,• •เป็น<200>หนึ่ง<200>ใน<200>แนว<200>โน้ม<200>ทาง<200>เทคโนโลยี<200>ซอฟต์แวร์<200>ระดับ<200>โลก<200>ที่<200>มี<200>ความ<200>สำคัญ<200>ที่สุด<200>.•
+•
+•การ<200>รวม<200> •Unicode<200> •เข้าไป<200>ใน<200>ระบบ<200>ไคลเอ็นต์<200>-•เซิร์ฟเวอร์<200> •หรือ<200>แอ็พ<200>พลิ<200>เค<200>ชัน<200>แบบ<200> •multi<200>-•tiered<200> •และ<200>เว็บไซต์<200> •จะ<200>ทำให้<200>เกิด<200>การ<200>ประหยัด<200>ค่า<200>ใช้<200>จ่าย<200>มากกว่า<200>การ<200>ใช้<200>ชุด<200>อักขระ<200>แบบ<200>เดิม<200>.• •Unicode<200> •ทำให้<200>ผลิตภัณฑ์<200>ซอฟต์แวร์<200>หนึ่ง<200>เดียว<200> •หรือ<200>เว็บไซต์<200>แห่ง<200>เดียว<200> •รองรับ<200>ได้<200>หลาย<200>แพ<200>ล็ต<200>ฟอร์ม<200>,• •หลาย<200>ภาษา<200>และ<200>หลาย<200>ประเทศ<200>โดย<200>ไม่<200>ต้อง<200>ทำการ<200>รื้อ<200>ปรับ<200>ระบบ<200>.• •Unicode<200> •ยัง<200>ทำให้<200>ข้อมูล<200>สามารถ<200>เคลื่อน<200>ย้าย<200>ไป<200>มา<200>ใน<200>หลายๆ<200> •ระบบ<200>โดย<200>ไม่<200>เกิด<200>ความ<200>ผิด<200>พลาด<200>เสีย<200>หาย<200>.•
+•
+•เกี่ยว<200>กับ<200> •Unicode<200> •Consortium<200>
+•
+•Unicode<200> •Consortium<200> •เป็น<200>องค์กร<200>ไม่<200>แสวงหา<200>กำไร<200>ที่<200>ก่อ<200>ตั้ง<200>ขึ้น<200>เพื่อ<200>พัฒนา<200>,• •ขยาย<200>และ<200>ส่ง<200>เสริม<200>การ<200>ใช้<200> •Unicode<200> •Standard<200>,• •ซึ่ง<200>กำหนด<200>รูป<200>แบบ<200>การ<200>แทน<200>ค่า<200>ของ<200>ข้อความ<200>ใน<200>ผลิตภัณฑ์<200>ซอฟต์แวร์<200>และ<200>มาตร<200>ฐาน<200>ใหม่ๆ<200>.• •สมาชิก<200>ของ<200>สมาคม<200>เป็น<200>ตัวแทน<200>จาก<200>บริษัท<200>และ<200>องค์กร<200>ใน<200>อุตสาหกรรม<200>คอมพิวเตอร์<200>และ<200>การ<200>ประมวล<200>ผล<200>สารสนเทศ<200>.• •สมาคม<200>ได้<200>รับ<200>การ<200>สนับสนุน<200>ทางการ<200>เงิน<200>ผ่าน<200>ทาง<200>ค่า<200>ธรรมเนียม<200>ของ<200>การ<200>เป็น<200>สมาชิก<200>เท่านั้น<200>.• •สมาชิก<200>ภาพ<200>ของ<200> •Unicode<200> •Consortium<200> •เปิด<200>กว้าง<200>สำหรับ<200>องค์กร<200>หรือ<200>บุคคล<200>ใดๆ<200> •ใน<200>โลก<200>ที่<200>ต้องการ<200>สนับสนุน<200> •Unicode<200> •Standard<200> •และ<200>ช่วย<200>เหลือ<200>การ<200>ขยาย<200>ตัว<200>และ<200>การนำ<200> •Unicode<200> •ไป<200>ใช้<200>งาน<200>.•
+•
+•สำหรับ<200>ข้อมูล<200>เพิ่ม<200>เติม<200>,• •ให้<200>ดู<200>ที่<200> •Glossary<200>,• •Sample<200> •Unicode<200>-•Enabled<200> •Products<200>,• •Technical<200> •Introduction<200> •และ<200> •Useful<200> •Resources<200>.•</data>
+
+<word>
+# Burmese
+<data>•အ<200>လော<200>င္<200>မ<200>င္<200>တရား<200>
+• • • • • •မဟာ<200>ဓမ္မရာဇာ<200>မိ<200>ပတိ<200>လ<200>က္<200>ထ<200>က္<200>တ္<200>ဝ<200>င္<200> •အ<200>င္<200>ဝ<200>နေ<200>ပ္<200>ရ<200>ည္<200>တော္<200>က္<200>ရီး<200>သ<200>ည္<200> •မ<200>င္<200>ရိ<200>မ္<200>မ<200>သ<200>က္<200>ဖ္<200>ရ<200>စ္<200>နေ<200>သ<200>ည္<200>။• •မဏိ<200>ပူ<200>ရ<200> •က<200>သ<200>ည္<200>မ္<200>ယား<200>က<200> •အ<200>င္<200>ဝ<200>နေ<200>ပ္<200>ရ<200>ည္<200>တော္<200>၏• •မ္<200>ရော<200>က္<200>ဘ<200>က္<200>တ<200>လ္<200>ဝ္<200>ဟား<200>ကုိ<200> •တုိ<200>က္<200>ခုိ<200>က္<200>ဖ္<200>ယ<200>က္<200>ဆီး<200>သ<200>ည္<200>။• •အော<200>က္<200>မ္<200>ရ<200>န္<200>မာ<200>နုိ<200>င္<200>ငံ<200> •ဟံ<200>သာ<200>ဝ<200>တီ<200>သား<200>တုိ့<200>က<200>လ<200>ည္<200> •ပု<200>န္<200>က<200>န္<200>သ<200>ည္<200>။• •မတ္တ<200>ရာ<200>အု<200>တ္<200>ဖုိ<200>ရ္<200>ဟိ<200> •က္<200>ဝေ့<200>ရ္<200>ဟ<200>မ္<200>မ္<200>ယား<200>က<200>လ<200>ည္<200> •ထ<200>က္<200>ရ္<200>ဝ<200>သ<200>ည္<200>။•
+•
+• • • • •ထုိ<200>အ<200>ခ္<200>ယိ<200>န္<200>တ္<200>ဝ<200>င္<200> •မု<200>ဆုိး<200>ဖုိ<200>ရ္<200>ဝာ<200>သူ<200>က္<200>ရီး<200> •အော<200>င္<200>ဇေ<200>ယ္<200>ယ<200>သ<200>ည္<200> •မိမိ<200>၏•ရ္<200>ဝာ<200>ကုိ<200> •လုံ<200>ခ္<200>ရုံ<200>အော<200>င္<200>ထ<200>န္<200>လုံး<200>တ<200>ပ္<200>မ္<200>ယား<200>ကာ<200>ရ<200>သ<200>ည္<200>။• •အနီး<200>အ<200>ပား<200> •က္<200>ယေး<200>ရ္<200>ဝာ<200> •လေး<200>ဆ<200>ယ့္<200>ခ္<200>ရော<200>က္<200>ရ္<200>ဝာ<200>ကုိ<200> •သိ<200>မ္း<200>သ္<200>ဝ<200>င္<200>ထား<200>သ<200>ည္<200>။• •မ<200>က္<200>ရာ<200>မီ<200>ပ<200>င္<200> •အ<200>င္<200>ဝ<200>နေ<200>ပ္<200>ရ<200>ည္<200>တော္<200>က္<200>ရီး<200>သ<200>ည္<200> •ဟံ<200>သာ<200>ဝ<200>တီ<200>တ<200>ပ္<200>မ္<200>ယား<200> •လ<200>က္<200>တ္<200>ဝ<200>င္<200>သ<200>က္<200>ဆ<200>င္<200>ရ<200>တော့<200>သ<200>ည္<200>။•
+•
+• • • • •အ<200>င္<200>ဝ<200>ကုိ<200> •သိ<200>မ္<200>ပုိ<200>က္<200>ပ္<200>ရီး<200>သော<200> •ဟံ<200>သာ<200>ဝ<200>တီ<200>တ<200>ပ္<200>မ္<200>ယား<200>သ<200>ည္<200> •မ္<200>ရော<200>က္<200>ဘ<200>က္<200>တ<200>လ္<200>ဝ္<200>ဟား<200>က္<200>ယေး<200>ရ္<200>ဝာ<200>မ္<200>ယား<200>ကုိ<200> •သစ္စာ<200>ခံ<200>ခုိ<200>င္<200>ရ<200>န္<200> •လာ<200>က္<200>ရ<200>ရာ<200> •မု<200>ဆုိး<200>ဖုိ<200>ရ္<200>ဝာ<200>သုိ့<200> •ရော<200>က္<200>ရ္<200>ဟိ<200>လာ<200>သ<200>ည္<200>။• •တ<200>ခ္<200>ယိ<200>န္<200>တ<200>ည္<200>မ္<200>ဟာ<200>ပ<200>င္<200> •က္<200>ဝေ့<200>ရ္<200>ဟ<200>မ္<200>မ္<200>ယား<200>က<200>လ<200>ည္<200> •သစ္စာ<200>ခံ<200>ခုိ<200>င္<200>ရ<200>န္<200> •ရော<200>က္<200>ရ္<200>ဟိ<200>လာ<200>သ<200>ည္<200>။• •ဦး<200>အော<200>င္<200>ဇေ<200>ယ္<200>ယ<200>သ<200>ည္<200> •အ<200>ဖ္<200>ဝဲ့<200>န္<200>ဟ<200>စ္<200>ဖ္<200>ဝဲ့<200>ကုိ<200> •ခ္<200>ရေ<200>ငံ<200>စ္<200>ဝာ<200> •ဆ<200>က္<200>ဆံ<200>သ<200>ည္<200>။• •မ<200>ည္<200>သူ့<200>သ<200>စ္<200>စာ<200>ကုိ<200>မ္<200>ယ္<200>ဟ<200> •ခံ<200>ယူ<200>ခ္<200>ရ<200>င္<200>မ<200>ပ္<200>ရု<200>ပေ<200>။• •ဟံ<200>သာ<200>ဝ<200>တီ<200>တ<200>ပ္<200>ဖ္<200>ဝဲ့<200>ကုိ<200> •အ<200>ပ္<200>ရ<200>န္<200>ခ<200>ရီး<200>တ္<200>ဝ<200>င္<200> •လ<200>မ္<200>မ္<200>ဟ<200>ဖ္<200>ရ<200>တ္<200>၍• •တုိ<200>က္<200>ခုိ<200>က္<200>သ<200>ည္<200>။• •ဟံ<200>သာ<200>ဝ<200>တီ<200>တ<200>ပ္<200>ဖ္<200>ဝဲ့<200>မ္<200>ယား<200> •အထိ<200>အ<200>ခုိ<200>က္<200>အ<200>က္<200>ယ<200>အ<200>ဆုံး<200>မ္<200>ယား<200>စ္<200>ဝာ<200>ဖ္<200>ရ<200>င္<200> •ပ္<200>ရ<200>န္<200>ရ<200>သ<200>ည္<200>။•
+•
+• • • • •ဟံ<200>သာ<200>ဝ<200>တီ<200>တ<200>ပ္<200>မ္<200>ယား<200>သ<200>ည္<200> •မု<200>ဆုိး<200>ဖုိ<200>ရ္<200>ဝာ<200>ကုိ<200> •လာ<200>ရော<200>က္<200>တုိ<200>က္<200>ခုိ<200>က္<200>က္<200>ရ<200>ပ္<200>ရ<200>န္<200>သ<200>ည္<200>။• •ဦး<200>အော<200>င္<200>ဇေ<200>ယ္<200>ယ<200>သ<200>ည္<200> •သ္<200>ဝေး<200>သော<200>က္<200>ရဲ<200>ဘော္<200> •ခ္<200>ရော<200>က္<200>က္<200>ယိ<200>ပ္<200>ရ္<200>ဟ<200>စ္<200>ယော<200>က္<200>န္<200>ဟ<200>င္<200>အတူ<200> •ဦးစီး<200>ကာ<200>အော<200>င္<200>မ္<200>ရ<200>င္<200>စ္<200>ဝာ<200>ခု<200>ခံ<200>တ္<200>ဝ<200>န္<200>လ္<200>ဟ<200>န္<200>နုိ<200>င္<200>ခဲ့<200>သ<200>ည္<200>။• •ထုိ့<200>နော<200>က္<200> •ဦး<200>အော<200>င္<200>ဇေ<200>ယ္<200>ယ<200>သ<200>ည္<200> •မ္<200>ရော<200>က္<200>ဘ<200>က္<200>တ<200>လ္<200>ဝ္<200>ဟား<200>ရ္<200>ဟိ<200> •ရ္<200>ဟ<200>မ္<200>မ္<200>ယား<200>န္<200>ဟ<200>င္<200> •မ္<200>ရ<200>န္<200>မာ<200>မ္<200>ယား<200>ကုိ<200>လ<200>ည္<200> •ဆ<200>က္<200>သ္<200>ဝ<200>ယ္<200>စ<200>ည္<200>ရုံး<200>နုိ<200>င္<200>ခဲ့<200>သ<200>ည္<200>။• •ဤ<200>သုိ့<200>ဖ္<200>ရ<200>င္<200> •ဦး<200>အော<200>င္<200>ဇေ<200>ယ္<200>ယ<200>၏• •အ<200>ရ္<200>ဟိ<200>န္<200>အ<200>ဝာ<200> •မ္<200>ရ<200>င္<200>မား<200>လာ<200>လေ<200>သ<200>ည္<200>။•
+•
+• • • • •ဦး<200>အော<200>င္<200>ဇေ<200>ယ္<200>ယ<200>သ<200>ည္<200> •အ<200>လော<200>င္<200>မ<200>င္<200>တ<200>ရား<200>ဘ္<200>ဝဲ့<200>ကုိ<200> •ခံယူ<200>ကာ<200> •ကု<200>န္<200>ဘော<200>င္<200>မ<200>င္<200>ဆ<200>က္<200>ကုိ<200>စ<200>တ<200>င္<200>တ<200>ည္<200>ထော<200>င္<200>သ<200>ည္<200>။• •မု<200>ဆုိး<200>ဖုိ<200>ရ္<200>ဝာ<200>ကုိ<200> •ရ္<200>ဝ္<200>ဟ<200>ဝေ<200>ဘုိ<200>ဟု<200> •သ<200>မု<200>တ္<200>ကာ<200> •မ္<200>ရုိ့<200>န<200>န္<200>တ<200>ည္<200>သ<200>ည္<200>။• •န<200>န္<200>တ<200>ည္<200>သ<200>က္<200>က<200>ရာ<200>ဇ္<200>ဖ္<200>ရ<200>စ္<200>သော<200> •၁၁၁၅<100> •ခု<200>ကုိ<200> •ဥ<200>ဩ<200>အော္<200>မ္<200>ရ<200>ည္<200> •ကု<200>န္<200>ဘော<200>င္<200>တ<200>ည္<200>ဟု<200> •အ<200>မ္<200>ဟ<200>တ္<200>အ<200>သား<200>ပ္<200>ရု<200>က္<200>ရ<200>သ<200>ည္<200>။•
+•
+• • • • •အ<200>လော<200>င္<200>မ<200>င္<200>တရား<200>သ<200>ည္<200> •ဧရာ<200>ဝ<200>တီ<200>န္<200>ဟ<200>င္<200>ခ္<200>ယ<200>င္<200>တ္<200>ဝ<200>င္<200> •မ္<200>ရ<200>စ္<200>န္<200>ဟ<200>စ္<200>သ္<200>ဝ<200>ယ္<200>အ<200>က္<200>ရား<200> •ဒေ<200>သ<200>မ္<200>ယား<200>ကုိ<200>အ<200>ခုိ<200>င္<200>အ<200>မာ<200> •စု<200>စ<200>ည္<200>ပ္<200>ရီး<200>နော<200>က္<200> •အ<200>င္<200>ဝ<200>ကုိ<200> •တုိ<200>က္<200>ခုိ<200>က္<200>အော<200>င္<200>မ္<200>ရ<200>င္<200>သ<200>ည္<200>။• •ထုိ<200>နော<200>က္<200>တ္<200>ဝ<200>င္<200>ပ္<200>ရ<200>ည္<200>၊• •လ္<200>ဝ<200>န္<200>ဆေး<200>၊• •ဒ<200>ဂုံ<200>မ္<200>ရုိ့<200>မ္<200>ယား<200>ကုိ<200> •သိ<200>မ္<200>ပုိ<200>က္<200>သ<200>ည္<200>။• •လ္<200>ဝ<200>န္<200>ဆေး<200> •ကုိ<200>မ္<200>ရ<200>န္<200>အော<200>င္<200>ဟူ<200>၍• •သ<200>မု<200>တ္<200>သ<200>ည္<200>။• •ဒ<200>ဂုံ<200>ကုိ<200>ရ<200>န္<200>ကု<200>န္<200>ဟူ<200>၍• •သ<200>မု<200>တ္<200>ထ<200>သ<200>ည္<200>။•</data>
+
+<word>
+# japanese
+<data>•ユニ<400>コード<400>と<400>は<400>何<400>か<400>？•
+•ユニ<400>コード<400>は<400>、•すべて<400>の<400>文字<400>に<400>固有<400>の<400>番号<400>を<400>付与<400>し<400>ます<400>
+•プラットフォーム<400>に<400>は<400>依存<400>しま<400>せん<400>
+•プログラム<400>に<400>も<400>依存<400>しま<400>せん<400>
+•言語<400>に<400>も<400>依存<400>しま<400>せん<400>
+•
+•コンピューター<400>は<400>、•本質<400>的<400>に<400>は<400>数字<400>しか<400>扱う<400>こと<400>が<400>でき<400>ま<400>せん<400>。•コンピューター<400>は<400>、•文字<400>や<400>記号<400>など<400>の<400>それぞれに<400>番号<400>を<400>割り振る<400>こと<400>によって<400>扱える<400>よう<400>にし<400>ます<400>。•ユニ<400>コード<400>が<400>出来る<400>まで<400>は<400>、•これらの<400>番号<400>を<400>割り振る<400>仕組み<400>が<400>何<400>百<400>種類<400>も<400>存在<400>しま<400>した<400>。•どの<400>一つ<400>を<400>とっても<400>、•十分<400>な<400>文字<400>を<400>含<400>んで<400>は<400>いま<400>せん<400>で<400>した<400>。•例えば<400>、•欧州<400>連合<400>一つ<400>を<400>見<400>て<400>も<400>、•その<400>すべて<400>の<400>言語<400>を<400>カバー<400>する<400>ため<400>に<400>は<400>、•いくつか<400>の<400>異なる<400>符号<400>化<400>の<400>仕組み<400>が<400>必要<400>で<400>した<400>。•英語<400>の<400>よう<400>な<400>一つ<400>の<400>言語<400>に<400>限<400>って<400>も<400>、•一つ<400>だけ<400>の<400>符号<400>化<400>の<400>仕組み<400>では<400>、•一般<400>的<400>に<400>使<400>われる<400>すべて<400>の<400>文字<400>、•句読点<400>、•技術<400>的<400>な<400>記号<400>など<400>を<400>扱う<400>に<400>は<400>不十分<400>で<400>した<400>。•
+•
+•これらの<400>符号<400>化<400>の<400>仕組み<400>は<400>、•相互<400>に<400>矛盾<400>する<400>もの<400>でも<400>ありま<400>した<400>。•二つ<400>の<400>異なる<400>符号<400>化<400>の<400>仕組み<400>が<400>、•二つ<400>の<400>異なる<400>文字<400>に<400>同一<400>の<400>番号<400>を<400>付ける<400>こと<400>も<400>できる<400>し<400>、•同じ<400>文字<400>に<400>異なる<400>番号<400>を<400>付ける<400>こと<400>も<400>できる<400>の<400>です<400>。•どの<400>よう<400>な<400>コンピューター<400>も<400>（•特に<400>サーバー<400>は<400>）•多く<400>の<400>異<400>な<400>っ<400>た<400>符号<400>化<400>の<400>仕組み<400>を<400>サポート<400>する<400>必要<400>が<400>あり<400>ます<400>。•たとえ<400>データ<400>が<400>異なる<400>符号<400>化<400>の<400>仕組み<400>や<400>プラットフォーム<400>を<400>通過<400>し<400>て<400>も<400>、•いつ<400>どこ<400>で<400>データ<400>が<400>乱れる<400>か<400>分<400>から<400>ない<400>危険<400>を<400>冒す<400>こと<400>の<400>なる<400>の<400>です<400>。•
+•
+•ユニ<400>コード<400>は<400>すべて<400>を<400>変<400>え<400>ます<400>
+•
+•ユニ<400>コード<400>は<400>、•プラットフォーム<400>に<400>係<400>わら<400>ず<400>、•プログラム<400>に<400>係<400>わら<400>ず<400>、•言語<400>に<400>係<400>わら<400>ず<400>、•すべて<400>の<400>文字<400>に<400>独立<400>した<400>番号<400>を<400>与<400>え<400>ます<400>。•ユニ<400>コード<400>標準<400>は<400>、•アップル<400>、•ヒュー<400>レット<400>パッ<400>カード<400>、•IBM<200>、•ジャスト<400>システム<400>、•マイクロ<400>ソフト<400>、•オラクル<400>、•SAＰ<200>、•サン<400>、•サイ<400>ベース<400>など<400>の<400>産業<400>界<400>の<400>主導<400>的<400>企業<400>と<400>他の<400>多く<400>の<400>企業<400>に<400>採用<400>さ<400>れ<400>てい<400>ます<400>。•ユニ<400>コード<400>は<400>、•XML<200>、•Java<200>、•ECMAScript<200>(•JavaScript<200>)•、•LDAP<200>、•CORBA<200> •3.0<100>など<400>の<400>最先端<400>の<400>標準<400>の<400>前提<400>と<400>な<400>って<400>おり<400>、•ユニ<400>コード<400>を<400>実装<400>す<400>れ<400>ば<400>、•ISO<200>/•IEC<200> •10646<100>に<400>適合<400>する<400>ことに<400>なり<400>ます<400>。•ユニ<400>コード<400>は<400>、•多く<400>の<400>オペレーティングシステム<400>と<400>すべて<400>の<400>最新<400>の<400>ブラウザー<400>と<400>他の<400>多く<400>の<400>製品<400>で<400>サポート<400>さ<400>れ<400>てい<400>ます<400>。•ユニ<400>コード<400>標準<400>の<400>出現<400>と<400>ユニ<400>コード<400>を<400>サポート<400>する<400>ツール<400>類<400>は<400>、•昨今<400>顕著<400>に<400>な<400>って<400>いる<400>ソフトウエア<400>技術<400>の<400>グローバル<400>化<400>の<400>流れ<400>に対して<400>、•特に<400>役<400>に<400>立<400>って<400>い<400>ます<400>。•
+•
+•ユニ<400>コード<400>を<400>ク<400>ライアン<400>ト<400>サーバー<400>型<400>の<400>アプリケーション<400>や<400>、•多層<400>構造<400>を<400>持つ<400>アプリケーション<400>、•ウェブサイト<400>など<400>に<400>組み込む<400>こと<400>で<400>、•従来<400>の<400>文字<400>コードセット<400>を<400>用いる<400>より<400>も<400>明らか<400>な<400>コスト<400>削減<400>が<400>可能<400>です<400>。•ユニ<400>コード<400>は<400>、•単一<400>の<400>ソフトウエア<400>製品<400>、•単一<400>の<400>ウェブサイト<400>に<400>、•何ら<400>手<400>を<400>加える<400>こと<400>なく<400>、•複数<400>の<400>プラットフォーム<400>、•複数<400>の<400>言語<400>、•複数<400>の<400>国<400>を<400>カバー<400>する<400>こと<400>が<400>出来る<400>の<400>です<400>。•ユニ<400>コード<400>は<400>、•データ<400>が<400>多く<400>の<400>異なる<400>システム<400>の<400>間<400>を<400>、•何<400>の<400>乱れ<400>も<400>なし<400>に<400>転送<400>する<400>こと<400>を<400>可能<400>と<400>する<400>の<400>です<400>。•
+•
+•ユニ<400>コード<400>コンソーシアム<400>について<400>
+•
+•ユニ<400>コード<400>コンソーシアム<400>は<400>、•最新<400>の<400>ソフトウエア<400>製品<400>と<400>標準<400>において<400>テキスト<400>を<400>表現<400>する<400>こと<400>を<400>意味<400>する<400>“•ユニ<400>コード<400>標準<400>”•の<400>構築<400>、•発展<400>、•普及<400>、•利用<400>促進<400>を<400>目的<400>として<400>設立<400>さ<400>れ<400>た<400>非<400>営利<400>組織<400>です<400>。•同<400>コンソーシアム<400>の<400>会員<400>は<400>、•コンピューター<400>と<400>情報処理<400>に<400>係わる<400>広汎<400>な<400>企業<400>や<400>組織<400>から<400>構成<400>さ<400>れ<400>てい<400>ます<400>。•同<400>コンソーシアム<400>は<400>、•財政<400>的<400>に<400>は<400>、•純粋<400>に<400>会費<400>のみ<400>によって<400>運営<400>さ<400>れ<400>てい<400>ます<400>。•ユニ<400>コード<400>標準<400>を<400>支持<400>し<400>、•その<400>拡張<400>と<400>実装<400>を<400>支援<400>する<400>世界中<400>の<400>組織<400>や<400>個人<400>は<400>、•だれ<400>も<400>が<400>ユニ<400>コード<400>コンソーシアム<400>の<400>会員<400>なる<400>こと<400>が<400>でき<400>ます<400>。•
+•
+•より<400>詳しい<400>こと<400>を<400>お<400>知<400>り<400>に<400>なり<400>たい<400>方<400>は<400>、•Glossary<200>,• •Technical<200> •Introduction<200> •および<400> •Useful<200> •Resources<200>を<400>ご<400>参照<400>くだ<400>さい<400>。•
+•</data>
projects / icu / commitdiff