ICU-21699 Fix CjkBreakEngine performance issue

1. vector.contains() uses sequential comparison, O(n).
   As the vector size is great, the performance will be impacted.
   Remove this unnecessary check, vector.contains(), in C++.

2. At Java's CjkBreakEngine, replace "vector.contains()" with "if(pos > previous)" to deal with duplicate breakpoint position.
   This way, C++ and Java implementation will be synchronous.
   Test: ant checkTest -Dtestclass='com.ibm.icu.dev.test.rbbi.RBBITest'
   (RBBTest#TestBreakAllChars() can generate duplicate position for word break. It could pass with this change)

diff --git a/icu4c/source/common/dictbe.cpp b/icu4c/source/common/dictbe.cpp
index 6b6d4297ad4e88011f6441c8e3e9994a15b87023..35d3cd48a7a6f6bfbb0e220f03cfb75883d3d79f 100644 (file)
--- a/icu4c/source/common/dictbe.cpp
+++ b/icu4c/source/common/dictbe.cpp
@@ -1370,7 +1370,7 @@ CjkBreakEngine::divideUpDictionaryRange( UText *inText,
         if (utextPos > prevUTextPos) {
             // Boundaries are added to foundBreaks output in ascending order.
             U_ASSERT(foundBreaks.size() == 0 || foundBreaks.peeki() < utextPos);
-            if (!(foundBreaks.contains(utextPos) || utextPos == rangeStart)) {
+            if (utextPos != rangeStart) {
                 foundBreaks.push(utextPos, status);
                 correctedNumBreaks++;
             }

diff --git a/icu4j/main/classes/core/src/com/ibm/icu/impl/breakiter/CjkBreakEngine.java b/icu4j/main/classes/core/src/com/ibm/icu/impl/breakiter/CjkBreakEngine.java
index a14c745e509946330ebe55a108980bab2126dc75..0404e031cc2367b7eea27c7d730d4269bea2ab30 100644 (file)
--- a/icu4j/main/classes/core/src/com/ibm/icu/impl/breakiter/CjkBreakEngine.java
+++ b/icu4j/main/classes/core/src/com/ibm/icu/impl/breakiter/CjkBreakEngine.java
@@ -209,12 +209,14 @@ public class CjkBreakEngine extends DictionaryBreakEngine {
         }
 
         int correctedNumBreaks = 0;
+        int previous = -1;
         for (int i = numBreaks - 1; i >= 0; i--) {
             int pos = charPositions[t_boundary[i]] + startPos;
-            if (!(foundBreaks.contains(pos) || pos == startPos)) {
-                foundBreaks.push(charPositions[t_boundary[i]] + startPos);
+            if (pos > previous && pos != startPos) {
+                foundBreaks.push(pos);
                 correctedNumBreaks++;
             }
+            previous = pos;
         }
 
         if (!foundBreaks.isEmpty() && foundBreaks.peek() == endPos) {