ICU-21699 Revise rule file

1. \u30fc doesn't belong to Hira, Kana nor Han. Add it into CJK dictionary
2. Include fullwidth char into ALPlus

diff --git a/icu4c/source/data/brkitr/rules/line_loose_phrase_cj.txt b/icu4c/source/data/brkitr/rules/line_loose_phrase_cj.txt
index a10b16897b74da64da9f193fd6b56b4484748691..3eab1f73bb19f0c50421a36c2756fe64c21f7dfa 100644 (file)
--- a/icu4c/source/data/brkitr/rules/line_loose_phrase_cj.txt
+++ b/icu4c/source/data/brkitr/rules/line_loose_phrase_cj.txt
@@ -93,9 +93,8 @@ $ZWJ = [:LineBreak = ZWJ:];
 # without a formal name. Because ICU rules require multiple uses of the expressions,
 # give them a single definition with a name
 
-$EAFWH = [\p{ea=F}\p{ea=W}\p{ea=H}];
-$OP30 = [$OP - $EAFWH];
-$CP30 = [$CP - $EAFWH];
+$OP30 = [$OP - [\p{ea=F}\p{ea=W}\p{ea=H}]];
+$CP30 = [$CP - [\p{ea=F}\p{ea=W}\p{ea=H}]];
 
 $ExtPictUnassigned = [\p{Extended_Pictographic} & \p{Cn}];
 
@@ -115,7 +114,7 @@ $Katakana = [:Katakana:];
 $Hiragana = [:Hiragana:];
 $HangulSyllable = [\uac00-\ud7a3];
 $ComplexContext = [:LineBreak = Complex_Context:];
-$KanaKanji      = [$Han $Hiragana $Katakana];
+$KanaKanji      = [$Han $Hiragana $Katakana \u30fc];
 $dictionaryCJK  = [$KanaKanji $HangulSyllable];
 $dictionary     = [$ComplexContext $dictionaryCJK];
 
@@ -127,7 +126,9 @@ $dictionary     = [$ComplexContext $dictionaryCJK];
 #                               XX  (Unknown, unassigned)
 #                         as $AL  (Alphabetic)
 #
-$ALPlus = [$AL $AI $SG $XX $EAFWH [$dictionary-[[:Mn:][:Mc:]]]];
+# Let fullwidth-ASCII digits and letters be part of words.
+$FW_alphanum = [\uff10-\uff19\uff21-\uff3a\uff41-\uff5a];
+$ALPlus = [$AL $AI $SG $XX $FW_alphanum [$dictionary-[[:Mn:][:Mc:]]]];
 
 
 ## -------------------------------------------------

diff --git a/icu4c/source/data/brkitr/rules/line_normal_phrase_cj.txt b/icu4c/source/data/brkitr/rules/line_normal_phrase_cj.txt
index eb51cdff54c29d384dde78ed40a37e8add42433c..55a12ff2bd7cd256f79b63714b08a347d060f86e 100644 (file)
--- a/icu4c/source/data/brkitr/rules/line_normal_phrase_cj.txt
+++ b/icu4c/source/data/brkitr/rules/line_normal_phrase_cj.txt
@@ -81,9 +81,8 @@ $ZWJ = [:LineBreak = ZWJ:];
 # without a formal name. Because ICU rules require multiple uses of the expressions,
 # give them a single definition with a name
 
-$EAFWH = [\p{ea=F}\p{ea=W}\p{ea=H}];
-$OP30 = [$OP - $EAFWH];
-$CP30 = [$CP - $EAFWH];
+$OP30 = [$OP - [\p{ea=F}\p{ea=W}\p{ea=H}]];
+$CP30 = [$CP - [\p{ea=F}\p{ea=W}\p{ea=H}]];
 
 $ExtPictUnassigned = [\p{Extended_Pictographic} & \p{Cn}];
 
@@ -103,7 +102,7 @@ $Katakana = [:Katakana:];
 $Hiragana = [:Hiragana:];
 $HangulSyllable = [\uac00-\ud7a3];
 $ComplexContext = [:LineBreak = Complex_Context:];
-$KanaKanji      = [$Han $Hiragana $Katakana];
+$KanaKanji      = [$Han $Hiragana $Katakana \u30fc];
 $dictionaryCJK  = [$KanaKanji $HangulSyllable];
 $dictionary     = [$ComplexContext $dictionaryCJK];
 
@@ -115,7 +114,9 @@ $dictionary     = [$ComplexContext $dictionaryCJK];
 #                               XX  (Unknown, unassigned)
 #                         as $AL  (Alphabetic)
 #
-$ALPlus = [$AL $AI $SG $XX $EAFWH [$dictionary-[[:Mn:][:Mc:]]]];
+# Let fullwidth-ASCII digits and letters be part of words.
+$FW_alphanum = [\uff10-\uff19\uff21-\uff3a\uff41-\uff5a];
+$ALPlus = [$AL $AI $SG $XX $FW_alphanum [$dictionary-[[:Mn:][:Mc:]]]];
 
 
 ## -------------------------------------------------

diff --git a/icu4c/source/data/brkitr/rules/line_phrase_cj.txt b/icu4c/source/data/brkitr/rules/line_phrase_cj.txt
index b9c713f13a124907b6c0b42c8c7a74d7c99c8410..290b9b8c83a7b7e346100bcd0203f476ae8ba4da 100644 (file)
--- a/icu4c/source/data/brkitr/rules/line_phrase_cj.txt
+++ b/icu4c/source/data/brkitr/rules/line_phrase_cj.txt
@@ -75,9 +75,8 @@ $ZWJ = [:LineBreak = ZWJ:];
 # without a formal name. Because ICU rules require multiple uses of the expressions,
 # give them a single definition with a name
 
-$EAFWH = [\p{ea=F}\p{ea=W}\p{ea=H}];
-$OP30 = [$OP - $EAFWH];
-$CP30 = [$CP - $EAFWH];
+$OP30 = [$OP - [\p{ea=F}\p{ea=W}\p{ea=H}]];
+$CP30 = [$CP - [\p{ea=F}\p{ea=W}\p{ea=H}]];
 
 $ExtPictUnassigned = [\p{Extended_Pictographic} & \p{Cn}];
 
@@ -97,7 +96,7 @@ $Katakana = [:Katakana:];
 $Hiragana = [:Hiragana:];
 $HangulSyllable = [\uac00-\ud7a3];
 $ComplexContext = [:LineBreak = Complex_Context:];
-$KanaKanji      = [$Han $Hiragana $Katakana];
+$KanaKanji      = [$Han $Hiragana $Katakana \u30fc];
 $dictionaryCJK  = [$KanaKanji $HangulSyllable];
 $dictionary     = [$ComplexContext $dictionaryCJK];
 
@@ -109,7 +108,9 @@ $dictionary     = [$ComplexContext $dictionaryCJK];
 #                               XX  (Unknown, unassigned)
 #                         as $AL  (Alphabetic)
 #
-$ALPlus = [$AL $AI $SG $XX $EAFWH [$dictionary-[[:Mn:][:Mc:]]]];
+# Let fullwidth-ASCII digits and letters be part of words.
+$FW_alphanum = [\uff10-\uff19\uff21-\uff3a\uff41-\uff5a];
+$ALPlus = [$AL $AI $SG $XX $FW_alphanum [$dictionary-[[:Mn:][:Mc:]]]];
 
 
 ## -------------------------------------------------

diff --git a/icu4c/source/test/testdata/rbbitst.txt b/icu4c/source/test/testdata/rbbitst.txt
index 702bb47903841635ccc0bc284f7c4141a36e1648..1b74e31a81008f31ff25f778f0a4efc70d73d4c5 100644 (file)
--- a/icu4c/source/test/testdata/rbbitst.txt
+++ b/icu4c/source/test/testdata/rbbitst.txt
@@ -1901,6 +1901,16 @@ Bangkok)•</data>
 #Kana supplement: 𛁈(U+1B048) -> \uD82C\uDC48, 𛀸(U+1B038) -> \uD82C\uDC38, 𛀙(U+1B019)-> \uD82C\uDC19</data>
 #𛁈る𛀸（しるこ）、あ𛀙よろし（あかよろし） -> 𛁈る𛀸•（しるこ）、•あ𛀙よろし•（あ•かよろし）
 <data>•\uD82C\uDC48\u308B\uD82C\uDC38•\uFF08\u3057\u308B\u3053\uFF09\u3001•\u3042\uD82C\uDC19\u3088\u308D\u3057•\uFF08\u3042•\u304B\u3088\u308D\u3057\uFF09•</data>
+#最初に目に入るのは、「許諾なき写真禁止」のサインである。 -> 最初に▁目に▁入るのは、▁「許諾なき▁写真▁禁止」▁の▁サインで▁ある。
+<data>•\u6700\u521D\u306B•\u76EE\u306B•\u5165\u308B\u306E\u306F\u3001•\u300C\u8A31\u8AFE\u306A\u304D•\u5199\u771F•\u7981\u6B62\u300D•\u306E•\u30B5\u30A4\u30F3\u3067•\u3042\u308B\u3002•</data>
+# ｄｏｃｏｍｏのサイト情報によると、７８０００パケット以上▁使うならパケ放題がいいとか -> ｄｏｃｏｍｏの▁サイト▁情報によると、▁７８０００パケット▁以上▁使う▁なら▁パケ▁放題が▁いい▁とか
+<data>•\uFF44\uFF4F\uFF43\uFF4F\uFF4D\uFF4F\u306E•\u30B5\u30A4\u30C8•\u60C5\u5831•\u306B\u3088\u308B\u3068\u3001•\uFF17\uFF18\uFF10\uFF10\uFF10\u30D1\u30B1\u30C3\u30C8•\u4EE5\u4E0A•\u4F7F\u3046•\u306A\u3089•\u30D1\u30B1•\u653E\u984C\u304C•\u3044\u3044•\u3068\u304B•</data>
+#日本の携帯はＣＤＭＡの形だといわれています -> 日本の▁携帯は▁ＣＤＭＡの▁形だと▁いわれ▁ています
+<data>•\u65E5\u672C\u306E•\u643A\u5E2F\u306F\uFF23\uFF24\uFF2D\uFF21\u306E•\u5F62\u3060\u3068•\u3044\u308F\u308C•\u3066\u3044\u307E\u3059•</data>
+#中国の携帯は約５００元から５０００元です -> 中国の▁携帯は▁約▁５００元から▁５０００元です
+<data>•\u4E2D\u56FD\u306E•\u643A\u5E2F\u306F•\u7D04•\uFF15\uFF10\uFF10\u5143\u304B\u3089•\uFF15\uFF10\uFF10\uFF10\u5143\u3067\u3059•</data>
+#プリペイドカード携帯布教 -> プリペイドカード▁携帯▁布教
+<data>•\u30D7\u30EA\u30DA\u30A4\u30C9\u30AB\u30FC\u30C9•\u643A\u5E2F•\u5E03\u6559•</data>
 
 ####################################################################################
 #

diff --git a/icu4j/main/shared/data/icudata.jar b/icu4j/main/shared/data/icudata.jar
index 1466f080d9148c2418575c1f29e9edc86e77d0cd..0dc31673b4c0f06aac19482bdcb1c6f47f4d9a33 100644 (file)
--- a/icu4j/main/shared/data/icudata.jar
+++ b/icu4j/main/shared/data/icudata.jar
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a53ab1162a9a25cf8f1e3abef3d4aa24acb11cb7677652be8d76e6843cbe5b35
-size 13871000
+oid sha256:658d41e63f73089111e306f0d90b2b3aa4757146d4eda963f8e181c0acf2b9ab
+size 13866832

diff --git a/icu4j/main/shared/data/icutzdata.jar b/icu4j/main/shared/data/icutzdata.jar
index 78ae065238d83298989752bf3bfe1165fc60da9a..7ce295810ed3333034c0500c0317ed983104d28c 100644 (file)
--- a/icu4j/main/shared/data/icutzdata.jar
+++ b/icu4j/main/shared/data/icutzdata.jar
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:57ada8a76a3d3d728a4ffd58fbcbe6aba7de2cf3b36d08ee95b2a9a222403880
-size 96439
+oid sha256:79e4e94b0fc31ed37701b94512b985c6bf80429bb3420adb645abc4f26c913d7
+size 96440

diff --git a/icu4j/main/shared/data/testdata.jar b/icu4j/main/shared/data/testdata.jar
index 5d0255244ba75cceb9cc35d861d3fcd4211797bf..a504f6883214d1df4afb74f5b250f38e10ab5118 100644 (file)
--- a/icu4j/main/shared/data/testdata.jar
+++ b/icu4j/main/shared/data/testdata.jar
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ccde7f7d4e6bf0a92d2c3d6999afabbb063a154b7d66b3fd79d6ab0707e1fad7
-size 826063
+oid sha256:c20f10955116ac255c504e8743a9c4b2eab9c0bac7dc2250f1fbe775100f3b8a
+size 826064

diff --git a/icu4j/main/tests/core/src/com/ibm/icu/dev/test/rbbi/rbbitst.txt b/icu4j/main/tests/core/src/com/ibm/icu/dev/test/rbbi/rbbitst.txt
index 2a238a80f9c3f9ae31e35c5386a4f0b56ba680ba..e8b07361ff864ed2a79908e92792d286fc4dc560 100644 (file)
--- a/icu4j/main/tests/core/src/com/ibm/icu/dev/test/rbbi/rbbitst.txt
+++ b/icu4j/main/tests/core/src/com/ibm/icu/dev/test/rbbi/rbbitst.txt
@@ -1901,7 +1901,17 @@ Bangkok)•</data>
 #Kana supplement: 𛁈(U+1B048) -> \uD82C\uDC48, 𛀸(U+1B038) -> \uD82C\uDC38, 𛀙(U+1B019)-> \uD82C\uDC19</data>
 #𛁈る𛀸（しるこ）、あ𛀙よろし（あかよろし） -> 𛁈る𛀸•（しるこ）、•あ𛀙よろし•（あ•かよろし）
 <data>•\uD82C\uDC48\u308B\uD82C\uDC38•\uFF08\u3057\u308B\u3053\uFF09\u3001•\u3042\uD82C\uDC19\u3088\u308D\u3057•\uFF08\u3042•\u304B\u3088\u308D\u3057\uFF09•</data>
-
+<data>•\uD82C\uDC48\u308B\uD82C\uDC38•\uFF08\u3057\u308B\u3053\uFF09\u3001•\u3042\uD82C\uDC19\u3088\u308D\u3057•\uFF08\u3042•\u304B\u3088\u308D\u3057\uFF09•</data>
+#最初に目に入るのは、「許諾なき写真禁止」のサインである。 -> 最初に▁目に▁入るのは、▁「許諾なき▁写真▁禁止」▁の▁サインで▁ある。
+<data>•\u6700\u521D\u306B•\u76EE\u306B•\u5165\u308B\u306E\u306F\u3001•\u300C\u8A31\u8AFE\u306A\u304D•\u5199\u771F•\u7981\u6B62\u300D•\u306E•\u30B5\u30A4\u30F3\u3067•\u3042\u308B\u3002•</data>
+# ｄｏｃｏｍｏのサイト情報によると、７８０００パケット以上▁使うならパケ放題がいいとか -> ｄｏｃｏｍｏの▁サイト▁情報によると、▁７８０００パケット▁以上▁使う▁なら▁パケ▁放題が▁いい▁とか
+<data>•\uFF44\uFF4F\uFF43\uFF4F\uFF4D\uFF4F\u306E•\u30B5\u30A4\u30C8•\u60C5\u5831•\u306B\u3088\u308B\u3068\u3001•\uFF17\uFF18\uFF10\uFF10\uFF10\u30D1\u30B1\u30C3\u30C8•\u4EE5\u4E0A•\u4F7F\u3046•\u306A\u3089•\u30D1\u30B1•\u653E\u984C\u304C•\u3044\u3044•\u3068\u304B•</data>
+#日本の携帯はＣＤＭＡの形だといわれています -> 日本の▁携帯は▁ＣＤＭＡの▁形だと▁いわれ▁ています
+<data>•\u65E5\u672C\u306E•\u643A\u5E2F\u306F\uFF23\uFF24\uFF2D\uFF21\u306E•\u5F62\u3060\u3068•\u3044\u308F\u308C•\u3066\u3044\u307E\u3059•</data>
+#中国の携帯は約５００元から５０００元です -> 中国の▁携帯は▁約▁５００元から▁５０００元です
+<data>•\u4E2D\u56FD\u306E•\u643A\u5E2F\u306F•\u7D04•\uFF15\uFF10\uFF10\u5143\u304B\u3089•\uFF15\uFF10\uFF10\uFF10\u5143\u3067\u3059•</data>
+#プリペイドカード携帯布教 -> プリペイドカード▁携帯▁布教
+<data>•\u30D7\u30EA\u30DA\u30A4\u30C9\u30AB\u30FC\u30C9•\u643A\u5E2F•\u5E03\u6559•</data>
 
 ####################################################################################
 #