Don't pass invalid JIS X 0208 characters through

Many Japanese encodings, such as JIS7/8, Shift JIS, ISO-2022-JP, EUC-JP, and
so on encode characters from the JIS X 0208 character set. JIS X 0208 is based
on the concept of a 94x94 table, with numbered rows and columns. However,
more than a thousand of the cells in that table are empty; JIS X 0208 does not
actually use all 94x94=8,836 possible kuten codes.

mbstring had a dubious feature whereby, if a Japanese string contained one of
these 'unmapped' kuten codes, and it was being converted to another Japanese
encoding which was also based on JIS X 0208, the non-existent character would
be silently passed through, and the unmapped kuten code would be re-encoded
using the normal encoding method of the target text encoding.

Again, this _only_ happened if converting the text with the funky kuten code
to a Japanese encoding. If one tried converting it to Unicode, mbstring would
treat that as an error.

If somebody, somewhere, made their own private extension to JIS X 0208, and
used the regular Japanese encodings like Shift JIS and EUC-JP to encode this
private character set, then this feature might conceivably be useful. But how
likely is that? If someone is using Shift JIS, EUC-JP, ISO-2022-JP, etc. to
encode a funky version of JIS X 0208 with extra characters added, then that
should be treated as a separate text encoding.

The code which flags such characters with MBFL_WCSPLANE_JIS0208 is retained
solely for error reporting in `mbfl_filt_conv_illegal_output`.

diff --git a/ext/mbstring/libmbfl/filters/mbfilter_cp5022x.c b/ext/mbstring/libmbfl/filters/mbfilter_cp5022x.c
index a4d1724e8fa5436a349db494319ffeb303d32799..b630d3fcb032de02bb40c5f0e62314ed9bfd91a2 100644 (file)
--- a/ext/mbstring/libmbfl/filters/mbfilter_cp5022x.c
+++ b/ext/mbstring/libmbfl/filters/mbfilter_cp5022x.c
@@ -409,9 +409,7 @@ mbfl_filt_conv_wchar_jis_ms(int c, mbfl_convert_filter *filter)
        /* do some transliteration */
        if (s <= 0) {
                c1 = c & ~MBFL_WCSPLANE_MASK;
-               if (c1 == MBFL_WCSPLANE_JIS0208) {
-                       s = c & MBFL_WCSPLANE_MASK;
-               } else if (c1 == MBFL_WCSPLANE_JIS0212) {
+               if (c1 == MBFL_WCSPLANE_JIS0212) {
                        s = c & MBFL_WCSPLANE_MASK;
                        s |= 0x8080;
                } else if (c == 0xa5) {         /* YEN SIGN */

diff --git a/ext/mbstring/libmbfl/filters/mbfilter_cp51932.c b/ext/mbstring/libmbfl/filters/mbfilter_cp51932.c
index cb95469408b55456876d658b84ac7167cb7b2996..25944a10dd5f229b562d7ed1b1546b64b8f974be 100644 (file)
--- a/ext/mbstring/libmbfl/filters/mbfilter_cp51932.c
+++ b/ext/mbstring/libmbfl/filters/mbfilter_cp51932.c
@@ -216,14 +216,6 @@ mbfl_filt_conv_wchar_cp51932(int c, mbfl_convert_filter *filter)
                        if (s1 >= ((85 + 0x20) << 8)) { /* 85ku - 120ku */
                                s1 = -1;
                        }
-               } else if (c1 == MBFL_WCSPLANE_JIS0208) {
-                       s1 = c & MBFL_WCSPLANE_MASK;
-                       if ((s1 >= ((85 + 0x20) << 8) &&  /* 85ku - 94ku */
-                             s1 <= ((88 + 0x20) << 8)) ||/* IBM extension */
-                           (s1 >= ((93 + 0x20) << 8) && /* 89ku - 92ku */
-                             s1 <= ((94 + 0x20) << 8))) {
-                               s1 = -1;
-                       }
                } else if (c == 0xa5) {         /* YEN SIGN */
                        s1 = 0x005c;                    /* YEN SIGN */
                } else if (c == 0x203e) {       /* OVER LINE */

diff --git a/ext/mbstring/libmbfl/filters/mbfilter_cp932.c b/ext/mbstring/libmbfl/filters/mbfilter_cp932.c
index c0fe18e0382b4871396722e1494b3014f5a81543..64ccd573611417807a30ba2d70815c0740726f27 100644 (file)
--- a/ext/mbstring/libmbfl/filters/mbfilter_cp932.c
+++ b/ext/mbstring/libmbfl/filters/mbfilter_cp932.c
@@ -255,8 +255,6 @@ mbfl_filt_conv_wchar_cp932(int c, mbfl_convert_filter *filter)
                if (c1 == MBFL_WCSPLANE_WINCP932) {
                        s1 = c & MBFL_WCSPLANE_MASK;
                        s2 = 1;
-               } else if (c1 == MBFL_WCSPLANE_JIS0208) {
-                       s1 = c & MBFL_WCSPLANE_MASK;
                } else if (c1 == MBFL_WCSPLANE_JIS0212) {
                        s1 = c & MBFL_WCSPLANE_MASK;
                        s1 |= 0x8080;

diff --git a/ext/mbstring/libmbfl/filters/mbfilter_euc_jp_win.c b/ext/mbstring/libmbfl/filters/mbfilter_euc_jp_win.c
index b93fc9101a4062bc75f854f7880d8a0c077bbb30..df51be01831413ca6f10023c8be36eecd5f6d48b 100644 (file)
--- a/ext/mbstring/libmbfl/filters/mbfilter_euc_jp_win.c
+++ b/ext/mbstring/libmbfl/filters/mbfilter_euc_jp_win.c
@@ -279,11 +279,6 @@ mbfl_filt_conv_wchar_eucjpwin(int c, mbfl_convert_filter *filter)
                        if (s1 >= ((85 + 0x20) << 8)) { /* 85ku - 120ku */
                                s1 = -1;
                        }
-               } else if (c1 == MBFL_WCSPLANE_JIS0208) {
-                       s1 = c & MBFL_WCSPLANE_MASK;
-                       if (s1 >= ((85 + 0x20) << 8)) { /* 85ku - 94ku */
-                               s1 = -1;
-                       }
                } else if (c1 == MBFL_WCSPLANE_JIS0212) {
                        s1 = c & MBFL_WCSPLANE_MASK;
                        if (s1 >= ((83 + 0x20) << 8)) { /* 83ku - 94ku */

diff --git a/ext/mbstring/libmbfl/filters/mbfilter_iso2022_jp_ms.c b/ext/mbstring/libmbfl/filters/mbfilter_iso2022_jp_ms.c
index 950365045c82e5e6cb7e6445bd0c4bc8ad4d3c26..af54a1c6051faf1c0b11aad626b1754ae689ae5f 100644 (file)
--- a/ext/mbstring/libmbfl/filters/mbfilter_iso2022_jp_ms.c
+++ b/ext/mbstring/libmbfl/filters/mbfilter_iso2022_jp_ms.c
@@ -306,8 +306,6 @@ mbfl_filt_conv_wchar_2022jpms(int c, mbfl_convert_filter *filter)
                if (c1 == MBFL_WCSPLANE_WINCP932) {
                        s1 = c & MBFL_WCSPLANE_MASK;
                        s2 = 1;
-               } else if (c1 == MBFL_WCSPLANE_JIS0208) {
-                       s1 = c & MBFL_WCSPLANE_MASK;
                } else if (c1 == MBFL_WCSPLANE_JIS0212) {
                        s1 = c & MBFL_WCSPLANE_MASK;
                        s1 |= 0x8080;

diff --git a/ext/mbstring/libmbfl/filters/mbfilter_iso2022jp_mobile.c b/ext/mbstring/libmbfl/filters/mbfilter_iso2022jp_mobile.c
index e6300675e81f177f1c51821049d10f2d47f506de..1b4857dfe25ee432257b322177fcc143d3867172 100644 (file)
--- a/ext/mbstring/libmbfl/filters/mbfilter_iso2022jp_mobile.c
+++ b/ext/mbstring/libmbfl/filters/mbfilter_iso2022jp_mobile.c
@@ -334,8 +334,6 @@ mbfl_filt_conv_wchar_2022jp_mobile(int c, mbfl_convert_filter *filter)
                if (c1 == MBFL_WCSPLANE_WINCP932) {
                        s1 = c & MBFL_WCSPLANE_MASK;
                        s2 = 1;
-               } else if (c1 == MBFL_WCSPLANE_JIS0208) {
-                       s1 = c & MBFL_WCSPLANE_MASK;
                } else if (c1 == MBFL_WCSPLANE_JIS0212) {
                        s1 = c & MBFL_WCSPLANE_MASK;
                        s1 |= 0x8080;

diff --git a/ext/mbstring/libmbfl/filters/mbfilter_jis.c b/ext/mbstring/libmbfl/filters/mbfilter_jis.c
index ba43872d83e5d59782df26f79743cc728e243e12..8ba46a5b9ace52936493e7c840cce922aebdd80a 100644 (file)
--- a/ext/mbstring/libmbfl/filters/mbfilter_jis.c
+++ b/ext/mbstring/libmbfl/filters/mbfilter_jis.c
@@ -288,9 +288,7 @@ mbfl_filt_conv_wchar_jis(int c, mbfl_convert_filter *filter)
        }
        if (s <= 0) {
                c1 = c & ~MBFL_WCSPLANE_MASK;
-               if (c1 == MBFL_WCSPLANE_JIS0208) {
-                       s = c & MBFL_WCSPLANE_MASK;
-               } else if (c1 == MBFL_WCSPLANE_JIS0212) {
+               if (c1 == MBFL_WCSPLANE_JIS0212) {
                        s = c & MBFL_WCSPLANE_MASK;
                        s |= 0x8080;
                } else if (c == 0xa5) {         /* YEN SIGN */

diff --git a/ext/mbstring/libmbfl/filters/mbfilter_sjis_mac.c b/ext/mbstring/libmbfl/filters/mbfilter_sjis_mac.c
index 45b87a8f989f171af194208bb4628dbb6e2e2ba0..6361195606d6322e00282a440516e6c7a31fd1ca 100644 (file)
--- a/ext/mbstring/libmbfl/filters/mbfilter_sjis_mac.c
+++ b/ext/mbstring/libmbfl/filters/mbfilter_sjis_mac.c
@@ -420,8 +420,6 @@ mbfl_filt_conv_wchar_sjis_mac(int c, mbfl_convert_filter *filter)
                        if (c1 == MBFL_WCSPLANE_WINCP932) {
                                s1 = c & MBFL_WCSPLANE_MASK;
                                s2 = 1;
-                       } else if (c1 == MBFL_WCSPLANE_JIS0208) {
-                               s1 = c & MBFL_WCSPLANE_MASK;
                        } else if (c1 == MBFL_WCSPLANE_JIS0212) {
                                s1 = c & MBFL_WCSPLANE_MASK;
                                s1 |= 0x8080;

diff --git a/ext/mbstring/libmbfl/filters/mbfilter_sjis_mobile.c b/ext/mbstring/libmbfl/filters/mbfilter_sjis_mobile.c
index 255a457c583640c7f6b8076097452c932d25b167..10ac7d9ce41cd307afc9d5d2adbbf19a40a75767 100644 (file)
--- a/ext/mbstring/libmbfl/filters/mbfilter_sjis_mobile.c
+++ b/ext/mbstring/libmbfl/filters/mbfilter_sjis_mobile.c
@@ -794,8 +794,6 @@ mbfl_filt_conv_wchar_sjis_mobile(int c, mbfl_convert_filter *filter)
                if (c1 == MBFL_WCSPLANE_WINCP932) {
                        s1 = c & MBFL_WCSPLANE_MASK;
                        s2 = 1;
-               } else if (c1 == MBFL_WCSPLANE_JIS0208) {
-                       s1 = c & MBFL_WCSPLANE_MASK;
                } else if (c1 == MBFL_WCSPLANE_JIS0212) {
                        s1 = c & MBFL_WCSPLANE_MASK;
                        s1 |= 0x8080;

diff --git a/ext/mbstring/libmbfl/filters/mbfilter_sjis_open.c b/ext/mbstring/libmbfl/filters/mbfilter_sjis_open.c
index d37f01568e281b7f29264d692e5e1f19055eb9af..ea19e6b1058d2958e005ce85455a9a5f1e913531 100644 (file)
--- a/ext/mbstring/libmbfl/filters/mbfilter_sjis_open.c
+++ b/ext/mbstring/libmbfl/filters/mbfilter_sjis_open.c
@@ -245,8 +245,6 @@ mbfl_filt_conv_wchar_sjis_open(int c, mbfl_convert_filter *filter)
                if (c1 == MBFL_WCSPLANE_WINCP932) {
                        s1 = c & MBFL_WCSPLANE_MASK;
                        s2 = 1;
-               } else if (c1 == MBFL_WCSPLANE_JIS0208) {
-                       s1 = c & MBFL_WCSPLANE_MASK;
                } else if (c1 == MBFL_WCSPLANE_JIS0212) {
                        s1 = c & MBFL_WCSPLANE_MASK;
                        s1 |= 0x8080;