added chinese HZ encoding support. fixed ascii area character conversion was not...

diff --git a/ext/mbstring/mbfilter.c b/ext/mbstring/mbfilter.c
index 4e335f12fb47da1b99a34f93bfc60c07e292d666..0968b44fc9cefff3de240ee8f8d062ac7c73602a 100644 (file)
--- a/ext/mbstring/mbfilter.c
+++ b/ext/mbstring/mbfilter.c
@@ -145,6 +145,16 @@ static mbfl_language mbfl_language_japanese = {
        mbfl_no_encoding_7bit
 };
 
+static mbfl_language mbfl_language_korean = {
+       mbfl_no_language_korean,
+       "Korean",
+       "ko",
+       NULL,
+       mbfl_no_encoding_2022kr,
+       mbfl_no_encoding_base64,
+       mbfl_no_encoding_7bit
+};
+
 static mbfl_language mbfl_language_english = {
        mbfl_no_language_english,
        "English",
@@ -155,12 +165,22 @@ static mbfl_language mbfl_language_english = {
        mbfl_no_encoding_8bit
 };
 
-static mbfl_language mbfl_language_chinese = {
-       mbfl_no_language_chinese,
-       "Chinese",
-       "zh",
+static mbfl_language mbfl_language_simplified_chinese = {
+       mbfl_no_language_simplified_chinese,
+       "Simplified Chinese",
+       "zh-cn",
        NULL,
-       mbfl_no_encoding_2022jp,
+       mbfl_no_encoding_hz,
+       mbfl_no_encoding_base64,
+       mbfl_no_encoding_7bit
+};
+
+static mbfl_language mbfl_language_traditional_chinese = {
+       mbfl_no_language_traditional_chinese,
+       "Traditional Chinese",
+       "zh-tw",
+       NULL,
+       mbfl_no_encoding_hz,
        mbfl_no_encoding_base64,
        mbfl_no_encoding_7bit
 };
@@ -168,7 +188,9 @@ static mbfl_language mbfl_language_chinese = {
 static mbfl_language *mbfl_language_ptr_table[] = {
        &mbfl_language_uni,
        &mbfl_language_japanese,
-       &mbfl_language_chinese,
+       &mbfl_language_korean,
+       &mbfl_language_simplified_chinese,
+       &mbfl_language_traditional_chinese,
        &mbfl_language_english,
        NULL
 };
@@ -707,6 +729,15 @@ static mbfl_encoding mbfl_encoding_cp936 = {
        MBFL_ENCTYPE_MBCS
 };
 
+static mbfl_encoding mbfl_encoding_hz = {
+       mbfl_no_encoding_hz,
+       "HZ",
+       "HZ-GB-2312",
+       NULL,
+       NULL,
+       MBFL_ENCTYPE_MBCS | MBFL_ENCTYPE_SHFTCODE
+};
+
 #endif /* HAVE_MBSTR_CN */
 
 #if defined(HAVE_MBSTR_TW)
@@ -967,6 +998,7 @@ static mbfl_encoding *mbfl_encoding_ptr_list[] = {
 #if defined(HAVE_MBSTR_CN)
        &mbfl_encoding_euc_cn,
        &mbfl_encoding_cp936,
+       &mbfl_encoding_hz,
 #endif
 #if defined(HAVE_MBSTR_TW)
        &mbfl_encoding_euc_tw,
@@ -1072,6 +1104,7 @@ static int mbfl_filt_ident_2022jp(int c, mbfl_identify_filter *filter TSRMLS_DC)
 #if defined(HAVE_MBSTR_CN)
 static int mbfl_filt_ident_euccn(int c, mbfl_identify_filter *filter TSRMLS_DC);
 static int mbfl_filt_ident_cp936(int c, mbfl_identify_filter *filter TSRMLS_DC);
+static int mbfl_filt_ident_hz(int c, mbfl_identify_filter *filter TSRMLS_DC);
 #endif /* HAVE_MBSTR_CN */
 
 #if defined(HAVE_MBSTR_TW)
@@ -1605,6 +1638,23 @@ static struct mbfl_convert_vtbl vtbl_wchar_cp936 = {
        mbfl_filt_conv_common_dtor,
        mbfl_filt_conv_wchar_cp936,
        mbfl_filt_conv_common_flush };
+
+static struct mbfl_convert_vtbl vtbl_hz_wchar = {
+       mbfl_no_encoding_hz,
+       mbfl_no_encoding_wchar,
+       mbfl_filt_conv_common_ctor,
+       mbfl_filt_conv_common_dtor,
+       mbfl_filt_conv_hz_wchar,
+       mbfl_filt_conv_common_flush };
+
+static struct mbfl_convert_vtbl vtbl_wchar_hz = {
+       mbfl_no_encoding_wchar,
+       mbfl_no_encoding_hz,
+       mbfl_filt_conv_common_ctor,
+       mbfl_filt_conv_common_dtor,
+       mbfl_filt_conv_wchar_hz,
+       mbfl_filt_conv_any_hz_flush };
+
 #endif /* HAVE_MBSTR_CN */
 
 #if defined(HAVE_MBSTR_TW)
@@ -1923,6 +1973,8 @@ static struct mbfl_convert_vtbl *mbfl_convert_filter_list[] = {
        &vtbl_wchar_euccn,
        &vtbl_cp936_wchar,
        &vtbl_wchar_cp936,
+       &vtbl_hz_wchar,
+       &vtbl_wchar_hz,
 #endif
 #if defined(HAVE_MBSTR_TW)
        &vtbl_euctw_wchar,
@@ -2083,6 +2135,13 @@ static struct mbfl_identify_vtbl vtbl_identify_cp936 = {
        mbfl_filt_ident_common_ctor,
        mbfl_filt_ident_common_dtor,
        mbfl_filt_ident_cp936 };
+
+static struct mbfl_identify_vtbl vtbl_identify_hz = {
+       mbfl_no_encoding_hz,
+       mbfl_filt_ident_common_ctor,
+       mbfl_filt_ident_common_dtor,
+       mbfl_filt_ident_hz };
+
 #endif /* HAVE_MBSTR_CN */
 
 #if defined(HAVE_MBSTR_TW)
@@ -2218,6 +2277,7 @@ static struct mbfl_identify_vtbl *mbfl_identify_filter_list[] = {
 #if defined(HAVE_MBSTR_CN)
        &vtbl_identify_euccn,
        &vtbl_identify_cp936,
+       &vtbl_identify_hz,
 #endif
 #if defined(HAVE_MBSTR_TW)
        &vtbl_identify_euctw,
@@ -5756,6 +5816,53 @@ mbfl_filt_ident_cp936(int c, mbfl_identify_filter *filter TSRMLS_DC)
        return c;
 }
 
+static int
+mbfl_filt_ident_hz(int c, mbfl_identify_filter *filter TSRMLS_DC)
+{
+       switch (filter->status & 0xf) {
+/*     case 0x00:       ASCII */
+/*     case 0x10:       GB2312 */
+       case 0:
+               if (c == 0x7e) {
+                       filter->status += 2;
+               } else if (filter->status == 0x10 && c > 0x20 && c < 0x7f) {            /* DBCS first char */
+                       filter->status += 1;
+               } else if (c >= 0 && c < 0x80) {                /* latin, CTLs */
+                       ;
+               } else {
+                       filter->flag = 1;       /* bad */
+               }
+               break;
+
+/*     case 0x11:       GB2312 second char */
+       case 1:
+               filter->status &= ~0xf;
+               if (c < 0x21 || c > 0x7e) {             /* bad */
+                       filter->flag = 1;
+               }
+               break;
+
+       case 2:
+               if (c == 0x7d) {                /* '}' */
+                       filter->status = 0;
+               } else if (c == 0x7b) {         /* '{' */
+                       filter->status = 0x10;
+               } else if (c == 0x7e) {         /* '~' */
+                       filter->status = 0;
+               } else {
+                       filter->flag = 1;       /* bad */
+                       filter->status &= ~0xf;
+               }
+               break;
+
+       default:
+               filter->status = 0;
+               break;
+       }
+
+       return c;
+}
+
 #endif /* HAVE_MBSTR_CN */
 
 #if defined(HAVE_MBSTR_TW)

diff --git a/ext/mbstring/mbfilter.h b/ext/mbstring/mbfilter.h
index a5077bd57877488583faaa24c8d405fb457c52e6..c63e6273ec62a60e5dcd3a1fef974e671697cdfc 100644 (file)
--- a/ext/mbstring/mbfilter.h
+++ b/ext/mbstring/mbfilter.h
@@ -111,7 +111,8 @@ enum mbfl_no_language {
        mbfl_no_language_polish,                /* pl */
        mbfl_no_language_portuguese,            /* pt */
        mbfl_no_language_swedish,               /* sv */
-       mbfl_no_language_chinese,               /* zh */
+       mbfl_no_language_simplified_chinese,            /* zh-cn */
+       mbfl_no_language_traditional_chinese,           /* zh-tw */
        mbfl_no_language_max
 };
 
@@ -172,7 +173,9 @@ enum mbfl_no_encoding {
        mbfl_no_encoding_euc_tw,
        mbfl_no_encoding_big5,
        mbfl_no_encoding_euc_kr,
+       mbfl_no_encoding_2022kr,
        mbfl_no_encoding_uhc,
+       mbfl_no_encoding_hz,
        mbfl_no_encoding_charset_max
 };
 

diff --git a/ext/mbstring/mbfilter_cn.c b/ext/mbstring/mbfilter_cn.c
index 41123b57c6107e8bfedd92c347f4bb296525a5ba..9dcdd7dfdb8c1cad03cf7cd35595cec2d3e95fc9 100644 (file)
--- a/ext/mbstring/mbfilter_cn.c
+++ b/ext/mbstring/mbfilter_cn.c
@@ -117,8 +117,8 @@ mbfl_filt_conv_wchar_euccn(int c, mbfl_convert_filter *filter TSRMLS_DC)
        c1 = (s >> 8) & 0xff;
        c2 = s & 0xff;
        
-       if (c1 < 0xa1 || c2 < 0xa1) { /* exclude CP932 extension */
-               s = 0;
+       if (c1 < 0xa1 || c2 < 0xa1) { /* exclude CP936 extension */
+               s = c;
        }
 
        if (s <= 0) {
@@ -259,6 +259,154 @@ mbfl_filt_conv_wchar_cp936(int c, mbfl_convert_filter *filter TSRMLS_DC)
        return c;
 }
 
+
+/*
+ * HZ => wchar
+ */
+int
+mbfl_filt_conv_hz_wchar(int c, mbfl_convert_filter *filter TSRMLS_DC)
+{
+       int c1, s, w;
+
+       switch (filter->status & 0xf) {
+/*     case 0x00:       ASCII */
+/*     case 0x10:       GB2312 */
+       case 0:
+               if (c == 0x7e) {
+                       filter->status += 2;
+               } else if (filter->status == 0x10 && c > 0x20 && c < 0x7f) {    /* DBCS first char */
+                       filter->cache = c;
+                       filter->status += 1;
+               } else if (c >= 0 && c < 0x80) {                /* latin, CTLs */
+                       CK((*filter->output_function)(c, filter->data TSRMLS_CC));
+               } else {
+                       w = c & MBFL_WCSGROUP_MASK;
+                       w |= MBFL_WCSGROUP_THROUGH;
+                       CK((*filter->output_function)(w, filter->data TSRMLS_CC));
+               }
+               break;
+
+/*     case 0x11:       GB2312 second char */
+       case 1:
+               filter->status &= ~0xf;
+               c1 = filter->cache;
+               if (c1 > 0x20 && c1 < 0x7f && c > 0x20 && c < 0x7f) {
+                       s = (c1 - 1)*192 + c + 0x40; /* GB2312 */
+                       if (s >= 0 && s < cp936_ucs_table_size) {
+                               w = cp936_ucs_table[s];
+                       } else {
+                               w = 0;
+                       }
+                       if (w <= 0) {
+                               w = (c1 << 8) | c;
+                               w &= MBFL_WCSPLANE_MASK;
+                               w |= MBFL_WCSPLANE_GB2312;
+                       }
+                       CK((*filter->output_function)(w, filter->data TSRMLS_CC));
+               } else if ((c >= 0 && c < 0x21) || c == 0x7f) {         /* CTLs */
+                       CK((*filter->output_function)(c, filter->data TSRMLS_CC));
+               } else {
+                       w = (c1 << 8) | c;
+                       w &= MBFL_WCSGROUP_MASK;
+                       w |= MBFL_WCSGROUP_THROUGH;
+                       CK((*filter->output_function)(w, filter->data TSRMLS_CC));
+               }
+               break;
+
+       /* '~' */
+       case 2:
+               if (c == 0x7d) {                /* '}' */
+                       filter->status = 0x0;
+               } else if (c == 0x7b) {         /* '{' */
+                       filter->status = 0x10;
+               } else if (c == 0x7e) { /* '~' */
+                       filter->status = 0x0;
+                       CK((*filter->output_function)(0x007e, filter->data TSRMLS_CC));
+               }
+               break;
+
+       default:
+               filter->status = 0;
+               break;
+       }
+
+       return c;
+}
+
+/*
+ * wchar => HZ
+ */
+int
+mbfl_filt_conv_wchar_hz(int c, mbfl_convert_filter *filter TSRMLS_DC)
+{
+       int s;
+
+       s = 0;
+       if (c >= ucs_a1_cp936_table_min && c < ucs_a1_cp936_table_max) {
+               s = ucs_a1_cp936_table[c - ucs_a1_cp936_table_min];
+       } else if (c >= ucs_a2_cp936_table_min && c < ucs_a2_cp936_table_max) {
+               s = ucs_a2_cp936_table[c - ucs_a2_cp936_table_min];
+       } else if (c >= ucs_a3_cp936_table_min && c < ucs_a3_cp936_table_max) {
+               s = ucs_a3_cp936_table[c - ucs_a3_cp936_table_min];
+       } else if (c >= ucs_i_cp936_table_min && c < ucs_i_cp936_table_max) {
+               s = ucs_i_cp936_table[c - ucs_i_cp936_table_min];
+       } else if (c >= ucs_hff_cp936_table_min && c < ucs_hff_cp936_table_max) {
+               s = ucs_hff_cp936_table[c - ucs_hff_cp936_table_min];
+       }
+       if (s >= 0x0080) {
+               s -= 0x8080;
+       }
+
+       if (s <= 0) {
+               if (c == 0) {
+                       s = 0;
+               } else if (s <= 0) {
+                       s = -1;
+               }
+       } else if ((s >= 0x80 && s < 0x2121) || (s > 0x8080)) {
+               s = -1;
+       }
+       if (s >= 0) {
+               if (s < 0x80) { /* ASCII */
+                       if ((filter->status & 0xff00) != 0) {
+                               CK((*filter->output_function)(0x7e, filter->data TSRMLS_CC));           /* '~' */
+                               CK((*filter->output_function)(0x7d, filter->data TSRMLS_CC));           /* '}' */
+                       }
+                       filter->status = 0;
+                       if (s == 0x7e){
+                               CK((*filter->output_function)(0x7e, filter->data TSRMLS_CC));
+                       }
+                       CK((*filter->output_function)(s, filter->data TSRMLS_CC));
+               } else { /* GB 2312-80 */
+                       if ((filter->status & 0xff00) != 0x200) {
+                               CK((*filter->output_function)(0x7e, filter->data TSRMLS_CC));           /* '~' */
+                               CK((*filter->output_function)(0x7b, filter->data TSRMLS_CC));           /* '{' */
+                       }
+                       filter->status = 0x200;
+                       CK((*filter->output_function)((s >> 8) & 0x7f, filter->data TSRMLS_CC));
+                       CK((*filter->output_function)(s & 0x7f, filter->data TSRMLS_CC));
+               }
+       } else {
+               if (filter->illegal_mode != MBFL_OUTPUTFILTER_ILLEGAL_MODE_NONE) {
+                       CK(mbfl_filt_conv_illegal_output(c, filter TSRMLS_CC));
+               }
+       }
+
+       return c;
+}
+
+int
+mbfl_filt_conv_any_hz_flush(mbfl_convert_filter *filter TSRMLS_DC)
+{
+       /* back to latin */
+       if ((filter->status & 0xff00) != 0) {
+               CK((*filter->output_function)(0x7e, filter->data TSRMLS_CC));           /* ~ */
+               CK((*filter->output_function)(0x7d, filter->data TSRMLS_CC));           /* '{' */
+       }
+       filter->status &= 0xff;
+       return 0;
+}
+
 #endif /* HAVE_MBSTR_CN */
 
 /*

diff --git a/ext/mbstring/mbfilter_cn.h b/ext/mbstring/mbfilter_cn.h
index a71168541b69ff509b566ca218e311e91a768c8c..be2541717418db46da729ce5cca03f8d828e9a30 100644 (file)
--- a/ext/mbstring/mbfilter_cn.h
+++ b/ext/mbstring/mbfilter_cn.h
@@ -26,5 +26,8 @@ int mbfl_filt_conv_euccn_wchar(int c, mbfl_convert_filter *filter TSRMLS_DC);
 int mbfl_filt_conv_wchar_euccn(int c, mbfl_convert_filter *filter TSRMLS_DC);
 int mbfl_filt_conv_cp936_wchar(int c, mbfl_convert_filter *filter TSRMLS_DC);
 int mbfl_filt_conv_wchar_cp936(int c, mbfl_convert_filter *filter TSRMLS_DC);
+int mbfl_filt_conv_hz_wchar(int c, mbfl_convert_filter *filter TSRMLS_DC);
+int mbfl_filt_conv_wchar_hz(int c, mbfl_convert_filter *filter TSRMLS_DC);
+int mbfl_filt_conv_any_hz_flush(mbfl_convert_filter *filter TSRMLS_DC);
 
 #endif /* MBFL_MBFILTER_CN_H */

diff --git a/ext/mbstring/mbfilter_kr.c b/ext/mbstring/mbfilter_kr.c
index 86f4bdca5dbd067de027870579ec158136bf4dbf..93f44274f067f34986cadbd66987818d74f56e9b 100644 (file)
--- a/ext/mbstring/mbfilter_kr.c
+++ b/ext/mbstring/mbfilter_kr.c
@@ -141,8 +141,8 @@ mbfl_filt_conv_wchar_euckr(int c, mbfl_convert_filter *filter TSRMLS_DC)
        c1 = (s >> 8) & 0xff;
        c2 = s & 0xff;
        /* exclude UHC extension area */
-       if (c1 < 0xa1 || c1 > 0xfe || c2 < 0xa1 && c2 > 0xfe){ 
-               s = 0;
+       if (c1 < 0xa1 || c2 < 0xa1){ 
+               s = c;
        }
 
        if (s <= 0) {