#include "unicode_table_jis.h"
int mbfl_filt_ident_eucjp(int c, mbfl_identify_filter *filter);
+static int mbfl_filt_conv_eucjp_wchar_flush(mbfl_convert_filter *filter);
const unsigned char mblen_table_eucjp[] = { /* 0xA1-0xFE */
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
mbfl_filt_conv_common_ctor,
NULL,
mbfl_filt_conv_eucjp_wchar,
- mbfl_filt_conv_common_flush,
+ mbfl_filt_conv_eucjp_wchar_flush,
NULL,
};
int
mbfl_filt_conv_eucjp_wchar(int c, mbfl_convert_filter *filter)
{
- int c1, s, w;
+ int c1, s, w = 0;
switch (filter->status) {
case 0:
} else if (c == 0x8f) { /* X 0212 first char */
filter->status = 3;
} else {
- w = c & MBFL_WCSGROUP_MASK;
- w |= MBFL_WCSGROUP_THROUGH;
- CK((*filter->output_function)(w, filter->data));
+ CK((*filter->output_function)(c | MBFL_WCSGROUP_THROUGH, filter->data));
}
break;
s = (c1 - 0xa1)*94 + c - 0xa1;
if (s >= 0 && s < jisx0208_ucs_table_size) {
w = jisx0208_ucs_table[s];
- } else {
- w = 0;
}
if (w <= 0) {
- w = ((c1 & 0x7f) << 8) | (c & 0x7f);
- w &= MBFL_WCSPLANE_MASK;
- w |= MBFL_WCSPLANE_JIS0208;
+ w = ((c1 & 0x7f) << 8) | (c & 0x7f) | MBFL_WCSPLANE_JIS0208;
}
CK((*filter->output_function)(w, filter->data));
- } else if ((c >= 0 && c < 0x21) || c == 0x7f) { /* CTLs */
- CK((*filter->output_function)(c, filter->data));
} else {
- w = (c1 << 8) | c;
- w &= MBFL_WCSGROUP_MASK;
- w |= MBFL_WCSGROUP_THROUGH;
+ w = (c1 << 8) | c | MBFL_WCSGROUP_THROUGH;
CK((*filter->output_function)(w, filter->data));
}
break;
if (c > 0xa0 && c < 0xe0) {
w = 0xfec0 + c;
CK((*filter->output_function)(w, filter->data));
- } else if ((c >= 0 && c < 0x21) || c == 0x7f) { /* CTLs */
- CK((*filter->output_function)(c, filter->data));
} else {
- w = 0x8e00 | c;
- w &= MBFL_WCSGROUP_MASK;
- w |= MBFL_WCSGROUP_THROUGH;
+ w = 0x8e00 | c | MBFL_WCSGROUP_THROUGH;
CK((*filter->output_function)(w, filter->data));
}
break;
- case 3: /* got 0x8f, X 0212 first char */
- if ((c >= 0 && c < 0x21) || c == 0x7f) { /* CTLs */
- CK((*filter->output_function)(c, filter->data));
- filter->status = 0;
- } else {
- filter->status++;
- filter->cache = c;
- }
+ case 3: /* got 0x8f, JIS X 0212 first byte */
+ filter->status++;
+ filter->cache = c;
break;
- case 4: /* got 0x8f, X 0212 second char */
+
+ case 4: /* got 0x8f, JIS X 0212 second byte */
filter->status = 0;
c1 = filter->cache;
- if (c1 > 0xa0 && c1 < 0xff && c > 0xa0 && c < 0xff) {
+ if (c > 0xA0 && c < 0xFF && c1 > 0xA0 && c1 < 0xFF) {
s = (c1 - 0xa1)*94 + c - 0xa1;
if (s >= 0 && s < jisx0212_ucs_table_size) {
w = jisx0212_ucs_table[s];
- } else {
- w = 0;
}
if (w <= 0) {
- w = ((c1 & 0x7f) << 8) | (c & 0x7f);
- w &= MBFL_WCSPLANE_MASK;
- w |= MBFL_WCSPLANE_JIS0212;
+ w = ((c1 & 0x7f) << 8) | (c & 0x7f) | MBFL_WCSPLANE_JIS0212;
}
CK((*filter->output_function)(w, filter->data));
- } else if ((c >= 0 && c < 0x21) || c == 0x7f) { /* CTLs */
- CK((*filter->output_function)(c, filter->data));
} else {
- w = (c1 << 8) | c | 0x8f0000;
- w &= MBFL_WCSGROUP_MASK;
- w |= MBFL_WCSGROUP_THROUGH;
+ w = (c1 << 8) | c | 0x8f0000 | MBFL_WCSGROUP_THROUGH;
CK((*filter->output_function)(w, filter->data));
}
break;
return c;
}
+static int mbfl_filt_conv_eucjp_wchar_flush(mbfl_convert_filter *filter)
+{
+ if (filter->status) {
+ (*filter->output_function)(filter->cache | MBFL_WCSGROUP_THROUGH, filter->data);
+ }
+
+ if (filter->flush_function) {
+ (*filter->flush_function)(filter->data);
+ }
+
+ return 0;
+}
+
/*
* wchar => EUC-JP
*/
int
mbfl_filt_conv_wchar_eucjp(int c, mbfl_convert_filter *filter)
{
- int c1, s;
+ int s;
s = 0;
if (c >= ucs_a1_jis_table_min && c < ucs_a1_jis_table_max) {
s = ucs_r_jis_table[c - ucs_r_jis_table_min];
}
if (s <= 0) {
- c1 = c & ~MBFL_WCSPLANE_MASK;
- if (c1 == MBFL_WCSPLANE_JIS0208) {
- s = c & MBFL_WCSPLANE_MASK;
- } else if (c1 == MBFL_WCSPLANE_JIS0212) {
- s = c & MBFL_WCSPLANE_MASK;
- s |= 0x8080;
- } else if (c == 0xff3c) { /* FULLWIDTH REVERSE SOLIDUS */
+ if (c == 0xff3c) { /* FULLWIDTH REVERSE SOLIDUS */
s = 0x2140;
} else if (c == 0xff5e) { /* FULLWIDTH TILDE */
s = 0x2141;
s = 0x2172;
} else if (c == 0xffe2) { /* FULLWIDTH NOT SIGN */
s = 0x224c;
- }
- if (c == 0) {
+ } else if (c == 0) {
s = 0;
- } else if (s <= 0) {
+ } else {
s = -1;
}
}
return c;
}
+/* Not all byte sequences in JIS X 0208 which would otherwise be valid are
+ * actually mapped to a character */
+static inline int in_unused_jisx0208_range(int c1, int c2)
+{
+ /* `c1`, `c2` are kuten codes */
+ unsigned int s = (c1 - 0x21)*94 + c2 - 0x21;
+ return s >= jisx0208_ucs_table_size || !jisx0208_ucs_table[s];
+}
+
+static inline int in_unused_jisx0212_range(int c1, int c2)
+{
+ unsigned int s = (c1 - 0x21)*94 + c2 - 0x21;
+ return s >= jisx0212_ucs_table_size || !jisx0212_ucs_table[s];
+}
+
int mbfl_filt_ident_eucjp(int c, mbfl_identify_filter *filter)
{
- switch (filter->status) {
- case 0: /* latin */
- if (c >= 0 && c < 0x80) { /* ok */
+ unsigned char ku, ten;
+
+ switch (filter->status & 0xF) {
+ case 0: /* latin */
+ if (c < 0x80) { /* ok */
;
- } else if (c > 0xa0 && c < 0xff) { /* kanji first char */
- filter->status = 1;
- } else if (c == 0x8e) { /* kana first char */
+ } else if (c > 0xa0 && c < 0xff) {
+ /* JIS X 0208, first byte
+ * In EUC-JP, each such byte ranges from 0xA1-0xFE; however,
+ * the bytes of JIS X 0208 kuten codes range from 0x21-0x7E */
+ filter->status = ((c - 0xA1 + 0x21) << 8) | 1;
+ } else if (c == 0x8e) { /* JIS X 0201 */
filter->status = 2;
- } else if (c == 0x8f) { /* X 0212 first char */
+ } else if (c == 0x8f) { /* JIS X 0212 */
filter->status = 3;
- } else { /* bad */
+ } else { /* bad */
filter->flag = 1;
}
break;
- case 1: /* got first half */
- if (c < 0xa1 || c > 0xfe) { /* bad */
+ case 1: /* 2nd byte of JIS X 0208 */
+ ku = filter->status >> 8;
+ ten = c - 0xA1 + 0x21;
+ if (c < 0xa1 || c > 0xfe || in_unused_jisx0208_range(ku, ten)) { /* bad */
filter->flag = 1;
}
filter->status = 0;
break;
- case 2: /* got 0x8e */
- if (c < 0xa1 || c > 0xdf) { /* bad */
+ case 2: /* JIS X 0201 */
+ if (c < 0xa1 || c > 0xdf) { /* bad */
filter->flag = 1;
}
filter->status = 0;
break;
- case 3: /* got 0x8f */
- if (c < 0xa1 || c > 0xfe) { /* bad */
+ case 3: /* JIS X 0212 */
+ if (c < 0xa1 || c > 0xfe) { /* bad */
filter->flag = 1;
+ } else {
+ filter->status = ((c - 0xA1 + 0x21) << 8) | 4;
}
- filter->status++;
break;
- case 4: /* got 0x8f */
- if (c < 0xa1 || c > 0xfe) { /* bad */
+
+ case 4: /* JIS X 0212, final byte */
+ ku = filter->status >> 8;
+ ten = c - 0xA1 + 0x21;
+ if (c < 0xa1 || c > 0xfe || in_unused_jisx0212_range(ku, ten)) { /* bad */
filter->flag = 1;
}
filter->status = 0;
--- /dev/null
+--TEST--
+Exhaustive test of EUC-JP encoding verification and conversion
+--SKIPIF--
+<?php
+extension_loaded('mbstring') or die('skip mbstring not available');
+if (getenv("SKIP_SLOW_TESTS")) die("skip slow test");
+?>
+--FILE--
+<?php
+srand(555); /* Make results consistent */
+include('encoding_tests.inc');
+mb_substitute_character(0x25); // '%'
+
+/* Read in the table of all characters in EUC-JP */
+readConversionTable(__DIR__ . '/data/EUC-JP.txt', $validChars, $fromUnicode, true);
+
+/* The JIS X 0208 character set does not have a single, straightforward
+ * mapping to the Unicode character set */
+
+/* Kuten code 0x2140 (EUC-JP 0xA1C0) is a backslash; this can be mapped to
+ * 0x005C for an ordinary backslash, or 0xFF3C for a _fullwidth_ one
+ * We go with fullwidth */
+$validChars["\xA1\xC0"] = "\x00\x00\xFF\x3C";
+$fromUnicode["\x00\x00\xFF\x3C"] = "\xA1\xC0";
+
+/* Unicode has both halfwidth and fullwidth NOT SIGN; convert both of them
+ * to JIS X 0208 NOT SIGN */
+$fromUnicode["\x00\x00\xFF\xE2"] = "\xA2\xCC";
+/* Likewise for fullwidth and halfwidth POUND SIGN */
+$fromUnicode["\x00\x00\xFF\xE1"] = "\xA1\xF2";
+/* Likewise for fullwidth and halfwidth CENT SIGN */
+$fromUnicode["\x00\x00\xFF\xE0"] = "\xA1\xF1";
+/* Convert Unicode FULLWIDTH TILDE to JIS X 0208 WAVE DASH */
+$fromUnicode["\x00\x00\xFF\x5E"] = "\xA1\xC1";
+/* Convert Unicode FULLWIDTH HYPHEN-MINUS to JIS X 0208 MINUS SIGN */
+$fromUnicode["\x00\x00\xFF\x0D"] = "\xA1\xDD";
+/* Convert Unicode PARALLEL TO to JIS X 0208 DOUBLE VERTICAL LINE */
+$fromUnicode["\x00\x00\x22\x25"] = "\xA1\xC2";
+
+/* Unicode 0x007E (tilde) can be represented in two different ways in EUC-JP
+ * When converting Unicode to EUC-JP, use the simpler representation */
+$fromUnicode["\x00\x00\x00\x7E"] = "\x7E";
+/* Likewise with 0x005C */
+$fromUnicode["\x00\x00\x00\x5C"] = "\x5C";
+
+findInvalidChars($validChars, $invalidChars, $truncated, map(range(0xA1, 0xFE), 2, array(0x8E => 2, 0x8F => 3)));
+
+/* In the JIS X 0212 character set, kuten code 0x2237 (EUC-JP 0x8FA2B7)
+ * is an ordinary tilde character
+ * This mapping is not reversible, because ASCII 0x7E also represents
+ * the same character */
+unset($validChars["\x8F\xA2\xB7"]);
+
+testAllValidChars($validChars, 'EUC-JP', 'UTF-32BE');
+echo "Encoding verification and conversion work for all valid characters\n";
+
+testAllInvalidChars($invalidChars, $validChars, 'EUC-JP', 'UTF-32BE', "\x00\x00\x00%");
+testTruncatedChars($truncated, 'EUC-JP', 'UTF-32BE', "\x00\x00\x00%");
+echo "Encoding verification and conversion work for all invalid characters\n";
+
+testValidString("\x8F\xA2\xB7", "\x00\x00\x00~", 'EUC-JP', 'UTF-32BE', false);
+echo "Irreversible mapping of 0x8FA2B7 follows JIS X 0212 correctly\n";
+
+$invalidChars = array();
+for ($cp = 0; $cp <= 0xFFFF; $cp++) {
+ $char = pack('N', $cp);
+ if (!isset($fromUnicode[$char]))
+ $invalidChars[$char] = true;
+}
+convertAllInvalidChars($invalidChars, $fromUnicode, 'UTF-32BE', 'EUC-JP', '%');
+echo "Unicode -> EUC-JP conversion works on all invalid characters\n";
+?>
+--EXPECT--
+Encoding verification and conversion work for all valid characters
+Encoding verification and conversion work for all invalid characters
+Irreversible mapping of 0x8FA2B7 follows JIS X 0212 correctly
+Unicode -> EUC-JP conversion works on all invalid characters
projects / php / commitdiff