Backport from trunk r51737:

Fixed a few bugs on cjkcodecs:
- gbk and gb18030 codec now handle U+30FB KATAKANA MIDDLE DOT correctly.
- iso2022_jp_2 codec now encodes into G0 for KS X 1001, GB2312
  codepoints to conform the standard.
- iso2022_jp_3 and iso2022_jp_2004 codec can encode JIS X 0213:2
  codepoints now.

diff --git a/Lib/test/test_codecencodings_cn.py b/Lib/test/test_codecencodings_cn.py
index 1bf85834bc8b371a6acd8a1b8d1c164c2af711c2..c558f1bb99b06c4f0d6177fe9aed5e8d78a70d77 100644 (file)
--- a/Lib/test/test_codecencodings_cn.py
+++ b/Lib/test/test_codecencodings_cn.py
@@ -32,6 +32,7 @@ class Test_GBK(test_multibytecodec_support.TestBase, unittest.TestCase):
         ("abc\x80\x80\xc1\xc4\xc8", "replace", u"abc\ufffd\u804a\ufffd"),
         ("abc\x80\x80\xc1\xc4", "ignore",  u"abc\u804a"),
         ("\x83\x34\x83\x31", "strict", None),
+        (u"\u30fb", "strict", None),
     )
 
 class Test_GB18030(test_multibytecodec_support.TestBase, unittest.TestCase):
@@ -45,6 +46,7 @@ class Test_GB18030(test_multibytecodec_support.TestBase, unittest.TestCase):
         ("abc\x80\x80\xc1\xc4\xc8", "replace", u"abc\ufffd\u804a\ufffd"),
         ("abc\x80\x80\xc1\xc4", "ignore",  u"abc\u804a"),
         ("abc\x84\x39\x84\x39\xc1\xc4", "replace", u"abc\ufffd\u804a"),
+        (u"\u30fb", "strict", "\x819\xa79"),
     )
     has_iso10646 = True
 

diff --git a/Lib/test/test_multibytecodec.py b/Lib/test/test_multibytecodec.py
index 397ebeb97a6f779c50909e8693fa34f19426a11f..800456e23b8bcdda6d4604e1de13e306c5582c20 100644 (file)
--- a/Lib/test/test_multibytecodec.py
+++ b/Lib/test/test_multibytecodec.py
@@ -202,6 +202,12 @@ class Test_ISO2022(unittest.TestCase):
         uni = u':hu4:unit\xe9 de famille'
         self.assertEqual(iso2022jp2.decode('iso2022-jp-2'), uni)
 
+    def test_iso2022_jp_g0(self):
+        self.failIf('\x0e' in u'\N{SOFT HYPHEN}'.encode('iso-2022-jp-2'))
+        for encoding in ('iso-2022-jp-2004', 'iso-2022-jp-3'):
+            e = u'\u3406'.encode(encoding)
+            self.failIf(filter(lambda x: x >= '\x80', e))
+
 def test_main():
     suite = unittest.TestSuite()
     suite.addTest(unittest.makeSuite(Test_MultibyteCodec))

diff --git a/Misc/NEWS b/Misc/NEWS
index 53708c1b9a436e83f706ffbbd31e440088b9629b..f4dd226f4dcb57722b3a74a8a8647eaf522350dd 100644 (file)
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -69,6 +69,12 @@ Extension Modules
 
 - Bug #1550714: fix SystemError from itertools.tee on negative value for n.
 
+- Fixed a few bugs on cjkcodecs:
+  - gbk and gb18030 codec now handle U+30FB KATAKANA MIDDLE DOT correctly.
+  - iso2022_jp_2 codec now encodes into G0 for KS X 1001, GB2312
+    codepoints to conform the standard.
+  - iso2022_jp_3 and iso2022_jp_2004 codec can encode JIS X 0213:2
+    codepoints now.
 
 Tests
 -----

diff --git a/Modules/cjkcodecs/_codecs_cn.c b/Modules/cjkcodecs/_codecs_cn.c
index fb51297ae06bf29828e10a764213340bce8344ac..c811a67eda00f72d5fde4839332e0c84bcae702c 100644 (file)
--- a/Modules/cjkcodecs/_codecs_cn.c
+++ b/Modules/cjkcodecs/_codecs_cn.c
@@ -15,14 +15,26 @@
 #undef hz
 #endif
 
-#define GBK_PREDECODE(dc1, dc2, assi) \
+/* GBK and GB2312 map differently in few codepoints that are listed below:
+ *
+ *             gb2312                          gbk
+ * A1A4                U+30FB KATAKANA MIDDLE DOT      U+00B7 MIDDLE DOT
+ * A1AA                U+2015 HORIZONTAL BAR           U+2014 EM DASH
+ * A844                undefined                       U+2015 HORIZONTAL BAR
+ */
+
+#define GBK_DECODE(dc1, dc2, assi) \
        if ((dc1) == 0xa1 && (dc2) == 0xaa) (assi) = 0x2014; \
        else if ((dc1) == 0xa8 && (dc2) == 0x44) (assi) = 0x2015; \
-       else if ((dc1) == 0xa1 && (dc2) == 0xa4) (assi) = 0x00b7;
-#define GBK_PREENCODE(code, assi) \
+       else if ((dc1) == 0xa1 && (dc2) == 0xa4) (assi) = 0x00b7; \
+       else TRYMAP_DEC(gb2312, assi, dc1 ^ 0x80, dc2 ^ 0x80); \
+       else TRYMAP_DEC(gbkext, assi, dc1, dc2);
+
+#define GBK_ENCODE(code, assi) \
        if ((code) == 0x2014) (assi) = 0xa1aa; \
        else if ((code) == 0x2015) (assi) = 0xa844; \
-       else if ((code) == 0x00b7) (assi) = 0xa1a4;
+       else if ((code) == 0x00b7) (assi) = 0xa1a4; \
+       else if ((code) != 0x30fb && TRYMAP_ENC_COND(gbcommon, assi, code));
 
 /*
  * GB2312 codec
@@ -99,8 +111,7 @@ ENCODER(gbk)
 
                REQUIRE_OUTBUF(2)
 
-               GBK_PREENCODE(c, code)
-               else TRYMAP_ENC(gbcommon, code, c);
+               GBK_ENCODE(c, code)
                else return 1;
 
                OUT1((code >> 8) | 0x80)
@@ -129,9 +140,7 @@ DECODER(gbk)
 
                REQUIRE_INBUF(2)
 
-               GBK_PREDECODE(c, IN2, **outbuf)
-               else TRYMAP_DEC(gb2312, **outbuf, c ^ 0x80, IN2 ^ 0x80);
-               else TRYMAP_DEC(gbkext, **outbuf, c, IN2);
+               GBK_DECODE(c, IN2, **outbuf)
                else return 2;
 
                NEXT(2, 1)
@@ -187,9 +196,7 @@ ENCODER(gb18030)
 
                REQUIRE_OUTBUF(2)
 
-               GBK_PREENCODE(c, code)
-               else TRYMAP_ENC(gbcommon, code, c);
-               else TRYMAP_ENC(gb18030ext, code, c);
+               GBK_ENCODE(c, code)
                else {
                        const struct _gb18030_to_unibmp_ranges *utrrange;
 
@@ -287,9 +294,7 @@ DECODER(gb18030)
                        return 4;
                }
 
-               GBK_PREDECODE(c, c2, **outbuf)
-               else TRYMAP_DEC(gb2312, **outbuf, c ^ 0x80, c2 ^ 0x80);
-               else TRYMAP_DEC(gbkext, **outbuf, c, c2);
+               GBK_DECODE(c, c2, **outbuf)
                else TRYMAP_DEC(gb18030ext, **outbuf, c, c2);
                else return 2;
 

diff --git a/Modules/cjkcodecs/_codecs_iso2022.c b/Modules/cjkcodecs/_codecs_iso2022.c
index 8a2ab7eb2f4e5658e9ec1bdccd77004041ef52d1..2a11e9a2747700da34e4984a4e5f7b6ff73e8213 100644 (file)
--- a/Modules/cjkcodecs/_codecs_iso2022.c
+++ b/Modules/cjkcodecs/_codecs_iso2022.c
@@ -854,7 +854,7 @@ jisx0213_2000_2_encoder(const ucs4_t *data, Py_ssize_t *length)
        if (coded == MAP_UNMAPPABLE || coded == MAP_MULTIPLE_AVAIL)
                return coded;
        else if (coded & 0x8000)
-               return coded;
+               return coded & 0x7fff;
        else
                return MAP_UNMAPPABLE;
 }
@@ -901,7 +901,7 @@ jisx0213_2004_2_encoder(const ucs4_t *data, Py_ssize_t *length)
        if (coded == MAP_UNMAPPABLE || coded == MAP_MULTIPLE_AVAIL)
                return coded;
        else if (coded & 0x8000)
-               return coded;
+               return coded & 0x7fff;
        else
                return MAP_UNMAPPABLE;
 }
@@ -992,7 +992,10 @@ dummy_encoder(const ucs4_t *data, Py_ssize_t *length)
 
 /*-*- registry tables -*-*/
 
-#define REGISTRY_KSX1001       { CHARSET_KSX1001, 1, 2,                \
+#define REGISTRY_KSX1001_G0    { CHARSET_KSX1001, 0, 2,                \
+                                 ksx1001_init,                         \
+                                 ksx1001_decoder, ksx1001_encoder }
+#define REGISTRY_KSX1001_G1    { CHARSET_KSX1001, 1, 2,                \
                                  ksx1001_init,                         \
                                  ksx1001_decoder, ksx1001_encoder }
 #define REGISTRY_JISX0201_R    { CHARSET_JISX0201_R, 0, 1,             \
@@ -1034,7 +1037,7 @@ dummy_encoder(const ucs4_t *data, Py_ssize_t *length)
                                  jisx0213_init,                        \
                                  jisx0213_2004_2_decoder,              \
                                  jisx0213_2004_2_encoder }
-#define REGISTRY_GB2312                { CHARSET_GB2312, 1, 2,                 \
+#define REGISTRY_GB2312                { CHARSET_GB2312, 0, 2,                 \
                                  gb2312_init,                          \
                                  gb2312_decoder, gb2312_encoder }
 #define REGISTRY_CNS11643_1    { CHARSET_CNS11643_1, 1, 2,             \
@@ -1054,7 +1057,7 @@ dummy_encoder(const ucs4_t *data, Py_ssize_t *length)
        };
 
 static const struct iso2022_designation iso2022_kr_designations[] = {
-       REGISTRY_KSX1001, REGISTRY_SENTINEL
+       REGISTRY_KSX1001_G1, REGISTRY_SENTINEL
 };
 CONFIGDEF(kr, 0)
 
@@ -1071,7 +1074,7 @@ static const struct iso2022_designation iso2022_jp_1_designations[] = {
 CONFIGDEF(jp_1, NO_SHIFT | USE_JISX0208_EXT)
 
 static const struct iso2022_designation iso2022_jp_2_designations[] = {
-       REGISTRY_JISX0208, REGISTRY_JISX0212, REGISTRY_KSX1001,
+       REGISTRY_JISX0208, REGISTRY_JISX0212, REGISTRY_KSX1001_G0,
        REGISTRY_GB2312, REGISTRY_JISX0201_R, REGISTRY_JISX0208_O,
        REGISTRY_ISO8859_1, REGISTRY_ISO8859_7, REGISTRY_SENTINEL
 };

diff --git a/Modules/cjkcodecs/cjkcodecs.h b/Modules/cjkcodecs/cjkcodecs.h
index b266c8f9b47c848d8979408324ae74dfda4b2c17..71c54f093fc96a24ef11f8fb3fa57453d5e64c56 100644 (file)
--- a/Modules/cjkcodecs/cjkcodecs.h
+++ b/Modules/cjkcodecs/cjkcodecs.h
@@ -159,29 +159,32 @@ static const struct dbcs_map *mapping_list;
 #endif
 
 #define _TRYMAP_ENC(m, assi, val)                              \
-       if ((m)->map != NULL && (val) >= (m)->bottom &&         \
+       ((m)->map != NULL && (val) >= (m)->bottom &&            \
            (val)<= (m)->top && ((assi) = (m)->map[(val) -      \
            (m)->bottom]) != NOCHAR)
-#define TRYMAP_ENC(charset, assi, uni)                         \
+#define TRYMAP_ENC_COND(charset, assi, uni)                    \
        _TRYMAP_ENC(&charset##_encmap[(uni) >> 8], assi, (uni) & 0xff)
+#define TRYMAP_ENC(charset, assi, uni)                         \
+       if TRYMAP_ENC_COND(charset, assi, uni)
+
 #define _TRYMAP_DEC(m, assi, val)                              \
-       if ((m)->map != NULL && (val) >= (m)->bottom &&         \
+       ((m)->map != NULL && (val) >= (m)->bottom &&            \
            (val)<= (m)->top && ((assi) = (m)->map[(val) -      \
            (m)->bottom]) != UNIINV)
 #define TRYMAP_DEC(charset, assi, c1, c2)                      \
-       _TRYMAP_DEC(&charset##_decmap[c1], assi, c2)
+       if _TRYMAP_DEC(&charset##_decmap[c1], assi, c2)
 
 #define _TRYMAP_ENC_MPLANE(m, assplane, asshi, asslo, val)     \
-       if ((m)->map != NULL && (val) >= (m)->bottom &&         \
+       ((m)->map != NULL && (val) >= (m)->bottom &&            \
            (val)<= (m)->top &&                                 \
            ((assplane) = (m)->map[((val) - (m)->bottom)*3]) != 0 && \
            (((asshi) = (m)->map[((val) - (m)->bottom)*3 + 1]), 1) && \
            (((asslo) = (m)->map[((val) - (m)->bottom)*3 + 2]), 1))
 #define TRYMAP_ENC_MPLANE(charset, assplane, asshi, asslo, uni)        \
-       _TRYMAP_ENC_MPLANE(&charset##_encmap[(uni) >> 8], \
+       if _TRYMAP_ENC_MPLANE(&charset##_encmap[(uni) >> 8], \
                           assplane, asshi, asslo, (uni) & 0xff)
 #define TRYMAP_DEC_MPLANE(charset, assi, plane, c1, c2)                \
-       _TRYMAP_DEC(&charset##_decmap[plane][c1], assi, c2)
+       if _TRYMAP_DEC(&charset##_decmap[plane][c1], assi, c2)
 
 #if Py_UNICODE_SIZE == 2
 #define DECODE_SURROGATE(c)                                    \