Patch #435971: UTF-7 codec by Brian Quinlan.

diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h
index d7540fb2260e8ba45c6ac8743735727a5722b340..41feae2ec66534d787ed3caa933f1f1bbe4ec4ec 100644 (file)
--- a/Include/unicodeobject.h
+++ b/Include/unicodeobject.h
@@ -607,6 +607,24 @@ extern DL_IMPORT(PyObject*) PyUnicode_AsEncodedString(
     const char *errors         /* error handling */
     );
 
+/* --- UTF-7 Codecs ------------------------------------------------------- */
+
+extern DL_IMPORT(PyObject*) PyUnicode_DecodeUTF7(
+    const char *string,        /* UTF-7 encoded string */
+    int length,                        /* size of string */
+    const char *errors         /* error handling */
+    );
+
+extern DL_IMPORT(PyObject*) PyUnicode_EncodeUTF7(
+    const Py_UNICODE *data,    /* Unicode char buffer */
+    int length,                        /* number of Py_UNICODE chars to encode */
+    int encodeSetO,             /* force the encoder to encode characters in
+                                   Set O, as described in RFC2152 */
+    int encodeWhiteSpace,       /* force the encoder to encode space, tab,
+                                   carriage return and linefeed characters */
+    const char *errors         /* error handling */
+    );
+
 /* --- UTF-8 Codecs ------------------------------------------------------- */
 
 extern DL_IMPORT(PyObject*) PyUnicode_DecodeUTF8(

diff --git a/Lib/encodings/aliases.py b/Lib/encodings/aliases.py
index c3c49b4d7f71e356a9dd4fe067742f3cf346a59b..65d8fef62571b7c35448d3b5a7cbee1da4f9bff2 100644 (file)
--- a/Lib/encodings/aliases.py
+++ b/Lib/encodings/aliases.py
@@ -14,6 +14,10 @@ aliases = {
     'latin': 'latin_1',
     'latin1': 'latin_1',
     
+    # UTF-7
+    'utf7': 'utf_7',
+    'u7': 'utf_7',
+    
     # UTF-8
     'utf': 'utf_8',
     'utf8': 'utf_8',

diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py
index dde16ef78510b0d331e830d39da3b47b13ad0a70..d57328d52c9f284dc322df4fb764011b8ca214a6 100644 (file)
--- a/Lib/test/test_unicode.py
+++ b/Lib/test/test_unicode.py
@@ -377,6 +377,32 @@ print 'done.'
 # Test builtin codecs
 print 'Testing builtin codecs...',
 
+# UTF-7 specific encoding tests:
+utfTests = [(u'A\u2262\u0391.', 'A+ImIDkQ.'),  # RFC2152 example
+ (u'Hi Mom -\u263a-!', 'Hi Mom -+Jjo--!'),     # RFC2152 example
+ (u'\u65E5\u672C\u8A9E', '+ZeVnLIqe-'),        # RFC2152 example
+ (u'Item 3 is \u00a31.', 'Item 3 is +AKM-1.'), # RFC2152 example
+ (u'+', '+-'),
+ (u'+-', '+--'),
+ (u'+?', '+-?'),
+ (u'\?', '+AFw?'),
+ (u'+?', '+-?'),
+ (ur'\\?', '+AFwAXA?'),
+ (ur'\\\?', '+AFwAXABc?'),
+ (ur'++--', '+-+---')]
+
+for x,y in utfTests:
+    verify( x.encode('utf-7') == y )
+
+try:        
+    unicode('+3ADYAA-', 'utf-7') # surrogates not supported
+except UnicodeError:
+    pass
+else:
+    raise TestFailed, "unicode('+3ADYAA-', 'utf-7') failed to raise an exception"
+
+verify(unicode('+3ADYAA-', 'utf-7', 'replace') == u'\ufffd')
+
 # UTF-8 specific encoding tests:
 verify(u'\u20ac'.encode('utf-8') == \
        ''.join((chr(0xe2), chr(0x82), chr(0xac))) )
@@ -439,6 +465,7 @@ verify(unicode('Andr\202 x','ascii','ignore') == u"Andr x")
 verify(unicode('Andr\202 x','ascii','replace') == u'Andr\uFFFD x')
 
 verify(u'hello'.encode('ascii') == 'hello')
+verify(u'hello'.encode('utf-7') == 'hello')
 verify(u'hello'.encode('utf-8') == 'hello')
 verify(u'hello'.encode('utf8') == 'hello')
 verify(u'hello'.encode('utf-16-le') == 'h\000e\000l\000l\000o\000')
@@ -447,7 +474,7 @@ verify(u'hello'.encode('latin-1') == 'hello')
 
 # Roundtrip safety for BMP (just the first 1024 chars)
 u = u''.join(map(unichr, range(1024)))
-for encoding in ('utf-8', 'utf-16', 'utf-16-le', 'utf-16-be',
+for encoding in ('utf-7', 'utf-8', 'utf-16', 'utf-16-le', 'utf-16-be',
                  'raw_unicode_escape', 'unicode_escape', 'unicode_internal'):
     verify(unicode(u.encode(encoding),encoding) == u)
 

diff --git a/Modules/_codecsmodule.c b/Modules/_codecsmodule.c
index a085bcf3ab99c4a5395b0b6d605f7b180506753e..29b068687c770142bd3a97d943d71b7636dbb234 100644 (file)
--- a/Modules/_codecsmodule.c
+++ b/Modules/_codecsmodule.c
@@ -123,6 +123,22 @@ unicode_internal_decode(PyObject *self,
     }
 }
 
+static PyObject *
+utf_7_decode(PyObject *self,
+           PyObject *args)
+{
+    const char *data;
+    int size;
+    const char *errors = NULL;
+    
+    if (!PyArg_ParseTuple(args, "t#|z:utf_7_decode",
+                         &data, &size, &errors))
+       return NULL;
+
+    return codec_tuple(PyUnicode_DecodeUTF7(data, size, errors),
+                      size);
+}
+
 static PyObject *
 utf_8_decode(PyObject *self,
            PyObject *args)
@@ -381,6 +397,30 @@ unicode_internal_encode(PyObject *self,
     }
 }
 
+static PyObject *
+utf_7_encode(PyObject *self,
+           PyObject *args)
+{
+    PyObject *str, *v;
+    const char *errors = NULL;
+
+    if (!PyArg_ParseTuple(args, "O|z:utf_7_encode",
+                         &str, &errors))
+       return NULL;
+
+    str = PyUnicode_FromObject(str);
+    if (str == NULL)
+       return NULL;
+    v = codec_tuple(PyUnicode_EncodeUTF7(PyUnicode_AS_UNICODE(str),
+                                        PyUnicode_GET_SIZE(str),
+                     0,
+                     0,
+                                        errors),
+                   PyUnicode_GET_SIZE(str));
+    Py_DECREF(str);
+    return v;
+}
+
 static PyObject *
 utf_8_encode(PyObject *self,
            PyObject *args)
@@ -632,6 +672,8 @@ static PyMethodDef _codecs_functions[] = {
 #ifdef Py_USING_UNICODE
     {"utf_8_encode",           utf_8_encode,                   1},
     {"utf_8_decode",           utf_8_decode,                   1},
+    {"utf_7_encode",           utf_7_encode,                   1},
+    {"utf_7_decode",           utf_7_decode,                   1},
     {"utf_16_encode",          utf_16_encode,                  1},
     {"utf_16_le_encode",       utf_16_le_encode,               1},
     {"utf_16_be_encode",       utf_16_be_encode,               1},

diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index 5080eb86661132f63dfa12e808b3b67be80dbc31..50f2f5c627349cd521cab1716686f24d64a4cd03 100644 (file)
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -635,6 +635,306 @@ int PyUnicode_SetDefaultEncoding(const char *encoding)
     return -1;
 }
 
+/* --- UTF-7 Codec -------------------------------------------------------- */
+
+/* see RFC2152 for details */
+
+static 
+char utf7_special[128] = {
+    /* indicate whether a UTF-7 character is special i.e. cannot be directly
+       encoded:
+          0 - not special
+          1 - special
+          2 - whitespace (optional)
+          3 - RFC2152 Set O (optional) */
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 1, 1, 2, 1, 1,
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+    2, 3, 3, 3, 3, 3, 3, 0, 0, 0, 3, 1, 0, 0, 0, 1,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3, 0,
+    3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 3, 3, 3,
+    3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 1, 1,
+
+};
+
+#define SPECIAL(c, encodeO, encodeWS) \
+       (((c)>127 || utf7_special[(c)] == 1) || \
+        (encodeWS && (utf7_special[(c)] == 2)) || \
+     (encodeO && (utf7_special[(c)] == 3)))
+
+#define B64(n)  ("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"[(n) & 0x3f])
+#define B64CHAR(c) (isalnum(c) || (c) == '+' || (c) == '/')
+#define UB64(c)        ((c) == '+' ? 62 : (c) == '/' ? 63 : (c) >= 'a' ? \
+                        (c) - 71 : (c) >= 'A' ? (c) - 65 : (c) + 4)
+
+#define ENCODE(out, ch, bits) \
+    while (bits >= 6) { \
+        *out++ = B64(ch >> (bits-6)); \
+        bits -= 6; \
+    }
+
+#define DECODE(out, ch, bits, surrogate) \
+    while (bits >= 16) { \
+        Py_UNICODE outCh = (Py_UNICODE) ((ch >> (bits-16)) & 0xffff); \
+        bits -= 16; \
+               if (surrogate) { \
+                       /* We have already generated an error for the high surrogate
+               so let's not bother seeing if the low surrogate is correct or not */\
+                       surrogate = 0; \
+               } else if (0xDC00 <= outCh && outCh <= 0xDFFF) { \
+            /* This is a surrogate pair. Unfortunately we can't represent \
+               it in a 16-bit character */ \
+                       surrogate = 1; \
+            errmsg = "code pairs are not supported"; \
+               goto utf7Error; \
+               } else { \
+                               *out++ = outCh; \
+               } \
+    } \
+
+static
+int utf7_decoding_error(Py_UNICODE **dest,
+                        const char *errors,
+                        const char *details) 
+{
+    if ((errors == NULL) ||
+        (strcmp(errors,"strict") == 0)) {
+        PyErr_Format(PyExc_UnicodeError,
+                     "UTF-7 decoding error: %.400s",
+                     details);
+        return -1;
+    }
+    else if (strcmp(errors,"ignore") == 0) {
+        return 0;
+    }
+    else if (strcmp(errors,"replace") == 0) {
+        if (dest != NULL) {
+            **dest = Py_UNICODE_REPLACEMENT_CHARACTER;
+            (*dest)++;
+        }
+        return 0;
+    }
+    else {
+        PyErr_Format(PyExc_ValueError,
+                     "UTF-7 decoding error; unknown error handling code: %.400s",
+                     errors);
+        return -1;
+    }
+}
+
+PyObject *PyUnicode_DecodeUTF7(const char *s,
+                              int size,
+                              const char *errors)
+{
+    const char *e;
+    PyUnicodeObject *unicode;
+    Py_UNICODE *p;
+    const char *errmsg = "";
+    int inShift = 0;
+    unsigned int bitsleft = 0;
+    unsigned long charsleft = 0;
+       int surrogate = 0;
+
+    unicode = _PyUnicode_New(size);
+    if (!unicode)
+        return NULL;
+    if (size == 0)
+        return (PyObject *)unicode;
+
+    p = unicode->str;
+    e = s + size;
+
+    while (s < e) {
+        Py_UNICODE ch = *s;
+
+        if (inShift) {
+            if ((ch == '-') || !B64CHAR(ch)) {
+                inShift = 0;
+                s++;
+                    
+                /* p, charsleft, bitsleft, surrogate = */ DECODE(p, charsleft, bitsleft, surrogate);
+                if (bitsleft >= 6) {
+                    /* The shift sequence has a partial character in it. If
+                       bitsleft < 6 then we could just classify it as padding
+                       but that is not the case here */
+
+                    errmsg = "partial character in shift sequence";
+                    goto utf7Error; 
+                }
+                /* According to RFC2152 the remaining bits should be zero. We
+                   choose to signal an error/insert a replacement character 
+                   here so indicate the potential of a misencoded character. */
+
+                /* On x86, a << b == a << (b%32) so make sure that bitsleft != 0 */
+                if (bitsleft && charsleft << (sizeof(charsleft) * 8 - bitsleft)) {
+                    errmsg = "non-zero padding bits in shift sequence";
+                    goto utf7Error; 
+                }
+
+                if (ch == '-') {
+                    if ((s < e) && (*(s) == '-')) {
+                        *p++ = '-';   
+                        inShift = 1;
+                    }
+                } else if (SPECIAL(ch,0,0)) {
+                    errmsg = "unexpected special character";
+                       goto utf7Error;  
+                } else  {
+                    *p++ = ch;
+                }
+            } else {
+                charsleft = (charsleft << 6) | UB64(ch);
+                bitsleft += 6;
+                s++;
+                /* p, charsleft, bitsleft, surrogate = */ DECODE(p, charsleft, bitsleft, surrogate);
+            }
+        }
+        else if ( ch == '+' ) {
+            s++;
+            if (s < e && *s == '-') {
+                s++;
+                *p++ = '+';
+            } else
+            {
+                inShift = 1;
+                bitsleft = 0;
+            }
+        }
+        else if (SPECIAL(ch,0,0)) {
+            errmsg = "unexpected special character";
+            s++;
+               goto utf7Error;  
+        }
+        else {
+            *p++ = ch;
+            s++;
+        }
+        continue;
+    utf7Error:
+      if (utf7_decoding_error(&p, errors, errmsg))
+          goto onError;
+    }
+
+    if (inShift) {
+        if (utf7_decoding_error(&p, errors, "unterminated shift sequence"))
+            goto onError;
+    }
+
+    if (_PyUnicode_Resize(&unicode, p - unicode->str))
+        goto onError;
+
+    return (PyObject *)unicode;
+
+onError:
+    Py_DECREF(unicode);
+    return NULL;
+}
+
+
+PyObject *PyUnicode_EncodeUTF7(const Py_UNICODE *s,
+                   int size,
+                   int encodeSetO,
+                   int encodeWhiteSpace,
+                   const char *errors)
+{
+    PyObject *v;
+    /* It might be possible to tighten this worst case */
+    unsigned int cbAllocated = 5 * size;
+    int inShift = 0;
+    int i = 0;
+    unsigned int bitsleft = 0;
+    unsigned long charsleft = 0;
+    char * out;
+    char * start;
+
+    if (size == 0)
+               return PyString_FromStringAndSize(NULL, 0);
+
+    v = PyString_FromStringAndSize(NULL, cbAllocated);
+    if (v == NULL)
+        return NULL;
+
+    start = out = PyString_AS_STRING(v);
+    for (;i < size; ++i) {
+        Py_UNICODE ch = s[i];
+
+        if (!inShift) {
+                       if (ch == '+') {
+                               *out++ = '+';
+                *out++ = '-';
+            } else if (SPECIAL(ch, encodeSetO, encodeWhiteSpace)) {
+                charsleft = ch;
+                bitsleft = 16;
+                *out++ = '+';
+                               /* out, charsleft, bitsleft = */ ENCODE(out, charsleft, bitsleft);
+                inShift = bitsleft > 0;
+                       } else {
+                               *out++ = (char) ch;
+                       }
+               } else {
+            if (!SPECIAL(ch, encodeSetO, encodeWhiteSpace)) {
+                *out++ = B64(charsleft << (6-bitsleft));
+                charsleft = 0;
+                bitsleft = 0;
+                /* Characters not in the BASE64 set implicitly unshift the sequence
+                   so no '-' is required, except if the character is itself a '-' */
+                if (B64CHAR(ch) || ch == '-') {
+                    *out++ = '-';
+                }
+                inShift = 0;
+                *out++ = (char) ch;
+            } else {
+                bitsleft += 16;
+                charsleft = (charsleft << 16) | ch;
+                /* out, charsleft, bitsleft = */ ENCODE(out, charsleft, bitsleft);
+
+                /* If the next character is special then we dont' need to terminate
+                   the shift sequence. If the next character is not a BASE64 character 
+                   or '-' then the shift sequence will be terminated implicitly and we
+                   don't have to insert a '-'. */
+
+                if (bitsleft == 0) {
+                    if (i + 1 < size) {
+                        Py_UNICODE ch2 = s[i+1];
+
+                        if (SPECIAL(ch2, encodeSetO, encodeWhiteSpace)) {
+                           
+                        } else if (B64CHAR(ch2) || ch2 == '-') {
+                            *out++ = '-';
+                            inShift = 0;
+                        } else {
+                            inShift = 0;
+                        }
+
+                    }
+                    else {
+                        *out++ = '-';
+                        inShift = 0;
+                    }
+                }
+            }            
+        }
+       }
+    if (bitsleft) {
+        *out++= B64(charsleft << (6-bitsleft) );
+        *out++ = '-';
+    }
+
+    if (_PyString_Resize(&v, out - start)) {
+        Py_DECREF(v);
+        return NULL;
+    }
+    return v;
+}
+
+#undef SPECIAL
+#undef B64
+#undef B64CHAR
+#undef UB64
+#undef ENCODE
+#undef DECODE
+
 /* --- UTF-8 Codec -------------------------------------------------------- */
 
 static