+++ /dev/null
-/*
- * _codecs_unicode.c: Codecs collection for Unicode encodings
- *
- * Written by Hye-Shik Chang <perky@FreeBSD.org>
- * $CJKCodecs: _codecs_unicode.c,v 1.5 2004/06/27 21:41:15 perky Exp $
- */
-
-#include "cjkcodecs.h"
-
-/*
- * UTF-7 codec
- */
-
-#define SET_DIRECT 1
-#define SET_OPTIONAL 2
-#define SET_WHITESPACE 3
-
-#define _D SET_DIRECT
-#define _O SET_OPTIONAL
-#define _W SET_WHITESPACE
-static const char utf7_sets[128] = {
- 0, 0, 0, 0, 0, 0, 0, 0, 0, _W, _W, 0, 0, _W, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- _W, _O, _O, _O, _O, _O, _O, _D, _D, _D, _O, 0, _D, _D, _D, 0,
- _D, _D, _D, _D, _D, _D, _D, _D, _D, _D, _D, _O, _O, _O, _O, _D,
- _O, _D, _D, _D, _D, _D, _D, _D, _D, _D, _D, _D, _D, _D, _D, _D,
- _D, _D, _D, _D, _D, _D, _D, _D, _D, _D, _D, _O, 0, _O, _O, _O,
- _O, _D, _D, _D, _D, _D, _D, _D, _D, _D, _D, _D, _D, _D, _D, _D,
- _D, _D, _D, _D, _D, _D, _D, _D, _D, _D, _D, _O, _O, _O, 0, 0,
-};
-#undef _W
-#undef _O
-#undef _D
-
-#define B64(n) ("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" \
- "0123456789+/"[(n) & 0x3f])
-#define B64CHAR(c) (((c) >= 'A' && (c) <= 'Z') || \
- ((c) >= 'a' && (c) <= 'z') || \
- ((c) >= '0' && (c) <= '9') || \
- (c) == '+' || (c) == '/')
-#define UB64(c) ((c) == '+' ? 62 : (c) == '/' ? 63 : (c) >= 'a' ? \
- (c) - 71 : (c) >= 'A' ? (c) - 65 : (c) + 4)
-
-#define UTF7_DENCODABLE_COMPATIBLE(c) (utf7_sets[c] != 0)
-#define UTF7_DENCODABLE_STRICT(c) (utf7_sets[c] == SET_DIRECT || \
- utf7_sets[c] == SET_WHITESPACE)
-
-#define ESTATE_INITIALIZE(state) \
- ESTATE_SETSTAGE(state, 0) \
- ESTATE_CLEARSHIFTED(state)
-
-#define ESTATE_SETPENDING(state, v) (state)->c[0] = (v);
-#define ESTATE_GETPENDING(state) (state)->c[0]
-
-#define ESTATE_SETSHIFTED(state) (state)->c[2] = 1;
-#define ESTATE_ISSHIFTED(state) ((state)->c[2])
-#define ESTATE_CLEARSHIFTED(state) (state)->c[2] = 0;
-
-#define ESTATE_SETSTAGE(state, v) (state)->c[3] = (v);
-#define ESTATE_GETSTAGE(state) ((state)->c[3])
-
-ENCODER_INIT(utf_7)
-{
- ESTATE_INITIALIZE(state)
- return 0;
-}
-
-ENCODER_RESET(utf_7)
-{
- if (ESTATE_ISSHIFTED(state)) {
- if (ESTATE_GETSTAGE(state) != 0) {
- unsigned char oc;
-
- oc = B64(ESTATE_GETPENDING(state));
- WRITE2(oc, '-')
- NEXT_OUT(2)
- }
- else {
- WRITE1('-')
- NEXT_OUT(1)
- }
- ESTATE_CLEARSHIFTED(state)
- }
- return 0;
-}
-
-ENCODER(utf_7)
-{
- while (inleft > 0) {
- Py_UNICODE c1 = IN1, c2 = 0;
- size_t insize = 1;
-
-#if Py_UNICODE_SIZE == 2
- if (c1 >> 10 == 0xd800 >> 10) { /* high surrogate */
- REQUIRE_INBUF(2)
- if (IN2 >> 10 != 0xdc00 >> 10) /* low surrogate */
- return 2; /* invalid surrogate pair */
- c2 = IN2;
- insize = 2;
- }
-#else
- if (c1 > 0x10ffff) /* UTF-16 unencodable */
- return 1;
- else if (c1 > 0xffff) {
- c2 = 0xdc00 | ((c1 - 0x10000) & 0x3ff);
- c1 = 0xd800 | ((c1 - 0x10000) >> 10);
- }
-#endif
-
- for (;;) {
- unsigned char oc1, oc2, oc3;
-
- if (ESTATE_ISSHIFTED(state)) {
- if (c1 < 128 && UTF7_DENCODABLE_STRICT(c1)) {
- if (ESTATE_GETSTAGE(state) != 0) {
- oc1 = B64(ESTATE_GETPENDING(
- state));
- WRITE3(oc1, '-',
- (unsigned char)c1)
- NEXT_OUT(3)
- } else {
- WRITE2('-',
- (unsigned char)c1)
- NEXT_OUT(2)
- }
- ESTATE_CLEARSHIFTED(state)
- } else {
- switch (ESTATE_GETSTAGE(state)) {
- case 0:
- oc1 = c1 >> 10;
- oc2 = (c1 >> 4) & 0x3f;
- WRITE2(B64(oc1), B64(oc2))
- ESTATE_SETPENDING(state,
- (c1 & 0x0f) << 2)
- ESTATE_SETSTAGE(state, 2)
- NEXT_OUT(2)
- break;
- case 1:
- oc1 = ESTATE_GETPENDING(state)
- | (c1 >> 12);
- oc2 = (c1 >> 6) & 0x3f;
- oc3 = c1 & 0x3f;
- WRITE3(B64(oc1), B64(oc2),
- B64(oc3))
- ESTATE_SETSTAGE(state, 0)
- NEXT_OUT(3)
- break;
- case 2:
- oc1 = ESTATE_GETPENDING(state)
- | (c1 >> 14);
- oc2 = (c1 >> 8) & 0x3f;
- oc3 = (c1 >> 2) & 0x3f;
- WRITE3(B64(oc1), B64(oc2),
- B64(oc3))
- ESTATE_SETPENDING(state,
- (c1 & 0x03) << 4)
- ESTATE_SETSTAGE(state, 1)
- NEXT_OUT(3)
- break;
- default:
- return MBERR_INTERNAL;
- }
- }
- }
- else {
- if (c1 < 128 && UTF7_DENCODABLE_STRICT(c1)) {
- WRITE1((unsigned char)c1)
- NEXT_OUT(1)
- }
- else if (c1 == '+') {
- WRITE2('+', '-')
- NEXT_OUT(2)
- }
- else {
- oc1 = c1 >> 10;
- oc2 = (c1 >> 4) & 0x3f;
- WRITE3('+', B64(oc1), B64(oc2))
- ESTATE_SETPENDING(state,
- (c1 & 0x0f) << 2)
- ESTATE_SETSTAGE(state, 2)
- ESTATE_SETSHIFTED(state)
- NEXT_OUT(3)
- }
- }
-
- if (c2 != 0) {
- c1 = c2;
- c2 = 0;
- }
- else
- break;
- }
-
- NEXT_IN(insize)
- }
-
- return 0;
-}
-
-#define DSTATE_INITIALIZE(state) \
- DSTATE_SETBSTAGE(state, 0) \
- DSTATE_CLEARSHIFTED(state) \
- DSTATE_SETULENGTH(state, 0) \
- DSTATE_SETUPENDING1(state, 0) \
- DSTATE_SETUPENDING2(state, 0)
-
-/* XXX: Type-mixed usage of a state union may be not so portable.
- * If you see any problem with this on your platfom. Please let
- * me know. */
-
-#define DSTATE_SETSHIFTED(state) (state)->c[0] = 1;
-#define DSTATE_ISSHIFTED(state) ((state)->c[0])
-#define DSTATE_CLEARSHIFTED(state) (state)->c[0] = 0;
-
-#define DSTATE_SETBSTAGE(state, v) (state)->c[1] = (v);
-#define DSTATE_GETBSTAGE(state) ((state)->c[1])
-
-#define DSTATE_SETBPENDING(state, v) (state)->c[2] = (v);
-#define DSTATE_GETBPENDING(state) ((state)->c[2])
-
-#define DSTATE_SETULENGTH(state, v) (state)->c[3] = (v);
-#define DSTATE_GETULENGTH(state) ((state)->c[3])
-
-#define DSTATE_SETUPENDING1(state, v) (state)->u2[2] = (v);
-#define DSTATE_GETUPENDING1(state) (state)->u2[2]
-
-#define DSTATE_SETUPENDING2(state, v) (state)->u2[3] = (v);
-#define DSTATE_GETUPENDING2(state) (state)->u2[3]
-
-#define DSTATE_UAPPEND(state, v) \
- (state)->u2[(state)->c[3] > 1 ? 3 : 2] |= \
- ((state)->c[3] & 1) ? (v) : ((ucs2_t)(v)) << 8; \
- (state)->c[3]++;
-
-DECODER_INIT(utf_7)
-{
- DSTATE_INITIALIZE(state)
- return 0;
-}
-
-static int
-utf_7_flush(MultibyteCodec_State *state,
- Py_UNICODE **outbuf, size_t *outleft)
-{
- switch (DSTATE_GETULENGTH(state)) {
- case 2: {
- ucs2_t uc;
-
- uc = DSTATE_GETUPENDING1(state);
-#if Py_UNICODE_SIZE == 4
- if (uc >> 10 == 0xd800 >> 10)
- return MBERR_TOOFEW;
-#endif
- OUT1(uc)
- (*outbuf)++;
- (*outleft)--;
- DSTATE_SETULENGTH(state, 0)
- DSTATE_SETUPENDING1(state, 0)
- break;
- }
-#if Py_UNICODE_SIZE == 4
- case 4:
- if (DSTATE_GETUPENDING2(state) >> 10 != 0xdc00 >> 10)
- return 1;
- OUT1(0x10000 + (((ucs4_t)DSTATE_GETUPENDING1(state) - 0xd800)
- << 10) + (DSTATE_GETUPENDING2(state) - 0xdc00))
- (*outbuf)++;
- (*outleft)--;
- DSTATE_SETULENGTH(state, 0)
- DSTATE_SETUPENDING1(state, 0)
- DSTATE_SETUPENDING2(state, 0)
- break;
-#endif
- case 0: /* FALLTHROUGH */
- case 1: /* FALLTHROUGH */
- case 3:
- return MBERR_TOOFEW;
- default:
- return MBERR_INTERNAL;
- }
-
- return 0;
-}
-
-DECODER_RESET(utf_7)
-{
- DSTATE_INITIALIZE(state)
- return 0;
-}
-
-DECODER(utf_7)
-{
- while (inleft > 0) {
- unsigned char c = IN1;
- int r;
-
- if (!DSTATE_ISSHIFTED(state)) {
- if (c == '+') {
- REQUIRE_INBUF(2)
- if (inleft >= 2 && IN2 == '-') {
- WRITE1('+')
- NEXT(2, 1)
- }
- else {
- DSTATE_SETSHIFTED(state)
- NEXT_IN(1)
- }
- }
- else if (c < 128 && UTF7_DENCODABLE_COMPATIBLE(c)) {
- WRITE1(c)
- NEXT(1, 1)
- }
- else
- return 1;
- }
- else if (B64CHAR(c)) {
- unsigned char tb;
-
- REQUIRE_OUTBUF(1)
- c = UB64(c);
- assert(DSTATE_GETULENGTH(state) < 4);
-
- switch (DSTATE_GETBSTAGE(state)) {
- case 0:
- DSTATE_SETBPENDING(state, c << 2)
- DSTATE_SETBSTAGE(state, 1)
- break;
- case 1:
- tb = DSTATE_GETBPENDING(state) | (c >> 4);
- DSTATE_SETBPENDING(state, c << 4)
- DSTATE_SETBSTAGE(state, 2)
- DSTATE_UAPPEND(state, tb)
- break;
- case 2:
- tb = DSTATE_GETBPENDING(state) | (c >> 2);
- DSTATE_SETBPENDING(state, c << 6)
- DSTATE_SETBSTAGE(state, 3)
- DSTATE_UAPPEND(state, tb)
- break;
- case 3:
- tb = DSTATE_GETBPENDING(state) | c;
- DSTATE_SETBSTAGE(state, 0)
- DSTATE_UAPPEND(state, tb)
- break;
- }
-
- r = utf_7_flush(state, outbuf, &outleft);
- if (r != 0 && r != MBERR_TOOFEW)
- return r;
- NEXT_IN(1)
- }
- else if (c == '-' || UTF7_DENCODABLE_COMPATIBLE(c)) {
- if (DSTATE_GETBSTAGE(state) != 0) {
- DSTATE_UAPPEND(state, DSTATE_GETBSTAGE(state))
- DSTATE_SETBSTAGE(state, 0)
- }
- r = utf_7_flush(state, outbuf, &outleft);
- if (r != 0 && r != MBERR_TOOFEW)
- return r;
- DSTATE_CLEARSHIFTED(state)
-
- if (c != '-') {
- WRITE1(c)
- NEXT_OUT(1)
- }
- NEXT_IN(1)
- }
- else
- return 1;
- }
-
- return 0;
-}
-
-
-/*
- * UTF-8 codec
- */
-
-ENCODER(utf_8)
-{
- while (inleft > 0) {
- ucs4_t c = **inbuf;
- size_t outsize, insize = 1;
-
- if (c < 0x80) outsize = 1;
- else if (c < 0x800) outsize = 2;
- else {
-#if Py_UNICODE_SIZE == 2
- if (c >> 10 == 0xd800 >> 10) { /* high surrogate */
- if (inleft < 2) {
- if (!(flags & MBENC_FLUSH))
- return MBERR_TOOFEW;
- }
- else if ((*inbuf)[1] >> 10 == 0xdc00 >> 10) {
- /* low surrogate */
- c = 0x10000 + ((c - 0xd800) << 10) +
- ((ucs4_t)((*inbuf)[1]) - 0xdc00);
- insize = 2;
- }
- }
-#endif
- if (c < 0x10000) outsize = 3;
- else if (c < 0x200000) outsize = 4;
- else if (c < 0x4000000) outsize = 5;
- else outsize = 6;
- }
-
- REQUIRE_OUTBUF(outsize)
-
- switch (outsize) {
- case 6:
- (*outbuf)[5] = 0x80 | (c & 0x3f);
- c = c >> 6;
- c |= 0x4000000;
- /* FALLTHROUGH */
- case 5:
- (*outbuf)[4] = 0x80 | (c & 0x3f);
- c = c >> 6;
- c |= 0x200000;
- /* FALLTHROUGH */
- case 4:
- (*outbuf)[3] = 0x80 | (c & 0x3f);
- c = c >> 6;
- c |= 0x10000;
- /* FALLTHROUGH */
- case 3:
- (*outbuf)[2] = 0x80 | (c & 0x3f);
- c = c >> 6;
- c |= 0x800;
- /* FALLTHROUGH */
- case 2:
- (*outbuf)[1] = 0x80 | (c & 0x3f);
- c = c >> 6;
- c |= 0xc0;
- /* FALLTHROUGH */
- case 1:
- (*outbuf)[0] = c;
- }
-
- NEXT(insize, outsize)
- }
-
- return 0;
-}
-
-DECODER(utf_8)
-{
- while (inleft > 0) {
- unsigned char c = **inbuf;
-
- REQUIRE_OUTBUF(1)
-
- if (c < 0x80) {
- (*outbuf)[0] = (unsigned char)c;
- NEXT(1, 1)
- }
- else if (c < 0xc2) {
- return 1;
- }
- else if (c < 0xe0) {
- unsigned char c2;
-
- REQUIRE_INBUF(2)
- c2 = (*inbuf)[1];
- if (!((c2 ^ 0x80) < 0x40))
- return 2;
- **outbuf = ((Py_UNICODE)(c & 0x1f) << 6) |
- (Py_UNICODE)(c2 ^ 0x80);
- NEXT(2, 1)
- }
- else if (c < 0xf0) {
- unsigned char c2, c3;
-
- REQUIRE_INBUF(3)
- c2 = (*inbuf)[1]; c3 = (*inbuf)[2];
- if (!((c2 ^ 0x80) < 0x40 &&
- (c3 ^ 0x80) < 0x40 && (c >= 0xe1 || c2 >= 0xa0)))
- return 3;
- **outbuf = ((Py_UNICODE)(c & 0x0f) << 12)
- | ((Py_UNICODE)(c2 ^ 0x80) << 6)
- | (Py_UNICODE)(c3 ^ 0x80);
- NEXT(3, 1)
- }
- else if (c < 0xf8) {
- unsigned char c2, c3, c4;
- ucs4_t code;
-
- REQUIRE_INBUF(4)
- c2 = (*inbuf)[1]; c3 = (*inbuf)[2];
- c4 = (*inbuf)[3];
- if (!((c2 ^ 0x80) < 0x40 &&
- (c3 ^ 0x80) < 0x40 && (c4 ^ 0x80) < 0x40 &&
- (c >= 0xf1 || c2 >= 0x90)))
- return 4;
- code = ((ucs4_t)(c & 0x07) << 18)
- | ((ucs4_t)(c2 ^ 0x80) << 12)
- | ((ucs4_t)(c3 ^ 0x80) << 6)
- | (ucs4_t)(c4 ^ 0x80);
- WRITEUCS4(code)
- NEXT_IN(4)
- }
- else if (c < 0xfc) {
- unsigned char c2, c3, c4, c5;
- ucs4_t code;
-
- REQUIRE_INBUF(5)
- c2 = (*inbuf)[1]; c3 = (*inbuf)[2];
- c4 = (*inbuf)[3]; c5 = (*inbuf)[4];
- if (!((c2 ^ 0x80) < 0x40 &&
- (c3 ^ 0x80) < 0x40 && (c4 ^ 0x80) < 0x40 &&
- (c5 ^ 0x80) < 0x40 && (c >= 0xf9 || c2 >= 0x88)))
- return 5;
- code = ((ucs4_t)(c & 0x03) << 24)
- | ((ucs4_t)(c2 ^ 0x80) << 18)
- | ((ucs4_t)(c3 ^ 0x80) << 12)
- | ((ucs4_t)(c4 ^ 0x80) << 6)
- | (ucs4_t)(c5 ^ 0x80);
- WRITEUCS4(code)
- NEXT_IN(5)
- }
- else if (c < 0xff) {
- unsigned char c2, c3, c4, c5, c6;
- ucs4_t code;
-
- REQUIRE_INBUF(6)
- c2 = (*inbuf)[1]; c3 = (*inbuf)[2];
- c4 = (*inbuf)[3]; c5 = (*inbuf)[4];
- c6 = (*inbuf)[5];
- if (!((c2 ^ 0x80) < 0x40 &&
- (c3 ^ 0x80) < 0x40 && (c4 ^ 0x80) < 0x40 &&
- (c5 ^ 0x80) < 0x40 && (c6 ^ 0x80) < 0x40 &&
- (c >= 0xfd || c2 >= 0x84)))
- return 6;
- code = ((ucs4_t)(c & 0x01) << 30)
- | ((ucs4_t)(c2 ^ 0x80) << 24)
- | ((ucs4_t)(c3 ^ 0x80) << 18)
- | ((ucs4_t)(c4 ^ 0x80) << 12)
- | ((ucs4_t)(c5 ^ 0x80) << 6)
- | (ucs4_t)(c6 ^ 0x80);
- WRITEUCS4(code)
- NEXT_IN(6)
- }
- else
- return 1;
- }
-
- return 0;
-}
-
-
-BEGIN_MAPPINGS_LIST
-END_MAPPINGS_LIST
-
-BEGIN_CODECS_LIST
- CODEC_STATEFUL(utf_7)
- CODEC_STATELESS(utf_8)
-END_CODECS_LIST
-
-I_AM_A_MODULE_FOR(unicode)
projects / python / commitdiff