-/*
+/*
Unicode implementation based on original code by Fredrik Lundh,
modified by Marc-Andre Lemburg <mal@lemburg.com> according to the
The implementation will keep allocated Unicode memory intact for
all objects on the free list having a size less than this
- limit. This reduces malloc() overhead for small Unicode objects.
+ limit. This reduces malloc() overhead for small Unicode objects.
At worst this will result in MAX_UNICODE_FREELIST_SIZE *
(sizeof(PyUnicodeObject) + KEEPALIVE_SIZE_LIMIT +
parameter; it is initialized by _PyUnicode_Init().
Always use the PyUnicode_SetDefaultEncoding() and
- PyUnicode_GetDefaultEncoding() APIs to access this global.
+ PyUnicode_GetDefaultEncoding() APIs to access this global.
*/
static char unicode_default_encoding[100];
int length)
{
void *oldstr;
-
+
/* Shortcut if there's nothing much to do. */
if (unicode->length == length)
goto reset;
/* Resizing shared object (unicode_empty or single character
objects) in-place is not allowed. Use PyUnicode_Resize()
instead ! */
- if (unicode == unicode_empty ||
- (unicode->length == 1 &&
- /* XXX Is unicode->str[] always unsigned? */
- unicode->str[0] < 256U &&
+ if (unicode == unicode_empty ||
+ (unicode->length == 1 &&
+ /* MvL said unicode->str[] may be signed. Python generally assumes
+ * an int contains at least 32 bits, and we don't use more than
+ * 32 bits even in a UCS4 build, so casting to unsigned int should
+ * be correct.
+ */
+ (unsigned int)unicode->str[0] < 256U &&
unicode_latin1[unicode->str[0]] == unicode)) {
PyErr_SetString(PyExc_SystemError,
"can't resize shared unicode objects");
unicode->defenc = NULL;
}
unicode->hash = -1;
-
+
return 0;
}
/* We allocate one more byte to make sure the string is
- Ux0000 terminated -- XXX is this needed ?
+ Ux0000 terminated -- XXX is this needed ?
XXX This allocator could further be enhanced by assuring that the
free list never reduces its size below 1.
{
register PyUnicodeObject *unicode;
- /* Optimization for empty strings */
+ /* Optimization fo empty strings */
if (length == 0 && unicode_empty != NULL) {
Py_INCREF(unicode_empty);
return unicode_empty;
if (unicode->str) {
/* Keep-Alive optimization: we only upsize the buffer,
never downsize it. */
- if ((unicode->length < length) &&
+ if ((unicode->length < length) &&
unicode_resize(unicode, length) < 0) {
PyMem_DEL(unicode->str);
goto onError;
goto onError;
}
/* Initialize the first element to guard against cases where
- the caller fails before initializing str.
- */
+ * the caller fails before initializing str -- unicode_resize()
+ * reads str[0], and the Keep-Alive optimization can keep memory
+ * allocated for str alive across a call to unicode_dealloc(unicode).
+ * We don't want unicode_resize to read uninitialized memory in
+ * that case.
+ */
unicode->str[0] = 0;
unicode->str[length] = 0;
unicode->length = length;
/* Resizing unicode_empty and single character objects is not
possible since these are being shared. We simply return a fresh
copy with the same Unicode content. */
- if (v->length != length &&
+ if (v->length != length &&
(v == unicode_empty || v->length == 1)) {
PyUnicodeObject *w = _PyUnicode_New(length);
if (w == NULL)
return (PyObject *)unicode;
}
}
-
+
unicode = _PyUnicode_New(size);
if (!unicode)
return NULL;
/* Copy the wchar_t data into the new object */
#ifdef HAVE_USABLE_WCHAR_T
memcpy(unicode->str, w, size * sizeof(wchar_t));
-#else
+#else
{
register Py_UNICODE *u;
register int i;
const char *s = NULL;
int len;
PyObject *v;
-
+
if (obj == NULL) {
PyErr_BadInternalCall();
return NULL;
obj->ob_type->tp_name);
goto onError;
}
-
+
/* Convert to Unicode */
if (len == 0) {
Py_INCREF(unicode_empty);
v = (PyObject *)unicode_empty;
}
- else
+ else
v = PyUnicode_Decode(s, len, encoding, errors);
return v;
}
Py_DECREF(buffer);
return unicode;
-
+
onError:
Py_XDECREF(buffer);
return NULL;
const char *errors)
{
PyObject *v, *unicode;
-
+
unicode = PyUnicode_FromUnicode(s, size);
if (unicode == NULL)
return NULL;
const char *errors)
{
PyObject *v;
-
+
if (!PyUnicode_Check(unicode)) {
PyErr_BadArgument();
goto onError;
}
- if (encoding == NULL)
+ if (encoding == NULL)
encoding = PyUnicode_GetDefaultEncoding();
/* Shortcuts for common default encodings */
goto onError;
}
return v;
-
+
onError:
return NULL;
}
int PyUnicode_SetDefaultEncoding(const char *encoding)
{
PyObject *v;
-
+
/* Make sure the encoding is valid. As side effect, this also
loads the encoding into the codec registry cache. */
v = _PyCodec_Lookup(encoding);
goto onError;
Py_DECREF(v);
strncpy(unicode_default_encoding,
- encoding,
+ encoding,
sizeof(unicode_default_encoding));
return 0;
/* see RFC2152 for details */
-static
+static
char utf7_special[128] = {
/* indicate whether a UTF-7 character is special i.e. cannot be directly
encoded:
if ((ch == '-') || !B64CHAR(ch)) {
inShift = 0;
s++;
-
+
/* p, charsleft, bitsleft, surrogate = */ DECODE(p, charsleft, bitsleft, surrogate);
if (bitsleft >= 6) {
/* The shift sequence has a partial character in it. If
but that is not the case here */
errmsg = "partial character in shift sequence";
- goto utf7Error;
+ goto utf7Error;
}
/* According to RFC2152 the remaining bits should be zero. We
- choose to signal an error/insert a replacement character
+ choose to signal an error/insert a replacement character
here so indicate the potential of a misencoded character. */
/* On x86, a << b == a << (b%32) so make sure that bitsleft != 0 */
if (bitsleft && charsleft << (sizeof(charsleft) * 8 - bitsleft)) {
errmsg = "non-zero padding bits in shift sequence";
- goto utf7Error;
+ goto utf7Error;
}
if (ch == '-') {
if ((s < e) && (*(s) == '-')) {
- *p++ = '-';
+ *p++ = '-';
inShift = 1;
}
} else if (SPECIAL(ch,0,0)) {
errmsg = "unexpected special character";
- goto utf7Error;
+ goto utf7Error;
} else {
*p++ = ch;
}
else if (SPECIAL(ch,0,0)) {
errmsg = "unexpected special character";
s++;
- goto utf7Error;
+ goto utf7Error;
}
else {
*p++ = ch;
/* out, charsleft, bitsleft = */ ENCODE(out, charsleft, bitsleft);
/* If the next character is special then we dont' need to terminate
- the shift sequence. If the next character is not a BASE64 character
+ the shift sequence. If the next character is not a BASE64 character
or '-' then the shift sequence will be terminated implicitly and we
don't have to insert a '-'. */
Py_UNICODE ch2 = s[i+1];
if (SPECIAL(ch2, encodeSetO, encodeWhiteSpace)) {
-
+
} else if (B64CHAR(ch2) || ch2 == '-') {
*out++ = '-';
inShift = 0;
inShift = 0;
}
}
- }
+ }
}
}
if (bitsleft) {
/* --- UTF-8 Codec -------------------------------------------------------- */
-static
+static
char utf8_code_length[256] = {
/* Map UTF-8 encoded prefix byte to sequence length. zero means
illegal prefix. see RFC 2279 for details */
break;
case 3:
- if ((s[1] & 0xc0) != 0x80 ||
+ if ((s[1] & 0xc0) != 0x80 ||
(s[2] & 0xc0) != 0x80) {
errmsg = "invalid data";
startinpos = s-starts;
ch = ((s[0] & 0x0f) << 12) + ((s[1] & 0x3f) << 6) + (s[2] & 0x3f);
if (ch < 0x0800) {
/* Note: UTF-8 encodings of surrogates are considered
- legal UTF-8 sequences;
+ legal UTF-8 sequences;
XXX For wide builds (UCS-4) we should probably try
to recombine the surrogates into a single code
*p++ = (Py_UNICODE)ch;
#else
/* compute and append the two surrogates: */
-
+
/* translate from 10000..10FFFF to 0..FFFF */
ch -= 0x10000;
-
+
/* high surrogate = top 10 bits added to D800 */
*p++ = (Py_UNICODE)(0xD800 + (ch >> 10));
-
+
/* low surrogate = bottom 10 bits added to DC00 */
*p++ = (Py_UNICODE)(0xDC00 + (ch & 0x03FF));
#endif
}
s += n;
continue;
-
+
utf8Error:
outpos = p-PyUnicode_AS_UNICODE(unicode);
if (unicode_decode_call_errorhandler(
q += 2;
bo = 1;
}
-#else
+#else
if (bom == 0xFEFF) {
q += 2;
bo = 1;
for (i = pairs = 0; i < size; i++)
if (s[i] >= 0x10000)
pairs++;
- v = PyString_FromStringAndSize(NULL,
+ v = PyString_FromStringAndSize(NULL,
2 * (size + pairs + (byteorder == 0)));
if (v == NULL)
return NULL;
if (quotes) {
*p++ = 'u';
- *p++ = (findchar(s, size, '\'') &&
+ *p++ = (findchar(s, size, '\'') &&
!findchar(s, size, '"')) ? '"' : '\'';
}
while (size-- > 0) {
Py_UNICODE ch = *s++;
/* Escape quotes */
- if (quotes &&
+ if (quotes &&
(ch == (Py_UNICODE) PyString_AS_STRING(repr)[1] || ch == '\\')) {
*p++ = '\\';
*p++ = (char) ch;
continue;
- }
+ }
#ifdef Py_UNICODE_WIDE
/* Map 21-bit characters to '\U00xxxxxx' */
else if (ch >= 0x10000) {
int offset = p - PyString_AS_STRING(repr);
-
+
/* Resize the string if necessary */
if (offset + 12 > PyString_GET_SIZE(repr)) {
if (_PyString_Resize(&repr, PyString_GET_SIZE(repr) + 100))
else if (ch >= 0xD800 && ch < 0xDC00) {
Py_UNICODE ch2;
Py_UCS4 ucs;
-
+
ch2 = *s++;
size--;
if (ch2 >= 0xDC00 && ch2 <= 0xDFFF) {
*p++ = 'x';
*p++ = hexdigit[(ch >> 4) & 0x000F];
*p++ = hexdigit[ch & 0x000F];
- }
+ }
/* Copy everything else as-is */
else
const char *bs;
PyObject *errorHandler = NULL;
PyObject *exc = NULL;
-
+
/* Escaped strings will always be longer than the resulting
Unicode string, so we start with size here and then reduce the
length after conversion to the true value. (But decoding error
Py_XDECREF(errorHandler);
Py_XDECREF(exc);
return (PyObject *)v;
-
+
onError:
Py_XDECREF(v);
Py_XDECREF(errorHandler);
*p++ = hexdigit[(ch >> 8) & 0xf];
*p++ = hexdigit[(ch >> 4) & 0xf];
*p++ = hexdigit[ch & 15];
- }
+ }
else
#endif
/* Map 16-bit characters to '\uxxxx' */
{
PyUnicodeObject *v;
Py_UNICODE *p;
-
+
/* Latin-1 is equivalent to the first 256 ordinals in Unicode. */
if (size == 1 && *(unsigned char*)s < 256) {
Py_UNICODE r = *(unsigned char*)s;
while (size-- > 0)
*p++ = (unsigned char)*s++;
return (PyObject *)v;
-
+
onError:
Py_XDECREF(v);
return NULL;
const char *e;
PyObject *errorHandler = NULL;
PyObject *exc = NULL;
-
+
/* ASCII is equivalent to the first 128 ordinals in Unicode. */
if (size == 1 && *(unsigned char*)s < 128) {
Py_UNICODE r = *(unsigned char*)s;
return PyUnicode_FromUnicode(&r, 1);
}
-
+
v = _PyUnicode_New(size);
if (v == NULL)
goto onError;
Py_XDECREF(errorHandler);
Py_XDECREF(exc);
return (PyObject *)v;
-
+
onError:
Py_XDECREF(v);
Py_XDECREF(errorHandler);
int extrachars = 0;
PyObject *errorHandler = NULL;
PyObject *exc = NULL;
-
+
/* Default to Latin-1 */
if (mapping == NULL)
return PyUnicode_DecodeLatin1(s, size, errors);
int needed = (targetsize - extrachars) + \
(targetsize << 2);
extrachars += needed;
- if (_PyUnicode_Resize(&v,
+ if (_PyUnicode_Resize(&v,
PyUnicode_GET_SIZE(v) + needed) < 0) {
Py_DECREF(x);
goto onError;
Py_XDECREF(errorHandler);
Py_XDECREF(exc);
return (PyObject *)v;
-
+
onError:
Py_XDECREF(errorHandler);
Py_XDECREF(exc);
const char *errors)
{
PyObject *result;
-
+
str = PyUnicode_FromObject(str);
if (str == NULL)
goto onError;
errors);
Py_DECREF(str);
return result;
-
+
onError:
Py_XDECREF(str);
return NULL;
}
-
+
/* --- Decimal Encoder ---------------------------------------------------- */
int PyUnicode_EncodeDecimal(Py_UNICODE *s,
Py_UNICODE *uni2;
Py_UNICODE *collstart;
Py_UNICODE *collend;
-
+
if (Py_UNICODE_ISSPACE(ch)) {
*output++ = ' ';
++p;
/* --- Helpers ------------------------------------------------------------ */
-static
+static
int count(PyUnicodeObject *self,
int start,
int end,
int end)
{
int result;
-
+
str = PyUnicode_FromObject(str);
if (str == NULL)
return -1;
Py_DECREF(str);
return -1;
}
-
+
result = count((PyUnicodeObject *)str,
start, end,
(PyUnicodeObject *)substr);
-
+
Py_DECREF(str);
Py_DECREF(substr);
return result;
}
-static
+static
int findstring(PyUnicodeObject *self,
PyUnicodeObject *substring,
int start,
int direction)
{
int result;
-
+
str = PyUnicode_FromObject(str);
if (str == NULL)
return -2;
Py_DECREF(str);
return -2;
}
-
+
result = findstring((PyUnicodeObject *)str,
(PyUnicodeObject *)substr,
start, end, direction);
return result;
}
-static
+static
int tailmatch(PyUnicodeObject *self,
PyUnicodeObject *substring,
int start,
int direction)
{
int result;
-
+
str = PyUnicode_FromObject(str);
if (str == NULL)
return -1;
Py_DECREF(substr);
return -1;
}
-
+
result = tailmatch((PyUnicodeObject *)str,
(PyUnicodeObject *)substr,
start, end, direction);
return result;
}
-static
+static
const Py_UNICODE *findchar(const Py_UNICODE *s,
int size,
Py_UNICODE ch)
/* Apply fixfct filter to the Unicode object self and return a
reference to the modified object */
-static
+static
PyObject *fixup(PyUnicodeObject *self,
int (*fixfct)(PyUnicodeObject *s))
{
return (PyObject*) u;
}
-static
+static
int fixupper(PyUnicodeObject *self)
{
int len = self->length;
Py_UNICODE *s = self->str;
int status = 0;
-
+
while (len-- > 0) {
register Py_UNICODE ch;
-
+
ch = Py_UNICODE_TOUPPER(*s);
if (ch != *s) {
status = 1;
return status;
}
-static
+static
int fixlower(PyUnicodeObject *self)
{
int len = self->length;
Py_UNICODE *s = self->str;
int status = 0;
-
+
while (len-- > 0) {
register Py_UNICODE ch;
-
+
ch = Py_UNICODE_TOLOWER(*s);
if (ch != *s) {
status = 1;
return status;
}
-static
+static
int fixswapcase(PyUnicodeObject *self)
{
int len = self->length;
Py_UNICODE *s = self->str;
int status = 0;
-
+
while (len-- > 0) {
if (Py_UNICODE_ISUPPER(*s)) {
*s = Py_UNICODE_TOLOWER(*s);
return status;
}
-static
+static
int fixcapitalize(PyUnicodeObject *self)
{
int len = self->length;
Py_UNICODE *s = self->str;
int status = 0;
-
+
if (len == 0)
return 0;
if (Py_UNICODE_ISLOWER(*s)) {
else
return 0;
}
-
+
e = p + PyUnicode_GET_SIZE(self);
previous_is_cased = 0;
for (; p < e; p++) {
register const Py_UNICODE ch = *p;
-
+
if (previous_is_cased)
*p = Py_UNICODE_TOLOWER(ch);
else
*p = Py_UNICODE_TOTITLE(ch);
-
- if (Py_UNICODE_ISLOWER(ch) ||
- Py_UNICODE_ISUPPER(ch) ||
+
+ if (Py_UNICODE_ISLOWER(ch) ||
+ Py_UNICODE_ISUPPER(ch) ||
Py_UNICODE_ISTITLE(ch))
previous_is_cased = 1;
else
sep = PyUnicode_AS_UNICODE(separator);
seplen = PyUnicode_GET_SIZE(separator);
}
-
+
res = _PyUnicode_New(sz);
if (res == NULL)
goto onError;
return NULL;
}
-static
-PyUnicodeObject *pad(PyUnicodeObject *self,
- int left,
+static
+PyUnicodeObject *pad(PyUnicodeObject *self,
+ int left,
int right,
Py_UNICODE fill)
{
for (i = j = 0; i < len; ) {
int eol;
-
+
/* Find a line and append it */
while (i < len && !Py_UNICODE_ISLINEBREAK(data[i]))
i++;
return NULL;
}
-static
+static
PyObject *split_char(PyUnicodeObject *self,
PyObject *list,
Py_UNICODE ch,
return NULL;
}
-static
+static
PyObject *split_substring(PyUnicodeObject *self,
PyObject *list,
PyUnicodeObject *substring,
return split_substring(self,list,substring,maxcount);
}
-static
+static
PyObject *replace(PyUnicodeObject *self,
PyUnicodeObject *str1,
PyUnicodeObject *str2,
} else {
Py_UNICODE u1 = str1->str[0];
Py_UNICODE u2 = str2->str[0];
-
+
u = (PyUnicodeObject*) PyUnicode_FromUnicode(
NULL,
self->length
);
if (u != NULL) {
- Py_UNICODE_COPY(u->str, self->str,
+ Py_UNICODE_COPY(u->str, self->str,
self->length);
for (i = 0; i < u->length; i++)
if (u->str[i] == u1) {
}
}
}
-
+
return (PyObject *) u;
}
static short utf16Fixup[32] =
{
- 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0x2000, -0x800, -0x800, -0x800, -0x800
};
len1 = str1->length;
len2 = str2->length;
-
+
while (len1 > 0 && len2 > 0) {
- Py_UNICODE c1, c2;
+ Py_UNICODE c1, c2;
c1 = *s1++;
c2 = *s2++;
if (c1 != c2)
return (c1 < c2) ? -1 : 1;
-
+
len1--; len2--;
}
len1 = str1->length;
len2 = str2->length;
-
+
while (len1 > 0 && len2 > 0) {
- Py_UNICODE c1, c2;
+ Py_UNICODE c1, c2;
c1 = *s1++;
c2 = *s2++;
(PyObject *)substring);
if (substring == NULL)
return NULL;
-
+
if (start < 0)
start += self->length;
if (start < 0)
if (!PyArg_ParseTuple(args, "O|O&O&:index", &substring,
_PyEval_SliceIndex, &start, _PyEval_SliceIndex, &end))
return NULL;
-
+
substring = (PyUnicodeObject *)PyUnicode_FromObject(
(PyObject *)substring);
if (substring == NULL)
cased = 0;
for (; p < e; p++) {
register const Py_UNICODE ch = *p;
-
+
if (Py_UNICODE_ISUPPER(ch) || Py_UNICODE_ISTITLE(ch))
return PyBool_FromLong(0);
else if (!cased && Py_UNICODE_ISLOWER(ch))
cased = 0;
for (; p < e; p++) {
register const Py_UNICODE ch = *p;
-
+
if (Py_UNICODE_ISLOWER(ch) || Py_UNICODE_ISTITLE(ch))
return PyBool_FromLong(0);
else if (!cased && Py_UNICODE_ISUPPER(ch))
previous_is_cased = 0;
for (; p < e; p++) {
register const Py_UNICODE ch = *p;
-
+
if (Py_UNICODE_ISUPPER(ch) || Py_UNICODE_ISTITLE(ch)) {
if (previous_is_cased)
return PyBool_FromLong(0);
Py_DECREF(str1);
return NULL;
}
- result = replace((PyUnicodeObject *)self,
- (PyUnicodeObject *)str1,
- (PyUnicodeObject *)str2,
+ result = replace((PyUnicodeObject *)self,
+ (PyUnicodeObject *)str1,
+ (PyUnicodeObject *)str2,
maxcount);
Py_DECREF(self);
Py_DECREF(str1);
int maxsplit)
{
PyObject *result;
-
+
s = PyUnicode_FromObject(s);
if (s == NULL)
return NULL;
static PyObject*
unicode_translate(PyUnicodeObject *self, PyObject *table)
{
- return PyUnicode_TranslateCharmap(self->str,
+ return PyUnicode_TranslateCharmap(self->str,
self->length,
- table,
+ table,
"ignore");
}
for (cur = start, i = 0; i < slicelength; cur += step, i++) {
result_buf[i] = source_buf[cur];
}
-
+
result = PyUnicode_FromUnicode(result_buf, slicelength);
PyMem_FREE(result_buf);
return result;
const void **ptr)
{
PyObject *str;
-
+
if (index != 0) {
PyErr_SetString(PyExc_SystemError,
"accessing non-existent unicode segment");
worst case length = 3 + 10 (len of INT_MAX) + 1 = 14 (use 20)*/
char fmt[20];
double x;
-
+
x = PyFloat_AsDouble(v);
if (x == -1.0 && PyErr_Occurred())
return -1;
len = 1 + 50 + 1 + prec = 52 + prec
If prec=0 the effective precision is 1 (the leading digit is
- always given), therefore increase the length by one.
+ always given), therefore increase the length by one.
*/
if ((type == 'g' && buflen <= (size_t)10 + (size_t)prec) ||
if ((flags & F_ALT) &&
(type == 'x' || type == 'X')) {
- /* When converting under %#x or %#X, there are a number
+ /* When converting under %#x or %#X, there are a number
* of issues that cause pain:
* - when 0 is being converted, the C standard leaves off
* the '0x' or '0X', which is inconsistent with other
* convert 0 with the '0x' or '0X'
* (Metrowerks, Compaq Tru64)
* - there are platforms that give '0x' when converting
- * under %#X, but convert 0 in accordance with the
+ * under %#X, but convert 0 in accordance with the
* standard (OS/2 EMX)
- *
+ *
* We can achieve the desired consistency by inserting our
* own '0x' or '0X' prefix, and substituting %x/%X in place
* of %#x/%#X.
* Note that this is the same approach as used in
* formatint() in stringobject.c
*/
- PyOS_snprintf(fmt, sizeof(fmt), "0%c%%.%dl%c",
+ PyOS_snprintf(fmt, sizeof(fmt), "0%c%%.%dl%c",
type, prec, type);
}
else {
PyOS_snprintf(fmt, sizeof(fmt), "%%%s.%dl%c",
- (flags&F_ALT) ? "#" : "",
+ (flags&F_ALT) ? "#" : "",
prec, type);
}
return usprintf(buf, fmt, x);
}
else if (PyString_Check(v)) {
- if (PyString_GET_SIZE(v) != 1)
+ if (PyString_GET_SIZE(v) != 1)
goto onError;
buf[0] = (Py_UNICODE)PyString_AS_STRING(v)[0];
}
PyUnicodeObject *result = NULL;
PyObject *dict = NULL;
PyObject *uformat;
-
+
if (format == NULL || args == NULL) {
PyErr_BadInternalCall();
return NULL;
if (dict == NULL) {
PyErr_SetString(PyExc_TypeError,
- "format requires a mapping");
+ "format requires a mapping");
goto onError;
}
++fmt;
PyErr_Format(PyExc_ValueError,
"unsupported format character '%c' (0x%x) "
"at index %i",
- (31<=c && c<=126) ? (char)c : '?',
+ (31<=c && c<=126) ? (char)c : '?',
(int)c,
(int)(fmt -1 - PyUnicode_AS_UNICODE(uformat)));
goto onError;