Use faster PyUnicode_FromEncodedObject() for bytes/bytearray.decode().
Add new PyCodec_KnownEncoding() API.
Add new PyUnicode_AsDecodedUnicode() and PyUnicode_AsEncodedUnicode() APIs.
Add missing PyUnicode_AsDecodedObject() to unicodeobject.h
Fix punicode codec to also work on memoryviews.
PyObject *search_function
);
-/* Codec register lookup API.
+/* Codec registry lookup API.
Looks up the given encoding and returns a CodecInfo object with
function attributes which implement the different aspects of
const char *encoding
);
+/* Codec registry encoding check API.
+
+ Returns 1/0 depending on whether there is a registered codec for
+ the given encoding.
+
+*/
+
+PyAPI_FUNC(int) PyCodec_KnownEncoding(
+ const char *encoding
+ );
+
/* Generic codec based encoding API.
object is passed through the encoder function found for the given
# define PyUnicode_AsASCIIString PyUnicodeUCS2_AsASCIIString
# define PyUnicode_AsCharmapString PyUnicodeUCS2_AsCharmapString
+# define PyUnicode_AsDecodedObject PyUnicodeUCS2_AsDecodedObject
+# define PyUnicode_AsDecodedUnicode PyUnicodeUCS2_AsDecodedUnicode
# define PyUnicode_AsEncodedObject PyUnicodeUCS2_AsEncodedObject
# define PyUnicode_AsEncodedString PyUnicodeUCS2_AsEncodedString
+# define PyUnicode_AsEncodedUnicode PyUnicodeUCS2_AsEncodedUnicode
# define PyUnicode_AsLatin1String PyUnicodeUCS2_AsLatin1String
# define PyUnicode_AsRawUnicodeEscapeString PyUnicodeUCS2_AsRawUnicodeEscapeString
# define PyUnicode_AsUTF32String PyUnicodeUCS2_AsUTF32String
# define PyUnicode_AsASCIIString PyUnicodeUCS4_AsASCIIString
# define PyUnicode_AsCharmapString PyUnicodeUCS4_AsCharmapString
+# define PyUnicode_AsDecodedObject PyUnicodeUCS4_AsDecodedObject
+# define PyUnicode_AsDecodedUnicode PyUnicodeUCS4_AsDecodedUnicode
# define PyUnicode_AsEncodedObject PyUnicodeUCS4_AsEncodedObject
# define PyUnicode_AsEncodedString PyUnicodeUCS4_AsEncodedString
+# define PyUnicode_AsEncodedUnicode PyUnicodeUCS4_AsEncodedUnicode
# define PyUnicode_AsLatin1String PyUnicodeUCS4_AsLatin1String
# define PyUnicode_AsRawUnicodeEscapeString PyUnicodeUCS4_AsRawUnicodeEscapeString
# define PyUnicode_AsUTF32String PyUnicodeUCS4_AsUTF32String
const char *errors /* error handling */
);
+/* Decode a Unicode object unicode and return the result as Python
+ object. */
+
+PyAPI_FUNC(PyObject*) PyUnicode_AsDecodedObject(
+ PyObject *unicode, /* Unicode object */
+ const char *encoding, /* encoding */
+ const char *errors /* error handling */
+ );
+
+/* Decode a Unicode object unicode and return the result as Unicode
+ object. */
+
+PyAPI_FUNC(PyObject*) PyUnicode_AsDecodedUnicode(
+ PyObject *unicode, /* Unicode object */
+ const char *encoding, /* encoding */
+ const char *errors /* error handling */
+ );
+
/* Encodes a Py_UNICODE buffer of the given size and returns a
Python string object. */
const char *errors /* error handling */
);
+/* Encodes a Unicode object and returns the result as Unicode
+ object. */
+
+PyAPI_FUNC(PyObject*) PyUnicode_AsEncodedUnicode(
+ PyObject *unicode, /* Unicode object */
+ const char *encoding, /* encoding */
+ const char *errors /* error handling */
+ );
+
+/* Build an encoding map. */
+
PyAPI_FUNC(PyObject*) PyUnicode_BuildEncodingMap(
PyObject* string /* 256 character map */
);
-
/* --- UTF-7 Codecs ------------------------------------------------------- */
PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF7(
def punycode_decode(text, errors):
if isinstance(text, str):
text = text.encode("ascii")
+ if isinstance(text, memoryview):
+ text = bytes(text)
pos = text.rfind(b"-")
if pos == -1:
base = ""
"string argument without an encoding");
return -1;
}
- encoded = PyCodec_Encode(arg, encoding, errors);
+ encoded = PyUnicode_AsEncodedString(arg, encoding, errors);
if (encoded == NULL)
return -1;
assert(PyBytes_Check(encoded));
return NULL;
if (encoding == NULL)
encoding = PyUnicode_GetDefaultEncoding();
- return PyCodec_Decode(self, encoding, errors);
+ return PyUnicode_FromEncodedObject(self, encoding, errors);
}
PyDoc_STRVAR(alloc_doc,
return NULL;
if (encoding == NULL)
encoding = PyUnicode_GetDefaultEncoding();
- return PyCodec_Decode(self, encoding, errors);
+ return PyUnicode_FromEncodedObject(self, encoding, errors);
}
"string argument without an encoding");
return NULL;
}
- new = PyCodec_Encode(x, encoding, errors);
+ new = PyUnicode_AsEncodedString(x, encoding, errors);
if (new == NULL)
return NULL;
assert(PyBytes_Check(new));
/* Coerce object */
if (PyBytes_Check(obj)) {
- s = PyBytes_AS_STRING(obj);
- len = PyBytes_GET_SIZE(obj);
- }
+ s = PyBytes_AS_STRING(obj);
+ len = PyBytes_GET_SIZE(obj);
+ }
+ else if (PyByteArray_Check(obj)) {
+ s = PyByteArray_AS_STRING(obj);
+ len = PyByteArray_GET_SIZE(obj);
+ }
else if (PyObject_AsCharBuffer(obj, &s, &len)) {
/* Overwrite the error message with something more useful in
case of a TypeError. */
if (PyErr_ExceptionMatches(PyExc_TypeError))
- PyErr_Format(PyExc_TypeError,
+ PyErr_Format(PyExc_TypeError,
"coercing to Unicode: need string or buffer, "
"%.80s found",
Py_TYPE(obj)->tp_name);
goto onError;
if (!PyUnicode_Check(unicode)) {
PyErr_Format(PyExc_TypeError,
- "decoder did not return an unicode object (type=%.400s)",
+ "decoder did not return a unicode object (type=%.400s)",
Py_TYPE(unicode)->tp_name);
Py_DECREF(unicode);
goto onError;
return NULL;
}
+PyObject *PyUnicode_AsDecodedUnicode(PyObject *unicode,
+ const char *encoding,
+ const char *errors)
+{
+ PyObject *v;
+
+ if (!PyUnicode_Check(unicode)) {
+ PyErr_BadArgument();
+ goto onError;
+ }
+
+ if (encoding == NULL)
+ encoding = PyUnicode_GetDefaultEncoding();
+
+ /* Decode via the codec registry */
+ v = PyCodec_Decode(unicode, encoding, errors);
+ if (v == NULL)
+ goto onError;
+ if (!PyUnicode_Check(v)) {
+ PyErr_Format(PyExc_TypeError,
+ "decoder did not return a unicode object (type=%.400s)",
+ Py_TYPE(v)->tp_name);
+ Py_DECREF(v);
+ goto onError;
+ }
+ return v;
+
+ onError:
+ return NULL;
+}
+
PyObject *PyUnicode_Encode(const Py_UNICODE *s,
Py_ssize_t size,
const char *encoding,
v = PyCodec_Encode(unicode, encoding, errors);
if (v == NULL)
goto onError;
- assert(PyBytes_Check(v));
+ if (PyByteArray_Check(v)) {
+ char msg[100];
+ PyOS_snprintf(msg, sizeof(msg),
+ "encoder %s returned buffer instead of bytes",
+ encoding);
+ if (PyErr_WarnEx(PyExc_RuntimeWarning, msg, 1) < 0) {
+ v = NULL;
+ goto onError;
+ }
+ v = PyBytes_FromStringAndSize(PyByteArray_AS_STRING(v), Py_SIZE(v));
+ }
+ else if (!PyBytes_Check(v)) {
+ PyErr_Format(PyExc_TypeError,
+ "encoder did not return a bytes object (type=%.400s)",
+ Py_TYPE(v)->tp_name);
+ v = NULL;
+ }
+ return v;
+
+ onError:
+ return NULL;
+}
+
+PyObject *PyUnicode_AsEncodedUnicode(PyObject *unicode,
+ const char *encoding,
+ const char *errors)
+{
+ PyObject *v;
+
+ if (!PyUnicode_Check(unicode)) {
+ PyErr_BadArgument();
+ goto onError;
+ }
+
+ if (encoding == NULL)
+ encoding = PyUnicode_GetDefaultEncoding();
+
+ /* Encode via the codec registry */
+ v = PyCodec_Encode(unicode, encoding, errors);
+ if (v == NULL)
+ goto onError;
+ if (!PyUnicode_Check(v)) {
+ PyErr_Format(PyExc_TypeError,
+ "encoder did not return an unicode object (type=%.400s)",
+ Py_TYPE(v)->tp_name);
+ Py_DECREF(v);
+ goto onError;
+ }
return v;
onError:
if (!PyArg_ParseTuple(args, "|ss:encode", &encoding, &errors))
return NULL;
- v = PyUnicode_AsEncodedObject((PyObject *)self, encoding, errors);
+ v = PyUnicode_AsEncodedString((PyObject *)self, encoding, errors);
if (v == NULL)
goto onError;
if (!PyBytes_Check(v)) {
return NULL;
}
+/* Codec registry encoding check API. */
+
+int PyCodec_KnownEncoding(const char *encoding)
+{
+ PyObject *codecs;
+
+ codecs = _PyCodec_Lookup(encoding);
+ if (!codecs) {
+ PyErr_Clear();
+ return 0;
+ }
+ else {
+ Py_DECREF(codecs);
+ return 1;
+ }
+}
+
static
PyObject *args_tuple(PyObject *object,
const char *errors)
"encoder must return a tuple (object, integer)");
goto onError;
}
- v = PyTuple_GET_ITEM(result, 0);
- if (PyByteArray_Check(v)) {
- char msg[100];
- PyOS_snprintf(msg, sizeof(msg),
- "encoder %s returned buffer instead of bytes",
- encoding);
- if (PyErr_WarnEx(PyExc_RuntimeWarning, msg, 1) < 0) {
- v = NULL;
- goto onError;
- }
- v = PyBytes_FromStringAndSize(PyByteArray_AS_STRING(v), Py_SIZE(v));
- }
- else if (PyBytes_Check(v))
- Py_INCREF(v);
- else {
- PyErr_SetString(PyExc_TypeError,
- "encoding must return a tuple(bytes, integer)");
- v = NULL;
- }
+ v = PyTuple_GET_ITEM(result,0);
+ Py_INCREF(v);
/* We don't check or use the second (integer) entry. */
+ Py_DECREF(args);
+ Py_DECREF(encoder);
+ Py_DECREF(result);
+ return v;
+
onError:
Py_XDECREF(result);
Py_XDECREF(args);
Py_XDECREF(encoder);
- return v;
+ return NULL;
}
/* Decode an object (usually a Python string) using the given encoding
codeset = nl_langinfo(CODESET);
if (codeset && *codeset) {
- PyObject *enc = PyCodec_Encoder(codeset);
- if (enc) {
- codeset = strdup(codeset);
- Py_DECREF(enc);
- } else {
- codeset = NULL;
- PyErr_Clear();
- }
+ if (PyCodec_KnownEncoding(codeset))
+ codeset = strdup(codeset);
+ else
+ codeset = NULL;
} else
codeset = NULL;
projects / python / commitdiff