Use AsCharBuffer to get a C string out of a Python string.

diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 0428e0154e61895c72e26b591b3d352082391603..a43094b2806adbb08b1b84ec338cf8c13f1029b8 100644 (file)
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -355,44 +355,31 @@ check_bom(int get_char(struct tok_state *),
 static char *
 fp_readl(char *s, int size, struct tok_state *tok)
 {
-       PyObject* utf8 = NULL;
-       PyObject* buf = tok->decoding_buffer;
-       char *str;
-       Py_ssize_t utf8len;
+       PyObject* bufobj = tok->decoding_buffer;
+       const char *buf;
+       Py_ssize_t buflen;
 
        /* Ask for one less byte so we can terminate it */
        assert(size > 0);
        size--;
 
-       if (buf == NULL) {
-               buf = PyObject_CallObject(tok->decoding_readline, NULL);
-               if (buf == NULL)
-                       return error_ret(tok);
-       } else {
-               tok->decoding_buffer = NULL;
-               if (PyString_CheckExact(buf))
-                       utf8 = buf;
-       }
-       if (utf8 == NULL) {
-               utf8 = PyUnicode_AsUTF8String(buf);
-               Py_DECREF(buf);
-               if (utf8 == NULL)
+       if (bufobj == NULL) {
+               bufobj = PyObject_CallObject(tok->decoding_readline, NULL);
+               if (bufobj == NULL)
                        return error_ret(tok);
        }
-       str = PyString_AsString(utf8);
-       utf8len = PyString_GET_SIZE(utf8);
-       if (utf8len > size) {
-               tok->decoding_buffer = PyString_FromStringAndSize(str+size, utf8len-size);
-               if (tok->decoding_buffer == NULL) {
-                       Py_DECREF(utf8);
+        if (PyObject_AsCharBuffer(bufobj, &buf, &buflen) < 0)
+               return error_ret(tok);
+       if (buflen > size) {
+               tok->decoding_buffer = PyBytes_FromStringAndSize(buf+size,
+                                                                buflen-size);
+               if (tok->decoding_buffer == NULL)
                        return error_ret(tok);
-               }
-               utf8len = size;
+               buflen = size;
        }
-       memcpy(s, str, utf8len);
-       s[utf8len] = '\0';
-       Py_DECREF(utf8);
-       if (utf8len == 0) return NULL; /* EOF */
+       memcpy(s, buf, buflen);
+       s[buflen] = '\0';
+       if (buflen == 0) return NULL; /* EOF */
        return s;
 }