From: Thomas Heller Date: Wed, 30 Apr 2008 17:11:46 +0000 (+0000) Subject: Merged revisions 60056-60071,60073-60127,60129-60261,60263-60284,60286-62589,62591... X-Git-Tag: v3.0a5~53 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=b041fdaf941e84513c643ac02febfb64a4c77435;p=python Merged revisions 60056-60071,60073-60127,60129-60261,60263-60284,60286-62589,62591-62594 via svnmerge from svn+ssh://pythondev@svn.python.org/python/branches/py3k-ctypes-pep3118 ........ r60059 | thomas.heller | 2008-01-18 22:17:05 +0100 (Fri, 18 Jan 2008) | 1 line Implement pep3118 format strings for SimpleCData types. ........ r60108 | thomas.heller | 2008-01-19 22:56:12 +0100 (Sat, 19 Jan 2008) | 3 lines Always use explicit endian specifiers for simple types, and a bug fix too. Add unittest. ........ r60112 | thomas.heller | 2008-01-19 23:25:14 +0100 (Sat, 19 Jan 2008) | 2 lines Fully implement tp_asbuffer for pointer types. ........ r60261 | thomas.heller | 2008-01-24 22:01:29 +0100 (Thu, 24 Jan 2008) | 4 lines Added shape and ndim field to StgDictObject. Implemented pep3118 format string, ndim, and shape for array types. Added a buffer_info(type_or_object) for testing. ........ r60278 | thomas.heller | 2008-01-25 11:53:33 +0100 (Fri, 25 Jan 2008) | 2 lines Implement pep3118 format strings for ctypes.Structure and ctypes.Union. ........ r60288 | thomas.heller | 2008-01-25 17:58:30 +0100 (Fri, 25 Jan 2008) | 2 lines All ctypes types now use the same CData_GetBuffer function. ........ r60289 | thomas.heller | 2008-01-25 19:59:45 +0100 (Fri, 25 Jan 2008) | 2 lines Fix format string for structures, and itemsize for arrays. ........ r60290 | thomas.heller | 2008-01-25 20:09:03 +0100 (Fri, 25 Jan 2008) | 2 lines Implement to format string for function pointers. ........ r60292 | thomas.heller | 2008-01-25 20:32:20 +0100 (Fri, 25 Jan 2008) | 3 lines Only structures with native packing implement the pep. Unions, or packed structures do not. ........ r60293 | thomas.heller | 2008-01-25 20:34:31 +0100 (Fri, 25 Jan 2008) | 2 lines Update the test. ........ r60295 | thomas.heller | 2008-01-25 20:44:41 +0100 (Fri, 25 Jan 2008) | 2 lines Fixed a few XXX markers. ........ r60298 | thomas.heller | 2008-01-25 21:11:08 +0100 (Fri, 25 Jan 2008) | 1 line Fix test for 64-bt platform. ........ r60299 | thomas.heller | 2008-01-25 21:34:11 +0100 (Fri, 25 Jan 2008) | 2 lines Add test for the readonly bit. ........ r60384 | thomas.heller | 2008-01-28 08:45:04 +0100 (Mon, 28 Jan 2008) | 4 lines Restructure the test so that it contains little endian format strings. On big endian machines, the format strings are converted by replacing '<' with '>'. ........ r60385 | thomas.heller | 2008-01-28 08:58:46 +0100 (Mon, 28 Jan 2008) | 1 line Bugfix and test for explicit big and little endian types. ........ r60428 | thomas.heller | 2008-01-29 22:00:37 +0100 (Tue, 29 Jan 2008) | 1 line Add comments to clarify the tests. ........ r62589 | thomas.heller | 2008-04-30 13:49:46 +0200 (Wed, 30 Apr 2008) | 1 line Fix compiler warnings. ........ --- diff --git a/Lib/ctypes/test/test_pep3118.py b/Lib/ctypes/test/test_pep3118.py new file mode 100644 index 0000000000..42a124eac9 --- /dev/null +++ b/Lib/ctypes/test/test_pep3118.py @@ -0,0 +1,170 @@ +import unittest +from ctypes import * +import re, struct, sys + +if sys.byteorder == "little": + THIS_ENDIAN = "<" + OTHER_ENDIAN = ">" +else: + THIS_ENDIAN = ">" + OTHER_ENDIAN = "<" + +def normalize(format): + # Remove current endian specifier and white space from a format + # string + format = format.replace(OTHER_ENDIAN, THIS_ENDIAN) + return re.sub(r"\s", "", format) + +class Test(unittest.TestCase): + + def test_native_types(self): + for tp, fmt, shape, itemtp in native_types: + ob = tp() + v = memoryview(ob) + try: + self.failUnlessEqual(normalize(v.format), normalize(fmt)) + self.failUnlessEqual(v.size, sizeof(ob)) + self.failUnlessEqual(v.itemsize, sizeof(itemtp)) + self.failUnlessEqual(v.shape, shape) + # ctypes object always have a non-strided memory block + self.failUnlessEqual(v.strides, None) + # they are always read/write + self.failIf(v.readonly) + + if v.shape: + n = 1 + for dim in v.shape: + n = n * dim + self.failUnlessEqual(v.itemsize * n, v.size) + except: + # so that we can see the failing type + print(tp) + raise + + def test_endian_types(self): + for tp, fmt, shape, itemtp in endian_types: + ob = tp() + v = memoryview(ob) + try: + self.failUnlessEqual(v.format, fmt) + self.failUnlessEqual(v.size, sizeof(ob)) + self.failUnlessEqual(v.itemsize, sizeof(itemtp)) + self.failUnlessEqual(v.shape, shape) + # ctypes object always have a non-strided memory block + self.failUnlessEqual(v.strides, None) + # they are always read/write + self.failIf(v.readonly) + + if v.shape: + n = 1 + for dim in v.shape: + n = n * dim + self.failUnlessEqual(v.itemsize * n, v.size) + except: + # so that we can see the failing type + print(tp) + raise + +# define some structure classes + +class Point(Structure): + _fields_ = [("x", c_long), ("y", c_long)] + +class PackedPoint(Structure): + _pack_ = 2 + _fields_ = [("x", c_long), ("y", c_long)] + +class Point2(Structure): + pass +Point2._fields_ = [("x", c_long), ("y", c_long)] + +class EmptyStruct(Structure): + _fields_ = [] + +class aUnion(Union): + _fields_ = [("a", c_int)] + +################################################################ +# +# This table contains format strings as they look on little endian +# machines. The test replaces '<' with '>' on big endian machines. +# +native_types = [ + # type format shape calc itemsize + + ## simple types + + (c_char, "l:x:>l:y:}", None, BEPoint), + (LEPoint, "T{l:x:>l:y:}", None, POINTER(BEPoint)), + (POINTER(LEPoint), "&T{format = alloc_format_string("&", itemdict->format); + if (stgdict->format == NULL) { + Py_DECREF((PyObject *)stgdict); + return NULL; + } + } + /* create the new instance (which is a class, since we are a metatype!) */ result = (PyTypeObject *)PyType_Type.tp_new(type, args, kwds); @@ -1101,6 +1141,7 @@ ArrayType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) long length; int overflow; Py_ssize_t itemsize, itemalign; + char buf[32]; typedict = PyTuple_GetItem(args, 2); if (!typedict) @@ -1140,6 +1181,28 @@ ArrayType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } + assert(itemdict->format); + if (itemdict->format[0] == '(') { + sprintf(buf, "(%ld,", length); + stgdict->format = alloc_format_string(buf, itemdict->format+1); + } else { + sprintf(buf, "(%ld)", length); + stgdict->format = alloc_format_string(buf, itemdict->format); + } + if (stgdict->format == NULL) { + Py_DECREF((PyObject *)stgdict); + return NULL; + } + stgdict->ndim = itemdict->ndim + 1; + stgdict->shape = PyMem_Malloc(sizeof(Py_ssize_t *) * stgdict->ndim); + if (stgdict->shape == NULL) { + Py_DECREF((PyObject *)stgdict); + return NULL; + } + stgdict->shape[0] = length; + memmove(&stgdict->shape[1], itemdict->shape, + sizeof(Py_ssize_t) * (stgdict->ndim - 1)); + itemsize = itemdict->size; if (length * itemsize < 0) { PyErr_SetString(PyExc_OverflowError, @@ -1691,6 +1754,16 @@ SimpleType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) stgdict->size = fmt->pffi_type->size; stgdict->setfunc = fmt->setfunc; stgdict->getfunc = fmt->getfunc; +#ifdef WORDS_BIGENDIAN + stgdict->format = alloc_format_string(">", proto_str); +#else + stgdict->format = alloc_format_string("<", proto_str); +#endif + if (stgdict->format == NULL) { + Py_DECREF(result); + Py_DECREF((PyObject *)stgdict); + return NULL; + } stgdict->paramfunc = SimpleType_paramfunc; /* @@ -1760,22 +1833,32 @@ SimpleType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (type == &SimpleType_Type && fmt->setfunc_swapped && fmt->getfunc_swapped) { PyObject *swapped = CreateSwappedType(type, args, kwds, proto, fmt); + StgDictObject *sw_dict; if (swapped == NULL) { Py_DECREF(result); return NULL; } + sw_dict = PyType_stgdict(swapped); #ifdef WORDS_BIGENDIAN PyObject_SetAttrString((PyObject *)result, "__ctype_le__", swapped); PyObject_SetAttrString((PyObject *)result, "__ctype_be__", (PyObject *)result); PyObject_SetAttrString(swapped, "__ctype_be__", (PyObject *)result); PyObject_SetAttrString(swapped, "__ctype_le__", swapped); + /* We are creating the type for the OTHER endian */ + sw_dict->format = alloc_format_string("<", stgdict->format+1); #else PyObject_SetAttrString((PyObject *)result, "__ctype_be__", swapped); PyObject_SetAttrString((PyObject *)result, "__ctype_le__", (PyObject *)result); PyObject_SetAttrString(swapped, "__ctype_le__", (PyObject *)result); PyObject_SetAttrString(swapped, "__ctype_be__", swapped); + /* We are creating the type for the OTHER endian */ + sw_dict->format = alloc_format_string(">", stgdict->format+1); #endif Py_DECREF(swapped); + if (PyErr_Occurred()) { + Py_DECREF(result); + return NULL; + } }; return (PyObject *)result; @@ -2025,6 +2108,13 @@ CFuncPtrType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; stgdict->paramfunc = CFuncPtrType_paramfunc; + /* We do NOT expose the function signature in the format string. It + is impossible, generally, because the only requirement for the + argtypes items is that they have a .from_param method - we do not + know the types of the arguments (although, in practice, most + argtypes would be a ctypes type). + */ + stgdict->format = alloc_format_string(NULL, "X{}"); stgdict->flags |= TYPEFLAG_ISPOINTER; /* create the new instance (which is a class, @@ -2240,7 +2330,31 @@ static PyMemberDef CData_members[] = { static int CData_GetBuffer(PyObject *_self, Py_buffer *view, int flags) { CDataObject *self = (CDataObject *)_self; - return PyBuffer_FillInfo(view, self->b_ptr, self->b_size, 0, flags); + StgDictObject *dict = PyObject_stgdict(_self); + Py_ssize_t i; + + if (view == NULL) return 0; + if (((flags & PyBUF_LOCK) == PyBUF_LOCK)) { + PyErr_SetString(PyExc_BufferError, + "Cannot lock this object."); + return -1; + } + + view->buf = self->b_ptr; + view->len = self->b_size; + view->readonly = 0; + /* use default format character if not set */ + view->format = dict->format ? dict->format : "B"; + view->ndim = dict->ndim; + view->shape = dict->shape; + view->itemsize = self->b_size; + for (i = 0; i < view->ndim; ++i) { + view->itemsize /= dict->shape[i]; + } + view->strides = NULL; + view->suboffsets = NULL; + view->internal = NULL; + return 0; } static PyBufferProcs CData_as_buffer = { diff --git a/Modules/_ctypes/callbacks.c b/Modules/_ctypes/callbacks.c index 45d190fa13..94c6096316 100644 --- a/Modules/_ctypes/callbacks.c +++ b/Modules/_ctypes/callbacks.c @@ -9,7 +9,8 @@ /**************************************************************/ -static CThunkObject_dealloc(PyObject *_self) +static void +CThunkObject_dealloc(PyObject *_self) { CThunkObject *self = (CThunkObject *)_self; Py_XDECREF(self->converters); diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 2e8635a87b..437cce710f 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1632,10 +1632,36 @@ pointer(PyObject *self, PyObject *arg) return result; } +static PyObject * +buffer_info(PyObject *self, PyObject *arg) +{ + StgDictObject *dict = PyType_stgdict(arg); + PyObject *shape; + Py_ssize_t i; + + if (dict == NULL) + dict = PyObject_stgdict(arg); + if (dict == NULL) { + PyErr_SetString(PyExc_TypeError, + "not a ctypes type or object"); + return NULL; + } + shape = PyTuple_New(dict->ndim); + for (i = 0; i < (int)dict->ndim; ++i) + PyTuple_SET_ITEM(shape, i, PyLong_FromSsize_t(dict->shape[i])); + + if (PyErr_Occurred()) { + Py_DECREF(shape); + return NULL; + } + return Py_BuildValue("siN", dict->format, dict->ndim, shape); +} + PyMethodDef module_methods[] = { {"POINTER", POINTER, METH_O }, {"pointer", pointer, METH_O }, {"_unpickle", unpickle, METH_VARARGS }, + {"buffer_info", buffer_info, METH_O, "Return buffer interface information"}, {"resize", resize, METH_VARARGS, "Resize the memory buffer of a ctypes instance"}, #ifdef CTYPES_UNICODE {"set_conversion_mode", set_conversion_mode, METH_VARARGS, set_conversion_mode_doc}, diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 467ace8f22..0982f761ea 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -204,6 +204,14 @@ typedef struct { PyObject *restype; /* CDataObject or NULL */ PyObject *checker; int flags; /* calling convention and such */ + + /* pep3118 fields, pointers neeed PyMem_Free */ + char *format; + int ndim; + Py_ssize_t *shape; +/* Py_ssize_t *strides; */ /* unused in ctypes */ +/* Py_ssize_t *suboffsets; */ /* unused in ctypes */ + } StgDictObject; /**************************************************************** @@ -342,6 +350,7 @@ extern void *MallocClosure(void); extern void _AddTraceback(char *, char *, int); extern PyObject *CData_FromBaseObj(PyObject *type, PyObject *base, Py_ssize_t index, char *adr); +extern char *alloc_format_string(const char *prefix, const char *suffix); /* XXX better name needed! */ extern int IsSimpleSubType(PyObject *obj); diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 0ad5840336..085b19f508 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -2,6 +2,7 @@ #include #ifdef MS_WIN32 #include +#include #endif #include "ctypes.h" @@ -20,6 +21,9 @@ StgDict_init(StgDictObject *self, PyObject *args, PyObject *kwds) { if (PyDict_Type.tp_init((PyObject *)self, args, kwds) < 0) return -1; + self->format = NULL; + self->ndim = 0; + self->shape = NULL; return 0; } @@ -38,6 +42,8 @@ static void StgDict_dealloc(StgDictObject *self) { StgDict_clear(self); + PyMem_Free(self->format); + PyMem_Free(self->shape); PyMem_Free(self->ffi_type_pointer.elements); PyDict_Type.tp_dealloc((PyObject *)self); } @@ -50,6 +56,10 @@ StgDict_clone(StgDictObject *dst, StgDictObject *src) StgDict_clear(dst); PyMem_Free(dst->ffi_type_pointer.elements); + PyMem_Free(dst->format); + dst->format = NULL; + PyMem_Free(dst->shape); + dst->shape = NULL; dst->ffi_type_pointer.elements = NULL; d = (char *)dst; @@ -64,6 +74,20 @@ StgDict_clone(StgDictObject *dst, StgDictObject *src) Py_XINCREF(dst->restype); Py_XINCREF(dst->checker); + if (src->format) { + dst->format = PyMem_Malloc(strlen(src->format) + 1); + if (dst->format == NULL) + return -1; + strcpy(dst->format, src->format); + } + if (src->shape) { + dst->shape = PyMem_Malloc(sizeof(Py_ssize_t) * src->ndim); + if (dst->shape == NULL) + return -1; + memcpy(dst->shape, src->shape, + sizeof(Py_ssize_t) * src->ndim); + } + if (src->ffi_type_pointer.elements == NULL) return 0; size = sizeof(ffi_type *) * (src->length + 1); @@ -341,6 +365,11 @@ StructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct) return -1; } + if (stgdict->format) { + PyMem_Free(stgdict->format); + stgdict->format = NULL; + } + if (stgdict->ffi_type_pointer.elements) PyMem_Free(stgdict->ffi_type_pointer.elements); @@ -379,6 +408,15 @@ StructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct) ffi_ofs = 0; } + if (isStruct && !isPacked) { + stgdict->format = alloc_format_string(NULL, "T{"); + } else { + /* PEP3118 doesn't support union, or packed structures (well, + only standard packing, but we dont support the pep for + that). Use 'B' for bytes. */ + stgdict->format = alloc_format_string(NULL, "B"); + } + #define realdict ((PyObject *)&stgdict->dict) for (i = 0; i < len; ++i) { PyObject *name = NULL, *desc = NULL; @@ -439,6 +477,24 @@ StructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct) } } else bitsize = 0; + if (isStruct && !isPacked) { + char *fieldfmt = dict->format ? dict->format : "B"; + char *fieldname = PyUnicode_AsString(name); + char *ptr; + Py_ssize_t len = strlen(fieldname) + strlen(fieldfmt); + char *buf = alloca(len + 2 + 1); + + sprintf(buf, "%s:%s:", fieldfmt, fieldname); + + ptr = stgdict->format; + stgdict->format = alloc_format_string(stgdict->format, buf); + PyMem_Free(ptr); + + if (stgdict->format == NULL) { + Py_DECREF(pair); + return -1; + } + } if (isStruct) { prop = CField_FromDesc(desc, i, &field_size, bitsize, &bitofs, @@ -469,6 +525,13 @@ StructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct) Py_DECREF(prop); } #undef realdict + + if (isStruct && !isPacked) { + stgdict->format = alloc_format_string(stgdict->format, "}"); + if (stgdict->format == NULL) + return -1; + } + if (!isStruct) size = union_size;