populate Py_buffer strides field in ctypes arrays

vanossj · vanossj · commit c13e305f4e56 · 2019-02-11T11:56:17.000-05:00
diff --git a/Lib/ctypes/test/test_pep3118.py b/Lib/ctypes/test/test_pep3118.py
@@ -1,6 +1,7 @@
 import unittest
 from ctypes import *
 import re, sys
+from _ctypes_test import buffer_info
 
 if sys.byteorder == "little":
     THIS_ENDIAN = "<"
@@ -20,7 +21,7 @@ def normalize(format):
 class Test(unittest.TestCase):
 
     def test_native_types(self):
-        for tp, fmt, shape, itemtp in native_types:
+        for tp, fmt, shape, stride, itemtp in native_types:
             ob = tp()
             v = memoryview(ob)
             try:
@@ -31,10 +32,7 @@ def test_native_types(self):
                     self.assertEqual(len(v) * sizeof(itemtp), sizeof(ob))
                 self.assertEqual(v.itemsize, sizeof(itemtp))
                 self.assertEqual(v.shape, shape)
-                # XXX Issue #12851: PyCData_NewGetBuffer() must provide strides
-                #     if requested. memoryview currently reconstructs missing
-                #     stride information, so this assert will fail.
-                # self.assertEqual(v.strides, ())
+                self.assertEqual(v.strides, stride)
 
                 # they are always read/write
                 self.assertFalse(v.readonly)
@@ -49,6 +47,23 @@ def test_native_types(self):
                 print(tp)
                 raise
 
+    def test_native_types_shape_strides(self):
+        # check that ctypes (not memoryview) correctly fills out shape and strides in buffer protocol
+        for tp, fmt, shape, stride, itemtp in native_types:
+            ob = tp()
+            v = buffer_info(ob)
+            try:
+                if v['ndim'] == 0:
+                    self.assertEqual(v['shape'], None)
+                    self.assertEqual(v['strides'], None)
+                else:
+                    self.assertEqual(v['shape'], shape)
+                    self.assertEqual(v['strides'], stride)
+            except:
+                # so that we can see the failing type
+                print(tp)
+                raise
+
     def test_endian_types(self):
         for tp, fmt, shape, itemtp in endian_types:
             ob = tp()
@@ -61,8 +76,7 @@ def test_endian_types(self):
                     self.assertEqual(len(v) * sizeof(itemtp), sizeof(ob))
                 self.assertEqual(v.itemsize, sizeof(itemtp))
                 self.assertEqual(v.shape, shape)
-                # XXX Issue #12851
-                # self.assertEqual(v.strides, ())
+                self.assertEqual(v.strides, ())
 
                 # they are always read/write
                 self.assertFalse(v.readonly)
@@ -141,73 +155,73 @@ class Complete(Structure):
 
 
 native_types = [
-    # type                      format                  shape           calc itemsize
+    # type                      format                  shape      stride       calc itemsize
 
     ## simple types
 
-    (c_char,                    "<c",                   (),           c_char),
-    (c_byte,                    "<b",                   (),           c_byte),
-    (c_ubyte,                   "<B",                   (),           c_ubyte),
-    (c_short,                   "<" + s_short,          (),           c_short),
-    (c_ushort,                  "<" + s_ushort,         (),           c_ushort),
+    (c_char,                    "<c",                   (),        (),   c_char),
+    (c_byte,                    "<b",                   (),        (),   c_byte),
+    (c_ubyte,                   "<B",                   (),        (),   c_ubyte),
+    (c_short,                   "<" + s_short,          (),        (),   c_short),
+    (c_ushort,                  "<" + s_ushort,         (),        (),   c_ushort),
 
-    (c_int,                     "<" + s_int,            (),           c_int),
-    (c_uint,                    "<" + s_uint,           (),           c_uint),
+    (c_int,                     "<" + s_int,            (),        (),   c_int),
+    (c_uint,                    "<" + s_uint,           (),        (),   c_uint),
 
-    (c_long,                    "<" + s_long,           (),           c_long),
-    (c_ulong,                   "<" + s_ulong,          (),           c_ulong),
+    (c_long,                    "<" + s_long,           (),        (),   c_long),
+    (c_ulong,                   "<" + s_ulong,          (),        (),   c_ulong),
 
-    (c_longlong,                "<" + s_longlong,       (),           c_longlong),
-    (c_ulonglong,               "<" + s_ulonglong,      (),           c_ulonglong),
+    (c_longlong,                "<" + s_longlong,       (),        (),   c_longlong),
+    (c_ulonglong,               "<" + s_ulonglong,      (),        (),   c_ulonglong),
 
-    (c_float,                   "<f",                   (),           c_float),
-    (c_double,                  "<d",                   (),           c_double),
+    (c_float,                   "<f",                   (),        (),   c_float),
+    (c_double,                  "<d",                   (),        (),   c_double),
 
-    (c_longdouble,              "<" + s_longdouble,     (),           c_longdouble),
+    (c_longdouble,              "<" + s_longdouble,     (),        (),   c_longdouble),
 
-    (c_bool,                    "<" + s_bool,           (),           c_bool),
-    (py_object,                 "<O",                   (),           py_object),
+    (c_bool,                    "<" + s_bool,           (),        (),   c_bool),
+    (py_object,                 "<O",                   (),        (),   py_object),
 
     ## pointers
 
-    (POINTER(c_byte),           "&<b",                  (),           POINTER(c_byte)),
-    (POINTER(POINTER(c_long)),  "&&<" + s_long,         (),           POINTER(POINTER(c_long))),
+    (POINTER(c_byte),           "&<b",                  (),        (),   POINTER(c_byte)),
+    (POINTER(POINTER(c_long)),  "&&<" + s_long,         (),        (),   POINTER(POINTER(c_long))),
 
     ## arrays and pointers
 
-    (c_double * 4,              "<d",                   (4,),           c_double),
-    (c_float * 4 * 3 * 2,       "<f",                   (2,3,4),        c_float),
-    (POINTER(c_short) * 2,      "&<" + s_short,         (2,),           POINTER(c_short)),
-    (POINTER(c_short) * 2 * 3,  "&<" + s_short,         (3,2,),         POINTER(c_short)),
-    (POINTER(c_short * 2),      "&(2)<" + s_short,      (),             POINTER(c_short)),
+    (c_double * 4,              "<d",                   (4,),         (sizeof(c_double),),   c_double),
+    (c_float * 4 * 3 * 2,       "<f",                   (2,3,4),      (3*4*sizeof(c_float), 4*sizeof(c_float), sizeof(c_float)),   c_float),
+    (POINTER(c_short) * 2,      "&<" + s_short,         (2,),         (sizeof(POINTER(c_short)),),   POINTER(c_short)),
+    (POINTER(c_short) * 2 * 3,  "&<" + s_short,         (3,2,),       (2*sizeof(POINTER(c_short),), sizeof(POINTER(c_short))),   POINTER(c_short)),
+    (POINTER(c_short * 2),      "&(2)<" + s_short,      (),           (),   POINTER(c_short)),
 
     ## structures and unions
 
-    (Point,                     "T{<l:x:<l:y:}".replace('l', s_long),  (),  Point),
+    (Point,                     "T{<l:x:<l:y:}".replace('l', s_long),  (),  (),  Point),
     # packed structures do not implement the pep
-    (PackedPoint,               "B",                                   (),  PackedPoint),
-    (Point2,                    "T{<l:x:<l:y:}".replace('l', s_long),  (),  Point2),
-    (EmptyStruct,               "T{}",                                 (),  EmptyStruct),
+    (PackedPoint,               "B",                                   (),  (),  PackedPoint),
+    (Point2,                    "T{<l:x:<l:y:}".replace('l', s_long),  (),  (),  Point2),
+    (EmptyStruct,               "T{}",                                 (),  (),  EmptyStruct),
     # the pep doesn't support unions
-    (aUnion,                    "B",                                   (),  aUnion),
+    (aUnion,                    "B",                                   (),  (),  aUnion),
     # structure with sub-arrays
-    (StructWithArrays, "T{(2,3)<l:x:(4)T{<l:x:<l:y:}:y:}".replace('l', s_long), (), StructWithArrays),
-    (StructWithArrays * 3, "T{(2,3)<l:x:(4)T{<l:x:<l:y:}:y:}".replace('l', s_long), (3,), StructWithArrays),
+    (StructWithArrays, "T{(2,3)<l:x:(4)T{<l:x:<l:y:}:y:}".replace('l', s_long), (), (), StructWithArrays),
+    (StructWithArrays * 3, "T{(2,3)<l:x:(4)T{<l:x:<l:y:}:y:}".replace('l', s_long), (3,), (sizeof(StructWithArrays),), StructWithArrays),
 
     ## pointer to incomplete structure
-    (Incomplete,                "B",                    (),           Incomplete),
-    (POINTER(Incomplete),       "&B",                   (),           POINTER(Incomplete)),
+    (Incomplete,                "B",                    (),     (),      Incomplete),
+    (POINTER(Incomplete),       "&B",                   (),     (),      POINTER(Incomplete)),
 
     # 'Complete' is a structure that starts incomplete, but is completed after the
     # pointer type to it has been created.
-    (Complete,                  "T{<l:a:}".replace('l', s_long), (), Complete),
+    (Complete,                  "T{<l:a:}".replace('l', s_long), (), (), Complete),
     # Unfortunately the pointer format string is not fixed...
-    (POINTER(Complete),         "&B",                   (),           POINTER(Complete)),
+    (POINTER(Complete),         "&B",                   (),      (),     POINTER(Complete)),
 
     ## other
 
     # function signatures are not implemented
-    (CFUNCTYPE(None),           "X{}",                  (),           CFUNCTYPE(None)),
+    (CFUNCTYPE(None),           "X{}",                  (),      (),     CFUNCTYPE(None)),
 
     ]
 
diff --git a/Misc/NEWS.d/next/Library/2017-12-08-19-58-16.bpo-12851.5H37Gx.rst b/Misc/NEWS.d/next/Library/2017-12-08-19-58-16.bpo-12851.5H37Gx.rst
@@ -0,0 +1,2 @@
+Fixed ctypes Arrays Py_buffer implimentation to provide strides if
+requested.
diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c
@@ -1495,7 +1495,7 @@ PyCArrayType_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
     if (stgdict->format == NULL)
         goto error;
     stgdict->ndim = itemdict->ndim + 1;
-    stgdict->shape = PyMem_Malloc(sizeof(Py_ssize_t) * stgdict->ndim);
+    stgdict->shape = PyMem_Malloc(sizeof(Py_ssize_t) * stgdict->ndim * 2);
     if (stgdict->shape == NULL) {
         PyErr_NoMemory();
         goto error;
@@ -1505,6 +1505,14 @@ PyCArrayType_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
         memmove(&stgdict->shape[1], itemdict->shape,
             sizeof(Py_ssize_t) * (stgdict->ndim - 1));
     }
+    stgdict->strides = stgdict->shape + stgdict->ndim;
+    if (stgdict->ndim > 1) {
+        memmove(&stgdict->strides[1], itemdict->strides,
+            sizeof(Py_ssize_t) * (stgdict->ndim - 1));
+        stgdict->strides[0] = stgdict->strides[1] * stgdict->shape[1];
+    } else {
+        stgdict->strides[0] = itemdict->size;
+    }
 
     itemsize = itemdict->size;
     if (length * itemsize < 0) {
@@ -2681,26 +2689,34 @@ static int PyCData_NewGetBuffer(PyObject *myself, Py_buffer *view, int flags)
 {
     CDataObject *self = (CDataObject *)myself;
     StgDictObject *dict = PyObject_stgdict(myself);
-    Py_ssize_t i;
 
     if (view == NULL) return 0;
 
+    if ((flags & PyBUF_F_CONTIGUOUS) == PyBUF_F_CONTIGUOUS) {
+        view->obj = NULL;
+        PyErr_Format(PyExc_TypeError, "Fortran contiguous buffer is not supported");
+        return -1;
+    }
+
     view->buf = self->b_ptr;
     view->obj = myself;
     Py_INCREF(myself);
     view->len = self->b_size;
     view->readonly = 0;
-    /* use default format character if not set */
-    view->format = dict->format ? dict->format : "B";
+    if ((flags & PyBUF_FORMAT) == PyBUF_FORMAT) {
+        /* use default format character if not set */
+        view->format = dict->format ? dict->format : "B";
+    } else {
+        view->format = NULL;
+    }
     view->ndim = dict->ndim;
-    view->shape = dict->shape;
-    view->itemsize = self->b_size;
-    if (view->itemsize) {
-        for (i = 0; i < view->ndim; ++i) {
-            view->itemsize /= dict->shape[i];
-        }
+    view->shape = ((flags & PyBUF_ND) == PyBUF_ND) ? dict->shape : NULL;
+    if (dict->strides) {
+        view->itemsize = dict->strides[dict->ndim - 1];
+    } else {
+        view->itemsize = self->b_size;
     }
-    view->strides = NULL;
+    view->strides = ((flags & PyBUF_STRIDES) == PyBUF_STRIDES) ? dict->strides : NULL;
     view->suboffsets = NULL;
     view->internal = NULL;
     return 0;
diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c
@@ -475,12 +475,121 @@ EXPORT(int) unpack_bitfields(struct BITS *bits, char name)
     return 999;
 }
 
+PyObject *py_getBufferInfo(PyObject *self, PyObject *obj)
+{
+    int flags = PyBUF_FULL;
+    int ii;
+    Py_buffer view;
+    PyObject *v;
+    PyObject *x;
+    PyObject *d;
+    int buffer_err;
+
+    d = PyDict_New();
+    if (NULL == d) Py_RETURN_NONE;
+    buffer_err = PyObject_GetBuffer(obj, &view, flags);
+    v = PyLong_FromLong(buffer_err);
+    if (NULL != v){
+        PyDict_SetItemString(d, "err", v);
+        Py_DECREF(v);
+    }
+
+    if (0 == buffer_err) {
+        v = PyLong_FromSsize_t(view.len);
+        if (NULL != v){
+            PyDict_SetItemString(d, "len", v);
+            Py_DECREF(v);
+        }
+        v = PyBool_FromLong(view.readonly);
+        if (NULL != v){
+            PyDict_SetItemString(d, "readonly", v);
+            Py_DECREF(v);
+        }
+        v = PyLong_FromSsize_t(view.itemsize);
+        if (NULL != v){
+            PyDict_SetItemString(d, "itemsize", v);
+            Py_DECREF(v);
+        }
+        if(NULL == view.format) {
+            v = PyUnicode_New(0, 0);
+        } else {
+            v = PyUnicode_FromString(view.format);
+        }
+        if (NULL != v){
+            PyDict_SetItemString(d, "format", v);
+            Py_DECREF(v);
+        }
+        v = PyLong_FromLong(view.ndim);
+        if (NULL != v){
+            PyDict_SetItemString(d, "ndim", v);
+            Py_DECREF(v);
+        }
+        if (NULL == view.shape) {
+            PyDict_SetItemString(d, "shape", Py_None);
+        } else {
+            v = PyTuple_New(view.ndim);
+            if (NULL != v){
+                for (ii=0; ii<view.ndim; ++ii) {
+                    x = PyLong_FromSsize_t(view.shape[ii]);
+                    if (NULL == x) {
+                        Py_DECREF(v);
+                        Py_DECREF(d);
+                        return NULL;
+                    }
+                    PyTuple_SetItem(v, ii, x);
+                }
+                PyDict_SetItemString(d, "shape", v);
+                Py_DECREF(v);
+            }
+        }
+        if (NULL == view.strides) {
+            PyDict_SetItemString(d, "strides", Py_None);
+        } else {
+            v = PyTuple_New(view.ndim);
+            if (NULL != v){
+                for (ii=0; ii<view.ndim; ++ii) {
+                    x = PyLong_FromSsize_t(view.strides[ii]);
+                    if (NULL == x) {
+                        Py_DECREF(v);
+                        Py_DECREF(d);
+                        return NULL;
+                    }
+                    PyTuple_SetItem(v, ii, x);
+                }
+                PyDict_SetItemString(d, "strides", v);
+                Py_DECREF(v);
+            }
+        }
+        if (NULL == view.suboffsets) {
+            PyDict_SetItemString(d, "suboffsets", Py_None);
+        } else {
+            v = PyTuple_New(view.ndim);
+            if (NULL != v){
+                for (ii=0; ii<view.ndim; ++ii) {
+                    x = PyLong_FromSsize_t(view.suboffsets[ii]);
+                    if (NULL == x) {
+                        Py_DECREF(v);
+                        Py_DECREF(d);
+                        return NULL;
+                    }
+                    PyTuple_SetItem(v, ii, x);
+                }
+                PyDict_SetItemString(d, "suboffsets", v);
+                Py_DECREF(v);
+            }
+        }
+        PyBuffer_Release(&view);
+    }
+    return d;
+}
+
 static PyMethodDef module_methods[] = {
 /*      {"get_last_tf_arg_s", get_last_tf_arg_s, METH_NOARGS},
     {"get_last_tf_arg_u", get_last_tf_arg_u, METH_NOARGS},
 */
     {"func_si", py_func_si, METH_VARARGS},
     {"func", py_func, METH_NOARGS},
+    {"buffer_info", py_getBufferInfo, METH_O},
     { NULL, NULL, 0, NULL},
 };
 
diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h
@@ -208,11 +208,11 @@ typedef struct {
     PyObject *checker;
     int flags;                  /* calling convention and such */
 
-    /* pep3118 fields, pointers neeed PyMem_Free */
+    /* pep3118 fields, pointers need PyMem_Free */
     char *format;
     int ndim;
     Py_ssize_t *shape;
-/*      Py_ssize_t *strides;    */ /* unused in ctypes */
+    Py_ssize_t *strides;  /* offset from *shape, not necessary to PyMem_Free */
 /*      Py_ssize_t *suboffsets; */ /* unused in ctypes */
 
 } StgDictObject;
diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+Fixed ctypes Arrays Py_buffer implimentation to provide strides if`
	`2`	`+requested.`