gh-112433: Add optional _align_ attribute to ctypes.Structure (GH-113790)

diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst
index ef3a9a0f5898af44a5a2c084432c537c95f45ed5..73779547b35a1f287c83d1c910bee0f5e45cd3da 100644 (file)
--- a/Doc/library/ctypes.rst
+++ b/Doc/library/ctypes.rst
@@ -670,6 +670,10 @@ compiler does it. It is possible to override this behavior by specifying a
 :attr:`~Structure._pack_` class attribute in the subclass definition.
 This must be set to a positive integer and specifies the maximum alignment for the fields.
 This is what ``#pragma pack(n)`` also does in MSVC.
+It is also possible to set a minimum alignment for how the subclass itself is packed in the
+same way ``#pragma align(n)`` works in MSVC.
+This can be achieved by specifying a ::attr:`~Structure._align_` class attribute
+in the subclass definition.
 
 :mod:`ctypes` uses the native byte order for Structures and Unions.  To build
 structures with non-native byte order, you can use one of the
@@ -2534,6 +2538,12 @@ fields, or any other data types containing pointer type fields.
       Setting this attribute to 0 is the same as not setting it at all.
 
 
+   .. attribute:: _align_
+
+      An optional small integer that allows overriding the alignment of
+      the structure when being packed or unpacked to/from memory.
+      Setting this attribute to 0 is the same as not setting it at all.
+
    .. attribute:: _anonymous_
 
       An optional sequence that lists the names of unnamed (anonymous) fields.

diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h
index 11755210d65432dbc9fa666af93678823751550c..3253b5271a9b7c8b06e7fe06c71bd67cc67a9aec 100644 (file)
--- a/Include/internal/pycore_global_objects_fini_generated.h
+++ b/Include/internal/pycore_global_objects_fini_generated.h
@@ -742,6 +742,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_abc_impl));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_abstract_));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_active));
+    _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_align_));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_annotation));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_anonymous_));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_argtypes_));

diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h
index 576ac703ca15081e90eaf8981c4e01f4b86efed9..8780f7ef9491651cdbc6c8f48aea236f21273bd8 100644 (file)
--- a/Include/internal/pycore_global_strings.h
+++ b/Include/internal/pycore_global_strings.h
@@ -231,6 +231,7 @@ struct _Py_global_strings {
         STRUCT_FOR_ID(_abc_impl)
         STRUCT_FOR_ID(_abstract_)
         STRUCT_FOR_ID(_active)
+        STRUCT_FOR_ID(_align_)
         STRUCT_FOR_ID(_annotation)
         STRUCT_FOR_ID(_anonymous_)
         STRUCT_FOR_ID(_argtypes_)

diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h
index e682c97e7c0248d5cb8b61a837f0e64eff06e2ad..a9d514856dab1ff29ebf55b43c5b520fa0569091 100644 (file)
--- a/Include/internal/pycore_runtime_init_generated.h
+++ b/Include/internal/pycore_runtime_init_generated.h
@@ -740,6 +740,7 @@ extern "C" {
     INIT_ID(_abc_impl), \
     INIT_ID(_abstract_), \
     INIT_ID(_active), \
+    INIT_ID(_align_), \
     INIT_ID(_annotation), \
     INIT_ID(_anonymous_), \
     INIT_ID(_argtypes_), \

diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h
index 739af0e73c23ff86fdd9902524c72398be6c1f2f..f3b064e2a2cb258407f546f30b89d9315956d70f 100644 (file)
--- a/Include/internal/pycore_unicodeobject_generated.h
+++ b/Include/internal/pycore_unicodeobject_generated.h
@@ -534,6 +534,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) {
     string = &_Py_ID(_active);
     assert(_PyUnicode_CheckConsistency(string, 1));
     _PyUnicode_InternInPlace(interp, &string);
+    string = &_Py_ID(_align_);
+    assert(_PyUnicode_CheckConsistency(string, 1));
+    _PyUnicode_InternInPlace(interp, &string);
     string = &_Py_ID(_annotation);
     assert(_PyUnicode_CheckConsistency(string, 1));
     _PyUnicode_InternInPlace(interp, &string);

diff --git a/Lib/test/test_ctypes/test_aligned_structures.py b/Lib/test/test_ctypes/test_aligned_structures.py
new file mode 100644 (file)
index 0000000..a208fb9
--- /dev/null
+++ b/Lib/test/test_ctypes/test_aligned_structures.py
@@ -0,0 +1,286 @@
+from ctypes import (
+    c_char, c_uint32, c_uint16, c_ubyte, c_byte, alignment, sizeof,
+    BigEndianStructure, LittleEndianStructure,
+    BigEndianUnion, LittleEndianUnion,
+)
+import struct
+import unittest
+
+
+class TestAlignedStructures(unittest.TestCase):
+    def test_aligned_string(self):
+        for base, e in (
+            (LittleEndianStructure, "<"),
+            (BigEndianStructure, ">"),
+        ):
+            data =  bytearray(struct.pack(f"{e}i12x16s", 7, b"hello world!"))
+            class Aligned(base):
+                _align_ = 16
+                _fields_ = [
+                    ('value', c_char * 12)
+                ]
+
+            class Main(base):
+                _fields_ = [
+                    ('first', c_uint32),
+                    ('string', Aligned),
+                ]
+
+            main = Main.from_buffer(data)
+            self.assertEqual(main.first, 7)
+            self.assertEqual(main.string.value, b'hello world!')
+            self.assertEqual(bytes(main.string), b'hello world!\0\0\0\0')
+            self.assertEqual(Main.string.offset, 16)
+            self.assertEqual(Main.string.size, 16)
+            self.assertEqual(alignment(main.string), 16)
+            self.assertEqual(alignment(main), 16)
+
+    def test_aligned_structures(self):
+        for base, data in (
+            (LittleEndianStructure, bytearray(b"\1\0\0\0\1\0\0\0\7\0\0\0")),
+            (BigEndianStructure, bytearray(b"\1\0\0\0\1\0\0\0\7\0\0\0")),
+        ):
+            class SomeBools(base):
+                _align_ = 4
+                _fields_ = [
+                    ("bool1", c_ubyte),
+                    ("bool2", c_ubyte),
+                ]
+            class Main(base):
+                _fields_ = [
+                    ("x", c_ubyte),
+                    ("y", SomeBools),
+                    ("z", c_ubyte),
+                ]
+
+            main = Main.from_buffer(data)
+            self.assertEqual(alignment(SomeBools), 4)
+            self.assertEqual(alignment(main), 4)
+            self.assertEqual(alignment(main.y), 4)
+            self.assertEqual(Main.x.size, 1)
+            self.assertEqual(Main.y.offset, 4)
+            self.assertEqual(Main.y.size, 4)
+            self.assertEqual(main.y.bool1, True)
+            self.assertEqual(main.y.bool2, False)
+            self.assertEqual(Main.z.offset, 8)
+            self.assertEqual(main.z, 7)
+
+    def test_oversized_structure(self):
+        data = bytearray(b"\0" * 8)
+        for base in (LittleEndianStructure, BigEndianStructure):
+            class SomeBoolsTooBig(base):
+                _align_ = 8
+                _fields_ = [
+                    ("bool1", c_ubyte),
+                    ("bool2", c_ubyte),
+                    ("bool3", c_ubyte),
+                ]
+            class Main(base):
+                _fields_ = [
+                    ("y", SomeBoolsTooBig),
+                    ("z", c_uint32),
+                ]
+            with self.assertRaises(ValueError) as ctx:
+                Main.from_buffer(data)
+                self.assertEqual(
+                    ctx.exception.args[0],
+                    'Buffer size too small (4 instead of at least 8 bytes)'
+                )
+
+    def test_aligned_subclasses(self):
+        for base, e in (
+            (LittleEndianStructure, "<"),
+            (BigEndianStructure, ">"),
+        ):
+            data = bytearray(struct.pack(f"{e}4i", 1, 2, 3, 4))
+            class UnalignedSub(base):
+                x: c_uint32
+                _fields_ = [
+                    ("x", c_uint32),
+                ]
+
+            class AlignedStruct(UnalignedSub):
+                _align_ = 8
+                _fields_ = [
+                    ("y", c_uint32),
+                ]
+
+            class Main(base):
+                _fields_ = [
+                    ("a", c_uint32),
+                    ("b", AlignedStruct)
+                ]
+
+            main = Main.from_buffer(data)
+            self.assertEqual(alignment(main.b), 8)
+            self.assertEqual(alignment(main), 8)
+            self.assertEqual(sizeof(main.b), 8)
+            self.assertEqual(sizeof(main), 16)
+            self.assertEqual(main.a, 1)
+            self.assertEqual(main.b.x, 3)
+            self.assertEqual(main.b.y, 4)
+            self.assertEqual(Main.b.offset, 8)
+            self.assertEqual(Main.b.size, 8)
+
+    def test_aligned_union(self):
+        for sbase, ubase, e in (
+            (LittleEndianStructure, LittleEndianUnion, "<"),
+            (BigEndianStructure, BigEndianUnion, ">"),
+        ):
+            data = bytearray(struct.pack(f"{e}4i", 1, 2, 3, 4))
+            class AlignedUnion(ubase):
+                _align_ = 8
+                _fields_ = [
+                    ("a", c_uint32),
+                    ("b", c_ubyte * 7),
+                ]
+
+            class Main(sbase):
+                _fields_ = [
+                    ("first", c_uint32),
+                    ("union", AlignedUnion),
+                ]
+
+            main = Main.from_buffer(data)
+            self.assertEqual(main.first, 1)
+            self.assertEqual(main.union.a, 3)
+            self.assertEqual(bytes(main.union.b), data[8:-1])
+            self.assertEqual(Main.union.offset, 8)
+            self.assertEqual(Main.union.size, 8)
+            self.assertEqual(alignment(main.union), 8)
+            self.assertEqual(alignment(main), 8)
+
+    def test_aligned_struct_in_union(self):
+        for sbase, ubase, e in (
+            (LittleEndianStructure, LittleEndianUnion, "<"),
+            (BigEndianStructure, BigEndianUnion, ">"),
+        ):
+            data = bytearray(struct.pack(f"{e}4i", 1, 2, 3, 4))
+            class Sub(sbase):
+                _align_ = 8
+                _fields_ = [
+                    ("x", c_uint32),
+                    ("y", c_uint32),
+                ]
+
+            class MainUnion(ubase):
+                _fields_ = [
+                    ("a", c_uint32),
+                    ("b", Sub),
+                ]
+
+            class Main(sbase):
+                _fields_ = [
+                    ("first", c_uint32),
+                    ("union", MainUnion),
+                ]
+
+            main = Main.from_buffer(data)
+            self.assertEqual(Main.first.size, 4)
+            self.assertEqual(alignment(main.union), 8)
+            self.assertEqual(alignment(main), 8)
+            self.assertEqual(Main.union.offset, 8)
+            self.assertEqual(Main.union.size, 8)
+            self.assertEqual(main.first, 1)
+            self.assertEqual(main.union.a, 3)
+            self.assertEqual(main.union.b.x, 3)
+            self.assertEqual(main.union.b.y, 4)
+
+    def test_smaller_aligned_subclassed_union(self):
+        for sbase, ubase, e in (
+            (LittleEndianStructure, LittleEndianUnion, "<"),
+            (BigEndianStructure, BigEndianUnion, ">"),
+        ):
+            data = bytearray(struct.pack(f"{e}H2xI", 1, 0xD60102D7))
+            class SubUnion(ubase):
+                _align_ = 2
+                _fields_ = [
+                    ("unsigned", c_ubyte),
+                    ("signed", c_byte),
+                ]
+
+            class MainUnion(SubUnion):
+                _fields_ = [
+                    ("num", c_uint32)
+                ]
+
+            class Main(sbase):
+                _fields_ = [
+                    ("first", c_uint16),
+                    ("union", MainUnion),
+                ]
+
+            main = Main.from_buffer(data)
+            self.assertEqual(main.union.num, 0xD60102D7)
+            self.assertEqual(main.union.unsigned, data[4])
+            self.assertEqual(main.union.signed, data[4] - 256)
+            self.assertEqual(alignment(main), 4)
+            self.assertEqual(alignment(main.union), 4)
+            self.assertEqual(Main.union.offset, 4)
+            self.assertEqual(Main.union.size, 4)
+            self.assertEqual(Main.first.size, 2)
+
+    def test_larger_aligned_subclassed_union(self):
+        for ubase, e in (
+            (LittleEndianUnion, "<"),
+            (BigEndianUnion, ">"),
+        ):
+            data = bytearray(struct.pack(f"{e}I4x", 0xD60102D6))
+            class SubUnion(ubase):
+                _align_ = 8
+                _fields_ = [
+                    ("unsigned", c_ubyte),
+                    ("signed", c_byte),
+                ]
+
+            class Main(SubUnion):
+                _fields_ = [
+                    ("num", c_uint32)
+                ]
+
+            main = Main.from_buffer(data)
+            self.assertEqual(alignment(main), 8)
+            self.assertEqual(sizeof(main), 8)
+            self.assertEqual(main.num, 0xD60102D6)
+            self.assertEqual(main.unsigned, 0xD6)
+            self.assertEqual(main.signed, -42)
+
+    def test_aligned_packed_structures(self):
+        for sbase, e in (
+            (LittleEndianStructure, "<"),
+            (BigEndianStructure, ">"),
+        ):
+            data = bytearray(struct.pack(f"{e}B2H4xB", 1, 2, 3, 4))
+
+            class Inner(sbase):
+                _align_ = 8
+                _fields_ = [
+                    ("x", c_uint16),
+                    ("y", c_uint16),
+                ]
+
+            class Main(sbase):
+                _pack_ = 1
+                _fields_ = [
+                    ("a", c_ubyte),
+                    ("b", Inner),
+                    ("c", c_ubyte),
+                ]
+
+            main = Main.from_buffer(data)
+            self.assertEqual(sizeof(main), 10)
+            self.assertEqual(Main.b.offset, 1)
+            # Alignment == 8 because _pack_ wins out.
+            self.assertEqual(alignment(main.b), 8)
+            # Size is still 8 though since inside this Structure, it will have
+            # effect.
+            self.assertEqual(sizeof(main.b), 8)
+            self.assertEqual(Main.c.offset, 9)
+            self.assertEqual(main.a, 1)
+            self.assertEqual(main.b.x, 2)
+            self.assertEqual(main.b.y, 3)
+            self.assertEqual(main.c, 4)
+
+
+if __name__ == '__main__':
+    unittest.main()

diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-28-02-46-12.gh-issue-112433.FUX-nT.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-28-02-46-12.gh-issue-112433.FUX-nT.rst
new file mode 100644 (file)
index 0000000..fdd11bd
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2024-01-28-02-46-12.gh-issue-112433.FUX-nT.rst
@@ -0,0 +1 @@
+Add ability to force alignment of :mod:`ctypes.Structure` by way of the new ``_align_`` attribute on the class.

diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c
index deafa696fdd0d03c692be38d46d923aee67a49cb..32ee414a7a0cddd008844f6b22101e3d7236ce55 100644 (file)
--- a/Modules/_ctypes/stgdict.c
+++ b/Modules/_ctypes/stgdict.c
@@ -384,6 +384,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct
     int bitofs;
     PyObject *tmp;
     int pack;
+    int forced_alignment = 1;
     Py_ssize_t ffi_ofs;
     int big_endian;
     int arrays_seen = 0;
@@ -424,6 +425,28 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct
         pack = 0;
     }
 
+    if (PyObject_GetOptionalAttr(type, &_Py_ID(_align_), &tmp) < 0) {
+        return -1;
+    }
+    if (tmp) {
+        forced_alignment = PyLong_AsInt(tmp);
+        Py_DECREF(tmp);
+        if (forced_alignment < 0) {
+            if (!PyErr_Occurred() ||
+                PyErr_ExceptionMatches(PyExc_TypeError) ||
+                PyErr_ExceptionMatches(PyExc_OverflowError))
+            {
+                PyErr_SetString(PyExc_ValueError,
+                                "_align_ must be a non-negative integer");
+            }
+            return -1;
+        }
+    }
+    else {
+        /* Setting `_align_ = 0` amounts to using the default alignment */
+        forced_alignment = 1;
+    }
+
     len = PySequence_Size(fields);
     if (len == -1) {
         if (PyErr_ExceptionMatches(PyExc_TypeError)) {
@@ -469,6 +492,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct
         align = basedict->align;
         union_size = 0;
         total_align = align ? align : 1;
+        total_align = max(total_align, forced_alignment);
         stgdict->ffi_type_pointer.type = FFI_TYPE_STRUCT;
         stgdict->ffi_type_pointer.elements = PyMem_New(ffi_type *, basedict->length + len + 1);
         if (stgdict->ffi_type_pointer.elements == NULL) {
@@ -488,7 +512,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct
         size = 0;
         align = 0;
         union_size = 0;
-        total_align = 1;
+        total_align = forced_alignment;
         stgdict->ffi_type_pointer.type = FFI_TYPE_STRUCT;
         stgdict->ffi_type_pointer.elements = PyMem_New(ffi_type *, len + 1);
         if (stgdict->ffi_type_pointer.elements == NULL) {