MEDIUM: ring: always allocate properly aligned ring structures

The rings were manually padded to place the various areas that compose
them into different cache lines, provided that the allocator returned
a cache-aligned address, which until now was not granted. By now
switching to the aligned API we can finally have this guarantee and
hope for more consistent ring performance between tests. Like previously
the few carefully crafted THREAD_PAD() could simply be replaced by
generic THREAD_ALIGN() that dictate the type's alignment.

This was the last user of THREAD_PAD() by the way.

diff --git a/include/haproxy/ring-t.h b/include/haproxy/ring-t.h
index f1fd477271bc26085be364b4f1c053905bbf86c5..7a9eeca4d5dc045337f21b5d60e9a194603d6ac1 100644 (file)
--- a/include/haproxy/ring-t.h
+++ b/include/haproxy/ring-t.h
@@ -130,11 +130,11 @@ struct ring_wait_cell {
 struct ring_storage {
        size_t size;         // storage size
        size_t rsvd;         // header length (used for file-backed maps)
-       THREAD_PAD(64 - 2 * sizeof(size_t));
+       THREAD_ALIGN(64);
        size_t tail;         // storage tail
-       THREAD_PAD(64 - sizeof(size_t));
+       THREAD_ALIGN(64);
        size_t head;         // storage head
-       THREAD_PAD(64 - sizeof(size_t));
+       THREAD_ALIGN(64);
        char area[0];        // storage area begins immediately here
 };
 
@@ -148,10 +148,9 @@ struct ring {
        uint waking;            // indicates a thread is currently waking up readers
 
        /* keep the queue in a separate cache line below */
-       THREAD_PAD(64 - 3*sizeof(void*) - 4*sizeof(int));
        struct {
+               THREAD_ALIGN(64);
                struct ring_wait_cell *ptr;
-               THREAD_PAD(64 - sizeof(void*));
        } queue[RING_WAIT_QUEUES + 1]; // wait queue + 1 spacer
 };
 

diff --git a/src/ring.c b/src/ring.c
index 9e3cd6e1cbcc95f1f97a221b069542455072fd72..83a3aefe9b8ed1ea51393d0c00d87156efd88c28 100644 (file)
--- a/src/ring.c
+++ b/src/ring.c
@@ -83,12 +83,12 @@ struct ring *ring_make_from_area(void *area, size_t size, int reset)
        if (size < sizeof(*ring->storage) + 2)
                return NULL;
 
-       ring = malloc(sizeof(*ring));
+       ring = ha_aligned_alloc_typed(1, typeof(*ring));
        if (!ring)
                goto fail;
 
        if (!area)
-               area = malloc(size);
+               area = ha_aligned_alloc(__alignof__(*ring->storage), size);
        else
                flags |= RING_FL_MAPPED;
 
@@ -99,7 +99,7 @@ struct ring *ring_make_from_area(void *area, size_t size, int reset)
        ring->flags |= flags;
        return ring;
  fail:
-       free(ring);
+       ha_aligned_free(ring);
        return NULL;
 }
 
@@ -125,7 +125,7 @@ struct ring *ring_resize(struct ring *ring, size_t size)
                return ring;
 
        old = ring->storage;
-       new = malloc(size);
+       new = ha_aligned_alloc(__alignof__(*ring->storage), size);
        if (!new)
                return NULL;
 
@@ -150,7 +150,7 @@ struct ring *ring_resize(struct ring *ring, size_t size)
        thread_release();
 
        /* free the unused one */
-       free(new);
+       ha_aligned_free(new);
        return ring;
 }
 
@@ -162,8 +162,8 @@ void ring_free(struct ring *ring)
 
        /* make sure it was not allocated by ring_make_from_area */
        if (!(ring->flags & RING_FL_MAPPED))
-               free(ring->storage);
-       free(ring);
+               ha_aligned_free(ring->storage);
+       ha_aligned_free(ring);
 }
 
 /* Tries to send <npfx> parts from <prefix> followed by <nmsg> parts from <msg>