MEDIUM: ring: align the head and tail fields in the ring_storage structure

We really want to let the readers and writers act on different areas, so
we want to have the tail and the head on separate cache lines, themselves
separate from the rest of the ring. Doing so improves the performance from
2.15 to 2.35M msg/s at 48 threads on a 24-core EPYC.

This increases the header space from 32 to 192 bytes when threads are
enabled. But since we already have the header size available in the file,
haring remains able to detect the aligned vs unaligned formats and call
dump_v2a() when aligned is detected.

diff --git a/dev/haring/haring.c b/dev/haring/haring.c
index 0a66238ad2e4799bc89a9d93cfe55eb74fa2792e..4e52b0a9e219cc1687a32479baf38392db52b743 100644 (file)
--- a/dev/haring/haring.c
+++ b/dev/haring/haring.c
@@ -55,6 +55,15 @@ struct ring_v2 {
        char area[0];        // storage area begins immediately here
 };
 
+// ring v2 format (thread aligned)
+struct ring_v2a {
+       size_t size;         // storage size
+       size_t rsvd;         // header length (used for file-backed maps)
+       size_t tail __attribute__((aligned(64)));         // storage tail
+       size_t head __attribute__((aligned(64)));         // storage head
+       char area[0] __attribute__((aligned(64)));        // storage area begins immediately here
+};
+
 /* display the message and exit with the code */
 __attribute__((noreturn)) void die(int code, const char *format, ...)
 {
@@ -215,6 +224,32 @@ int dump_ring_v2(struct ring_v2 *ring, size_t ofs, int flags)
        return dump_ring_as_buf(buf, ofs, flags);
 }
 
+/* This function dumps all events from the ring <ring> from offset <ofs> and
+ * with flags <flags>.
+ */
+int dump_ring_v2a(struct ring_v2a *ring, size_t ofs, int flags)
+{
+       size_t size, head, tail, data;
+       struct buffer buf;
+
+       /* In ring v2 format, we have in this order:
+        *    - size
+        *    - hdr len (reserved bytes)
+        *    - tail
+        *    - head
+        * We can rebuild an equivalent buffer from these info for the function
+        * to dump.
+        */
+
+       /* Now make our own buffer pointing to that area */
+       size = ring->size;
+       head = ring->head;
+       tail = ring->tail & ~RING_TAIL_LOCK;
+       data = (head <= tail ? 0 : size) + tail - head;
+       buf = b_make((void *)ring + ring->rsvd, size, head, data);
+       return dump_ring_as_buf(buf, ofs, flags);
+}
+
 int main(int argc, char **argv)
 {
        void *ring;
@@ -260,8 +295,12 @@ int main(int argc, char **argv)
        }
 
        if (((struct ring_v2 *)ring)->rsvd < 4096 && // not a pointer (v1), must be ringv2's rsvd
-           ((struct ring_v2 *)ring)->rsvd + ((struct ring_v2 *)ring)->size == statbuf.st_size)
-               return dump_ring_v2(ring, 0, 0);
+           ((struct ring_v2 *)ring)->rsvd + ((struct ring_v2 *)ring)->size == statbuf.st_size) {
+               if (((struct ring_v2 *)ring)->rsvd < 192)
+                       return dump_ring_v2(ring, 0, 0);
+               else
+                       return dump_ring_v2a(ring, 0, 0); // thread-aligned version
+       }
        else
                return dump_ring_v1(ring, 0, 0);
 }

diff --git a/include/haproxy/ring-t.h b/include/haproxy/ring-t.h
index aa0973be59e217e54fcd4b455b0ea6470f529d4c..0acbee8e726f9a15c1ad5799dbec7c3437180cbc 100644 (file)
--- a/include/haproxy/ring-t.h
+++ b/include/haproxy/ring-t.h
@@ -107,8 +107,11 @@
 struct ring_storage {
        size_t size;         // storage size
        size_t rsvd;         // header length (used for file-backed maps)
+       THREAD_PAD(64 - 2 * sizeof(size_t));
        size_t tail;         // storage tail
+       THREAD_PAD(64 - sizeof(size_t));
        size_t head;         // storage head
+       THREAD_PAD(64 - sizeof(size_t));
        char area[0];        // storage area begins immediately here
 };