lib: Avoid declaring zero-length VLAs in various messaging functions

In messaging_rec_create(), messaging_recv_cb() and
messaging_dispatch_rec(), variable length arrays of file descriptors are
declared using an incoming num_fds parameter.

However, there are several scenarios where num_fds can be zero, and
declaring a zero-length VLA is undefined behavior. This can lead to
segmentation faults and/or other crashes when compiling with recent
versions of clang at high optimization levels.

To avoid ever using zero as the length for these declarations, use
MAX(1, length) instead.

BUG: https://bugzilla.samba.org/show_bug.cgi?id=14605

Signed-off-by: Dimitry Andric <dimitry@andric.com>
Reviewed-by: Volker Lendecke <vl@samba.org>
Reviewed-by: Ralph Boehme <slow@samba.org>
Autobuild-User(master): Ralph Böhme <slow@samba.org>
Autobuild-Date(master): Mon Jan  4 10:50:07 UTC 2021 on sn-devel-184

(cherry picked from commit 3e96c95d41e4ccd0bf43b3ee78af644e2bc32e30)

Autobuild-User(v4-12-test): Karolin Seeger <kseeger@samba.org>
Autobuild-Date(v4-12-test): Tue Jan  5 09:11:44 UTC 2021 on sn-devel-184

diff --git a/source3/lib/messages.c b/source3/lib/messages.c
index 63d6362e0c975c7edad8ed69596e5a0a1d5a8ef1..2b7026f60c44cc85fc56f2124a8e144d635b84ee 100644 (file)
--- a/source3/lib/messages.c
+++ b/source3/lib/messages.c
@@ -157,7 +157,7 @@ struct messaging_rec *messaging_rec_create(
 
        {
                struct messaging_rec rec;
-               int64_t fds64[num_fds];
+               int64_t fds64[MAX(1, num_fds)];
                size_t i;
 
                for (i=0; i<num_fds; i++) {
@@ -391,7 +391,7 @@ static void messaging_recv_cb(struct tevent_context *ev,
                private_data, struct messaging_context);
        struct server_id_buf idbuf;
        struct messaging_rec rec;
-       int64_t fds64[MIN(num_fds, INT8_MAX)];
+       int64_t fds64[MAX(1, MIN(num_fds, INT8_MAX))];
        size_t i;
 
        if (msg_len < MESSAGE_HDR_LENGTH) {
@@ -1375,7 +1375,7 @@ static void messaging_dispatch_rec(struct messaging_context *msg_ctx,
 
        if (ev != msg_ctx->event_ctx) {
                struct iovec iov;
-               int fds[rec->num_fds];
+               int fds[MAX(1, rec->num_fds)];
                int ret;
 
                /*