From: John Wolfe <jwolfe@vmware.com>
Date: Thu, 17 Feb 2022 22:51:25 +0000 (-0800)
Subject: asyncsocket.c: Use size_t in place of int type for array size and indexing.
X-Git-Tag: stable-12.1.0~134
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=de6d129476724668b8903e2a87654f50ba21b1b2;p=thirdparty%2Fopen-vm-tools.git

asyncsocket.c: Use size_t in place of int type for array size and indexing.

Glibc 2.35 with GCC 11 and 12 produces additional warnings about strings
and array bounds.  Switching from "int" to "size_t" type for variable
used for the array size and element indexing.

GCC warned when an integer value is passed as the size of the
struct pollfd array to poll().

Fixes https://github.com/vmware/open-vm-tools/issues/570
---

diff --git a/open-vm-tools/lib/asyncsocket/asyncSocketVTable.h b/open-vm-tools/lib/asyncsocket/asyncSocketVTable.h
index a69b65677..c068ff505 100644
--- a/open-vm-tools/lib/asyncsocket/asyncSocketVTable.h
+++ b/open-vm-tools/lib/asyncsocket/asyncSocketVTable.h
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2011,2014-2017,2019-2021 VMware, Inc. All rights reserved.
+ * Copyright (C) 2011,2014-2017,2019-2022 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU Lesser General Public License as published
@@ -131,8 +131,8 @@ typedef struct AsyncSocketVTable {
                        int timeoutMS);
    int (*doOneMsg)(AsyncSocket *s, Bool read, int timeoutMS);
    int (*waitForConnection)(AsyncSocket *s, int timeoutMS);
-   int (*waitForReadMultiple)(AsyncSocket **asock, int numSock, int timeoutMS,
-                              int *outIdx);
+   int (*waitForReadMultiple)(AsyncSocket **asock, size_t numSock,
+                              int timeoutMS, int *outIdx);
    int (*peek)(AsyncSocket *asock, void *buf, int len, void *cb, void *cbData);
 
    /*
diff --git a/open-vm-tools/lib/asyncsocket/asyncsocket.c b/open-vm-tools/lib/asyncsocket/asyncsocket.c
index ecb5a933f..2bf97b549 100644
--- a/open-vm-tools/lib/asyncsocket/asyncsocket.c
+++ b/open-vm-tools/lib/asyncsocket/asyncsocket.c
@@ -370,7 +370,7 @@ static int AsyncTCPSocketRecvPartialBlocking(AsyncSocket *s, void *buf, int len,
 static int AsyncTCPSocketSendBlocking(AsyncSocket *s, void *buf, int len,
                                       int *sent, int timeoutMS);
 static int AsyncTCPSocketDoOneMsg(AsyncSocket *s, Bool read, int timeoutMS);
-static int AsyncTCPSocketWaitForReadMultiple(AsyncSocket **asock, int numSock,
+static int AsyncTCPSocketWaitForReadMultiple(AsyncSocket **asock, size_t numSock,
                                              int timeoutMS, int *outIdx);
 static int AsyncTCPSocketSetOption(AsyncSocket *asyncSocket,
                                    AsyncSocketOpts_Layer layer,
@@ -2807,7 +2807,7 @@ AsyncTCPSocketPeek(AsyncSocket *base,   // IN:
 
 static int
 AsyncTCPSocketPollWork(AsyncTCPSocket **asock,     // IN:
-                       int numSock,                // IN:
+                       size_t numSock,             // IN:
                        void *p,                    // IN:
                        Bool read,                  // IN:
                        int timeoutMS,              // IN:
@@ -2827,11 +2827,11 @@ AsyncTCPSocketPollWork(AsyncTCPSocket **asock,     // IN:
    struct fd_set rwfds;
    struct fd_set exceptfds;
 #endif
-   int i;
+   size_t i;
    int retval;
 
    ASSERT(outAsock != NULL && *outAsock == NULL && asock != NULL &&
-          numSock > 0);
+          numSock != 0);
 
    for (i = 0; i < numSock; i++) {
       if (read && SSL_Pending(asock[i]->sslSock)) {
@@ -2852,7 +2852,7 @@ AsyncTCPSocketPollWork(AsyncTCPSocket **asock,     // IN:
          retval = poll(pfd, numSock, timeoutMS);
          AsyncTCPSocketLock(parentSock);
       } else {
-         for (i = numSock - 1; i >= 0; i--) {
+         for (i = numSock; i-- > 0; ) {
             AsyncTCPSocketUnlock(asock[i]);
          }
          retval = poll(pfd, numSock, timeoutMS);
@@ -2878,7 +2878,7 @@ AsyncTCPSocketPollWork(AsyncTCPSocket **asock,     // IN:
                          &exceptfds, timeoutMS >= 0 ? &tv : NULL);
          AsyncTCPSocketLock(parentSock);
       } else {
-         for (i = numSock - 1; i >= 0; i--) {
+         for (i = numSock; i-- > 0; ) {
             AsyncTCPSocketUnlock(asock[i]);
          }
          retval = select(1, read ? &rwfds : NULL, read ? NULL : &rwfds,
@@ -3032,7 +3032,7 @@ AsyncTCPSocketPoll(AsyncTCPSocket *s,          // IN:
 #else
    void *p = NULL;
 #endif
-   int numSock = 0;
+   size_t numSock = 0;
 
    if (read && s->fd == -1) {
       if (!s->listenAsock4 && !s->listenAsock6) {
@@ -3078,11 +3078,11 @@ AsyncTCPSocketPoll(AsyncTCPSocket *s,          // IN:
 
 static int
 AsyncTCPSocketWaitForReadMultiple(AsyncSocket **asock,   // IN:
-                                  int numSock,           // IN:
+                                  size_t numSock,        // IN:
                                   int timeoutMS,         // IN:
                                   int *outIdx)           // OUT:
 {
-   int i;
+   size_t i;
    int err;
    AsyncTCPSocket *outAsock  = NULL;
 #ifndef _WIN32
@@ -3096,7 +3096,7 @@ AsyncTCPSocketWaitForReadMultiple(AsyncSocket **asock,   // IN:
    }
    err = AsyncTCPSocketPollWork((AsyncTCPSocket **)asock, numSock, p, TRUE,
                                 timeoutMS, NULL, &outAsock);
-   for (i = numSock - 1; i >= 0; i--) {
+   for (i = numSock; i-- > 0; ) {
       AsyncTCPSocket *tcpAsock = TCPSocket(asock[i]);
       if (outAsock == tcpAsock) {
          *outIdx = i;