AsyncSocket: Fix DoOneMsg's inBlockingRecv bookkeeping

For the I[Vmdb]Poll support in AsyncTCPSocket, inBlockingRecv has been
used to avoid conflict with AsyncSocketDoOneMsg.  There is an issue
with where inBlocking Recv is decremented, which is right after poll,
where the thread would drop the lock.  The lock may also be dropped
during AsyncTCPSocketFillRecvBuffer, and for that there is a separate
inDoOneMsg flag.  This change unifies the two flags and makes
inBlockingRecv cover both functions that may drop the lock.  This fixes
the race that would allow AsyncTCPSocketRecvCallback to run when
AsyncTCPSocketDoOneMsg is in progress.  A couple places where we
previously checked inBlockingRecv to report an error now have to also
check inRecvLoop (TRUE during AsyncTCPSocketFillRecvBuffer) -- those
checks cannot be enforced during FillRecvBuffer because the client
callback is allowed to change the callback or cancel it.  The lone
usage of inDoOneMsg is switched to inBlockingRecv.

diff --git a/open-vm-tools/lib/asyncsocket/asyncsocket.c b/open-vm-tools/lib/asyncsocket/asyncsocket.c
index 57341d631e57051dba7cb39be9c4a6854cf7e2ad..758dd88f42d3c8bed86b36bc98cc5f0eb4f10465 100644 (file)
--- a/open-vm-tools/lib/asyncsocket/asyncsocket.c
+++ b/open-vm-tools/lib/asyncsocket/asyncsocket.c
@@ -220,7 +220,6 @@ typedef struct AsyncTCPSocket {
 
    uint8 inIPollCb;
    Bool inRecvLoop;
-   Bool inDoOneMsg;
    uint32 inBlockingRecv;
 
    struct AsyncTCPSocket *listenAsock4;
@@ -2411,7 +2410,7 @@ AsyncTCPSocketRecv(AsyncSocket *base,   // IN:
       return ASOCKERR_NOTCONNECTED;
    }
 
-   if (asock->inBlockingRecv) {
+   if (asock->inBlockingRecv && !asock->inRecvLoop) {
       TCPSOCKWARN(asock, ("Recv called while a blocking recv is pending.\n"));
       return ASOCKERR_INVAL;
    }
@@ -3935,7 +3934,6 @@ AsyncTCPSocketDoOneMsg(AsyncSocket *base, // IN
 
       s->inBlockingRecv++;
       retVal = AsyncTCPSocketPoll(s, read, timeoutMS, &asock);
-      s->inBlockingRecv--;
       if (retVal != ASOCKERR_SUCCESS) {
          if (retVal == ASOCKERR_GENERIC) {
             TCPSOCKWARN(s, ("%s: failed to poll on the socket during read.\n",
@@ -3943,10 +3941,9 @@ AsyncTCPSocketDoOneMsg(AsyncSocket *base, // IN
          }
       } else {
          ASSERT(asock == s);
-         s->inDoOneMsg = TRUE;
          retVal = AsyncTCPSocketFillRecvBuffer(s);
-         s->inDoOneMsg = FALSE;
       }
+      s->inBlockingRecv--;
 
       /*
        * If socket got closed in AsyncTCPSocketFillRecvBuffer, we
@@ -4250,7 +4247,7 @@ AsyncTCPSocketCancelCbForClose(AsyncSocket *base)  // IN:
                                          asock->internalRecvFn);
       /* Callback might be temporarily removed in AsyncSocket_DoOneMsg. */
       ASSERT_NOT_TESTED(removed ||
-                        asock->inDoOneMsg ||
+                        asock->inBlockingRecv ||
                         AsyncTCPSocketPollParams(asock)->iPoll);
 
       asock->recvCb = FALSE;
@@ -5184,7 +5181,7 @@ AsyncTCPSocketCancelRecv(AsyncSocket *base,          // IN
       return ASOCKERR_INVAL;
    }
 
-   if (asock->inBlockingRecv) {
+   if (asock->inBlockingRecv && !asock->inRecvLoop) {
       Warning(ASOCKPREFIX "Cannot cancel request while a blocking recv is "
                           "pending.\n");
       return ASOCKERR_INVAL;