[vgauth] Retry on EBADF when getting pam entry (getpwuid_r, getpwnam_r)

Below are the changes made as part of this update
o resetting the errno value before calling the getpw*_r functions in
  usercheck.c. Some man pages mention this is required as the errno might
  be left unchanged by the function. We're performing a few retries,
  therefore resetting errno at the start of a retry loop ensures we're
  not 'stuck' on the error.
o implementing retry in impersonateLinux.c which includes the reset of errno.
o logging 'errno' in impersonateLinux.c and usercheck.c where relevant.
o On EBADF error, adding g_thread_yield in the retry loop to add some
  delay to the retry.

diff --git a/open-vm-tools/vgauth/common/usercheck.c b/open-vm-tools/vgauth/common/usercheck.c
index 2b4e424da45d8b4f35bd4ec4bd1180d612758a1f..73adfa14866dfc37916364a2600b635ba237bf38 100644 (file)
--- a/open-vm-tools/vgauth/common/usercheck.c
+++ b/open-vm-tools/vgauth/common/usercheck.c
@@ -89,6 +89,178 @@
 #define MAX_RETRIES  5
 
 #ifndef _WIN32
+/*
+ * ******************************************************************************
+ * UsercheckRetryGetpwuid_r --                                             */ /**
+ *
+ * Wrapper for calls to getpwuid_r().
+ *
+ * Handle retryable errors from calls to getpwuid_r.
+ *
+ * @param[in]     uid, the user id to find.
+ * @param[in/out] struct passwd to use.
+ * @param[in/out] buf, buffer for the entry data.
+ * @param[in]     bufLen, buffer size in bytes.
+ * @param[out]    result, where to store the struct passwd address if found.
+ * @param[in]     retry, the maximum number of retries:
+ *                   < 0: use default (MAX_RETRIES),
+ *                     0: no retries (do once),
+ *                   > 0: use value for max retries
+ *
+ * @return On success (found), error is 0 and *result == ppw;
+ *         On not found, error is 0 or ENOENT and *result == NULL;
+ *         On error, error != 0 and *result == NULL.
+ *
+ *         In case retries occurred, the last attempt error and result values
+ *         are returned.
+ *
+ * Note: retries not implemented for Sun OS.
+ */
+
+int
+UsercheckRetryGetpwuid_r(const uid_t uid,
+                         struct passwd *ppw,
+                         char *buf,
+                         const size_t bufLen,
+                         struct passwd **result,
+                         const int retry) {
+#ifdef sun /* sun */
+   /*
+    * Retry loop for EBADF is not implemented for Sun OS.
+    * Adapt the getpwuid_r call's outcome to match the expectation.
+    * Use errno as the returned error value.
+    */
+   errno = 0;
+   if ((ppw = getpwuid_r(uid, ppw, buf, bufLen)) == NULL) {
+      /* Fall through, set *result to ppw (NULL), and return errno. */
+   }
+   *result = ppw;
+   return errno;
+
+#else /* !sun */
+   int error;
+   int maxRetries = retry < 0 ? MAX_RETRIES : retry;
+   int retryCount = 0;
+   int saveErrno;
+
+   /*
+    * Retry loop for EBADF.
+    */
+retry:
+   saveErrno = errno;
+   errno = 0; /* PR3105769 - reset errno before the call. Per man page */
+   if ((error = getpwuid_r(uid, ppw, buf, bufLen, result)) != 0 ||
+       !result) {
+      /*
+       * According to POSIX 1003.1-2003
+       *   - On not found: error == 0 or ENOENT && result == NULL
+       *   - On error:     error != 0 && result == NULL
+       */
+      if (EBADF == errno) {
+         retryCount++;
+         if (retryCount < maxRetries) {
+            g_debug("%s: getpwuid_r(%lu) failed %d (%d) (was: %d), try #%d\n",
+                  __FUNCTION__, (unsigned long int)uid, error, errno, saveErrno,
+                  retryCount);
+            g_thread_yield(); /* XXX: if adds too much delay, use g_usleep(X) */
+            goto retry;
+         }
+         /* Else: fall through and return the last attempt error. */
+         g_warning("%s: getpwuid_r(%lu) failed %d (%d) (was: %d), try #%d\n",
+                 __FUNCTION__, (unsigned long int)uid, error, errno, saveErrno,
+                 retryCount);
+      }
+      /* Else: fall through and return the current error. */
+   }
+   return error;
+#endif /* !sun */
+}
+
+
+/*
+ * ******************************************************************************
+ * UsercheckRetryGetpwnam_r --                                             */ /**
+ *
+ * Wrapper for calls to getpwnam_r().
+ *
+ * Handle retryable errors from calls to getpwnam_r.
+ *
+ * @param[in]     name, the user name to find.
+ * @param[in/out] struct passwd to use.
+ * @param[in/out] buf, buffer for the entry data.
+ * @param[in]     bufLen, buffer size in bytes.
+ * @param[out]    result, where to store the struct passwd address if found.
+ * @param[in]     retry, the maximum number of retries:
+ *                   < 0: use default (MAX_RETRIES),
+ *                     0: no retries (do once),
+ *                   > 0: use value for max retries
+ *
+ * @return On success (found), error is 0 and *result == ppw;
+ *         On not found, error is 0 or ENOENT and *result == NULL;
+ *         On error, error != 0 and *result == NULL.
+ *
+ *         In case retries occurred, the last attempt error and result values
+ *         are returned.
+ *
+ * Note: retries not implemented for Sun OS.
+ */
+int UsercheckRetryGetpwnam_r(const char *name,
+                             struct passwd *ppw,
+                             char *buf,
+                             const size_t bufLen,
+                             struct passwd **result,
+                             const int retry) {
+#ifdef sun /* sun */
+   /*
+    * Retry loop for EBADF is not implemented for Sun OS.
+    * Adapt the getpwnam_r call's outcome to match the expectation.
+    * Use errno as the returned error value.
+    */
+   errno = 0;
+   if ((ppw = getpwnam_r(name, ppw, buf, bufLen)) == NULL) {
+      /* Fall through, set *result to ppw (NULL), and return errno. */
+   }
+   *result = ppw;
+   return errno;
+
+#else /* !sun */
+   int error;
+   int maxRetries = retry < 0 ? MAX_RETRIES : retry;
+   int retryCount = 0;
+   int saveErrno;
+
+   /*
+    * Retry loop for EBADF.
+    */
+retry:
+   saveErrno = errno;
+   errno = 0; /* PR3105769 - reset errno before the call. Per man page */
+   if ((error = getpwnam_r(name, ppw, buf, bufLen, result)) != 0 ||
+       !result) {
+      /*
+       * According to POSIX 1003.1-2003
+       *   - On not found: error == 0 or ENOENT && result == NULL
+       *   - On error:     error != 0 && result == NULL
+       */
+      if (EBADF == errno) {
+         retryCount++;
+         if (retryCount < maxRetries) {
+            g_debug("%s: getpwnam_r(%s) failed %d (%d) (was: %d), try #%d\n",
+                  __FUNCTION__, name, error, errno, saveErrno, retryCount);
+            g_thread_yield(); /* XXX: if adds too much delay, use g_usleep(X) */
+            goto retry;
+         }
+         /* Else: fall through and return the last attempt error. */
+         g_warning("%s: getpwnam_r(%s) failed %d (%d) (was: %d), try #%d\n",
+                 __FUNCTION__, name, error, errno, saveErrno, retryCount);
+      }
+      /* Else: fall through and return the current error. */
+   }
+   return error;
+#endif /* !sun */
+}
+
+
 /*
  ******************************************************************************
  * UsercheckLookupUser --                                                */ /**
@@ -113,28 +285,13 @@ UsercheckLookupUser(const gchar *userName,
    struct passwd pw;
    struct passwd *ppw = &pw;
    char buffer[BUFSIZ];
-#ifndef sun
-   int ret;
-   int retryCount = 0;
+   int error;
 
-   /*
-    * XXX Retry kludge -- see above.
-    */
-retry:
-   if ((ret = getpwnam_r(userName, &pw, buffer, sizeof buffer, &ppw)) != 0 ||
-                         !ppw) {
-      if ((EBADF == errno) && (++retryCount < MAX_RETRIES)) {
-         g_debug("%s: getpwnam_r(%s) failed %d (%d), try #%d\n",
-                 __FUNCTION__, userName, ret, errno, retryCount);
-         goto retry;
-      }
+   error = UsercheckRetryGetpwnam_r(userName, &pw, buffer, sizeof buffer, &ppw,
+                                    -1);
+   if (error != 0 || !ppw) {
       return VGAUTH_E_NO_SUCH_USER;
    }
-#else
-   if ((ppw = getpwnam_r(userName, &pw, buffer, sizeof buffer)) == NULL) {
-      return VGAUTH_E_NO_SUCH_USER;
-   }
-#endif
 
    *uid = ppw->pw_uid;
    *gid = ppw->pw_gid;
@@ -164,33 +321,12 @@ UsercheckLookupUid(uid_t uid,
    struct passwd pw;
    struct passwd *ppw = &pw;
    char buffer[BUFSIZ];
-#ifndef sun
    int error;
-   int retryCount = 0;
 
-   /*
-    * XXX Retry kludge -- see above.
-    */
-retry:
-   if ((error = getpwuid_r(uid, &pw, buffer, sizeof buffer, &ppw)) != 0 ||
-       !ppw) {
-      /*
-       * getpwuid_r() and getpwnam_r() can return a 0 (success) but not
-       * set the return pointer (ppw) if there's no entry for the user,
-       * according to POSIX 1003.1-2003.
-       */
-      if ((EBADF == errno) && (++retryCount < MAX_RETRIES)) {
-         g_debug("%s: getpwuid_r(%d) failed %d (%d), try #%d\n",
-                 __FUNCTION__, uid, error, errno, retryCount);
-         goto retry;
-      }
+   error = UsercheckRetryGetpwuid_r(uid, &pw, buffer, sizeof buffer, &ppw, -1);
+   if (error != 0 || !ppw) {
       return VGAUTH_E_NO_SUCH_USER;
    }
-#else
-   if ((ppw = getpwuid_r(uid, &pw, buffer, sizeof buffer)) == NULL) {
-      return VGAUTH_E_NO_SUCH_USER;
-   }
-#endif
 
    // XXX locale issue lurking here
    *userName = g_strdup(ppw->pw_name);

diff --git a/open-vm-tools/vgauth/common/usercheck.h b/open-vm-tools/vgauth/common/usercheck.h
index e232cdeaadd26f0e4eade9f35243788f55cdc6ed..a799c1b46026157898e79c47cc8511eea82f8cae 100644 (file)
--- a/open-vm-tools/vgauth/common/usercheck.h
+++ b/open-vm-tools/vgauth/common/usercheck.h
@@ -1,5 +1,6 @@
 /*********************************************************
- * Copyright (C) 2011-2016 VMware, Inc. All rights reserved.
+ * Copyright (c) 2011-2025 Broadcom. All Rights Reserved.
+ * The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU Lesser General Public License as published
@@ -22,7 +23,7 @@
 /*
  * @file usercheck.h
  *
- * User existance support.
+ * User existence support.
  */
 
 #include <glib.h>
@@ -36,6 +37,20 @@
 
 gboolean UsercheckUserExists(const gchar *userName);
 #ifndef _WIN32
+int UsercheckRetryGetpwuid_r(const uid_t uid,
+                             struct passwd *ppw,
+                             char *buf,
+                             const size_t size,
+                             struct passwd **result,
+                             const int retry);
+
+int UsercheckRetryGetpwnam_r(const char *name,
+                             struct passwd *ppw,
+                             char *buf,
+                             const size_t size,
+                             struct passwd **result,
+                             const int retry);
+
 VGAuthError UsercheckLookupUser(const gchar *userName,
                                 uid_t *uid,                       // OUT
                                 gid_t *gid);                      // OUT

diff --git a/open-vm-tools/vgauth/lib/impersonateLinux.c b/open-vm-tools/vgauth/lib/impersonateLinux.c
index 240a96d5cf90bc5509c6144b5fc699321be48e7c..2fe853d0857aadd4edfbf7b746614c5386815f54 100644 (file)
--- a/open-vm-tools/vgauth/lib/impersonateLinux.c
+++ b/open-vm-tools/vgauth/lib/impersonateLinux.c
@@ -1,5 +1,6 @@
 /*********************************************************
- * Copyright (C) 2011-2017 VMware, Inc. All rights reserved.
+ * Copyright (c) 2011-2025 Broadcom. All Rights Reserved.
+ * The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU Lesser General Public License as published
@@ -41,6 +42,7 @@
 #include <errno.h>
 
 #include "VGAuthInt.h"
+#include "usercheck.h"
 
 
 #if (__GLIBC__ == 2) && (__GLIBC_MINOR__ < 3)
@@ -101,25 +103,21 @@ VGAuthImpersonateImpl(VGAuthContext *ctx,
    struct passwd pw;
    struct passwd *ppw = &pw;
    gid_t root_gid;
-   int error;
    int ret;
 
-   if ((error = getpwuid_r(0, &pw, buffer, sizeof buffer, &ppw)) != 0 ||
-       !ppw) {
-      /*
-       * getpwuid_r() and getpwnam_r() can return a 0 (success) but not
-       * set the return pointer (ppw) if there's no entry for the user,
-       * according to POSIX 1003.1-2003.
-       */
-      Warning("Failed to lookup root (%d)\n", error);
+   ret = UsercheckRetryGetpwuid_r(0, &pw, buffer, sizeof buffer, &ppw, -1);
+   if (ret != 0 || !ppw) {
+      Warning("Failed to lookup root %d (%d)\n", ret, errno);
       return VGAUTH_E_INVALID_ARGUMENT;
    }
 
    root_gid = ppw->pw_gid;
 
-   if ((error = getpwnam_r(handle->userName, &pw, buffer, sizeof buffer, &ppw)) != 0 ||
-       !ppw) {
-      Warning("Failed to lookup user '%s' (%d)\n", handle->userName, error);
+   ret = UsercheckRetryGetpwnam_r(handle->userName, &pw, buffer, sizeof buffer,
+                                  &ppw, -1);
+   if (ret != 0 || !ppw) {
+      Warning("Failed to lookup user '%s' %d (%d)\n",
+              handle->userName, ret, errno);
       // XXX add VGAUTH_E_INVALIDUSER ???
       return VGAUTH_E_INVALID_ARGUMENT;
    }
@@ -177,12 +175,11 @@ VGAuthEndImpersonationImpl(VGAuthContext *ctx)
    char buffer[BUFSIZ];
    struct passwd pw;
    struct passwd *ppw = &pw;
-   int error;
    int ret;
 
-   if ((error = getpwuid_r(0, &pw, buffer, sizeof buffer, &ppw)) != 0 ||
-       !ppw) {
-      Warning("Failed to lookup root (%d)\n", error);
+   ret = UsercheckRetryGetpwuid_r(0, &pw, buffer, sizeof buffer, &ppw, -1);
+   if (ret != 0 || !ppw) {
+      Warning("Failed to lookup root %d (%d)\n", ret, errno);
       return VGAUTH_E_INVALID_ARGUMENT;
    }
 
@@ -235,9 +232,10 @@ VGAuth_GetCurrentUsername(void)
    int error;
    gchar *userName = NULL;
 
-   if ((error = getpwuid_r(uid, &pw, buffer, sizeof buffer, &ppw)) != 0 ||
-       !ppw) {
-      Warning("Failed to look up username for current uid (%d)\n", error);
+   error = UsercheckRetryGetpwuid_r(uid, &pw, buffer, sizeof buffer, &ppw, -1);
+   if (error != 0 || !ppw) {
+      Warning("Failed to look up username for current uid %d (%d)\n",
+              error, errno);
       return userName;
    }