From: John Wolfe <jwolfe@vmware.com>
Date: Fri, 13 Aug 2021 18:35:58 +0000 (-0700)
Subject: VmCheck_IsVirtualWorld(): assume VMware hypervisor if USE_VALGRIND is defined.
X-Git-Tag: stable-11.3.5~4
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=07aa3359ea6c734afe9cb5907019f16e9fedc59e;p=thirdparty%2Fopen-vm-tools.git

VmCheck_IsVirtualWorld(): assume VMware hypervisor if USE_VALGRIND is defined.

Updating the VmCheck_IsVirtualWorld() function to assume that if the
open-vm-tools build was configured with "--enable-valgrind", the tools
will be used on a VMware hypervisor for memory leak detection.  In this
case, simply return TRUE and avoid touching the backdoor.

The "backdoor" touch test cannot be handled by Valgrind.
---

diff --git a/open-vm-tools/lib/vmCheck/vmcheck.c b/open-vm-tools/lib/vmCheck/vmcheck.c
index ca1482662..06fc3acff 100644
--- a/open-vm-tools/lib/vmCheck/vmcheck.c
+++ b/open-vm-tools/lib/vmCheck/vmcheck.c
@@ -249,7 +249,8 @@ VmCheck_GetVersion(uint32 *version, // OUT
  *    environment.
  *
  * Return value:
- *    TRUE if we're in a virtual machine, FALSE otherwise.
+ *    TRUE if we're in a virtual machine or a Linux compilation using Valgrind,
+ *    FALSE otherwise.
  *
  * Side effects:
  *    None.
@@ -264,6 +265,12 @@ VmCheck_IsVirtualWorld(void)
    uint32 dummy;
 
 #if !defined(WINNT_DDK)
+#ifdef USE_VALGRIND
+   /*
+    * Valgrind can't handle the backdoor check.
+    */
+   return TRUE;
+#endif
 #if defined VM_X86_ANY
    char *hypervisorSig;
    uint32 i;
diff --git a/open-vm-tools/services/vmtoolsd/mainLoop.c b/open-vm-tools/services/vmtoolsd/mainLoop.c
index 119f974d6..b5d3d37d3 100644
--- a/open-vm-tools/services/vmtoolsd/mainLoop.c
+++ b/open-vm-tools/services/vmtoolsd/mainLoop.c
@@ -1201,14 +1201,7 @@ ToolsCore_Setup(ToolsServiceState *state)
 #else
    state->ctx.mainLoop = g_main_loop_new(gctx, FALSE);
 #endif
-   /*
-    * Valgrind can't handle the backdoor check.
-    */
-#ifdef USE_VALGRIND
-   state->ctx.isVMware = TRUE;
-#else
    state->ctx.isVMware = VmCheck_IsVirtualWorld();
-#endif
    g_main_context_unref(gctx);
 
    g_type_init();
diff --git a/open-vm-tools/toolbox/toolbox-cmd.c b/open-vm-tools/toolbox/toolbox-cmd.c
index 6bde3747d..121104add 100644
--- a/open-vm-tools/toolbox/toolbox-cmd.c
+++ b/open-vm-tools/toolbox/toolbox-cmd.c
@@ -476,18 +476,11 @@ main(int argc,    // IN: length of command line arguments
    VMTools_ConfigLogging("toolboxcmd", conf, FALSE, FALSE);
    VMTools_BindTextDomain(VMW_TEXT_DOMAIN, NULL, NULL);
 
-   /*
-    * Check if we are in a VM
-    *
-    * Valgrind can't handle the backdoor check, so don't bother.
-    */
-#ifndef USE_VALGRIND
    if (!VmCheck_IsVirtualWorld()) {
       g_printerr(SU_(error.novirtual, "%s must be run inside a virtual machine.\n"),
                  argv[0]);
       goto exit;
    }
-#endif
 
    /*
     * Parse the command line optional arguments