VmCheck_IsVirtualWorld(): assume VMware hypervisor if USE_VALGRIND is defined.

Updating the VmCheck_IsVirtualWorld() function to assume that if the
open-vm-tools build was configured with "--enable-valgrind", the tools
will be used on a VMware hypervisor for memory leak detection.  In this
case, simply return TRUE and avoid touching the backdoor.

The "backdoor" touch test cannot be handled by Valgrind.

diff --git a/open-vm-tools/lib/vmCheck/vmcheck.c b/open-vm-tools/lib/vmCheck/vmcheck.c
index ca148266267dbb5332931131379cc40331bfbb70..06fc3acff275618fbce1311a804c438665c61506 100644 (file)
--- a/open-vm-tools/lib/vmCheck/vmcheck.c
+++ b/open-vm-tools/lib/vmCheck/vmcheck.c
@@ -249,7 +249,8 @@ VmCheck_GetVersion(uint32 *version, // OUT
  *    environment.
  *
  * Return value:
- *    TRUE if we're in a virtual machine, FALSE otherwise.
+ *    TRUE if we're in a virtual machine or a Linux compilation using Valgrind,
+ *    FALSE otherwise.
  *
  * Side effects:
  *    None.
@@ -264,6 +265,12 @@ VmCheck_IsVirtualWorld(void)
    uint32 dummy;
 
 #if !defined(WINNT_DDK)
+#ifdef USE_VALGRIND
+   /*
+    * Valgrind can't handle the backdoor check.
+    */
+   return TRUE;
+#endif
 #if defined VM_X86_ANY
    char *hypervisorSig;
    uint32 i;

diff --git a/open-vm-tools/services/vmtoolsd/mainLoop.c b/open-vm-tools/services/vmtoolsd/mainLoop.c
index 119f974d6f7fb64d744a22ce293c5e87d808004c..b5d3d37d353a76b2592d48d82aa839b31c35e662 100644 (file)
--- a/open-vm-tools/services/vmtoolsd/mainLoop.c
+++ b/open-vm-tools/services/vmtoolsd/mainLoop.c
@@ -1201,14 +1201,7 @@ ToolsCore_Setup(ToolsServiceState *state)
 #else
    state->ctx.mainLoop = g_main_loop_new(gctx, FALSE);
 #endif
-   /*
-    * Valgrind can't handle the backdoor check.
-    */
-#ifdef USE_VALGRIND
-   state->ctx.isVMware = TRUE;
-#else
    state->ctx.isVMware = VmCheck_IsVirtualWorld();
-#endif
    g_main_context_unref(gctx);
 
    g_type_init();

diff --git a/open-vm-tools/toolbox/toolbox-cmd.c b/open-vm-tools/toolbox/toolbox-cmd.c
index 6bde3747df16d116ec3090b408e14808ee4324b0..121104addf932f05c44182933206ab1678b7b36f 100644 (file)
--- a/open-vm-tools/toolbox/toolbox-cmd.c
+++ b/open-vm-tools/toolbox/toolbox-cmd.c
@@ -476,18 +476,11 @@ main(int argc,    // IN: length of command line arguments
    VMTools_ConfigLogging("toolboxcmd", conf, FALSE, FALSE);
    VMTools_BindTextDomain(VMW_TEXT_DOMAIN, NULL, NULL);
 
-   /*
-    * Check if we are in a VM
-    *
-    * Valgrind can't handle the backdoor check, so don't bother.
-    */
-#ifndef USE_VALGRIND
    if (!VmCheck_IsVirtualWorld()) {
       g_printerr(SU_(error.novirtual, "%s must be run inside a virtual machine.\n"),
                  argv[0]);
       goto exit;
    }
-#endif
 
    /*
     * Parse the command line optional arguments