From: Oliver Kurth <okurth@vmware.com>
Date: Fri, 15 Sep 2017 18:23:49 +0000 (-0700)
Subject: Avoid loading resolutionSet when support for resolutionKMS available
X-Git-Tag: stable-10.2.0~118
X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=dec55c31e7dcdbb747647123ad60cc2cde4e0635;p=thirdparty%2Fopen-vm-tools.git

Avoid loading resolutionSet when support for resolutionKMS available

Loading resolutionSet with display NULL will cause a segmentation
fault in vmtoolsd user process. This change only avoids the
SEG fault. Still need to channel the resolution/topology request
to correct channel presenting the capabilities.
---

diff --git a/open-vm-tools/services/plugins/resolutionSet/resolutionX11.c b/open-vm-tools/services/plugins/resolutionSet/resolutionX11.c
index d2a2a7a40..cdb9988e5 100644
--- a/open-vm-tools/services/plugins/resolutionSet/resolutionX11.c
+++ b/open-vm-tools/services/plugins/resolutionSet/resolutionX11.c
@@ -105,17 +105,28 @@ ResolutionBackendInit(InitHandle handle)
    if (resInfoX->canUseResolutionKMS == TRUE) {
       resInfo->canSetResolution = FALSE;
       resInfo->canSetTopology = FALSE;
-      return TRUE;
+      return FALSE;
    }
 
    XSetErrorHandler(ResolutionX11ErrorHandler);
    resInfoX->display = XOpenDisplay(NULL);
 
+   /*
+    * In case display is NULL, we do not load resolutionSet
+    * as it serve no purpose. Also avoids SEGFAULT issue
+    * like BZ1880932.
+    *
+    * VMX currently remembers the settings across a reboot,
+    * so let's say someone replaces our Xorg driver with
+    * xf86-video-modesetting, and then rebooted, we'd end up here,
+    * but the VMX would still send resolution / topology events
+    * and we'd hit the same segfault.
+    */
    if (resInfoX->display == NULL) {
       g_error("%s: Invalid display detected.\n", __func__);
       resInfo->canSetResolution = FALSE;
       resInfo->canSetTopology = FALSE;
-      return TRUE;
+      return FALSE;
    }
 
    resInfoX->rootWindow = DefaultRootWindow(resInfoX->display);