Unity/X11: Ignore XComposite composite overlay windows.

XComposite since 0.3 provides composite overlay windows, or windows whose
geometry match the root windows', are stacked above applications but below
the screensaver, don't appear in QueryTree requests, etc.  They're intended
for use by compositing managers to redirect contents to and draw upon.

When Unity/X11 encountered such a window, it would treat it as a regular
override redirect window.  In other words, it'd tell the host UI that the
VM has a window whose geometry takes up the entire screen.  This resulted
in bug 690447, where the guest's entire desktop appears onscreen.

My fix was to ask XComposite for the IDs of these windows, and then just
ignore any events related to them.

Signed-off-by: Marcelo Vanzin <mvanzin@vmware.com>

diff --git a/open-vm-tools/configure.ac b/open-vm-tools/configure.ac
index eabab1cd791ab71c763a7566bf48a13b4b86e6e2..0b8ffad2fd9781e41f99d08abee4d5ea9762a49c 100644 (file)
--- a/open-vm-tools/configure.ac
+++ b/open-vm-tools/configure.ac
@@ -515,11 +515,24 @@ else
       have_xsm="yes"
    fi
 
+   AC_CHECK_LIB(
+      [Xcomposite],
+      [XCompositeQueryExtension],
+      [XCOMPOSITE_LIBS="-lXcomposite"],
+      [have_xcomposite="no"]
+      [])
+   AC_CHECK_HEADERS([X11/extensions/Xcomposite.h],
+                    [],
+                    [have_xcomposite="no"],
+                    [])
+   if test "$have_xcomposite" != "no"; then
+      have_xcomposite="yes"
+   fi
+
    # If we're building with support for Unity, we'll need a few additional
    # libraries.
    if test "$enable_unity" != "no"; then
-
-      # Unity needs the X11 Screen Saver extension library. It should be
+           # Unity needs the X11 Screen Saver extension library. It should be
           # in the same place as the X11 libraries, so no need for any fancy
           # path checking.
           AC_VMW_CHECK_X11_LIB(
@@ -1086,6 +1099,7 @@ AM_CONDITIONAL(HAVE_X11, test "$have_x" = "yes")
 AM_CONDITIONAL(HAVE_ICU, test "$with_icu" = "yes")
 AM_CONDITIONAL(WITH_KERNEL_MODULES, test "$with_kernel_modules" = "yes")
 AM_CONDITIONAL(HAVE_XSM, test "$have_xsm" = "yes")
+AM_CONDITIONAL(HAVE_XCOMPOSITE, test "$have_xcomposite" = "yes")
 AM_CONDITIONAL(ENABLE_UNITY, test "$enable_unity" = "yes")
 AM_CONDITIONAL(ENABLE_TESTS, test "$have_cunit" = "yes")
 AM_CONDITIONAL(WITH_ROOT_PRIVILEGES, test "$with_root_privileges" = "yes")
@@ -1101,6 +1115,10 @@ if test "$have_xsm" != "yes"; then
 AC_DEFINE([NO_XSM], 1, [])
 fi
 
+if test "$have_xcomposite" != "yes"; then
+   AC_DEFINE([NO_XCOMPOSITE])
+fi
+
 ### Feature-specific flags / actions
 # Combine where possible
 
@@ -1162,6 +1180,7 @@ AC_SUBST([MODULES_DIR])
 AC_SUBST([MODULES])
 AC_SUBST([COMMON_XLIBS])
 AC_SUBST([XSM_LIBS])
+AC_SUBST([XCOMPOSITE_LIBS])
 AC_SUBST([PAM_PREFIX])
 AC_SUBST([PLUGIN_CPPFLAGS])
 AC_SUBST([PLUGIN_LDFLAGS])

diff --git a/open-vm-tools/services/plugins/unity/Makefile.am b/open-vm-tools/services/plugins/unity/Makefile.am
index 5e4a48ce700e7e1bb8edeb6c73e4b5dc191d3741..ec64feb132549eff1d9f0452c966339390b24c72 100644 (file)
--- a/open-vm-tools/services/plugins/unity/Makefile.am
+++ b/open-vm-tools/services/plugins/unity/Makefile.am
@@ -37,6 +37,7 @@ libunity_la_LDFLAGS += @PLUGIN_LDFLAGS@
 
 libunity_la_LIBADD =
 libunity_la_LIBADD += @COMMON_XLIBS@
+libunity_la_LIBADD += @XCOMPOSITE_LIBS@
 libunity_la_LIBADD += @GTK_LIBS@
 libunity_la_LIBADD += @GTKMM_LIBS@
 libunity_la_LIBADD += @GIO_LIBS@

diff --git a/open-vm-tools/services/plugins/unity/unitylib/unityPlatformX11.cc b/open-vm-tools/services/plugins/unity/unitylib/unityPlatformX11.cc
index ba658a66ae99b7f5e3d4290c46dc62fd28afd098..ceea43bba3d86fe4b53398953274b433aa7959d7 100644 (file)
--- a/open-vm-tools/services/plugins/unity/unitylib/unityPlatformX11.cc
+++ b/open-vm-tools/services/plugins/unity/unitylib/unityPlatformX11.cc
@@ -34,6 +34,10 @@ extern "C" {
 
 #include <X11/extensions/Xinerama.h>
 #include <X11/extensions/XTest.h>
+
+#ifndef NO_XCOMPOSITE
+#   include <X11/extensions/Xcomposite.h>
+#endif
 }
 
 typedef struct {
@@ -85,6 +89,7 @@ static Bool GetRelevantWMWindow(UnityPlatform *up,
 static Bool SetWindowStickiness(UnityPlatform *up,
                                 UnityWindowId windowId,
                                 Bool wantSticky);
+static UnitySpecialWindow *MakeCompositeOverlaysObject(UnityPlatform *up);
 
 static const GuestCapabilities platformUnityCaps[] = {
    UNITY_CAP_WORK_AREA,
@@ -945,6 +950,7 @@ UnityPlatformEnterUnity(UnityPlatform *up) // IN
 
    XSync(up->display, TRUE);
    up->rootWindows = UnityPlatformMakeRootWindowsObject(up);
+   MakeCompositeOverlaysObject(up);
    up->isRunning = TRUE;
    up->eventTimeDiff = 0;
 
@@ -3340,6 +3346,66 @@ UnityPlatformSendMouseWheel(UnityPlatform *up,    // IN
 }
 
 
+/*
+ *-----------------------------------------------------------------------------
+ *
+ * MakeCompositeOverlaysObject --
+ *
+ *      Probes X server for any composite overlay windows.  If found, they're
+ *      monitored as UnitySpecialWindows whereby we won't mistakenly place them
+ *      in the global window tracker.
+ *
+ * Results:
+ *      Returns pointer to new UnitySpecialWindow on success, NULL on failure.
+ *
+ * Side effects:
+ *      If overlay windows haven't yet been mapped, they will be (temporarily).
+ *
+ *      Caller doesn't need to free returned UnitySpecialWindow*.  That's handled
+ *      automatically when exiting Unity.
+ *
+ *-----------------------------------------------------------------------------
+ */
+
+static UnitySpecialWindow *
+MakeCompositeOverlaysObject(UnityPlatform *up)  // IN
+{
+   ASSERT(up);
+   ASSERT(up->rootWindows);
+
+#ifndef NO_XCOMPOSITE
+   int eventBase;
+   int errorBase;
+   if (XCompositeQueryExtension(up->display, &eventBase, &errorBase)) {
+      /*
+       * XCompositeGetOverlayWindow didn't appear until XComposite 0.3.
+       */
+      int major = 0;
+      int minor = 0;
+      XCompositeQueryVersion(up->display, &major, &minor);
+      if (major > 0 || (major == 0 && minor >= 3)) {
+         size_t nWindows = up->rootWindows->numWindows;
+         Window *overlays = (Window*)Util_SafeCalloc(nWindows, sizeof *overlays);
+         for (unsigned int i = 0; i < nWindows; i++) {
+            overlays[i] = XCompositeGetOverlayWindow(up->display,
+                                                     up->rootWindows->windows[i]);
+            XCompositeReleaseOverlayWindow(up->display, overlays[i]);
+         }
+
+         /*
+          * Note 1: See above. Caller doesn't need to need to track this explicitly.
+          * Note 2: The NULL event handler parameter is how we tell the X event handling
+          *         pieces to ignore this window, thereby keeping it out of the window
+          *         tracker.
+          */
+         return USWindowCreate(up, NULL, overlays, nWindows);
+      }
+   }
+#endif // ifndef NO_XCOMPOSITE
+
+   return NULL;
+}
+
 /*
  *
  * End file-scope functions.