Tools Logging: fix overzealous redirecting to console on Windows

The change that introduced automatic redirect of standard output
and standard error streams to a console on Windows completely broke
use of "vmtoolsd --cmd <xxx>" in scripts: redirect to files stopped
working.

Change VMTools_AttachConsole() to check if stdout/stderr point to
an on-disk file or a pipe before redirecting them to CONOUT.

Signed-off-by: Marcelo Vanzin <mvanzin@vmware.com>

diff --git a/open-vm-tools/libvmtools/vmtoolsLog.c b/open-vm-tools/libvmtools/vmtoolsLog.c
index 9830810b0031c8f737cf1141404e0fea85dbb8ea..77c0972515693ab3199b00e27771d2e3d88f8e6a 100644 (file)
--- a/open-vm-tools/libvmtools/vmtoolsLog.c
+++ b/open-vm-tools/libvmtools/vmtoolsLog.c
@@ -643,6 +643,28 @@ VMToolsRestoreLogging(LogHandlerData *oldDefault,
 
 
 #if defined(_WIN32)
+
+
+/**
+ * Checks whether given standard device (standard input, standard output,
+ * or standard error) has been redirected to an on-disk file/pipe.
+ * Win32-only.
+ *
+ * @param[in] nStdHandle          The standard device number.
+ *
+ * @return TRUE if device redirected to a file/pipe.
+ */
+
+static gboolean
+VMToolsIsRedirected(DWORD nStdHandle)
+{
+   HANDLE handle = GetStdHandle(nStdHandle);
+   DWORD type = handle ? GetFileType(handle) : FILE_TYPE_UNKNOWN;
+
+   return type == FILE_TYPE_DISK || type == FILE_TYPE_PIPE;
+}
+
+
 /**
  * Attaches a console to the current process. If the parent process already has
  * a console open, reuse it. Otherwise, create a new console for the current
@@ -661,34 +683,45 @@ gboolean
 VMTools_AttachConsole(void)
 {
    typedef BOOL (WINAPI *AttachConsoleFn)(DWORD);
-   gboolean ret = FALSE;
+   gboolean ret = TRUE;
    AttachConsoleFn _AttachConsole;
+   BOOL reopenStdout, reopenStderr;
 
    if (GetConsoleWindow() != NULL) {
-      return TRUE;
+      goto exit;
+   }
+
+   reopenStdout = !VMToolsIsRedirected(STD_OUTPUT_HANDLE);
+   reopenStderr = !VMToolsIsRedirected(STD_ERROR_HANDLE);
+   if (!reopenStdout && !reopenStderr) {
+      goto exit;
    }
 
    _AttachConsole = (AttachConsoleFn) GetProcAddress(GetModuleHandleW(L"kernel32.dll"),
                                                      "AttachConsole");
    if ((_AttachConsole != NULL && _AttachConsole(ATTACH_PARENT_PROCESS)) ||
        AllocConsole()) {
-      FILE* fptr;
-
-      fptr = _wfreopen(L"CONOUT$", L"a", stdout);
-      if (fptr == NULL) {
-         g_warning("_wfreopen failed for stdout/CONOUT$: %d (%s)",
-                   errno, strerror(errno));
-         goto exit;
+      FILE *fptr;
+
+      if (reopenStdout) {
+         fptr = _wfreopen(L"CONOUT$", L"a", stdout);
+         if (fptr == NULL) {
+            g_warning("_wfreopen failed for stdout/CONOUT$: %d (%s)",
+                      errno, strerror(errno));
+            ret = FALSE;
+         }
       }
 
-      fptr = _wfreopen(L"CONOUT$", L"a", stderr);
-      if (fptr == NULL) {
-         g_warning("_wfreopen failed for stderr/CONOUT$: %d (%s)",
-                   errno, strerror(errno));
-         goto exit;
+      if (reopenStderr) {
+         fptr = _wfreopen(L"CONOUT$", L"a", stderr);
+         if (fptr == NULL) {
+            g_warning("_wfreopen failed for stderr/CONOUT$: %d (%s)",
+                      errno, strerror(errno));
+            ret = FALSE;
+         } else {
+            setvbuf(fptr, NULL, _IONBF, 0);
+         }
       }
-      setvbuf(fptr, NULL, _IONBF, 0);
-      ret = TRUE;
    }
 
 exit: