xz: Create separate is_tty() function.

The new is_tty() will report if a file descriptor is a terminal or not.
On POSIX systems, it is a wrapper around isatty(). However, the native
Windows implementation of isatty() will return true for all character
devices, not just terminals. So is_tty() has a special case for Windows
so it can use alternative Windows API functions to determine if a file
descriptor is a terminal.

This fixes a bug with MSVC and MinGW-w64 builds that refused to read from
or write to non-terminal character devices because xz thought it was a
terminal. For instance:

    xz foo -c > /dev/null

would fail because /dev/null was assumed to be a terminal.

(cherry picked from commit 0ecfaa6df91f7c37510f370295f593b9c0b88b98)

diff --git a/src/xz/util.c b/src/xz/util.c
index a1339f4fdf3cf09f558a6af0b533db4fd3b65355..69e0cd3c63cede701525100d4e9cf9e4c3ca2f56 100644 (file)
--- a/src/xz/util.c
+++ b/src/xz/util.c
@@ -272,10 +272,31 @@ is_empty_filename(const char *filename)
 }
 
 
+extern bool
+is_tty(int fd)
+{
+#if defined(_WIN32) && !defined(__CYGWIN__)
+       // There is no need to check if handle == INVALID_HANDLE_VALUE
+       // because it will return false anyway when used in GetConsoleMode().
+       // The resulting HANDLE is owned by the file descriptor.
+       // The HANDLE must not be closed here.
+       intptr_t handle = _get_osfhandle(fd);
+       DWORD mode;
+
+       // GetConsoleMode() is an easy way to tell if the HANDLE is a
+       // console or not. We do not care about the value of mode since we
+       // do not plan to use any further Windows console functions.
+       return GetConsoleMode((HANDLE)handle, &mode);
+#else
+       return isatty(fd);
+#endif
+}
+
+
 extern bool
 is_tty_stdin(void)
 {
-       const bool ret = isatty(STDIN_FILENO);
+       const bool ret = is_tty(STDIN_FILENO);
 
        if (ret)
                message_error(_("Compressed data cannot be read from "
@@ -288,7 +309,7 @@ is_tty_stdin(void)
 extern bool
 is_tty_stdout(void)
 {
-       const bool ret = isatty(STDOUT_FILENO);
+       const bool ret = is_tty(STDOUT_FILENO);
 
        if (ret)
                message_error(_("Compressed data cannot be written to "

diff --git a/src/xz/util.h b/src/xz/util.h
index 7aa9caee6471880c9680f07429674c76d54ed2db..995c2444db4361caf18a8df3cd9234a5a730503d 100644 (file)
--- a/src/xz/util.h
+++ b/src/xz/util.h
@@ -109,6 +109,20 @@ extern void my_snprintf(char **pos, size_t *left, const char *fmt, ...);
 extern bool is_empty_filename(const char *filename);
 
 
+/// \brief      Test if file descriptor is a terminal
+///
+/// For POSIX systems, this is a simple wrapper around isatty(). However on
+/// Windows, isatty() returns true for all character devices, not just
+/// terminals.
+///
+/// \param      fd    File descriptor to test
+///
+/// \return     bool:
+///             - true if file descriptor is a terminal
+///             - false otherwise
+extern bool is_tty(int fd);
+
+
 /// \brief      Test if stdin is a terminal
 ///
 /// If stdin is a terminal, an error message is printed and exit status set