openat2: port O_TMPFILE check to non-GNU/Linux

* lib/openat2.c (openat2): Don’t assume !!(O_TMPFILE & O_DIRECTORY).
Although true on GNU/Linux, it is a hack and might not be true on
other systems such as Cygwin.

diff --git a/ChangeLog b/ChangeLog
index 3ccd074a4cb27233528948123eadc939025e73bc..b475160780f288cb95971fe891db2bc7407ae059 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,10 @@
 2025-10-28  Paul Eggert  <eggert@cs.ucla.edu>
 
+       openat2: port O_TMPFILE check to non-GNU/Linux
+       * lib/openat2.c (openat2): Don’t assume !!(O_TMPFILE & O_DIRECTORY).
+       Although true on GNU/Linux, it is a hack and might not be true on
+       other systems such as Cygwin.
+
        openat2-tests: port to FreeBSD, NetBSD
        Problem reported by Bruno Haible in:
        https://lists.gnu.org/r/bug-gnulib/2025-10/msg00117.html

diff --git a/lib/openat2.c b/lib/openat2.c
index ba5f579716ee31e6f2856d53d398b0bdabd3d7c9..ee64d058365c371bee32401371df46f7627dec19 100644 (file)
--- a/lib/openat2.c
+++ b/lib/openat2.c
@@ -512,13 +512,11 @@ openat2 (int dfd, char const *filename,
                        | O_NOLINK | O_NOLINKS | O_NONBLOCK | O_NOTRANS
                        | O_RSYNC | O_SYNC
                        | O_TEXT | O_TMPFILE | O_TRUNC | O_TTY_INIT))))
-           | ((how->flags & (O_DIRECTORY | O_CREAT))
-              == (O_DIRECTORY | O_CREAT))
+           | (!!(how->flags & O_CREAT)
+              & !!(how->flags & (O_DIRECTORY | O_TMPFILE)))
            | (!!(how->flags & O_TMPFILE & ~O_DIRECTORY)
-              & ((how->flags & (O_ACCMODE | O_DIRECTORY))
-                 != (O_WRONLY | O_DIRECTORY))
-              & ((how->flags & (O_ACCMODE | O_DIRECTORY))
-                 != (O_RDWR | O_DIRECTORY)))
+              & ((how->flags & (O_ACCMODE | O_PATH)) != O_WRONLY)
+              & ((how->flags & (O_ACCMODE | O_PATH)) != O_RDWR))
            | (how->mode
               & ~ (how->flags & (O_CREAT | (O_TMPFILE & ~O_DIRECTORY))
                    ? (S_ISUID | S_ISGID | S_ISVTX