Single threaded stdio optimization

Locking overhead can be significant in some stdio operations
that are common in single threaded applications.

This patch adds the _IO_FLAGS2_NEED_LOCK flag to indicate if
an _IO_FILE object needs to be locked and some of the stdio
functions just jump to their _unlocked variant when not.  The
flag is set on all _IO_FILE objects when the first thread is
created.  A new GLIBC_PRIVATE libc symbol, _IO_enable_locks,
was added to do this from libpthread.

The optimization can be applied to more stdio functions,
currently it is only applied to single flag check or single
non-wide-char standard operations.  The flag should probably
be never set for files with _IO_USER_LOCK, but that's just a
further optimization, not a correctness requirement.

The optimization is valid in a single thread because stdio
operations are non-as-safe (so lock state is not observable
from a signal handler) and stdio locks are recursive (so lock
state is not observable via deadlock).  The optimization is not
valid if a thread may be created while an stdio lock is taken
and thus it should be disabled if any user code may run during
an stdio operation (interposed malloc, printf hooks, etc).
This makes the optimization more complicated for some stdio
operations (e.g. printf), but those are bigger and thus less
important to optimize so this patch does not try to do that.

	* libio/libio.h (_IO_FLAGS2_NEED_LOCK, _IO_need_lock): Define.
	* libio/libioP.h (_IO_enable_locks): Declare.
	* libio/Versions (_IO_enable_locks): New symbol.
	* libio/genops.c (_IO_enable_locks): Define.
	(_IO_old_init): Initialize flags2.
	* libio/feof.c.c (_IO_feof): Avoid locking when not needed.
	* libio/ferror.c (_IO_ferror): Likewise.
	* libio/fputc.c (fputc): Likewise.
	* libio/putc.c (_IO_putc): Likewise.
	* libio/getc.c (_IO_getc): Likewise.
	* libio/getchar.c (getchar): Likewise.
	* libio/ioungetc.c (_IO_ungetc): Likewise.
	* nptl/pthread_create.c (__pthread_create_2_1): Enable stdio locks.
	* libio/iofopncook.c (_IO_fopencookie): Enable locking for the file.
	* sysdeps/pthread/flockfile.c (__flockfile): Likewise.

diff --git a/ChangeLog b/ChangeLog
index 314c69f6645e7723a233e79bc68770155a306f29..7f67b658f3e26f0f5eb74a28e40753c2def92885 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,21 @@
+2017-07-04  Szabolcs Nagy  <szabolcs.nagy@arm.com>
+
+       * libio/libio.h (_IO_FLAGS2_NEED_LOCK, _IO_need_lock): Define.
+       * libio/libioP.h (_IO_enable_locks): Declare.
+       * libio/Versions (_IO_enable_locks): New symbol.
+       * libio/genops.c (_IO_enable_locks): Define.
+       (_IO_old_init): Initialize flags2.
+       * libio/feof.c.c (_IO_feof): Avoid locking when not needed.
+       * libio/ferror.c (_IO_ferror): Likewise.
+       * libio/fputc.c (fputc): Likewise.
+       * libio/putc.c (_IO_putc): Likewise.
+       * libio/getc.c (_IO_getc): Likewise.
+       * libio/getchar.c (getchar): Likewise.
+       * libio/ioungetc.c (_IO_ungetc): Likewise.
+       * nptl/pthread_create.c (__pthread_create_2_1): Enable stdio locks.
+       * libio/iofopncook.c (_IO_fopencookie): Enable locking for the file.
+       * sysdeps/pthread/flockfile.c (__flockfile): Likewise.
+
 2017-07-04  Florian Weimer  <fweimer@redhat.com>
 
        [BZ #21542]

diff --git a/libio/Versions b/libio/Versions
index 2a1d6e6c8524da7eaf42c3aec854c00bb8b9f742..77123347e3cb19f57cbf4f16f76f5e80114c58a1 100644 (file)
--- a/libio/Versions
+++ b/libio/Versions
@@ -155,5 +155,8 @@ libc {
   GLIBC_PRIVATE {
     # Used by NPTL and librt
     __libc_fatal;
+
+    # Used by NPTL
+    _IO_enable_locks;
   }
 }

diff --git a/libio/feof.c b/libio/feof.c
index 9712a81e783f9c5ae8dfaa0149dfe4856c2f3b43..8890a5f51fe9ad4d440802a4a6992758ddf7b2f2 100644 (file)
--- a/libio/feof.c
+++ b/libio/feof.c
@@ -32,6 +32,8 @@ _IO_feof (_IO_FILE *fp)
 {
   int result;
   CHECK_FILE (fp, EOF);
+  if (!_IO_need_lock (fp))
+    return _IO_feof_unlocked (fp);
   _IO_flockfile (fp);
   result = _IO_feof_unlocked (fp);
   _IO_funlockfile (fp);

diff --git a/libio/ferror.c b/libio/ferror.c
index 01e3bd8e2b8b81f507fca7430620196b52968e49..d10fcd9fff2d12334d6d5ec434cfa4acc2d796f9 100644 (file)
--- a/libio/ferror.c
+++ b/libio/ferror.c
@@ -32,6 +32,8 @@ _IO_ferror (_IO_FILE *fp)
 {
   int result;
   CHECK_FILE (fp, EOF);
+  if (!_IO_need_lock (fp))
+    return _IO_ferror_unlocked (fp);
   _IO_flockfile (fp);
   result = _IO_ferror_unlocked (fp);
   _IO_funlockfile (fp);

diff --git a/libio/fputc.c b/libio/fputc.c
index a7cd682fe2b8020ea16a92e197209bb691c34e13..b72305c06f17d525354561f7a870173f7ce76d5e 100644 (file)
--- a/libio/fputc.c
+++ b/libio/fputc.c
@@ -32,6 +32,8 @@ fputc (int c, _IO_FILE *fp)
 {
   int result;
   CHECK_FILE (fp, EOF);
+  if (!_IO_need_lock (fp))
+    return _IO_putc_unlocked (c, fp);
   _IO_acquire_lock (fp);
   result = _IO_putc_unlocked (c, fp);
   _IO_release_lock (fp);

diff --git a/libio/genops.c b/libio/genops.c
index a466cfa3370601b60160244756fdb210068f61f5..6ad7346cae5c169d638dabac86291447ef666386 100644 (file)
--- a/libio/genops.c
+++ b/libio/genops.c
@@ -570,11 +570,39 @@ _IO_init (_IO_FILE *fp, int flags)
   _IO_init_internal (fp, flags);
 }
 
+static int stdio_needs_locking;
+
+/* In a single-threaded process most stdio locks can be omitted.  After
+   _IO_enable_locks is called, locks are not optimized away any more.
+   It must be first called while the process is still single-threaded.
+
+   This lock optimization can be disabled on a per-file basis by setting
+   _IO_FLAGS2_NEED_LOCK, because a file can have user-defined callbacks
+   or can be locked with flockfile and then a thread may be created
+   between a lock and unlock, so omitting the lock is not valid.
+
+   Here we have to make sure that the flag is set on all existing files
+   and files created later.  */
+void
+_IO_enable_locks (void)
+{
+  _IO_ITER i;
+
+  if (stdio_needs_locking)
+    return;
+  stdio_needs_locking = 1;
+  for (i = _IO_iter_begin (); i != _IO_iter_end (); i = _IO_iter_next (i))
+    _IO_iter_file (i)->_flags2 |= _IO_FLAGS2_NEED_LOCK;
+}
+libc_hidden_def (_IO_enable_locks)
+
 void
 _IO_old_init (_IO_FILE *fp, int flags)
 {
   fp->_flags = _IO_MAGIC|flags;
   fp->_flags2 = 0;
+  if (stdio_needs_locking)
+    fp->_flags2 |= _IO_FLAGS2_NEED_LOCK;
   fp->_IO_buf_base = NULL;
   fp->_IO_buf_end = NULL;
   fp->_IO_read_base = NULL;

diff --git a/libio/getc.c b/libio/getc.c
index b58fd6230867e6af2d1828e42ef5450d34e84af8..fd66ef93cf49beb8a37da7184d914b6091d5dac6 100644 (file)
--- a/libio/getc.c
+++ b/libio/getc.c
@@ -34,6 +34,8 @@ _IO_getc (FILE *fp)
 {
   int result;
   CHECK_FILE (fp, EOF);
+  if (!_IO_need_lock (fp))
+    return _IO_getc_unlocked (fp);
   _IO_acquire_lock (fp);
   result = _IO_getc_unlocked (fp);
   _IO_release_lock (fp);

diff --git a/libio/getchar.c b/libio/getchar.c
index 5b41595d17f2b858d5f344ef0fb65e3753b658ad..d79932114ee7d7e9ac4f3f7d2d64fd82c3d70773 100644 (file)
--- a/libio/getchar.c
+++ b/libio/getchar.c
@@ -33,6 +33,8 @@ int
 getchar (void)
 {
   int result;
+  if (!_IO_need_lock (_IO_stdin))
+    return _IO_getc_unlocked (_IO_stdin);
   _IO_acquire_lock (_IO_stdin);
   result = _IO_getc_unlocked (_IO_stdin);
   _IO_release_lock (_IO_stdin);

diff --git a/libio/iofopncook.c b/libio/iofopncook.c
index a08dfdaa42502775a42e620329d8c0b604bb5bfc..982f464a685e0eafea92684e619421dd95ec36fe 100644 (file)
--- a/libio/iofopncook.c
+++ b/libio/iofopncook.c
@@ -172,6 +172,8 @@ _IO_cookie_init (struct _IO_cookie_file *cfile, int read_write,
   _IO_mask_flags (&cfile->__fp.file, read_write,
                  _IO_NO_READS+_IO_NO_WRITES+_IO_IS_APPENDING);
 
+  cfile->__fp.file._flags2 |= _IO_FLAGS2_NEED_LOCK;
+
   /* We use a negative number different from -1 for _fileno to mark that
      this special stream is not associated with a real file, but still has
      to be treated as such.  */

diff --git a/libio/ioungetc.c b/libio/ioungetc.c
index 951064fa1275cb4967bc92116f1201f1d33cc10f..917cad8abb27e8ab5da687dff7f06564c382924a 100644 (file)
--- a/libio/ioungetc.c
+++ b/libio/ioungetc.c
@@ -33,6 +33,8 @@ _IO_ungetc (int c, _IO_FILE *fp)
   CHECK_FILE (fp, EOF);
   if (c == EOF)
     return EOF;
+  if (!_IO_need_lock (fp))
+    return _IO_sputbackc (fp, (unsigned char) c);
   _IO_acquire_lock (fp);
   result = _IO_sputbackc (fp, (unsigned char) c);
   _IO_release_lock (fp);

diff --git a/libio/libio.h b/libio/libio.h
index 518ffd8e44fb70059c96abfe7ce6418275c1eee8..14bcb92332c391212fdc3e756bcf038e3ebe3a50 100644 (file)
--- a/libio/libio.h
+++ b/libio/libio.h
@@ -119,6 +119,7 @@
 # define _IO_FLAGS2_SCANF_STD 16
 # define _IO_FLAGS2_NOCLOSE 32
 # define _IO_FLAGS2_CLOEXEC 64
+# define _IO_FLAGS2_NEED_LOCK 128
 #endif
 
 /* These are "formatting flags" matching the iostream fmtflags enum values. */
@@ -451,6 +452,9 @@ extern int _IO_ftrylockfile (_IO_FILE *) __THROW;
 #define _IO_cleanup_region_end(_Doit) /**/
 #endif
 
+#define _IO_need_lock(_fp) \
+  (((_fp)->_flags2 & _IO_FLAGS2_NEED_LOCK) != 0)
+
 extern int _IO_vfscanf (_IO_FILE * __restrict, const char * __restrict,
                        _IO_va_list, int *__restrict);
 extern int _IO_vfprintf (_IO_FILE *__restrict, const char *__restrict,

diff --git a/libio/libioP.h b/libio/libioP.h
index eb93418c8d3373353fa0208e8c2c97e8d61412ca..1832b44cc7f009aa9e72af6e3a6365e49d747a38 100644 (file)
--- a/libio/libioP.h
+++ b/libio/libioP.h
@@ -444,6 +444,8 @@ extern void _IO_list_unlock (void) __THROW;
 libc_hidden_proto (_IO_list_unlock)
 extern void _IO_list_resetlock (void) __THROW;
 libc_hidden_proto (_IO_list_resetlock)
+extern void _IO_enable_locks (void) __THROW;
+libc_hidden_proto (_IO_enable_locks)
 
 /* Default jumptable functions. */
 

diff --git a/libio/putc.c b/libio/putc.c
index b591c5538b87542eb29d76363af9665ed5fc2b96..6e1fdeef3a22b800412f762bf55a4c363be8f99b 100644 (file)
--- a/libio/putc.c
+++ b/libio/putc.c
@@ -25,6 +25,8 @@ _IO_putc (int c, _IO_FILE *fp)
 {
   int result;
   CHECK_FILE (fp, EOF);
+  if (!_IO_need_lock (fp))
+    return _IO_putc_unlocked (c, fp);
   _IO_acquire_lock (fp);
   result = _IO_putc_unlocked (c, fp);
   _IO_release_lock (fp);

diff --git a/nptl/pthread_create.c b/nptl/pthread_create.c
index 7a970ffc5bc6123bbad7d2e38e974ffd2bf859f9..2f8ada34d6070880c88f14a0091d0da07d6ca4c1 100644 (file)
--- a/nptl/pthread_create.c
+++ b/nptl/pthread_create.c
@@ -32,6 +32,7 @@
 #include <exit-thread.h>
 #include <default-sched.h>
 #include <futex-internal.h>
+#include "libioP.h"
 
 #include <shlib-compat.h>
 
@@ -756,6 +757,9 @@ __pthread_create_2_1 (pthread_t *newthread, const pthread_attr_t *attr,
         collect_default_sched (pd);
     }
 
+  if (__glibc_unlikely (__nptl_nthreads == 1))
+    _IO_enable_locks ();
+
   /* Pass the descriptor to the caller.  */
   *newthread = (pthread_t) pd;
 

diff --git a/sysdeps/pthread/flockfile.c b/sysdeps/pthread/flockfile.c
index 7fe8e991616b0e7e174a89e204c12e2a9b6ec55e..a8e6c28ed94d8ed8e1ac59ed1f6276051650aea3 100644 (file)
--- a/sysdeps/pthread/flockfile.c
+++ b/sysdeps/pthread/flockfile.c
@@ -25,6 +25,7 @@
 void
 __flockfile (FILE *stream)
 {
+  stream->_flags2 |= _IO_FLAGS2_NEED_LOCK;
   _IO_lock_lock (*stream->_lock);
 }
 strong_alias (__flockfile, _IO_flockfile)