#include <unistd.h>
#include <sysdep-cancel.h>
+static inline ssize_t
+getrandom_syscall (void *buffer, size_t length, unsigned int flags,
+ bool cancel)
+{
+ return cancel
+ ? SYSCALL_CANCEL (getrandom, buffer, length, flags)
+ : INLINE_SYSCALL_CALL (getrandom, buffer, length, flags);
+}
+
+#ifdef HAVE_GETRANDOM_VSYSCALL
+# include <assert.h>
+# include <ldsodefs.h>
+# include <libc-lock.h>
+# include <list.h>
+# include <setvmaname.h>
+# include <sys/mman.h>
+# include <sys/sysinfo.h>
+# include <tls-internal.h>
+
+/* These values will be initialized at loading time by calling the
+ _dl_vdso_getrandom with a special value. The 'state_size' is the opaque
+ state size per-thread allocated with a mmap using 'mmap_prot' and
+ 'mmap_flags' argument. */
+static uint32_t state_size;
+static uint32_t state_size_cache_aligned;
+static uint32_t mmap_prot;
+static uint32_t mmap_flags;
+
+/* The function below are used on reentracy handling with (i.e. SA_NODEFER).
+ Before allocating a new state or issue the vDSO, atomically read the
+ current thread buffer, and if this is already reserved (is_reserved_ptr)
+ fallback to the syscall. Otherwise, reserve the buffer by atomically
+ setting the LSB of the opaque state pointer. The bit is cleared after the
+ vDSO is called, or before issuing the fallback syscall. */
+
+static inline void *reserve_ptr (void *p)
+{
+ return (void *) ((uintptr_t) (p) | 1UL);
+}
+
+static inline void *release_ptr (void *p)
+{
+ return (void *) ((uintptr_t) (p) & ~1UL);
+}
+
+static inline bool is_reserved_ptr (void *p)
+{
+ return (uintptr_t) (p) & 1UL;
+}
+
+static struct
+{
+ __libc_lock_define (, lock);
+
+ void **states; /* Queue of opaque states allocated with the kernel
+ provided flags and used on getrandom vDSO call. */
+ size_t len; /* Number of available free states in the queue. */
+ size_t total; /* Number of states allocated from the kernel. */
+ size_t cap; /* Total number of states that 'states' can hold before
+ needed to be resized. */
+} grnd_alloc = {
+ .lock = LLL_LOCK_INITIALIZER
+};
+
+static bool
+vgetrandom_get_state_alloc (void)
+{
+ /* Start by allocating one page for the opaque states. */
+ size_t block_size = ALIGN_UP (state_size_cache_aligned, GLRO(dl_pagesize));
+ size_t states_per_page = GLRO (dl_pagesize) / state_size_cache_aligned;
+ void *block = __mmap (NULL, GLRO(dl_pagesize), mmap_prot, mmap_flags, -1, 0);
+ if (block == MAP_FAILED)
+ return false;
+ __set_vma_name (block, block_size, " glibc: getrandom");
+
+ if (grnd_alloc.total + states_per_page > grnd_alloc.cap)
+ {
+ /* Use a new mmap instead of trying to mremap. It avoids a
+ potential multithread fork issue where fork is called just after
+ mremap returns but before assigning to the grnd_alloc.states,
+ thus making the its value invalid in the child. */
+ void *old_states = grnd_alloc.states;
+ size_t new_states_size = ALIGN_UP ((grnd_alloc.total + states_per_page)
+ * sizeof (*grnd_alloc.states),
+ GLRO(dl_pagesize));
+
+ /* There is no need to memcpy any opaque state information because
+ all the allocated opaque states are assigned to running threads
+ (meaning that if we iterate over them we can reconstruct the state
+ list). */
+ void **states = __mmap (NULL, new_states_size, PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+ if (states == MAP_FAILED)
+ {
+ __munmap (block, block_size);
+ return false;
+ }
+
+ /* Atomically replace the old state, so if a fork happens the child
+ process will see a consistent free state buffer. The size might
+ not be updated, but it does not really matter since the buffer is
+ always increased. */
+ grnd_alloc.states = states;
+ atomic_thread_fence_seq_cst ();
+ if (old_states != NULL)
+ __munmap (old_states, grnd_alloc.cap * sizeof (*grnd_alloc.states));
+
+ __set_vma_name (states, new_states_size, " glibc: getrandom states");
+ grnd_alloc.cap = new_states_size / sizeof (*grnd_alloc.states);
+ atomic_thread_fence_seq_cst ();
+ }
+
+ for (size_t i = 0; i < states_per_page; ++i)
+ {
+ /* There is no need to handle states that straddle a page because
+ we allocate only one page. */
+ grnd_alloc.states[i] = block;
+ block += state_size_cache_aligned;
+ }
+ /* Concurrent fork should not observe the previous pointer value. */
+ grnd_alloc.len = states_per_page;
+ grnd_alloc.total += states_per_page;
+ atomic_thread_fence_seq_cst ();
+
+ return true;
+}
+
+/* Allocate an opaque state for vgetrandom. If the grnd_alloc does not have
+ any, mmap() another page of them using the vgetrandom parameters. */
+static void *
+vgetrandom_get_state (void)
+{
+ void *state = NULL;
+
+ /* The signal blocking avoid the potential issue where _Fork() (which is
+ async-signal-safe) is called with the lock taken. The function is
+ called only once during thread lifetime, so the overhead should be
+ minimal. */
+ internal_sigset_t set;
+ internal_signal_block_all (&set);
+ __libc_lock_lock (grnd_alloc.lock);
+
+ if (grnd_alloc.len > 0 || vgetrandom_get_state_alloc ())
+ state = grnd_alloc.states[--grnd_alloc.len];
+
+ __libc_lock_unlock (grnd_alloc.lock);
+ internal_signal_restore_set (&set);
+
+ return state;
+}
+
+/* Returns true when vgetrandom is used successfully. Returns false if the
+ syscall fallback should be issued in the case the vDSO is not present, in
+ the case of reentrancy, or if any memory allocation fails. */
+static ssize_t
+getrandom_vdso (void *buffer, size_t length, unsigned int flags, bool cancel)
+{
+ if (__glibc_unlikely (state_size == 0))
+ return getrandom_syscall (buffer, length, flags, cancel);
+
+ struct pthread *self = THREAD_SELF;
+
+ void *state = atomic_load_relaxed (&self->getrandom_buf);
+ if (is_reserved_ptr (state))
+ return getrandom_syscall (buffer, length, flags, cancel);
+ atomic_store_relaxed (&self->getrandom_buf, reserve_ptr (state));
+ __atomic_signal_fence (__ATOMIC_ACQ_REL);
+
+ bool r = false;
+ if (state == NULL)
+ {
+ state = vgetrandom_get_state ();
+ if (state == NULL)
+ goto out;
+ }
+
+ /* Since the vDSO implementation does not issue the syscall with the
+ cancellation bridge (__syscall_cancel_arch), use GRND_NONBLOCK so there
+ is no potential unbounded blocking in the kernel. It should be a rare
+ situation, only at system startup when RNG is not initialized. */
+ ssize_t ret = GLRO (dl_vdso_getrandom) (buffer,
+ length,
+ flags | GRND_NONBLOCK,
+ state,
+ state_size);
+ if (INTERNAL_SYSCALL_ERROR_P (ret))
+ {
+ /* Fallback to the syscall if the kernel would block. */
+ int err = INTERNAL_SYSCALL_ERRNO (ret);
+ if (err == EAGAIN && !(flags & GRND_NONBLOCK))
+ goto out;
+
+ __set_errno (err);
+ ret = -1;
+ }
+ r = true;
+
+out:
+ __atomic_signal_fence (__ATOMIC_ACQ_REL);
+ atomic_store_relaxed (&self->getrandom_buf, state);
+ return r ? ret : getrandom_syscall (buffer, length, flags, cancel);
+}
+#endif
+
+void
+__getrandom_early_init (_Bool initial)
+{
+#ifdef HAVE_GETRANDOM_VSYSCALL
+ /* libcs loaded for audit modules, dlmopen, etc. fallback to syscall. */
+ if (initial && (GLRO (dl_vdso_getrandom) != NULL))
+ {
+ /* Used to query the vDSO for the required mmap flags and the opaque
+ per-thread state size. Defined by linux/random.h. */
+ struct vgetrandom_opaque_params
+ {
+ uint32_t size_of_opaque_state;
+ uint32_t mmap_prot;
+ uint32_t mmap_flags;
+ uint32_t reserved[13];
+ } params;
+ if (GLRO(dl_vdso_getrandom) (NULL, 0, 0, ¶ms, ~0UL) == 0)
+ {
+ /* Align each opaque state to L1 data cache size to avoid false
+ sharing. If the size can not be obtained, use the kernel
+ provided one. */
+ state_size = params.size_of_opaque_state;
+
+ long int ld1sz = __sysconf (_SC_LEVEL1_DCACHE_LINESIZE);
+ if (ld1sz <= 0)
+ ld1sz = 1;
+ state_size_cache_aligned = ALIGN_UP (state_size, ld1sz);
+ /* Do not enable vDSO if the required opaque state size is larger
+ than a page because we only allocate one page per time to hold
+ the states. */
+ if (state_size_cache_aligned > GLRO(dl_pagesize))
+ {
+ state_size = 0;
+ return;
+ }
+ mmap_prot = params.mmap_prot;
+ mmap_flags = params.mmap_flags;
+ }
+ }
+#endif
+}
+
+/* Re-add the state state from CURP on the free list. This function is
+ called after fork returns in the child, so no locking is required. */
+void
+__getrandom_reset_state (struct pthread *curp)
+{
+#ifdef HAVE_GETRANDOM_VSYSCALL
+ if (grnd_alloc.states == NULL || curp->getrandom_buf == NULL)
+ return;
+ assert (grnd_alloc.len < grnd_alloc.cap);
+ grnd_alloc.states[grnd_alloc.len++] = release_ptr (curp->getrandom_buf);
+ curp->getrandom_buf = NULL;
+#endif
+}
+
+/* Called when a thread terminates, and adds its random buffer back into the
+ allocator pool for use in a future thread. This is called by
+ pthread_create during thread termination, and after signal has been
+ blocked. */
+void
+__getrandom_vdso_release (struct pthread *curp)
+{
+#ifdef HAVE_GETRANDOM_VSYSCALL
+ if (curp->getrandom_buf == NULL)
+ return;
+
+ __libc_lock_lock (grnd_alloc.lock);
+ grnd_alloc.states[grnd_alloc.len++] = curp->getrandom_buf;
+ __libc_lock_unlock (grnd_alloc.lock);
+#endif
+}
+
+/* Reset the internal lock state in case another thread has locked while
+ this thread calls fork. The stale thread states will be handled by
+ reclaim_stacks which calls __getrandom_reset_state on each thread. */
+void
+__getrandom_fork_subprocess (void)
+{
+#ifdef HAVE_GETRANDOM_VSYSCALL
+ grnd_alloc.lock = LLL_LOCK_INITIALIZER;
+#endif
+}
+
+ssize_t
+__getrandom_nocancel (void *buffer, size_t length, unsigned int flags)
+{
+#ifdef HAVE_GETRANDOM_VSYSCALL
+ return getrandom_vdso (buffer, length, flags, false);
+#else
+ return getrandom_syscall (buffer, length, flags, false);
+#endif
+}
+
/* Write up to LENGTH bytes of randomness starting at BUFFER.
Return the number of bytes written, or -1 on error. */
ssize_t
__getrandom (void *buffer, size_t length, unsigned int flags)
{
- return SYSCALL_CANCEL (getrandom, buffer, length, flags);
+#ifdef HAVE_GETRANDOM_VSYSCALL
+ return getrandom_vdso (buffer, length, flags, true);
+#else
+ return getrandom_syscall (buffer, length, flags, true);
+#endif
}
libc_hidden_def (__getrandom)
weak_alias (__getrandom, getrandom)
projects / thirdparty / glibc.git / commitdiff
