From: Witold Kręcicki Date: Tue, 5 Nov 2019 21:28:50 +0000 (-0800) Subject: implement fetch-and-add array queue data structure X-Git-Tag: v9.15.6~21^2~12 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=402969bf950507e9a4da81749db6dd417fb804f5;p=thirdparty%2Fbind9.git implement fetch-and-add array queue data structure this is a lockless queue based on hazard pointers. --- diff --git a/config.h.in b/config.h.in index 5f66011d538..88f46849b8d 100644 --- a/config.h.in +++ b/config.h.in @@ -366,6 +366,9 @@ /* define if struct stat has st_mtim.tv_nsec field */ #undef HAVE_STAT_NSEC +/* Define to 1 if you have the header file. */ +#undef HAVE_STDALIGN_H + /* Define to 1 if you have the header file. */ #undef HAVE_STDATOMIC_H diff --git a/configure b/configure index cd2b3540d56..fbbc4b6ec8d 100755 --- a/configure +++ b/configure @@ -848,6 +848,7 @@ infodir docdir oldincludedir includedir +runstatedir localstatedir sharedstatedir sysconfdir @@ -1018,6 +1019,7 @@ datadir='${datarootdir}' sysconfdir='${prefix}/etc' sharedstatedir='${prefix}/com' localstatedir='${prefix}/var' +runstatedir='${localstatedir}/run' includedir='${prefix}/include' oldincludedir='/usr/include' docdir='${datarootdir}/doc/${PACKAGE_TARNAME}' @@ -1270,6 +1272,15 @@ do | -silent | --silent | --silen | --sile | --sil) silent=yes ;; + -runstatedir | --runstatedir | --runstatedi | --runstated \ + | --runstate | --runstat | --runsta | --runst | --runs \ + | --run | --ru | --r) + ac_prev=runstatedir ;; + -runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \ + | --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \ + | --run=* | --ru=* | --r=*) + runstatedir=$ac_optarg ;; + -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb) ac_prev=sbindir ;; -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \ @@ -1407,7 +1418,7 @@ fi for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \ datadir sysconfdir sharedstatedir localstatedir includedir \ oldincludedir docdir infodir htmldir dvidir pdfdir psdir \ - libdir localedir mandir + libdir localedir mandir runstatedir do eval ac_val=\$$ac_var # Remove trailing slashes. @@ -1560,6 +1571,7 @@ Fine tuning of the installation directories: --sysconfdir=DIR read-only single-machine data [PREFIX/etc] --sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com] --localstatedir=DIR modifiable single-machine data [PREFIX/var] + --runstatedir=DIR modifiable per-process data [LOCALSTATEDIR/run] --libdir=DIR object code libraries [EPREFIX/lib] --includedir=DIR C header files [PREFIX/include] --oldincludedir=DIR C header files for non-gcc [/usr/include] @@ -4000,7 +4012,7 @@ else We can't simply define LARGE_OFF_T to be 9223372036854775807, since some C++ compilers masquerading as C compilers incorrectly reject 9223372036854775807. */ -#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62)) +#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31)) int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721 && LARGE_OFF_T % 2147483647 == 1) ? 1 : -1]; @@ -4046,7 +4058,7 @@ else We can't simply define LARGE_OFF_T to be 9223372036854775807, since some C++ compilers masquerading as C compilers incorrectly reject 9223372036854775807. */ -#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62)) +#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31)) int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721 && LARGE_OFF_T % 2147483647 == 1) ? 1 : -1]; @@ -4070,7 +4082,7 @@ rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext We can't simply define LARGE_OFF_T to be 9223372036854775807, since some C++ compilers masquerading as C compilers incorrectly reject 9223372036854775807. */ -#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62)) +#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31)) int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721 && LARGE_OFF_T % 2147483647 == 1) ? 1 : -1]; @@ -4115,7 +4127,7 @@ else We can't simply define LARGE_OFF_T to be 9223372036854775807, since some C++ compilers masquerading as C compilers incorrectly reject 9223372036854775807. */ -#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62)) +#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31)) int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721 && LARGE_OFF_T % 2147483647 == 1) ? 1 : -1]; @@ -4139,7 +4151,7 @@ rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext We can't simply define LARGE_OFF_T to be 9223372036854775807, since some C++ compilers masquerading as C compilers incorrectly reject 9223372036854775807. */ -#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62)) +#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31)) int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721 && LARGE_OFF_T % 2147483647 == 1) ? 1 : -1]; @@ -19672,6 +19684,19 @@ done LIBS="$LIBS $ISC_ATOMIC_LIBS" +for ac_header in stdalign.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "stdalign.h" "ac_cv_header_stdalign_h" "$ac_includes_default" +if test "x$ac_cv_header_stdalign_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_STDALIGN_H 1 +_ACEOF + +fi + +done + + for ac_header in uchar.h do : ac_fn_c_check_header_mongrel "$LINENO" "uchar.h" "ac_cv_header_uchar_h" "$ac_includes_default" diff --git a/configure.ac b/configure.ac index 385d715182c..dfdecb192af 100644 --- a/configure.ac +++ b/configure.ac @@ -1796,6 +1796,8 @@ AC_CHECK_HEADERS( ]) LIBS="$LIBS $ISC_ATOMIC_LIBS" +AC_CHECK_HEADERS([stdalign.h]) + AC_CHECK_HEADERS([uchar.h]) # diff --git a/lib/isc/Makefile.in b/lib/isc/Makefile.in index 4006606263e..d30031faeb2 100644 --- a/lib/isc/Makefile.in +++ b/lib/isc/Makefile.in @@ -55,9 +55,9 @@ OBJS = pk11.@O@ pk11_result.@O@ \ lex.@O@ lfsr.@O@ lib.@O@ log.@O@ \ md.@O@ mem.@O@ mutexblock.@O@ \ netaddr.@O@ netscope.@O@ nonce.@O@ openssl_shim.@O@ pool.@O@ \ - parseint.@O@ portset.@O@ quota.@O@ radix.@O@ random.@O@ \ - ratelimiter.@O@ region.@O@ regex.@O@ result.@O@ \ - rwlock.@O@ \ + parseint.@O@ portset.@O@ queue.@O@ quota.@O@ \ + radix.@O@ random.@O@ ratelimiter.@O@ \ + region.@O@ regex.@O@ result.@O@ rwlock.@O@ \ serial.@O@ siphash.@O@ sockaddr.@O@ stats.@O@ \ string.@O@ symtab.@O@ task.@O@ taskpool.@O@ \ tm.@O@ timer.@O@ version.@O@ \ @@ -74,7 +74,7 @@ SRCS = pk11.c pk11_result.c \ lex.c lfsr.c lib.c log.c \ md.c mem.c mutexblock.c \ netaddr.c netscope.c nonce.c openssl_shim.c pool.c \ - parseint.c portset.c quota.c radix.c random.c \ + parseint.c portset.c queue.c quota.c radix.c random.c \ ratelimiter.c region.c regex.c result.c rwlock.c \ serial.c siphash.c sockaddr.c stats.c string.c \ symtab.c task.c taskpool.c timer.c \ diff --git a/lib/isc/include/isc/queue.h b/lib/isc/include/isc/queue.h index a23f7c5da07..6e3b8a3e4e8 100644 --- a/lib/isc/include/isc/queue.h +++ b/lib/isc/include/isc/queue.h @@ -9,7 +9,46 @@ * information regarding copyright ownership. */ -#ifndef ISC_QUEUE_H -#define ISC_QUEUE_H 1 +#pragma once +#include -#endif /* ISC_QUEUE_H */ +typedef struct isc_queue isc_queue_t; + +isc_queue_t * +isc_queue_new(isc_mem_t *mctx, int max_threads); +/*%< + * Create a new fetch-and-add array queue. + * + * 'max_threads' is currently unused. In the future it can be used + * to pass a maximum threads parameter when creating hazard pointers, + * but currently `isc_hp_t` uses a hard-coded value. + */ + +void +isc_queue_enqueue(isc_queue_t *queue, uintptr_t item); +/*%< + * Enqueue an object pointer 'item' at the tail of the queue. + * + * Requires: + * \li 'item' is not null. + */ + +uintptr_t +isc_queue_dequeue(isc_queue_t *queue); +/*%< + * Remove an object pointer from the head of the queue and return the + * pointer. If the queue is empty, return `nulluintptr` (the uintptr_t + * representation of NULL). + * + * Requires: + * \li 'queue' is not null. + */ + +void +isc_queue_destroy(isc_queue_t *queue); +/*%< + * Destroy a queue. + * + * Requires: + * \li 'queue' is not null. + */ diff --git a/lib/isc/queue.c b/lib/isc/queue.c new file mode 100644 index 00000000000..17290799067 --- /dev/null +++ b/lib/isc/queue.c @@ -0,0 +1,219 @@ +/* + * Copyright (C) Internet Systems Consortium, Inc. ("ISC") + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + * + * See the COPYRIGHT file distributed with this work for additional + * information regarding copyright ownership. + */ + +#include + +#include +#include +#include +#include +#include +#include + +#define BUFFER_SIZE 1024 + +#define MAX_THREADS 128 + +static uintptr_t nulluintptr = (uintptr_t)NULL; + +typedef struct node { + atomic_uint_fast32_t deqidx; + atomic_uintptr_t items[BUFFER_SIZE]; + atomic_uint_fast32_t enqidx; + atomic_uintptr_t next; + isc_mem_t *mctx; +} node_t; + +/* we just need one Hazard Pointer */ +#define HP_TAIL 0 +#define HP_HEAD 0 + +struct isc_queue { + alignas(128) atomic_uintptr_t head; + alignas(128) atomic_uintptr_t tail; + isc_mem_t *mctx; + int max_threads; + int taken; + isc_hp_t *hp; +}; + +static node_t * +node_new(isc_mem_t *mctx, uintptr_t item) { + node_t *node = isc_mem_get(mctx, sizeof(*node)); + *node = (node_t){ + .mctx = NULL + }; + + atomic_init(&node->deqidx, 0); + atomic_init(&node->enqidx, 1); + atomic_init(&node->next, 0); + atomic_init(&node->items[0], item); + + for (int i = 1; i < BUFFER_SIZE; i++) { + atomic_init(&node->items[i], 0); + } + + isc_mem_attach(mctx, &node->mctx); + + return (node); +} + +static void +node_destroy(void *node0) { + node_t *node = (node_t *)node0; + + isc_mem_putanddetach(&node->mctx, node, sizeof(*node)); +} + +static bool +node_cas_next(node_t *node, node_t *cmp, const node_t *val) { + return (atomic_compare_exchange_strong(&node->next, + (uintptr_t *)&cmp, + (uintptr_t)val)); +} + +static bool +queue_cas_tail(isc_queue_t *queue, node_t *cmp, const node_t *val) { + return (atomic_compare_exchange_strong(&queue->tail, + (uintptr_t *)&cmp, + (uintptr_t)val)); +} + +static bool +queue_cas_head(isc_queue_t *queue, node_t *cmp, const node_t *val) { + return (atomic_compare_exchange_strong(&queue->head, + (uintptr_t *)&cmp, + (uintptr_t)val)); +} + +isc_queue_t * +isc_queue_new(isc_mem_t *mctx, int max_threads) { + isc_queue_t *queue = isc_mem_get(mctx, sizeof(*queue)); + node_t *sentinel = node_new(mctx, nulluintptr); + + if (max_threads == 0) { + max_threads = MAX_THREADS; + } + + *queue = (isc_queue_t){ + .max_threads = max_threads, + }; + + isc_mem_attach(mctx, &queue->mctx); + + queue->hp = isc_hp_new(mctx, 1, node_destroy); + + atomic_init(&sentinel->enqidx, 0); + atomic_init(&queue->head, (uintptr_t)sentinel); + atomic_init(&queue->tail, (uintptr_t)sentinel); + + return (queue); +} + +void +isc_queue_enqueue(isc_queue_t *queue, uintptr_t item) { + REQUIRE(item != nulluintptr); + + while (true) { + node_t *lt = NULL; + uint_fast32_t idx; + uintptr_t n = nulluintptr; + + lt = (node_t *)isc_hp_protect(queue->hp, 0, &queue->tail); + idx = atomic_fetch_add(<->enqidx, 1); + if (idx > BUFFER_SIZE-1) { + node_t *lnext = NULL; + + if (lt != (node_t *)atomic_load(&queue->tail)) { + continue; + } + + lnext = (node_t *)atomic_load(<->next); + if (lnext == NULL) { + node_t *newnode = node_new(queue->mctx, item); + if (node_cas_next(lt, NULL, newnode)) { + queue_cas_tail(queue, lt, newnode); + isc_hp_clear(queue->hp); + return; + } + node_destroy(newnode); + } else { + queue_cas_tail(queue, lt, lnext); + } + + continue; + } + + if (atomic_compare_exchange_strong(<->items[idx], &n, item)) { + isc_hp_clear(queue->hp); + return; + } + } +} + +uintptr_t +isc_queue_dequeue(isc_queue_t *queue) { + REQUIRE(queue != NULL); + + while (true) { + node_t *lh = NULL; + uint_fast32_t idx; + uintptr_t item; + + lh = (node_t *)isc_hp_protect(queue->hp, 0, &queue->head); + if (atomic_load(&lh->deqidx) >= atomic_load(&lh->enqidx) && + atomic_load(&lh->next) == nulluintptr) + { + break; + } + + idx = atomic_fetch_add(&lh->deqidx, 1); + if (idx > BUFFER_SIZE-1) { + node_t *lnext = (node_t *)atomic_load(&lh->next); + if (lnext == NULL) { + break; + } + if (queue_cas_head(queue, lh, lnext)) { + isc_hp_retire(queue->hp, (uintptr_t)lh); + } + + continue; + } + + item = atomic_exchange(&(lh->items[idx]), + (uintptr_t)&queue->taken); + if (item == nulluintptr) { + continue; + } + + isc_hp_clear(queue->hp); + return (item); + } + + isc_hp_clear(queue->hp); + return (nulluintptr); +} + +void +isc_queue_destroy(isc_queue_t *queue) { + node_t *last = NULL; + + REQUIRE(queue != NULL); + + while (isc_queue_dequeue(queue) != nulluintptr) { + /* do nothing */ + } + + last = (node_t *)atomic_load_relaxed(&queue->head); + node_destroy(last); + isc_hp_destroy(queue->hp); + isc_mem_putanddetach(&queue->mctx, queue, sizeof(*queue)); +} diff --git a/lib/isc/unix/include/isc/align.h b/lib/isc/unix/include/isc/align.h new file mode 100644 index 00000000000..9cadbec345c --- /dev/null +++ b/lib/isc/unix/include/isc/align.h @@ -0,0 +1,18 @@ +/* + * Copyright (C) Internet Systems Consortium, Inc. ("ISC") + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + * + * See the COPYRIGHT file distributed with this work for additional + * information regarding copyright ownership. + */ + +#pragma once + +#ifdef HAVE_STDALIGN_H +#include +#else +#define alignas(x) __attribute__ ((__aligned__ (x))) +#endif diff --git a/lib/isc/win32/include/isc/align.h b/lib/isc/win32/include/isc/align.h new file mode 100644 index 00000000000..d5b02a32b6d --- /dev/null +++ b/lib/isc/win32/include/isc/align.h @@ -0,0 +1,13 @@ +/* + * Copyright (C) Internet Systems Consortium, Inc. ("ISC") + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + * + * See the COPYRIGHT file distributed with this work for additional + * information regarding copyright ownership. + */ + +#pragma once +#define alignas(x) __declspec(align(x)) diff --git a/lib/isc/win32/libisc.def.in b/lib/isc/win32/libisc.def.in index db49db737a6..89a46daa1ab 100644 --- a/lib/isc/win32/libisc.def.in +++ b/lib/isc/win32/libisc.def.in @@ -452,6 +452,10 @@ isc_portset_isset isc_portset_nports isc_portset_remove isc_portset_removerange +isc_queue_enqueue +isc_queue_dequeue +isc_queue_destroy +isc_queue_new isc_quota_attach isc_quota_destroy isc_quota_detach diff --git a/lib/isc/win32/libisc.vcxproj.filters.in b/lib/isc/win32/libisc.vcxproj.filters.in index 98993ee9b28..36bfa472586 100644 --- a/lib/isc/win32/libisc.vcxproj.filters.in +++ b/lib/isc/win32/libisc.vcxproj.filters.in @@ -554,6 +554,9 @@ Library Source Files + + Library Source Files + Library Source Files diff --git a/lib/isc/win32/libisc.vcxproj.in b/lib/isc/win32/libisc.vcxproj.in index 332aa2aaf15..552443fd506 100644 --- a/lib/isc/win32/libisc.vcxproj.in +++ b/lib/isc/win32/libisc.vcxproj.in @@ -440,6 +440,7 @@ copy InstallFiles ..\Build\Release\ + diff --git a/util/copyrights b/util/copyrights index e07e9972b38..7f6dd8774c0 100644 --- a/util/copyrights +++ b/util/copyrights @@ -2270,6 +2270,7 @@ ./lib/isc/pthreads/include/isc/thread.h C 1998,1999,2000,2001,2004,2005,2007,2013,2016,2017,2018,2019 ./lib/isc/pthreads/mutex.c C 2000,2001,2002,2004,2005,2007,2008,2011,2012,2014,2015,2016,2018,2019 ./lib/isc/pthreads/thread.c C 2000,2001,2003,2004,2005,2007,2013,2016,2017,2018,2019 +./lib/isc/queue.c C 2019 ./lib/isc/quota.c C 2000,2001,2004,2005,2007,2016,2018,2019 ./lib/isc/radix.c C 2007,2008,2009,2011,2012,2013,2014,2015,2016,2018,2019 ./lib/isc/random.c C 1999,2000,2001,2002,2003,2004,2005,2007,2009,2013,2014,2016,2017,2018,2019 @@ -2331,6 +2332,7 @@ ./lib/isc/unix/file.c C 2000,2001,2002,2004,2005,2007,2009,2011,2012,2013,2014,2015,2016,2017,2018,2019 ./lib/isc/unix/fsaccess.c C 2000,2001,2004,2005,2006,2007,2016,2018,2019 ./lib/isc/unix/ifiter_getifaddrs.c C 2003,2004,2005,2007,2008,2009,2014,2016,2018,2019 +./lib/isc/unix/include/isc/align.h C 2019 ./lib/isc/unix/include/isc/dir.h C 1999,2000,2001,2004,2005,2007,2016,2018,2019 ./lib/isc/unix/include/isc/net.h C 1999,2000,2001,2002,2003,2004,2005,2007,2008,2012,2013,2014,2016,2017,2018,2019 ./lib/isc/unix/include/isc/netdb.h C 1999,2000,2001,2004,2005,2007,2016,2018,2019 @@ -2363,6 +2365,7 @@ ./lib/isc/win32/fsaccess.c C 2000,2001,2002,2004,2007,2013,2016,2017,2018,2019 ./lib/isc/win32/include/Makefile.in MAKE 1999,2000,2001,2004,2007,2012,2014,2016,2018,2019 ./lib/isc/win32/include/isc/Makefile.in MAKE 1999,2000,2001,2004,2007,2012,2013,2014,2015,2016,2018,2019 +./lib/isc/win32/include/isc/align.h C 2019 ./lib/isc/win32/include/isc/bind_registry.h C 2001,2004,2007,2016,2018,2019 ./lib/isc/win32/include/isc/bindevt.h C 2001,2004,2007,2016,2018,2019 ./lib/isc/win32/include/isc/condition.h C 1998,1999,2000,2001,2004,2007,2016,2018,2019