From: Wouter Wijngaards Date: Mon, 26 Mar 2007 10:33:41 +0000 (+0000) Subject: config file settings for message cache. X-Git-Tag: release-0.2~31 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=0280e3446a0e1ce642fe0f3d061df43c18709b77;p=thirdparty%2Funbound.git config file settings for message cache. git-svn-id: file:///svn/unbound/trunk@196 be551aaa-1e26-0410-a405-d3ace91eadb9 --- diff --git a/daemon/daemon.c b/daemon/daemon.c index 53722540d..5cfea217b 100644 --- a/daemon/daemon.c +++ b/daemon/daemon.c @@ -48,6 +48,7 @@ #include "util/log.h" #include "util/config_file.h" #include "util/data/msgreply.h" +#include "util/storage/slabhash.h" #include "services/listen_dnsport.h" #include @@ -117,7 +118,7 @@ daemon_init() signal_handling_record(); checklock_start(); daemon->need_to_exit = 0; - daemon->msg_cache = lruhash_create(HASH_DEFAULT_STARTARRAY, + daemon->msg_cache = slabhash_create(4, HASH_DEFAULT_STARTARRAY, HASH_DEFAULT_MAXMEM, msgreply_sizefunc, query_info_compare, query_entry_delete, reply_info_delete, NULL); if(!daemon->msg_cache) { @@ -311,7 +312,7 @@ daemon_delete(struct daemon* daemon) if(!daemon) return; listening_ports_free(daemon->ports); - lruhash_delete(daemon->msg_cache); + slabhash_delete(daemon->msg_cache); alloc_clear(&daemon->superalloc); free(daemon->cwd); free(daemon->pidfile); diff --git a/daemon/daemon.h b/daemon/daemon.h index a52fbe739..022623f87 100644 --- a/daemon/daemon.h +++ b/daemon/daemon.h @@ -47,7 +47,7 @@ struct config_file; struct worker; struct listen_port; -struct lruhash; +struct slabhash; /** * Structure holding worker list. @@ -73,7 +73,7 @@ struct daemon { /** master allocation cache */ struct alloc_cache superalloc; /** the message cache, content is struct msgreply_entry* */ - struct lruhash* msg_cache; + struct slabhash* msg_cache; }; /** diff --git a/daemon/unbound.c b/daemon/unbound.c index 6ab21609e..197e5d074 100644 --- a/daemon/unbound.c +++ b/daemon/unbound.c @@ -44,6 +44,8 @@ #include "util/log.h" #include "daemon/daemon.h" #include "util/config_file.h" +#include "util/storage/slabhash.h" +#include "util/data/msgreply.h" #include #include #include @@ -80,6 +82,17 @@ apply_dir(struct daemon* daemon, struct config_file* cfg, int cmdline_verbose) fatal_exit("malloc failed"); } } + if(cfg->msg_cache_size != slabhash_get_size(daemon->msg_cache) || + cfg->msg_cache_slabs != daemon->msg_cache->size) { + slabhash_delete(daemon->msg_cache); + daemon->msg_cache = slabhash_create(cfg->msg_cache_slabs, + HASH_DEFAULT_STARTARRAY, cfg->msg_cache_size, + msgreply_sizefunc, query_info_compare, + query_entry_delete, reply_info_delete, NULL); + if(!daemon->msg_cache) { + fatal_exit("malloc failure updating config settings"); + } + } } /** Read existing pid from pidfile. */ diff --git a/daemon/worker.c b/daemon/worker.c index 03497f433..0806b9bc6 100644 --- a/daemon/worker.c +++ b/daemon/worker.c @@ -47,6 +47,7 @@ #include "daemon/daemon.h" #include "util/netevent.h" #include "util/config_file.h" +#include "util/storage/slabhash.h" #include "services/listen_dnsport.h" #include "services/outside_network.h" @@ -159,7 +160,7 @@ worker_handle_reply(struct comm_point* c, void* arg, int error, log_err("out of memory"); return 0; } - lruhash_insert(worker->daemon->msg_cache, worker->query_hash, + slabhash_insert(worker->daemon->msg_cache, worker->query_hash, &e->entry, rep); return 0; } @@ -290,7 +291,7 @@ worker_handle_request(struct comm_point* c, void* arg, int error, return 1; } h = query_info_hash(&worker->qinfo); - if((e=lruhash_lookup(worker->daemon->msg_cache, h, &worker->qinfo, + if((e=slabhash_lookup(worker->daemon->msg_cache, h, &worker->qinfo, 0))) { /* answer from cache */ log_info("answer from the cache"); diff --git a/doc/Changelog b/doc/Changelog index f6727a0c1..8ce4df11d 100644 --- a/doc/Changelog +++ b/doc/Changelog @@ -1,3 +1,6 @@ +26 March 2007: Wouter + - config settings for slab hash message cache. + 23 March 2007: Wouter - review of yesterday's commits. - covered up memory leak of the entry locks. diff --git a/doc/example.conf b/doc/example.conf index 81869f03d..84d633d24 100644 --- a/doc/example.conf +++ b/doc/example.conf @@ -36,6 +36,14 @@ server: # But also takes more system resources (for open sockets). # outgoing-range: 16 + # the amount of memory to use for the message cache. + # in bytes. default is 4 Mb + # msg-cache-size: 4194304 + + # the number of slabs to use for the message cache. + # more slabs reduce lock contention, but fracture memory usage. + # msg-cache-slabs: 4 + # Enable IPv4, "yes" or "no". # do-ip4: yes diff --git a/doc/unbound.conf.5 b/doc/unbound.conf.5 index bafc6f8a6..7fa0929e7 100644 --- a/doc/unbound.conf.5 +++ b/doc/unbound.conf.5 @@ -63,6 +63,12 @@ Number of ports to open. This number is opened per thread for every outgoing query interface. Must be at least 1. Default is 16. Larger numbers give more protection against spoofing attempts, but need extra resources from the operating system. +.It \fBmsg-cache-size:\fR +Number of bytes size of the message cache. Default is 4 megabytes. +.It \fBmsg-cache-slabs:\fR +Number of slabs in the message cache. Slabs reduce lock contention by threads. +Must be set to a power of 2. Setting (close) to the number of cpus is a +reasonable guess. .It \fBdo-ip4:\fR Enable or disable whether ip4 queries are answered. Default is yes. .It \fBdo-ip6:\fR diff --git a/testcode/unitmain.c b/testcode/unitmain.c index 479023507..833af9265 100644 --- a/testcode/unitmain.c +++ b/testcode/unitmain.c @@ -97,6 +97,26 @@ net_test() unit_assert( !str_is_ip6("213.154.224.12") ); unit_assert( !str_is_ip6("213.154.224.255") ); unit_assert( !str_is_ip6("255.255.255.0") ); + unit_assert( is_pow2(0) ); + unit_assert( is_pow2(1) ); + unit_assert( is_pow2(2) ); + unit_assert( is_pow2(4) ); + unit_assert( is_pow2(8) ); + unit_assert( is_pow2(16) ); + unit_assert( is_pow2(1024) ); + unit_assert( is_pow2(1024*1024) ); + unit_assert( is_pow2(1024*1024*1024) ); + unit_assert( !is_pow2(3) ); + unit_assert( !is_pow2(5) ); + unit_assert( !is_pow2(6) ); + unit_assert( !is_pow2(7) ); + unit_assert( !is_pow2(9) ); + unit_assert( !is_pow2(10) ); + unit_assert( !is_pow2(11) ); + unit_assert( !is_pow2(17) ); + unit_assert( !is_pow2(23) ); + unit_assert( !is_pow2(257) ); + unit_assert( !is_pow2(259) ); } /** put dname into buffer */ diff --git a/util/config_file.c b/util/config_file.c index 8e05e0da2..026c6eb61 100644 --- a/util/config_file.c +++ b/util/config_file.c @@ -77,6 +77,8 @@ config_create() cfg->do_tcp = 1; cfg->outgoing_base_port = cfg->port + 1000; cfg->outgoing_num_ports = 16; + cfg->msg_cache_size = 4 * 1024 * 1024; + cfg->msg_cache_slabs = 4; if(!(cfg->fwd_address = strdup(""))) {config_delete(cfg); return NULL;} if(!(cfg->username = strdup(""))) {config_delete(cfg); return NULL;} if(!(cfg->chrootdir = strdup(""))) {config_delete(cfg); return NULL;} diff --git a/util/config_file.h b/util/config_file.h index f75568331..4b94e333f 100644 --- a/util/config_file.h +++ b/util/config_file.h @@ -69,6 +69,11 @@ struct config_file { /** outgoing port range number of ports (per thread, per if) */ int outgoing_num_ports; + /** size of the message cache */ + size_t msg_cache_size; + /** slabs in the message cache. */ + size_t msg_cache_slabs; + /** forwarder address. string. If not NULL fwder mode is enabled. */ char* fwd_address; /** forwarder port */ diff --git a/util/configlexer.lex b/util/configlexer.lex index e104a0a59..73739efeb 100644 --- a/util/configlexer.lex +++ b/util/configlexer.lex @@ -114,6 +114,8 @@ username{COLON} { LEXOUT(("v(%s) ", yytext)); return VAR_USERNAME;} directory{COLON} { LEXOUT(("v(%s) ", yytext)); return VAR_DIRECTORY;} logfile{COLON} { LEXOUT(("v(%s) ", yytext)); return VAR_LOGFILE;} pidfile{COLON} { LEXOUT(("v(%s) ", yytext)); return VAR_PIDFILE;} +msg-cache-size{COLON} { LEXOUT(("v(%s) ", yytext)); return VAR_MSG_CACHE_SIZE;} +msg-cache-slabs{COLON} { LEXOUT(("v(%s) ", yytext)); return VAR_MSG_CACHE_SLABS;} {NEWLINE} { LEXOUT(("NL\n")); cfg_parser->line++;} /* Quoted strings. Strip leading and ending quotes */ diff --git a/util/configparser.y b/util/configparser.y index 6d3759750..4ddcce2b1 100644 --- a/util/configparser.y +++ b/util/configparser.y @@ -46,6 +46,7 @@ #include "util/configyyrename.h" #include "util/config_file.h" +#include "util/net_help.h" int ub_c_lex(void); void ub_c_error(const char *message); @@ -71,6 +72,7 @@ extern struct config_parser_state* cfg_parser; %token VAR_DO_IP4 VAR_DO_IP6 VAR_DO_UDP VAR_DO_TCP %token VAR_FORWARD_TO VAR_FORWARD_TO_PORT VAR_CHROOT %token VAR_USERNAME VAR_DIRECTORY VAR_LOGFILE VAR_PIDFILE +%token VAR_MSG_CACHE_SIZE VAR_MSG_CACHE_SLABS %% toplevelvars: /* empty */ | toplevelvars toplevelvar ; @@ -90,7 +92,8 @@ content_server: server_num_threads | server_verbosity | server_port | server_outgoing_port | server_outgoing_range | server_do_ip4 | server_do_ip6 | server_do_udp | server_do_tcp | server_forward_to | server_forward_to_port | server_interface | server_chroot | - server_username | server_directory | server_logfile | server_pidfile; + server_username | server_directory | server_logfile | server_pidfile | + server_msg_cache_size | server_msg_cache_slabs; server_num_threads: VAR_NUM_THREADS STRING { OUTYY(("P(server_num_threads:%s)\n", $2)); @@ -236,6 +239,28 @@ server_pidfile: VAR_PIDFILE STRING cfg_parser->cfg->pidfile = $2; } ; +server_msg_cache_size: VAR_MSG_CACHE_SIZE STRING + { + OUTYY(("P(server_msg_cache_size:%s)\n", $2)); + if(atoi($2) == 0) + yyerror("number expected"); + else cfg_parser->cfg->msg_cache_size = atoi($2); + free($2); + } + ; +server_msg_cache_slabs: VAR_MSG_CACHE_SLABS STRING + { + OUTYY(("P(server_msg_cache_slabs:%s)\n", $2)); + if(atoi($2) == 0) + yyerror("number expected"); + else { + cfg_parser->cfg->msg_cache_slabs = atoi($2); + if(!is_pow2(cfg_parser->cfg->msg_cache_slabs)) + yyerror("must be a power of 2"); + } + free($2); + } + ; %% /* parse helper routines could be here */ diff --git a/util/net_help.c b/util/net_help.c index 2ffd41d3f..a42f2af93 100644 --- a/util/net_help.c +++ b/util/net_help.c @@ -87,3 +87,10 @@ fd_set_nonblock(int s) } return 1; } + +int +is_pow2(size_t num) +{ + if(num == 0) return 1; + return (num & (num-1)) == 0; +} diff --git a/util/net_help.h b/util/net_help.h index 78d432f76..6ea82ba2a 100644 --- a/util/net_help.h +++ b/util/net_help.h @@ -66,4 +66,11 @@ write_socket(int s, const void *buf, size_t size); */ int fd_set_nonblock(int s); +/** + * See if number is a power of 2. + * @param num: the value. + * @return: true if the number is a power of 2. + */ +int is_pow2(size_t num); + #endif /* NET_HELP_H */ diff --git a/util/storage/slabhash.c b/util/storage/slabhash.c index d0d2e5626..929e5e170 100644 --- a/util/storage/slabhash.c +++ b/util/storage/slabhash.c @@ -131,3 +131,14 @@ void slabhash_status(struct slabhash* sl, const char* id, int extended) lruhash_status(sl->array[i], num, extended); } } + +size_t slabhash_get_size(struct slabhash* sl) +{ + size_t i, total = 0; + for(i=0; isize; i++) { + lock_quick_lock(&sl->array[i]->lock); + total += sl->array[i]->space_max; + lock_quick_unlock(&sl->array[i]->lock); + } + return total; +} diff --git a/util/storage/slabhash.h b/util/storage/slabhash.h index 2e9a74029..7fde7caf4 100644 --- a/util/storage/slabhash.h +++ b/util/storage/slabhash.h @@ -134,4 +134,10 @@ void slabhash_remove(struct slabhash* table, hashvalue_t hash, void* key); */ void slabhash_status(struct slabhash* table, const char* id, int extended); +/** + * Retrieve slab hash total size. + * @param table: hash table. + */ +size_t slabhash_get_size(struct slabhash* table); + #endif /* UTIL_STORAGE_SLABHASH_H */