From: Wouter Wijngaards Date: Thu, 30 Aug 2007 08:36:41 +0000 (+0000) Subject: support extremely small memory footprints. X-Git-Tag: release-0.5~76 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=341492391e75c5c2f61dfb2207f9c104bb8819be;p=thirdparty%2Funbound.git support extremely small memory footprints. git-svn-id: file:///svn/unbound/trunk@565 be551aaa-1e26-0410-a405-d3ace91eadb9 --- diff --git a/daemon/daemon.c b/daemon/daemon.c index 944bd7da5..0db21b917 100644 --- a/daemon/daemon.c +++ b/daemon/daemon.c @@ -39,9 +39,6 @@ * The daemon consists of global settings and a number of workers. */ -/** buffer size for network connections */ -#define BUFSZ 65552 - #include "config.h" #include "daemon/daemon.h" #include "daemon/worker.h" @@ -338,8 +335,7 @@ thread_start(void* arg) worker->cmd_send_fd = -1; close_other_pipes(worker->daemon, worker->thread_num); #endif - if(!worker_init(worker, worker->daemon->cfg, worker->daemon->ports, - BUFSZ, 0)) + if(!worker_init(worker, worker->daemon->cfg, worker->daemon->ports, 0)) fatal_exit("Could not initialize thread"); worker_work(worker); @@ -414,8 +410,7 @@ daemon_fork(struct daemon* daemon) /* Special handling for the main thread. This is the thread * that handles signals. */ - if(!worker_init(daemon->workers[0], daemon->cfg, daemon->ports, - BUFSZ, 1)) + if(!worker_init(daemon->workers[0], daemon->cfg, daemon->ports, 1)) fatal_exit("Could not initialize main thread"); signal_handling_playback(daemon->workers[0]); diff --git a/daemon/unbound.c b/daemon/unbound.c index ffcc4c094..6ff00b108 100644 --- a/daemon/unbound.c +++ b/daemon/unbound.c @@ -77,7 +77,8 @@ static void checkrlimits(struct config_file* cfg) { int list = ((cfg->do_ip4?1:0) + (cfg->do_ip6?1:0)) * - ((cfg->do_udp?1:0) + (cfg->do_tcp?1 + TCP_ACCEPT_COUNT:0)); + ((cfg->do_udp?1:0) + (cfg->do_tcp?1 + + (int)cfg->incoming_num_tcp:0)); size_t ifs = (size_t)(cfg->num_ifs==0?1:cfg->num_ifs); size_t listen_num = list*ifs; size_t outnum = cfg->outgoing_num_ports*ifs + cfg->outgoing_num_tcp; diff --git a/daemon/worker.c b/daemon/worker.c index b7bcb0a45..91b993c4e 100644 --- a/daemon/worker.c +++ b/daemon/worker.c @@ -838,7 +838,7 @@ worker_create(struct daemon* daemon, int id) int worker_init(struct worker* worker, struct config_file *cfg, - struct listen_port* ports, size_t buffer_size, int do_sigs) + struct listen_port* ports, int do_sigs) { unsigned int seed; int startport; @@ -885,7 +885,8 @@ worker_init(struct worker* worker, struct config_file *cfg, return 0; } worker->front = listen_create(worker->base, ports, - buffer_size, worker_handle_request, worker); + cfg->msg_buffer_size, (int)cfg->incoming_num_tcp, + worker_handle_request, worker); if(!worker->front) { log_err("could not create listening sockets"); worker_delete(worker); @@ -894,8 +895,8 @@ worker_init(struct worker* worker, struct config_file *cfg, startport = cfg->outgoing_base_port + cfg->outgoing_num_ports * worker->thread_num; worker->back = outside_network_create(worker->base, - buffer_size, (size_t)cfg->outgoing_num_ports, cfg->ifs, - cfg->num_ifs, cfg->do_ip4, cfg->do_ip6, startport, + cfg->msg_buffer_size, (size_t)cfg->outgoing_num_ports, + cfg->ifs, cfg->num_ifs, cfg->do_ip4, cfg->do_ip6, startport, cfg->do_tcp?cfg->outgoing_num_tcp:0, worker->daemon->env->infra_cache, worker->rndstate); if(!worker->back) { @@ -906,15 +907,17 @@ worker_init(struct worker* worker, struct config_file *cfg, if(worker->thread_num != 0) { /* start listening to commands */ if(!(worker->cmd_com=comm_point_create_local(worker->base, - worker->cmd_recv_fd, buffer_size, + worker->cmd_recv_fd, cfg->msg_buffer_size, worker_handle_control_cmd, worker))) { log_err("could not create control compt."); worker_delete(worker); return 0; } } + /* we use the msg_buffer_size as a good estimate for what the + * user wants for memory usage sizes */ worker->scratchpad = region_create_custom(malloc, free, - 65536, 8192, 32, 1); + cfg->msg_buffer_size, cfg->msg_buffer_size/4, 32, 1); if(!worker->scratchpad) { log_err("malloc failure"); worker_delete(worker); @@ -936,7 +939,7 @@ worker_init(struct worker* worker, struct config_file *cfg, worker->env.attach_sub = &mesh_attach_sub; worker->env.kill_sub = &mesh_state_delete; worker->env.detect_cycle = &mesh_detect_cycle; - worker->env.scratch_buffer = ldns_buffer_new(65536); + worker->env.scratch_buffer = ldns_buffer_new(cfg->msg_buffer_size); if(!worker->env.mesh || !worker->env.scratch_buffer) { worker_delete(worker); return 0; diff --git a/daemon/worker.h b/daemon/worker.h index 19820343b..371a01195 100644 --- a/daemon/worker.h +++ b/daemon/worker.h @@ -127,12 +127,11 @@ struct worker* worker_create(struct daemon* daemon, int id); * @param worker: worker to initialize, created with worker_create. * @param cfg: configuration settings. * @param ports: list of shared query ports. - * @param buffer_size: size of datagram buffer. * @param do_sigs: if true, worker installs signal handlers. * @return: false on error. */ int worker_init(struct worker* worker, struct config_file *cfg, - struct listen_port* ports, size_t buffer_size, int do_sigs); + struct listen_port* ports, int do_sigs); /** * Make worker work. diff --git a/doc/Changelog b/doc/Changelog index 60f3886aa..b1dbdbb04 100644 --- a/doc/Changelog +++ b/doc/Changelog @@ -1,3 +1,7 @@ +30 August 2007: Wouter + - fixup override date config option. + - config options to control memory usage. + 29 August 2007: Wouter - test tool to sign rrsets for testing validator with. - added RSA and DSA test keys, public and private pairs, 512 bits. diff --git a/doc/example.conf b/doc/example.conf index 37ec62cd2..8ee8ade1b 100644 --- a/doc/example.conf +++ b/doc/example.conf @@ -42,6 +42,13 @@ server: # number of outgoing simultaneous tcp buffers to hold per thread. # outgoing-num-tcp: 10 + # number of incoming simultaneous tcp buffers to hold per thread. + # incoming-num-tcp: 10 + + # buffer size for handling DNS data. No messages larger than this + # size can be sent or received, by UDP or TCP. In bytes. + # msg-buffer-size: 65552 + # the amount of memory to use for the message cache. # in bytes. default is 4 Mb # msg-cache-size: 4194304 diff --git a/doc/unbound.conf.5 b/doc/unbound.conf.5 index dc442bb53..b85348321 100644 --- a/doc/unbound.conf.5 +++ b/doc/unbound.conf.5 @@ -91,6 +91,15 @@ extra resources from the operating system. .It \fBoutgoing-num-tcp:\fR Number of outgoing TCP buffers to allocate per thread. Default is 10. If set to 0, or if do_tcp is "no", no TCP queries to authoritative servers are done. +.It \fBincoming-num-tcp:\fR +Number of incoming TCP buffers to allocate per thread. Default is 10. If set +to 0, or if do_tcp is "no", no TCP queries from clients are accepted. +.It \fBmsg-buffer-size:\fR +Number of bytes size of the message buffers. Default is 65552 bytes, enough +for 64 Kb packets, the maximum DNS message size. No message larger than this +can be sent or received. Can be reduced to use less memory, but some requests +for DNS data, such as for huge resource records, will result in a SERVFAIL +reply to the client. .It \fBmsg-cache-size:\fR Number of bytes size of the message cache. Default is 4 megabytes. .It \fBmsg-cache-slabs:\fR @@ -258,6 +267,37 @@ IP address of server to forward to. Can be IP 4 or IP 6. To use a nondefault port for DNS communication append '@' with the port number. .El +.Sh MEMORY CONTROL EXAMPLE +In the example config settings below memory usage is reduced. Some service +levels are lower, notable very large data and a high TCP load are no longer +supported. Very large data and high TCP loads are exceptional for the DNS. +DNSSEC validation is enabled, just add trust anchors. +If you do not have to worry about programs using more than 1 meg of memory, +the below example is not for you. Use the defaults to receive full service. +.nf + +# example settings that reduce memory usage +server: + num-threads: 1 + outgoing-num-tcp: 1 # this limits TCP service, uses less buffers. + incoming-num-tcp: 1 + outgoing-range: 1 # uses less memory, but less port randomness. + msg-buffer-size: 8192 # note this limits service, 'no huge stuff'. + msg-cache-size: 102400 # 100 Kb. + msg-cache-slabs: 1 + rrset-cache-size: 102400 # 100 Kb. + rrset-cache-slabs: 1 + infra-cache-numhosts: 200 + infra-cache-numlame: 10 + num-queries-per-thread: 30 + target-fetch-policy: "2 1 0 0 0 0" + harden-large-queries: "yes" + harden-short-bufsize: "yes" + do-ip6: no # save a bit of memory if not used. +.fi + +.El + .Sh FILES .Bl -tag -width indent .It Pa /etc/unbound diff --git a/services/listen_dnsport.c b/services/listen_dnsport.c index 125685d15..48f7c5e1a 100644 --- a/services/listen_dnsport.c +++ b/services/listen_dnsport.c @@ -288,7 +288,8 @@ listen_cp_insert(struct comm_point* c, struct listen_dnsport* front) struct listen_dnsport* listen_create(struct comm_base* base, struct listen_port* ports, - size_t bufsize, comm_point_callback_t* cb, void *cb_arg) + size_t bufsize, int tcp_accept_count, + comm_point_callback_t* cb, void *cb_arg) { struct listen_dnsport* front = (struct listen_dnsport*) malloc(sizeof(struct listen_dnsport)); @@ -308,7 +309,7 @@ listen_create(struct comm_base* base, struct listen_port* ports, cp = comm_point_create_udp(base, ports->fd, front->udp_buff, cb, cb_arg); else cp = comm_point_create_tcp(base, ports->fd, - TCP_ACCEPT_COUNT, bufsize, cb, cb_arg); + tcp_accept_count, bufsize, cb, cb_arg); if(!cp) { log_err("can't create commpoint"); listen_delete(front); @@ -381,10 +382,14 @@ listening_ports_open(struct config_file* cfg) struct listen_port* list = NULL; struct addrinfo hints; int i, do_ip4, do_ip6; + int do_tcp; char portbuf[32]; snprintf(portbuf, sizeof(portbuf), "%d", cfg->port); do_ip4 = cfg->do_ip4; do_ip6 = cfg->do_ip6; + do_tcp = cfg->do_tcp; + if(cfg->incoming_num_tcp == 0) + do_tcp = 0; /* getaddrinfo */ memset(&hints, 0, sizeof(hints)); @@ -403,7 +408,7 @@ listening_ports_open(struct config_file* cfg) if(cfg->num_ifs == 0) { if(do_ip6) { hints.ai_family = AF_INET6; - if(!ports_create_if(NULL, cfg->do_udp, cfg->do_tcp, + if(!ports_create_if(NULL, cfg->do_udp, do_tcp, &hints, portbuf, &list)) { listening_ports_free(list); return NULL; @@ -411,7 +416,7 @@ listening_ports_open(struct config_file* cfg) } if(do_ip4) { hints.ai_family = AF_INET; - if(!ports_create_if(NULL, cfg->do_udp, cfg->do_tcp, + if(!ports_create_if(NULL, cfg->do_udp, do_tcp, &hints, portbuf, &list)) { listening_ports_free(list); return NULL; @@ -423,7 +428,7 @@ listening_ports_open(struct config_file* cfg) continue; hints.ai_family = AF_INET6; if(!ports_create_if(cfg->ifs[i], cfg->do_udp, - cfg->do_tcp, &hints, portbuf, &list)) { + do_tcp, &hints, portbuf, &list)) { listening_ports_free(list); return NULL; } @@ -432,7 +437,7 @@ listening_ports_open(struct config_file* cfg) continue; hints.ai_family = AF_INET; if(!ports_create_if(cfg->ifs[i], cfg->do_udp, - cfg->do_tcp, &hints, portbuf, &list)) { + do_tcp, &hints, portbuf, &list)) { listening_ports_free(list); return NULL; } diff --git a/services/listen_dnsport.h b/services/listen_dnsport.h index 4fabea50b..fb11a6ade 100644 --- a/services/listen_dnsport.h +++ b/services/listen_dnsport.h @@ -48,9 +48,6 @@ struct listen_list; struct addrinfo; struct config_file; -/** number of simultaneous open TCP connections for queries */ -#define TCP_ACCEPT_COUNT 10 - /** * Listening for queries structure. * Contains list of query-listen sockets. @@ -111,13 +108,15 @@ void listening_ports_free(struct listen_port* list); * for default all ifs. * @param ports: the list of shared ports. * @param bufsize: size of datagram buffer. + * @param tcp_accept_count: max number of simultaneous TCP connections + * from clients. * @param cb: callback function when a request arrives. It is passed * the packet and user argument. Return true to send a reply. * @param cb_arg: user data argument for callback function. * @return: the malloced listening structure, ready for use. NULL on error. */ struct listen_dnsport* listen_create(struct comm_base* base, - struct listen_port* ports, size_t bufsize, + struct listen_port* ports, size_t bufsize, int tcp_accept_count, comm_point_callback_t* cb, void* cb_arg); /** diff --git a/testcode/fake_event.c b/testcode/fake_event.c index 75de4655a..187e4a4ad 100644 --- a/testcode/fake_event.c +++ b/testcode/fake_event.c @@ -520,7 +520,8 @@ run_scenario(struct replay_runtime* runtime) struct listen_dnsport* listen_create(struct comm_base* base, struct listen_port* ATTR_UNUSED(ports), - size_t bufsize, comm_point_callback_t* cb, void* cb_arg) + size_t bufsize, int ATTR_UNUSED(tcp_accept_count), + comm_point_callback_t* cb, void* cb_arg) { struct replay_runtime* runtime = (struct replay_runtime*)base; struct listen_dnsport* l= calloc(1, sizeof(struct listen_dnsport)); diff --git a/util/config_file.c b/util/config_file.c index 4c147c438..31db0a770 100644 --- a/util/config_file.c +++ b/util/config_file.c @@ -78,6 +78,8 @@ config_create() cfg->outgoing_base_port = cfg->port + 2000; cfg->outgoing_num_ports = 16; cfg->outgoing_num_tcp = 10; + cfg->incoming_num_tcp = 10; + cfg->msg_buffer_size = 65552; /* 64 k + a small margin */ cfg->msg_cache_size = 4 * 1024 * 1024; cfg->msg_cache_slabs = 4; cfg->num_queries_per_thread = 1024; diff --git a/util/config_file.h b/util/config_file.h index bedc74689..c4bf9208e 100644 --- a/util/config_file.h +++ b/util/config_file.h @@ -72,7 +72,11 @@ struct config_file { int outgoing_num_ports; /** number of outgoing tcp buffers per (per thread) */ size_t outgoing_num_tcp; + /** number of incoming tcp buffers per (per thread) */ + size_t incoming_num_tcp; + /** number of bytes buffer size for DNS messages */ + size_t msg_buffer_size; /** size of the message cache */ size_t msg_cache_size; /** slabs in the message cache. */ diff --git a/util/configlexer.lex b/util/configlexer.lex index c77800855..08443d635 100644 --- a/util/configlexer.lex +++ b/util/configlexer.lex @@ -105,6 +105,7 @@ port{COLON} { YDOUT; return VAR_PORT;} outgoing-port{COLON} { YDOUT; return VAR_OUTGOING_PORT;} outgoing-range{COLON} { YDOUT; return VAR_OUTGOING_RANGE;} outgoing-num-tcp{COLON} { YDOUT; return VAR_OUTGOING_NUM_TCP;} +incoming-num-tcp{COLON} { YDOUT; return VAR_INCOMING_NUM_TCP;} do-ip4{COLON} { YDOUT; return VAR_DO_IP4;} do-ip6{COLON} { YDOUT; return VAR_DO_IP6;} do-udp{COLON} { YDOUT; return VAR_DO_UDP;} @@ -115,6 +116,7 @@ username{COLON} { YDOUT; return VAR_USERNAME;} directory{COLON} { YDOUT; return VAR_DIRECTORY;} logfile{COLON} { YDOUT; return VAR_LOGFILE;} pidfile{COLON} { YDOUT; return VAR_PIDFILE;} +msg-buffer-size{COLON} { YDOUT; return VAR_MSG_BUFFER_SIZE;} msg-cache-size{COLON} { YDOUT; return VAR_MSG_CACHE_SIZE;} msg-cache-slabs{COLON} { YDOUT; return VAR_MSG_CACHE_SLABS;} rrset-cache-size{COLON} { YDOUT; return VAR_RRSET_CACHE_SIZE;} diff --git a/util/configparser.y b/util/configparser.y index 525e6c546..75bfb438a 100644 --- a/util/configparser.y +++ b/util/configparser.y @@ -82,6 +82,7 @@ extern struct config_parser_state* cfg_parser; %token VAR_IDENTITY VAR_VERSION VAR_HARDEN_GLUE VAR_MODULE_CONF %token VAR_TRUST_ANCHOR_FILE VAR_TRUST_ANCHOR VAR_VAL_OVERRIDE_DATE %token VAR_BOGUS_TTL VAR_VAL_CLEAN_ADDITIONAL VAR_VAL_PERMISSIVE_MODE +%token VAR_INCOMING_NUM_TCP VAR_MSG_BUFFER_SIZE %% toplevelvars: /* empty */ | toplevelvars toplevelvar ; @@ -116,7 +117,8 @@ content_server: server_num_threads | server_verbosity | server_port | server_hide_version | server_identity | server_version | server_harden_glue | server_module_conf | server_trust_anchor_file | server_trust_anchor | server_val_override_date | server_bogus_ttl | - server_val_clean_additional | server_val_permissive_mode + server_val_clean_additional | server_val_permissive_mode | + server_incoming_num_tcp | server_msg_buffer_size ; stubstart: VAR_STUB_ZONE { @@ -217,6 +219,15 @@ server_outgoing_num_tcp: VAR_OUTGOING_NUM_TCP STRING free($2); } ; +server_incoming_num_tcp: VAR_INCOMING_NUM_TCP STRING + { + OUTYY(("P(server_incoming_num_tcp:%s)\n", $2)); + if(atoi($2) == 0 && strcmp($2, "0") != 0) + yyerror("number expected"); + else cfg_parser->cfg->incoming_num_tcp = atoi($2); + free($2); + } + ; server_do_ip4: VAR_DO_IP4 STRING { OUTYY(("P(server_do_ip4:%s)\n", $2)); @@ -335,6 +346,17 @@ server_version: VAR_VERSION STRING cfg_parser->cfg->version = $2; } ; +server_msg_buffer_size: VAR_MSG_BUFFER_SIZE STRING + { + OUTYY(("P(server_msg_buffer_size:%s)\n", $2)); + if(atoi($2) == 0) + yyerror("number expected"); + else if (atoi($2) < 4096) + yyerror("message buffer size too small (use 4096)"); + else cfg_parser->cfg->msg_buffer_size = atoi($2); + free($2); + } + ; server_msg_cache_size: VAR_MSG_CACHE_SIZE STRING { OUTYY(("P(server_msg_cache_size:%s)\n", $2)); @@ -501,7 +523,7 @@ server_val_override_date: VAR_VAL_OVERRIDE_DATE STRING } else { if(atoi($2) == 0) yyerror("number expected"); - cfg_parser->cfg->outgoing_num_ports = atoi($2); + cfg_parser->cfg->val_date_override = atoi($2); } free($2); }