From: Olivier Houchard Date: Wed, 5 Apr 2017 23:05:05 +0000 (+0200) Subject: MINOR: proxy: Don't close FDs if not our proxy. X-Git-Tag: v1.8-dev2~79 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=1fc0516516defd57574efb677d7f698757e62d68;p=thirdparty%2Fhaproxy.git MINOR: proxy: Don't close FDs if not our proxy. When running with multiple process, if some proxies are just assigned to some processes, the other processes will just close the file descriptors for the listening sockets. However, we may still have to provide those sockets when reloading, so instead we just try hard to pretend those proxies are dead, while keeping the sockets opened. A new global option, no-reused-socket", has been added, to restore the old behavior of closing the sockets not bound to this process. --- diff --git a/doc/configuration.txt b/doc/configuration.txt index 05f07019f9..e6ea2cfc89 100644 --- a/doc/configuration.txt +++ b/doc/configuration.txt @@ -587,6 +587,7 @@ The following keywords are supported in the "global" section : - nosplice - nogetaddrinfo - noreuseport + - no-unused-socket - spread-checks - server-state-base - server-state-file @@ -1250,6 +1251,12 @@ noreuseport Disables the use of SO_REUSEPORT - see socket(7). It is equivalent to the command line argument "-dR". +no-unused-socket + By default, each haproxy process keeps all sockets opened, event those that + are only used by another processes, so that any process can provide all the + sockets, to make reloads seamless. This option disables this, and close all + unused sockets, to save some file descriptors. + spread-checks <0..50, in percent> Sometimes it is desirable to avoid sending agent and health checks to servers at exact intervals, for instance when many logical servers are diff --git a/include/proto/fd.h b/include/proto/fd.h index 87309bf0a3..1efe32399a 100644 --- a/include/proto/fd.h +++ b/include/proto/fd.h @@ -41,6 +41,11 @@ extern int fd_nbupdt; // number of updates in the list */ void fd_delete(int fd); +/* Deletes an FD from the fdsets, and recomputes the maxfd limit. + * The file descriptor is kept open. + */ +void fd_remove(int fd); + /* disable the specified poller */ void disable_poller(const char *poller_name); diff --git a/include/proto/listener.h b/include/proto/listener.h index 552d695954..079d976a30 100644 --- a/include/proto/listener.h +++ b/include/proto/listener.h @@ -88,6 +88,11 @@ void dequeue_all_listeners(struct list *list); */ int unbind_listener(struct listener *listener); +/* This function pretends the listener is dead, but keeps the FD opened, so + * that we can provide it, for conf reloading. + */ +int unbind_listener_no_close(struct listener *listener); + /* This function closes all listening sockets bound to the protocol , * and the listeners end in LI_ASSIGNED state if they were higher. It does not * detach them from the protocol. It always returns ERR_NONE. diff --git a/include/proto/proxy.h b/include/proto/proxy.h index 72f1e1d332..e5d20c4f5d 100644 --- a/include/proto/proxy.h +++ b/include/proto/proxy.h @@ -42,6 +42,7 @@ void soft_stop(void); int pause_proxy(struct proxy *p); int resume_proxy(struct proxy *p); void stop_proxy(struct proxy *p); +void zombify_proxy(struct proxy *p); void pause_proxies(void); void resume_proxies(void); int stream_set_backend(struct stream *s, struct proxy *be); diff --git a/include/types/global.h b/include/types/global.h index df8e2c69aa..57b969dd14 100644 --- a/include/types/global.h +++ b/include/types/global.h @@ -62,6 +62,8 @@ #define GTUNE_USE_REUSEPORT (1<<6) #define GTUNE_RESOLVE_DONTFAIL (1<<7) +#define GTUNE_SOCKET_TRANSFER (1<<8) + /* Access level for a stats socket */ #define ACCESS_LVL_NONE 0 #define ACCESS_LVL_USER 1 diff --git a/include/types/listener.h b/include/types/listener.h index 227cc284d4..2b8f5feb64 100644 --- a/include/types/listener.h +++ b/include/types/listener.h @@ -47,6 +47,7 @@ enum li_state { LI_INIT, /* all parameters filled in, but not assigned yet */ LI_ASSIGNED, /* assigned to the protocol, but not listening yet */ LI_PAUSED, /* listener was paused, it's bound but not listening */ + LI_ZOMBIE, /* The listener doesn't belong to the process, but is kept opened */ LI_LISTEN, /* started, listening but not enabled */ LI_READY, /* started, listening and enabled */ LI_FULL, /* reached its connection limit */ diff --git a/src/cfgparse.c b/src/cfgparse.c index 47d33cf1b0..348b9e8866 100644 --- a/src/cfgparse.c +++ b/src/cfgparse.c @@ -659,6 +659,11 @@ int cfg_parse_global(const char *file, int linenum, char **args, int kwm) goto out; global.tune.options &= ~GTUNE_USE_REUSEPORT; } + else if (!strcmp(args[0], "no-unused-socket")) { + if (alertif_too_many_args(0, file, linenum, args, &err_code)) + goto out; + global.tune.options &= ~GTUNE_SOCKET_TRANSFER; + } else if (!strcmp(args[0], "quiet")) { if (alertif_too_many_args(0, file, linenum, args, &err_code)) goto out; diff --git a/src/cli.c b/src/cli.c index 54fb43892e..20e143b965 100644 --- a/src/cli.c +++ b/src/cli.c @@ -1063,10 +1063,11 @@ static int _getsocks(char **args, struct appctx *appctx, void *private) struct listener *l; list_for_each_entry(l, &px->conf.listeners, by_fe) { - /* Only transfer IPv4/IPv6 sockets */ - if (l->proto->sock_family == AF_INET || + /* Only transfer IPv4/IPv6/UNIX sockets */ + if (l->state >= LI_ZOMBIE && + (l->proto->sock_family == AF_INET || l->proto->sock_family == AF_INET6 || - l->proto->sock_family == AF_UNIX) + l->proto->sock_family == AF_UNIX)) tot_fd_nb++; } px = px->next; @@ -1117,7 +1118,7 @@ static int _getsocks(char **args, struct appctx *appctx, void *private) list_for_each_entry(l, &px->conf.listeners, by_fe) { int ret; /* Only transfer IPv4/IPv6 sockets */ - if (l->state >= LI_LISTEN && + if (l->state >= LI_ZOMBIE && (l->proto->sock_family == AF_INET || l->proto->sock_family == AF_INET6 || l->proto->sock_family == AF_UNIX)) { diff --git a/src/fd.c b/src/fd.c index aeee602610..1a62f9a669 100644 --- a/src/fd.c +++ b/src/fd.c @@ -175,7 +175,7 @@ int fd_nbupdt = 0; // number of updates in the list /* Deletes an FD from the fdsets, and recomputes the maxfd limit. * The file descriptor is also closed. */ -void fd_delete(int fd) +static void fd_dodelete(int fd, int do_close) { if (fdtab[fd].linger_risk) { /* this is generally set when connecting to servers */ @@ -190,7 +190,8 @@ void fd_delete(int fd) port_range_release_port(fdinfo[fd].port_range, fdinfo[fd].local_port); fdinfo[fd].port_range = NULL; - close(fd); + if (do_close) + close(fd); fdtab[fd].owner = NULL; fdtab[fd].new = 0; @@ -198,6 +199,22 @@ void fd_delete(int fd) maxfd--; } +/* Deletes an FD from the fdsets, and recomputes the maxfd limit. + * The file descriptor is also closed. + */ +void fd_delete(int fd) +{ + fd_dodelete(fd, 1); +} + +/* Deletes an FD from the fdsets, and recomputes the maxfd limit. + * The file descriptor is kept open. + */ +void fd_remove(int fd) +{ + fd_dodelete(fd, 0); +} + /* Scan and process the cached events. This should be called right after * the poller. The loop may cause new entries to be created, for example * if a listener causes an accept() to initiate a new incoming connection diff --git a/src/haproxy.c b/src/haproxy.c index bf176df789..01969c90d1 100644 --- a/src/haproxy.c +++ b/src/haproxy.c @@ -857,6 +857,7 @@ static void init(int argc, char **argv) #if defined(SO_REUSEPORT) global.tune.options |= GTUNE_USE_REUSEPORT; #endif + global.tune.options |= GTUNE_SOCKET_TRANSFER; pid = getpid(); progname = *argv; @@ -1668,6 +1669,15 @@ void deinit(void) }/* end while(s) */ list_for_each_entry_safe(l, l_next, &p->conf.listeners, by_fe) { + /* + * Zombie proxy, the listener just pretend to be up + * because they still hold an opened fd. + * Close it and give the listener its real state. + */ + if (p->state == PR_STSTOPPED && l->state >= LI_ZOMBIE) { + close(l->fd); + l->state = LI_INIT; + } unbind_listener(l); delete_listener(l); LIST_DEL(&l->by_fe); @@ -2148,8 +2158,12 @@ int main(int argc, char **argv) px = proxy; while (px != NULL) { if (px->bind_proc && px->state != PR_STSTOPPED) { - if (!(px->bind_proc & (1UL << proc))) - stop_proxy(px); + if (!(px->bind_proc & (1UL << proc))) { + if (global.tune.options & GTUNE_SOCKET_TRANSFER) + zombify_proxy(px); + else + stop_proxy(px); + } } px = px->next; } diff --git a/src/listener.c b/src/listener.c index 7a1df0e412..a99e4c0d35 100644 --- a/src/listener.c +++ b/src/listener.c @@ -59,7 +59,12 @@ void enable_listener(struct listener *listener) /* we don't want to enable this listener and don't * want any fd event to reach it. */ - unbind_listener(listener); + if (!(global.tune.options & GTUNE_SOCKET_TRANSFER)) + unbind_listener(listener); + else { + unbind_listener_no_close(listener); + listener->state = LI_LISTEN; + } } else if (listener->nbconn < listener->maxconn) { fd_want_recv(listener->fd); @@ -95,7 +100,7 @@ void disable_listener(struct listener *listener) */ int pause_listener(struct listener *l) { - if (l->state <= LI_PAUSED) + if (l->state <= LI_ZOMBIE) return 1; if (l->proto->pause) { @@ -149,7 +154,7 @@ int resume_listener(struct listener *l) return 0; } - if (l->state < LI_PAUSED) + if (l->state < LI_PAUSED || l->state == LI_ZOMBIE) return 0; if (l->proto->sock_prot == IPPROTO_TCP && @@ -242,12 +247,7 @@ void dequeue_all_listeners(struct list *list) } } -/* This function closes the listening socket for the specified listener, - * provided that it's already in a listening state. The listener enters the - * LI_ASSIGNED state. It always returns ERR_NONE. This function is intended - * to be used as a generic function for standard protocols. - */ -int unbind_listener(struct listener *listener) +static int do_unbind_listener(struct listener *listener, int do_close) { if (listener->state == LI_READY) fd_stop_recv(listener->fd); @@ -256,13 +256,35 @@ int unbind_listener(struct listener *listener) LIST_DEL(&listener->wait_queue); if (listener->state >= LI_PAUSED) { - fd_delete(listener->fd); - listener->fd = -1; + if (do_close) { + fd_delete(listener->fd); + listener->fd = -1; + } + else + fd_remove(listener->fd); listener->state = LI_ASSIGNED; } return ERR_NONE; } +/* This function closes the listening socket for the specified listener, + * provided that it's already in a listening state. The listener enters the + * LI_ASSIGNED state. It always returns ERR_NONE. This function is intended + * to be used as a generic function for standard protocols. + */ +int unbind_listener(struct listener *listener) +{ + return do_unbind_listener(listener, 1); +} + +/* This function pretends the listener is dead, but keeps the FD opened, so + * that we can provide it, for conf reloading. + */ +int unbind_listener_no_close(struct listener *listener) +{ + return do_unbind_listener(listener, 0); +} + /* This function closes all listening sockets bound to the protocol , * and the listeners end in LI_ASSIGNED state if they were higher. It does not * detach them from the protocol. It always returns ERR_NONE. diff --git a/src/proxy.c b/src/proxy.c index d158fac03f..dc702139ad 100644 --- a/src/proxy.c +++ b/src/proxy.c @@ -991,6 +991,19 @@ void soft_stop(void) p = proxy; tv_update_date(0,1); /* else, the old time before select will be used */ while (p) { + /* Zombie proxy, let's close the file descriptors */ + if (p->state == PR_STSTOPPED && + !LIST_ISEMPTY(&p->conf.listeners) && + LIST_ELEM(p->conf.listeners.n, + struct listener *, by_fe)->state >= LI_ZOMBIE) { + struct listener *l; + list_for_each_entry(l, &p->conf.listeners, by_fe) { + if (l->state >= LI_ZOMBIE) + close(l->fd); + l->state = LI_INIT; + } + } + if (p->state != PR_STSTOPPED) { Warning("Stopping %s %s in %d ms.\n", proxy_cap_str(p->cap), p->id, p->grace); send_log(p, LOG_WARNING, "Stopping %s %s in %d ms.\n", proxy_cap_str(p->cap), p->id, p->grace); @@ -1051,6 +1064,45 @@ int pause_proxy(struct proxy *p) return 1; } +/* This function makes the proxy unusable, but keeps the listening sockets + * opened, so that if any process requests them, we are able to serve them. + * This should only be called early, before we started accepting requests. + */ +void zombify_proxy(struct proxy *p) +{ + struct listener *l; + struct listener *first_to_listen = NULL; + + list_for_each_entry(l, &p->conf.listeners, by_fe) { + enum li_state oldstate = l->state; + + unbind_listener_no_close(l); + if (l->state >= LI_ASSIGNED) { + delete_listener(l); + listeners--; + jobs--; + } + /* + * Pretend we're still up and running so that the fd + * will be sent if asked. + */ + l->state = LI_ZOMBIE; + if (!first_to_listen && oldstate >= LI_LISTEN) + first_to_listen = l; + } + /* Quick hack : at stop time, to know we have to close the sockets + * despite the proxy being marked as stopped, make the first listener + * of the listener list an active one, so that we don't have to + * parse the whole list to be sure. + */ + if (first_to_listen && LIST_ELEM(p->conf.listeners.n, + struct listener *, by_fe) != first_to_listen) { + LIST_DEL(&l->by_fe); + LIST_ADD(&p->conf.listeners, &l->by_fe); + } + + p->state = PR_STSTOPPED; +} /* * This function completely stops a proxy and releases its listeners. It has