2 * Copyright (C) 1996-2021 The Squid Software Foundation and contributors
4 * Squid software is distributed under GPLv2+ license and includes
5 * contributions from numerous individuals and organizations.
6 * Please see the COPYING and CONTRIBUTORS files for details.
9 /* DEBUG: section 05 Socket Functions */
14 #include "anyp/PortCfg.h"
15 #include "comm/Connection.h"
16 #include "comm/Loops.h"
19 #include "mgr/Registration.h"
20 #include "SquidTime.h"
21 #include "StatCounters.h"
27 static int MAX_POLL_TIME
= 1000; /* see also Comm::QuickPollRequired() */
30 #define howmany(x, y) (((x)+((y)-1))/(y))
35 #define FD_MASK_BYTES sizeof(fd_mask)
36 #define FD_MASK_BITS (FD_MASK_BYTES*NBBY)
39 static int examine_select(fd_set
*, fd_set
*);
40 static int fdIsTcpListener(int fd
);
41 static int fdIsUdpListener(int fd
);
42 static int fdIsDns(int fd
);
43 static OBJH commIncomingStats
;
44 static int comm_check_incoming_select_handlers(int nfds
, int *fds
);
45 static void comm_select_dns_incoming(void);
46 static void commUpdateReadBits(int fd
, PF
* handler
);
47 static void commUpdateWriteBits(int fd
, PF
* handler
);
49 static struct timeval zero_tv
;
50 static fd_set global_readfds
;
51 static fd_set global_writefds
;
56 * Automatic tuning for incoming requests:
58 * INCOMING sockets are the ICP and HTTP ports. We need to check these
59 * fairly regularly, but how often? When the load increases, we
60 * want to check the incoming sockets more often. If we have a lot
61 * of incoming ICP, then we need to check these sockets more than
62 * if we just have HTTP.
64 * The variables 'incoming_udp_interval' and 'incoming_tcp_interval'
65 * determine how many normal I/O events to process before checking
66 * incoming sockets again. Note we store the incoming_interval
67 * multiplied by a factor of (2^INCOMING_FACTOR) to have some
68 * pseudo-floating point precision.
70 * The variable 'udp_io_events' and 'tcp_io_events' counts how many normal
71 * I/O events have been processed since the last check on the incoming
72 * sockets. When io_events > incoming_interval, its time to check incoming
75 * Every time we check incoming sockets, we count how many new messages
76 * or connections were processed. This is used to adjust the
77 * incoming_interval for the next iteration. The new incoming_interval
78 * is calculated as the current incoming_interval plus what we would
79 * like to see as an average number of events minus the number of
80 * events just processed.
82 * incoming_interval = incoming_interval + target_average - number_of_events_processed
84 * There are separate incoming_interval counters for DNS, UDP and TCP events
86 * You can see the current values of the incoming_interval's, as well as
87 * a histogram of 'incoming_events' by asking the cache manager
88 * for 'comm_incoming', e.g.:
90 * % ./client mgr:comm_incoming
94 * - We have MAX_INCOMING_INTEGER as a magic upper limit on
95 * incoming_interval for both types of sockets. At the
96 * largest value the cache will effectively be idling.
98 * - The higher the INCOMING_FACTOR, the slower the algorithm will
99 * respond to load spikes/increases/decreases in demand. A value
100 * between 3 and 8 is recommended.
103 #define MAX_INCOMING_INTEGER 256
104 #define INCOMING_FACTOR 5
105 #define MAX_INCOMING_INTERVAL (MAX_INCOMING_INTEGER << INCOMING_FACTOR)
106 static int udp_io_events
= 0;
107 static int dns_io_events
= 0;
108 static int tcp_io_events
= 0;
109 static int incoming_udp_interval
= 16 << INCOMING_FACTOR
;
110 static int incoming_dns_interval
= 16 << INCOMING_FACTOR
;
111 static int incoming_tcp_interval
= 16 << INCOMING_FACTOR
;
112 #define commCheckUdpIncoming (++udp_io_events > (incoming_udp_interval>> INCOMING_FACTOR))
113 #define commCheckDnsIncoming (++dns_io_events > (incoming_dns_interval>> INCOMING_FACTOR))
114 #define commCheckTcpIncoming (++tcp_io_events > (incoming_tcp_interval>> INCOMING_FACTOR))
117 Comm::SetSelect(int fd
, unsigned int type
, PF
* handler
, void *client_data
, time_t timeout
)
119 fde
*F
= &fd_table
[fd
];
121 assert(F
->flags
.open
|| (!handler
&& !client_data
&& !timeout
));
122 debugs(5, 5, HERE
<< "FD " << fd
<< ", type=" << type
<<
123 ", handler=" << handler
<< ", client_data=" << client_data
<<
124 ", timeout=" << timeout
);
126 if (type
& COMM_SELECT_READ
) {
127 F
->read_handler
= handler
;
128 F
->read_data
= client_data
;
129 commUpdateReadBits(fd
, handler
);
132 if (type
& COMM_SELECT_WRITE
) {
133 F
->write_handler
= handler
;
134 F
->write_data
= client_data
;
135 commUpdateWriteBits(fd
, handler
);
139 F
->timeout
= squid_curtime
+ timeout
;
143 fdIsUdpListener(int fd
)
145 if (icpIncomingConn
!= NULL
&& fd
== icpIncomingConn
->fd
)
148 if (icpOutgoingConn
!= NULL
&& fd
== icpOutgoingConn
->fd
)
157 if (fd
== DnsSocketA
)
160 if (fd
== DnsSocketB
)
167 fdIsTcpListener(int fd
)
169 for (AnyP::PortCfgPointer s
= HttpPortList
; s
!= NULL
; s
= s
->next
) {
170 if (s
->listenConn
!= NULL
&& s
->listenConn
->fd
== fd
)
178 comm_check_incoming_select_handlers(int nfds
, int *fds
)
187 FD_ZERO(&write_mask
);
188 incoming_sockets_accepted
= 0;
190 for (i
= 0; i
< nfds
; ++i
) {
193 if (fd_table
[fd
].read_handler
) {
194 FD_SET(fd
, &read_mask
);
200 if (fd_table
[fd
].write_handler
) {
201 FD_SET(fd
, &write_mask
);
213 ++ statCounter
.syscalls
.selects
;
215 if (select(maxfd
, &read_mask
, &write_mask
, NULL
, &zero_tv
) < 1)
216 return incoming_sockets_accepted
;
218 for (i
= 0; i
< nfds
; ++i
) {
221 if (FD_ISSET(fd
, &read_mask
)) {
222 if ((hdl
= fd_table
[fd
].read_handler
) != NULL
) {
223 fd_table
[fd
].read_handler
= NULL
;
224 commUpdateReadBits(fd
, NULL
);
225 hdl(fd
, fd_table
[fd
].read_data
);
227 debugs(5, DBG_IMPORTANT
, "comm_select_incoming: FD " << fd
<< " NULL read handler");
231 if (FD_ISSET(fd
, &write_mask
)) {
232 if ((hdl
= fd_table
[fd
].write_handler
) != NULL
) {
233 fd_table
[fd
].write_handler
= NULL
;
234 commUpdateWriteBits(fd
, NULL
);
235 hdl(fd
, fd_table
[fd
].write_data
);
237 debugs(5, DBG_IMPORTANT
, "comm_select_incoming: FD " << fd
<< " NULL write handler");
242 return incoming_sockets_accepted
;
246 comm_select_udp_incoming(void)
253 if (Comm::IsConnOpen(icpIncomingConn
)) {
254 fds
[nfds
] = icpIncomingConn
->fd
;
258 if (Comm::IsConnOpen(icpOutgoingConn
) && icpIncomingConn
!= icpOutgoingConn
) {
259 fds
[nfds
] = icpOutgoingConn
->fd
;
266 nevents
= comm_check_incoming_select_handlers(nfds
, fds
);
268 incoming_udp_interval
+= Config
.comm_incoming
.udp
.average
- nevents
;
270 if (incoming_udp_interval
< 0)
271 incoming_udp_interval
= 0;
273 if (incoming_udp_interval
> MAX_INCOMING_INTERVAL
)
274 incoming_udp_interval
= MAX_INCOMING_INTERVAL
;
276 if (nevents
> INCOMING_UDP_MAX
)
277 nevents
= INCOMING_UDP_MAX
;
279 statCounter
.comm_udp_incoming
.count(nevents
);
283 comm_select_tcp_incoming(void)
286 int fds
[MAXTCPLISTENPORTS
];
290 // XXX: only poll sockets that won't be deferred. But how do we identify them?
292 for (AnyP::PortCfgPointer s
= HttpPortList
; s
!= NULL
; s
= s
->next
) {
293 if (Comm::IsConnOpen(s
->listenConn
)) {
294 fds
[nfds
] = s
->listenConn
->fd
;
299 nevents
= comm_check_incoming_select_handlers(nfds
, fds
);
300 incoming_tcp_interval
+= Config
.comm_incoming
.tcp
.average
- nevents
;
302 if (incoming_tcp_interval
< 0)
303 incoming_tcp_interval
= 0;
305 if (incoming_tcp_interval
> MAX_INCOMING_INTERVAL
)
306 incoming_tcp_interval
= MAX_INCOMING_INTERVAL
;
308 if (nevents
> INCOMING_TCP_MAX
)
309 nevents
= INCOMING_TCP_MAX
;
311 statCounter
.comm_tcp_incoming
.count(nevents
);
314 #define DEBUG_FDBITS 0
315 /* Select on all sockets; call handlers for those that are ready. */
317 Comm::DoSelect(int msec
)
328 int calldns
= 0, calludp
= 0, calltcp
= 0;
334 struct timeval poll_time
;
335 double timeout
= current_dtime
+ (msec
/ 1000.0);
345 start
= current_dtime
;
347 if (commCheckUdpIncoming
)
348 comm_select_udp_incoming();
350 if (commCheckDnsIncoming
)
351 comm_select_dns_incoming();
353 if (commCheckTcpIncoming
)
354 comm_select_tcp_incoming();
356 calldns
= calludp
= calltcp
= 0;
358 maxfd
= Biggest_FD
+ 1;
360 memcpy(&readfds
, &global_readfds
, sizeof(global_readfds
));
362 memcpy(&writefds
, &global_writefds
, sizeof(global_writefds
));
364 memcpy(&errfds
, &global_writefds
, sizeof(global_writefds
));
366 /* remove stalled FDs, and deal with pending descriptors */
369 FD_ZERO(&pendingfds
);
371 for (j
= 0; j
< (int) readfds
.fd_count
; ++j
) {
372 register int readfds_handle
= readfds
.fd_array
[j
];
375 for ( fd
= Biggest_FD
; fd
; --fd
) {
376 if ( fd_table
[fd
].win32
.handle
== readfds_handle
) {
377 if (fd_table
[fd
].flags
.open
) {
387 if (FD_ISSET(fd
, &readfds
) && fd_table
[fd
].flags
.read_pending
) {
388 FD_SET(fd
, &pendingfds
);
394 for (i
= 0; i
< maxfd
; ++i
) {
395 /* Check each open socket for a handler. */
397 if (fd_table
[i
].read_handler
) {
398 assert(FD_ISSET(i
, readfds
));
401 if (fd_table
[i
].write_handler
) {
402 assert(FD_ISSET(i
, writefds
));
407 if (nreadfds
+ nwritefds
== 0) {
408 assert(shutting_down
);
409 return Comm::SHUTDOWN
;
412 if (msec
> MAX_POLL_TIME
)
413 msec
= MAX_POLL_TIME
;
419 poll_time
.tv_sec
= msec
/ 1000;
420 poll_time
.tv_usec
= (msec
% 1000) * 1000;
421 ++ statCounter
.syscalls
.selects
;
422 num
= select(maxfd
, &readfds
, &writefds
, &errfds
, &poll_time
);
424 ++ statCounter
.select_loops
;
426 if (num
>= 0 || pending
> 0)
429 if (ignoreErrno(xerrno
))
432 debugs(5, DBG_CRITICAL
, MYNAME
<< "WARNING: select failure: " << xstrerr(xerrno
));
434 examine_select(&readfds
, &writefds
);
436 return Comm::COMM_ERROR
;
441 if (num
< 0 && !pending
)
446 debugs(5, num
? 5 : 8, "comm_select: " << num
<< "+" << pending
<< " FDs ready");
448 statCounter
.select_fds_hist
.count(num
);
450 if (num
== 0 && pending
== 0)
453 /* Scan return fd masks for ready descriptors */
454 assert(readfds
.fd_count
<= (unsigned int) Biggest_FD
);
455 assert(pendingfds
.fd_count
<= (unsigned int) Biggest_FD
);
457 for (j
= 0; j
< (int) readfds
.fd_count
; ++j
) {
458 register int readfds_handle
= readfds
.fd_array
[j
];
459 register int pendingfds_handle
= pendingfds
.fd_array
[j
];
460 register int osfhandle
;
463 for ( fd
= Biggest_FD
; fd
; --fd
) {
464 osfhandle
= fd_table
[fd
].win32
.handle
;
466 if (( osfhandle
== readfds_handle
) ||
467 ( osfhandle
== pendingfds_handle
)) {
468 if (fd_table
[fd
].flags
.open
) {
480 debugs(5, 9, "FD " << fd
<< " bit set for reading");
482 assert(FD_ISSET(fd
, readfds
));
486 if (fdIsUdpListener(fd
)) {
496 if (fdIsTcpListener(fd
)) {
502 debugs(5, 6, "comm_select: FD " << fd
<< " ready for reading");
504 if ((hdl
= F
->read_handler
)) {
505 F
->read_handler
= NULL
;
506 commUpdateReadBits(fd
, NULL
);
507 hdl(fd
, F
->read_data
);
508 ++ statCounter
.select_fds
;
510 if (commCheckUdpIncoming
)
511 comm_select_udp_incoming();
513 if (commCheckDnsIncoming
)
514 comm_select_dns_incoming();
516 if (commCheckTcpIncoming
)
517 comm_select_tcp_incoming();
521 assert(errfds
.fd_count
<= (unsigned int) Biggest_FD
);
523 for (j
= 0; j
< (int) errfds
.fd_count
; ++j
) {
524 register int errfds_handle
= errfds
.fd_array
[j
];
526 for ( fd
= Biggest_FD
; fd
; --fd
) {
527 if ( fd_table
[fd
].win32
.handle
== errfds_handle
)
531 if (fd_table
[fd
].flags
.open
) {
534 if ((hdl
= F
->write_handler
)) {
535 F
->write_handler
= NULL
;
536 commUpdateWriteBits(fd
, NULL
);
537 hdl(fd
, F
->write_data
);
538 ++ statCounter
.select_fds
;
543 assert(writefds
.fd_count
<= (unsigned int) Biggest_FD
);
545 for (j
= 0; j
< (int) writefds
.fd_count
; ++j
) {
546 register int writefds_handle
= writefds
.fd_array
[j
];
549 for ( fd
= Biggest_FD
; fd
; --fd
) {
550 if ( fd_table
[fd
].win32
.handle
== writefds_handle
) {
551 if (fd_table
[fd
].flags
.open
) {
563 debugs(5, 9, "FD " << fd
<< " bit set for writing");
565 assert(FD_ISSET(fd
, writefds
));
569 if (fdIsUdpListener(fd
)) {
579 if (fdIsTcpListener(fd
)) {
585 debugs(5, 6, "comm_select: FD " << fd
<< " ready for writing");
587 if ((hdl
= F
->write_handler
)) {
588 F
->write_handler
= NULL
;
589 commUpdateWriteBits(fd
, NULL
);
590 hdl(fd
, F
->write_data
);
591 ++ statCounter
.select_fds
;
593 if (commCheckUdpIncoming
)
594 comm_select_udp_incoming();
596 if (commCheckDnsIncoming
)
597 comm_select_dns_incoming();
599 if (commCheckTcpIncoming
)
600 comm_select_tcp_incoming();
605 comm_select_udp_incoming();
608 comm_select_dns_incoming();
611 comm_select_tcp_incoming();
615 statCounter
.select_time
+= (current_dtime
- start
);
618 } while (timeout
> current_dtime
);
619 debugs(5, 8, "comm_select: time out: " << squid_curtime
);
621 return Comm::TIMEOUT
;
625 comm_select_dns_incoming(void)
632 if (DnsSocketA
< 0 && DnsSocketB
< 0)
635 if (DnsSocketA
>= 0) {
636 fds
[nfds
] = DnsSocketA
;
640 if (DnsSocketB
>= 0) {
641 fds
[nfds
] = DnsSocketB
;
645 nevents
= comm_check_incoming_select_handlers(nfds
, fds
);
650 incoming_dns_interval
+= Config
.comm_incoming
.dns
.average
- nevents
;
652 if (incoming_dns_interval
< Config
.comm_incoming
.dns
.min_poll
)
653 incoming_dns_interval
= Config
.comm_incoming
.dns
.min_poll
;
655 if (incoming_dns_interval
> MAX_INCOMING_INTERVAL
)
656 incoming_dns_interval
= MAX_INCOMING_INTERVAL
;
658 if (nevents
> INCOMING_DNS_MAX
)
659 nevents
= INCOMING_DNS_MAX
;
661 statCounter
.comm_dns_incoming
.count(nevents
);
665 Comm::SelectLoopInit(void)
669 FD_ZERO(&global_readfds
);
670 FD_ZERO(&global_writefds
);
671 nreadfds
= nwritefds
= 0;
673 Mgr::RegisterAction("comm_select_incoming",
674 "comm_incoming() stats",
675 commIncomingStats
, 0, 1);
679 * examine_select - debug routine.
681 * I spend the day chasing this core dump that occurs when both the client
682 * and the server side of a cache fetch simultaneoulsy abort the
683 * connection. While I haven't really studied the code to figure out how
684 * it happens, the snippet below may prevent the cache from exitting:
686 * Call this from where the select loop fails.
689 examine_select(fd_set
* readfds
, fd_set
* writefds
)
696 AsyncCall::Pointer ch
= NULL
;
700 debugs(5, DBG_CRITICAL
, "examine_select: Examining open file descriptors...");
702 for (fd
= 0; fd
< Squid_MaxFD
; ++fd
) {
705 tv
.tv_sec
= tv
.tv_usec
= 0;
707 if (FD_ISSET(fd
, readfds
))
709 else if (FD_ISSET(fd
, writefds
))
710 FD_SET(fd
, &write_x
);
714 ++ statCounter
.syscalls
.selects
;
717 if (!fstat(fd
, &sb
)) {
718 debugs(5, 5, "FD " << fd
<< " is valid.");
724 debugs(5, DBG_CRITICAL
, "fstat(FD " << fd
<< "): " << xstrerr(xerrno
));
725 debugs(5, DBG_CRITICAL
, "WARNING: FD " << fd
<< " has handlers, but it's invalid.");
726 debugs(5, DBG_CRITICAL
, "FD " << fd
<< " is a " << fdTypeStr
[F
->type
] << " called '" << F
->desc
<< "'");
727 debugs(5, DBG_CRITICAL
, "tmout:" << F
->timeoutHandler
<< " read:" << F
->read_handler
<< " write:" << F
->write_handler
);
729 for (ch
= F
->closeHandler
; ch
!= NULL
; ch
= ch
->Next())
730 debugs(5, DBG_CRITICAL
, " close handler: " << ch
);
732 if (F
->closeHandler
!= NULL
) {
733 commCallCloseHandlers(fd
);
734 } else if (F
->timeoutHandler
!= NULL
) {
735 debugs(5, DBG_CRITICAL
, "examine_select: Calling Timeout Handler");
736 ScheduleCallHere(F
->timeoutHandler
);
739 F
->closeHandler
= NULL
;
740 F
->timeoutHandler
= NULL
;
741 F
->read_handler
= NULL
;
742 F
->write_handler
= NULL
;
744 FD_CLR(fd
, writefds
);
751 commIncomingStats(StoreEntry
* sentry
)
753 storeAppendPrintf(sentry
, "Current incoming_udp_interval: %d\n",
754 incoming_udp_interval
>> INCOMING_FACTOR
);
755 storeAppendPrintf(sentry
, "Current incoming_dns_interval: %d\n",
756 incoming_dns_interval
>> INCOMING_FACTOR
);
757 storeAppendPrintf(sentry
, "Current incoming_tcp_interval: %d\n",
758 incoming_tcp_interval
>> INCOMING_FACTOR
);
759 storeAppendPrintf(sentry
, "\n");
760 storeAppendPrintf(sentry
, "Histogram of events per incoming socket type\n");
761 storeAppendPrintf(sentry
, "ICP Messages handled per comm_select_udp_incoming() call:\n");
762 statCounter
.comm_udp_incoming
.dump(sentry
, statHistIntDumper
);
763 storeAppendPrintf(sentry
, "DNS Messages handled per comm_select_dns_incoming() call:\n");
764 statCounter
.comm_dns_incoming
.dump(sentry
, statHistIntDumper
);
765 storeAppendPrintf(sentry
, "HTTP Messages handled per comm_select_tcp_incoming() call:\n");
766 statCounter
.comm_tcp_incoming
.dump(sentry
, statHistIntDumper
);
770 commUpdateReadBits(int fd
, PF
* handler
)
772 if (handler
&& !FD_ISSET(fd
, &global_readfds
)) {
773 FD_SET(fd
, &global_readfds
);
775 } else if (!handler
&& FD_ISSET(fd
, &global_readfds
)) {
776 FD_CLR(fd
, &global_readfds
);
782 commUpdateWriteBits(int fd
, PF
* handler
)
784 if (handler
&& !FD_ISSET(fd
, &global_writefds
)) {
785 FD_SET(fd
, &global_writefds
);
787 } else if (!handler
&& FD_ISSET(fd
, &global_writefds
)) {
788 FD_CLR(fd
, &global_writefds
);
793 /* Called by async-io or diskd to speed up the polling */
795 Comm::QuickPollRequired(void)
800 #endif /* USE_SELECT_WIN32 */