2 * Copyright (C) 1996-2021 The Squid Software Foundation and contributors
4 * Squid software is distributed under GPLv2+ license and includes
5 * contributions from numerous individuals and organizations.
6 * Please see the COPYING and CONTRIBUTORS files for details.
9 /* DEBUG: section 35 FQDN Cache */
13 #include "dns/forward.h"
14 #include "dns/LookupDetails.h"
15 #include "dns/rfc1035.h"
17 #include "fqdncache.h"
19 #include "mgr/Registration.h"
20 #include "snmp_agent.h"
21 #include "SquidConfig.h"
22 #include "SquidTime.h"
23 #include "StatCounters.h"
28 #include "snmp_core.h"
32 \defgroup FQDNCacheAPI FQDN Cache API
34 \section FQDNCacheIntroduction Introduction
36 * The FQDN cache is a built-in component of squid providing
37 * Hostname to IP-Number translation functionality and managing
38 * the involved data-structures. Efficiency concerns require
39 * mechanisms that allow non-blocking access to these mappings.
40 * The FQDN cache usually doesn't block on a request except for
41 * special cases where this is desired (see below).
45 \defgroup FQDNCacheInternal FQDN Cache Internals
48 * Internally, the execution flow is as follows:
49 * On a miss, fqdncache_nbgethostbyaddr() checks whether a request
50 * for this name is already pending, and if positive, it creates a
51 * new entry using fqdncacheAddEntry(). Then it calls
52 * fqdncacheAddPending() to add a request to the queue together
53 * with data and handler. Else, ifqdncache_dnsDispatch() is called
54 * to directly create a DNS query or to fqdncacheEnqueue() if all
55 * no DNS port is free.
58 * fqdncacheCallback() is called regularly to walk down the pending
59 * list and call handlers.
62 * LRU clean-up is performed through fqdncache_purgelru() according
63 * to the fqdncache_high threshold.
66 /// \ingroup FQDNCacheInternal
67 #define FQDN_LOW_WATER 90
69 /// \ingroup FQDNCacheInternal
70 #define FQDN_HIGH_WATER 95
73 * The data structure used for storing name-address mappings
74 * is a small hashtable (static hash_table *fqdn_table),
75 * where structures of type fqdncache_entry whose most
76 * interesting members are:
80 MEMPROXY_CLASS(fqdncache_entry
);
83 fqdncache_entry(const char *name
);
86 hash_link hash
; /* must be first */
89 unsigned char name_count
;
90 char *names
[FQDN_MAX_NAMES
+ 1];
95 struct timeval request_time
;
100 Flags() : negcached(false), fromhosts(false) {}
106 int age() const; ///< time passed since request_time or -1 if unknown
109 /// \ingroup FQDNCacheInternal
110 static struct _fqdn_cache_stats
{
118 /// \ingroup FQDNCacheInternal
119 static dlink_list lru_list
;
121 static IDNSCB fqdncacheHandleReply
;
122 static int fqdncacheParse(fqdncache_entry
*, const rfc1035_rr
*, int, const char *error_message
);
123 static void fqdncacheRelease(fqdncache_entry
*);
124 static void fqdncacheCallback(fqdncache_entry
*, int wait
);
125 static fqdncache_entry
*fqdncache_get(const char *);
126 static int fqdncacheExpiredEntry(const fqdncache_entry
*);
127 static void fqdncacheLockEntry(fqdncache_entry
* f
);
128 static void fqdncacheUnlockEntry(fqdncache_entry
* f
);
129 static FREE fqdncacheFreeEntry
;
130 static void fqdncacheAddEntry(fqdncache_entry
* f
);
132 /// \ingroup FQDNCacheInternal
133 static hash_table
*fqdn_table
= NULL
;
135 /// \ingroup FQDNCacheInternal
136 static long fqdncache_low
= 180;
138 /// \ingroup FQDNCacheInternal
139 static long fqdncache_high
= 200;
141 /// \ingroup FQDNCacheInternal
142 inline int fqdncacheCount() { return fqdn_table
? fqdn_table
->count
: 0; }
145 fqdncache_entry::age() const
147 return request_time
.tv_sec
? tvSubMsec(request_time
, current_time
) : -1;
151 \ingroup FQDNCacheInternal
152 * Removes the given fqdncache entry
155 fqdncacheRelease(fqdncache_entry
* f
)
157 hash_remove_link(fqdn_table
, (hash_link
*) f
);
158 debugs(35, 5, "fqdncacheRelease: Released FQDN record for '" << hashKeyStr(&f
->hash
) << "'.");
159 dlinkDelete(&f
->lru
, &lru_list
);
164 \ingroup FQDNCacheInternal
165 \param name FQDN hash string.
166 \retval Match for given name
168 static fqdncache_entry
*
169 fqdncache_get(const char *name
)
172 static fqdncache_entry
*f
;
176 if ((e
= (hash_link
*)hash_lookup(fqdn_table
, name
)) != NULL
)
177 f
= (fqdncache_entry
*) e
;
183 /// \ingroup FQDNCacheInternal
185 fqdncacheExpiredEntry(const fqdncache_entry
* f
)
187 /* all static entries are locked, so this takes care of them too */
192 if (f
->expires
> squid_curtime
)
198 /// \ingroup FQDNCacheAPI
200 fqdncache_purgelru(void *)
203 dlink_node
*prev
= NULL
;
206 eventAdd("fqdncache_purgelru", fqdncache_purgelru
, NULL
, 10.0, 1);
208 for (m
= lru_list
.tail
; m
; m
= prev
) {
209 if (fqdncacheCount() < fqdncache_low
)
214 f
= (fqdncache_entry
*)m
->data
;
224 debugs(35, 9, "fqdncache_purgelru: removed " << removed
<< " entries");
227 /// \ingroup FQDNCacheAPI
229 purge_entries_fromhosts(void)
231 dlink_node
*m
= lru_list
.head
;
232 fqdncache_entry
*i
= NULL
;
236 if (i
!= NULL
) { /* need to delay deletion */
237 fqdncacheRelease(i
); /* we just override locks */
241 t
= (fqdncache_entry
*)m
->data
;
243 if (t
->flags
.fromhosts
)
253 fqdncache_entry::fqdncache_entry(const char *name
) :
255 expires(squid_curtime
+ Config
.negativeDnsTtl
),
258 handlerData(nullptr),
259 error_message(nullptr),
260 locks(0) // XXX: use Lock
262 hash
.key
= xstrdup(name
);
264 memset(&request_time
, 0, sizeof(request_time
));
265 memset(&names
, 0, sizeof(names
));
268 /// \ingroup FQDNCacheInternal
270 fqdncacheAddEntry(fqdncache_entry
* f
)
272 hash_link
*e
= (hash_link
*)hash_lookup(fqdn_table
, f
->hash
.key
);
275 /* avoid collision */
276 fqdncache_entry
*q
= (fqdncache_entry
*) e
;
280 hash_join(fqdn_table
, &f
->hash
);
281 dlinkAdd(f
, &f
->lru
, &lru_list
);
282 f
->lastref
= squid_curtime
;
286 \ingroup FQDNCacheInternal
288 * Walks down the pending list, calling handlers
291 fqdncacheCallback(fqdncache_entry
* f
, int wait
)
295 f
->lastref
= squid_curtime
;
300 fqdncacheLockEntry(f
);
302 callback
= f
->handler
;
306 if (cbdataReferenceValidDone(f
->handlerData
, &cbdata
)) {
307 const Dns::LookupDetails
details(f
->error_message
, wait
);
308 callback(f
->name_count
? f
->names
[0] : NULL
, details
, cbdata
);
311 fqdncacheUnlockEntry(f
);
314 /// \ingroup FQDNCacheInternal
316 fqdncacheParse(fqdncache_entry
*f
, const rfc1035_rr
* answers
, int nr
, const char *error_message
)
320 const char *name
= (const char *)f
->hash
.key
;
321 f
->expires
= squid_curtime
+ Config
.negativeDnsTtl
;
322 f
->flags
.negcached
= true;
325 debugs(35, 3, "fqdncacheParse: Lookup of '" << name
<< "' failed (" << error_message
<< ")");
326 f
->error_message
= xstrdup(error_message
);
331 debugs(35, 3, "fqdncacheParse: No DNS records for '" << name
<< "'");
332 f
->error_message
= xstrdup("No DNS records");
336 debugs(35, 3, "fqdncacheParse: " << nr
<< " answers for '" << name
<< "'");
339 for (k
= 0; k
< nr
; ++k
) {
340 if (answers
[k
]._class
!= RFC1035_CLASS_IN
)
343 if (answers
[k
].type
== RFC1035_TYPE_PTR
) {
344 if (!answers
[k
].rdata
[0]) {
345 debugs(35, 2, "fqdncacheParse: blank PTR record for '" << name
<< "'");
349 if (strchr(answers
[k
].rdata
, ' ')) {
350 debugs(35, 2, "fqdncacheParse: invalid PTR record '" << answers
[k
].rdata
<< "' for '" << name
<< "'");
354 f
->names
[f
->name_count
] = xstrdup(answers
[k
].rdata
);
356 } else if (answers
[k
].type
!= RFC1035_TYPE_CNAME
)
359 if (ttl
== 0 || (int) answers
[k
].ttl
< ttl
)
360 ttl
= answers
[k
].ttl
;
362 if (f
->name_count
>= FQDN_MAX_NAMES
)
366 if (f
->name_count
== 0) {
367 debugs(35, DBG_IMPORTANT
, "fqdncacheParse: No PTR record for '" << name
<< "'");
371 if (ttl
> Config
.positiveDnsTtl
)
372 ttl
= Config
.positiveDnsTtl
;
374 if (ttl
< Config
.negativeDnsTtl
)
375 ttl
= Config
.negativeDnsTtl
;
377 f
->expires
= squid_curtime
+ ttl
;
379 f
->flags
.negcached
= false;
381 return f
->name_count
;
385 \ingroup FQDNCacheAPI
387 * Callback for handling DNS results.
390 fqdncacheHandleReply(void *data
, const rfc1035_rr
* answers
, int na
, const char *error_message
, const bool lastAnswer
)
392 assert(lastAnswer
); // reverse DNS lookups do not generate multiple queries
394 static_cast<generic_cbdata
*>(data
)->unwrap(&f
);
395 ++FqdncacheStats
.replies
;
396 const int age
= f
->age();
397 statCounter
.dns
.svcTime
.count(age
);
398 fqdncacheParse(f
, answers
, na
, error_message
);
399 fqdncacheAddEntry(f
);
400 fqdncacheCallback(f
, age
);
404 \ingroup FQDNCacheAPI
406 \param addr IP address of domain to resolve.
407 \param handler A pointer to the function to be called when
408 * the reply from the FQDN cache
409 * (or the DNS if the FQDN cache misses)
410 \param handlerData Information that is passed to the handler
411 * and does not affect the FQDN cache.
414 fqdncache_nbgethostbyaddr(const Ip::Address
&addr
, FQDNH
* handler
, void *handlerData
)
416 fqdncache_entry
*f
= NULL
;
417 char name
[MAX_IPSTRLEN
];
419 addr
.toStr(name
,MAX_IPSTRLEN
);
420 debugs(35, 4, "fqdncache_nbgethostbyaddr: Name '" << name
<< "'.");
421 ++FqdncacheStats
.requests
;
423 if (name
[0] == '\0') {
424 debugs(35, 4, "fqdncache_nbgethostbyaddr: Invalid name!");
425 const Dns::LookupDetails
details("Invalid hostname", -1); // error, no lookup
427 handler(NULL
, details
, handlerData
);
431 f
= fqdncache_get(name
);
436 } else if (fqdncacheExpiredEntry(f
)) {
437 /* hit, but expired -- bummer */
442 debugs(35, 4, "fqdncache_nbgethostbyaddr: HIT for '" << name
<< "'");
444 if (f
->flags
.negcached
)
445 ++ FqdncacheStats
.negative_hits
;
447 ++ FqdncacheStats
.hits
;
449 f
->handler
= handler
;
451 f
->handlerData
= cbdataReference(handlerData
);
453 fqdncacheCallback(f
, -1); // no lookup
458 debugs(35, 5, "fqdncache_nbgethostbyaddr: MISS for '" << name
<< "'");
459 ++ FqdncacheStats
.misses
;
460 f
= new fqdncache_entry(name
);
461 f
->handler
= handler
;
462 f
->handlerData
= cbdataReference(handlerData
);
463 f
->request_time
= current_time
;
464 c
= new generic_cbdata(f
);
465 idnsPTRLookup(addr
, fqdncacheHandleReply
, c
);
469 \ingroup FQDNCacheAPI
471 * Is different in that it only checks if an entry exists in
472 * it's data-structures and does not by default contact the
473 * DNS, unless this is requested, by setting the flags
474 * to FQDN_LOOKUP_IF_MISS.
476 \param addr address of the FQDN being resolved
477 \param flags values are NULL or FQDN_LOOKUP_IF_MISS. default is NULL.
481 fqdncache_gethostbyaddr(const Ip::Address
&addr
, int flags
)
483 char name
[MAX_IPSTRLEN
];
484 fqdncache_entry
*f
= NULL
;
486 if (addr
.isAnyAddr() || addr
.isNoAddr()) {
490 addr
.toStr(name
,MAX_IPSTRLEN
);
491 ++ FqdncacheStats
.requests
;
492 f
= fqdncache_get(name
);
496 } else if (fqdncacheExpiredEntry(f
)) {
499 } else if (f
->flags
.negcached
) {
500 ++ FqdncacheStats
.negative_hits
;
501 // ignore f->error_message: the caller just checks FQDN cache presence
504 ++ FqdncacheStats
.hits
;
505 f
->lastref
= squid_curtime
;
506 // ignore f->error_message: the caller just checks FQDN cache presence
510 /* no entry [any more] */
512 ++ FqdncacheStats
.misses
;
514 if (flags
& FQDN_LOOKUP_IF_MISS
) {
515 fqdncache_nbgethostbyaddr(addr
, NULL
, NULL
);
522 \ingroup FQDNCacheInternal
524 * Process objects list
527 fqdnStats(StoreEntry
* sentry
)
529 fqdncache_entry
*f
= NULL
;
533 if (fqdn_table
== NULL
)
536 storeAppendPrintf(sentry
, "FQDN Cache Statistics:\n");
538 storeAppendPrintf(sentry
, "FQDNcache Entries In Use: %d\n",
539 fqdncache_entry::UseCount());
541 storeAppendPrintf(sentry
, "FQDNcache Entries Cached: %d\n",
544 storeAppendPrintf(sentry
, "FQDNcache Requests: %d\n",
545 FqdncacheStats
.requests
);
547 storeAppendPrintf(sentry
, "FQDNcache Hits: %d\n",
548 FqdncacheStats
.hits
);
550 storeAppendPrintf(sentry
, "FQDNcache Negative Hits: %d\n",
551 FqdncacheStats
.negative_hits
);
553 storeAppendPrintf(sentry
, "FQDNcache Misses: %d\n",
554 FqdncacheStats
.misses
);
556 storeAppendPrintf(sentry
, "FQDN Cache Contents:\n\n");
558 storeAppendPrintf(sentry
, "%-45.45s %3s %3s %3s %s\n",
559 "Address", "Flg", "TTL", "Cnt", "Hostnames");
561 hash_first(fqdn_table
);
563 while ((f
= (fqdncache_entry
*) hash_next(fqdn_table
))) {
564 ttl
= (f
->flags
.fromhosts
? -1 : (f
->expires
- squid_curtime
));
565 storeAppendPrintf(sentry
, "%-45.45s %c%c %3.3d % 3d",
566 hashKeyStr(&f
->hash
),
567 f
->flags
.negcached
? 'N' : ' ',
568 f
->flags
.fromhosts
? 'H' : ' ',
570 (int) f
->name_count
);
572 for (k
= 0; k
< (int) f
->name_count
; ++k
)
573 storeAppendPrintf(sentry
, " %s", f
->names
[k
]);
575 storeAppendPrintf(sentry
, "\n");
579 /// \ingroup FQDNCacheInternal
581 fqdncacheLockEntry(fqdncache_entry
* f
)
583 if (f
->locks
++ == 0) {
584 dlinkDelete(&f
->lru
, &lru_list
);
585 dlinkAdd(f
, &f
->lru
, &lru_list
);
589 /// \ingroup FQDNCacheInternal
591 fqdncacheUnlockEntry(fqdncache_entry
* f
)
593 assert(f
->locks
> 0);
596 if (fqdncacheExpiredEntry(f
))
600 /// \ingroup FQDNCacheInternal
602 fqdncacheFreeEntry(void *data
)
604 fqdncache_entry
*f
= (fqdncache_entry
*)data
;
608 fqdncache_entry::~fqdncache_entry()
610 for (int k
= 0; k
< (int)name_count
; ++k
)
614 xfree(error_message
);
617 /// \ingroup FQDNCacheAPI
619 fqdncacheFreeMemory(void)
621 hashFreeItems(fqdn_table
, fqdncacheFreeEntry
);
622 hashFreeMemory(fqdn_table
);
627 \ingroup FQDNCacheAPI
629 * Recalculate FQDN cache size upon reconfigure.
630 * Is called to clear the FQDN cache's data structures,
631 * cancel all pending requests.
634 fqdncache_restart(void)
636 fqdncache_high
= (long) (((float) Config
.fqdncache
.size
*
637 (float) FQDN_HIGH_WATER
) / (float) 100);
638 fqdncache_low
= (long) (((float) Config
.fqdncache
.size
*
639 (float) FQDN_LOW_WATER
) / (float) 100);
640 purge_entries_fromhosts();
644 * Adds a "static" entry from /etc/hosts.
646 \param addr FQDN name to be added.
647 \param hostnames list of hostnames for the addr
650 fqdncacheAddEntryFromHosts(char *addr
, SBufList
&hostnames
)
652 fqdncache_entry
*fce
= fqdncache_get(addr
);
654 if (1 == fce
->flags
.fromhosts
) {
655 fqdncacheUnlockEntry(fce
);
656 } else if (fce
->locks
> 0) {
657 debugs(35, DBG_IMPORTANT
, "WARNING: can't add static entry for locked address '" << addr
<< "'");
660 fqdncacheRelease(fce
);
664 fce
= new fqdncache_entry(addr
);
667 for (auto &h
: hostnames
) {
668 fce
->names
[j
] = xstrdup(h
.c_str());
669 Tolower(fce
->names
[j
]);
672 if (j
>= FQDN_MAX_NAMES
)
677 fce
->names
[j
] = NULL
; /* it's safe */
678 fce
->flags
.fromhosts
= true;
679 fqdncacheAddEntry(fce
);
680 fqdncacheLockEntry(fce
);
683 /// \ingroup FQDNCacheInternal
685 fqdncacheRegisterWithCacheManager(void)
687 Mgr::RegisterAction("fqdncache", "FQDN Cache Stats and Contents",
693 \ingroup FQDNCacheAPI
695 * Initialize the fqdncache.
696 * Called after IP cache initialization.
703 fqdncacheRegisterWithCacheManager();
708 debugs(35, 3, "Initializing FQDN Cache...");
710 memset(&FqdncacheStats
, '\0', sizeof(FqdncacheStats
));
711 lru_list
= dlink_list();
713 fqdncache_high
= (long) (((float) Config
.fqdncache
.size
*
714 (float) FQDN_HIGH_WATER
) / (float) 100);
716 fqdncache_low
= (long) (((float) Config
.fqdncache
.size
*
717 (float) FQDN_LOW_WATER
) / (float) 100);
719 n
= hashPrime(fqdncache_high
/ 4);
721 fqdn_table
= hash_create((HASHCMP
*) strcmp
, n
, hash4
);
726 * \ingroup FQDNCacheAPI
727 * The function to return the FQDN statistics via SNMP
730 snmp_netFqdnFn(variable_list
* Var
, snint
* ErrP
)
732 variable_list
*Answer
= NULL
;
734 debugs(49, 5, "snmp_netFqdnFn: Processing request:" << snmpDebugOid(Var
->name
, Var
->name_length
, tmp
));
735 *ErrP
= SNMP_ERR_NOERROR
;
737 switch (Var
->name
[LEN_SQ_NET
+ 1]) {
740 Answer
= snmp_var_new_integer(Var
->name
, Var
->name_length
,
746 Answer
= snmp_var_new_integer(Var
->name
, Var
->name_length
,
747 FqdncacheStats
.requests
,
752 Answer
= snmp_var_new_integer(Var
->name
, Var
->name_length
,
758 /* this is now worthless */
759 Answer
= snmp_var_new_integer(Var
->name
, Var
->name_length
,
765 Answer
= snmp_var_new_integer(Var
->name
, Var
->name_length
,
766 FqdncacheStats
.negative_hits
,
771 Answer
= snmp_var_new_integer(Var
->name
, Var
->name_length
,
772 FqdncacheStats
.misses
,
777 Answer
= snmp_var_new_integer(Var
->name
, Var
->name_length
,
783 *ErrP
= SNMP_ERR_NOSUCHNAME
;
790 #endif /*SQUID_SNMP */