src/icmp/net_db.cc

   1 /*
   2  * Copyright (C) 1996-2023 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 38    Network Measurement Database */
  10
  11 /*
  12  * XXX XXX XXX
  13  *
  14  * This code may be slightly broken now. If you're getting consistent
  15  * (sometimes working) corrupt data exchanges, please contact adrian
  16  * (adrian@squid-cache.org) to sort them out.
  17  */
  18
  19 #include "squid.h"
  20 #include "CachePeer.h"
  21 #include "CachePeers.h"
  22 #include "cbdata.h"
  23 #include "event.h"
  24 #include "fde.h"
  25 #include "fs_io.h"
  26 #include "FwdState.h"
  27 #include "HttpReply.h"
  28 #include "icmp/net_db.h"
  29 #include "internal.h"
  30 #include "ip/Address.h"
  31 #include "log/File.h"
  32 #include "MemObject.h"
  33 #include "mgr/Registration.h"
  34 #include "mime_header.h"
  35 #include "neighbors.h"
  36 #include "PeerSelectState.h"
  37 #include "sbuf/SBuf.h"
  38 #include "SquidConfig.h"
  39 #include "Store.h"
  40 #include "StoreClient.h"
  41 #include "tools.h"
  42 #include "wordlist.h"
  43
  44 #if HAVE_SYS_STAT_H
  45 #include <sys/stat.h>
  46 #endif
  47
  48 #if USE_ICMP
  49 #include "icmp/IcmpSquid.h"
  50 #include "ipcache.h"
  51 #include "StoreClient.h"
  52
  53 typedef enum {
  54     STATE_NONE,
  55     STATE_HEADER,
  56     STATE_BODY
  57 } netdb_conn_state_t;
  58
  59 class netdbExchangeState
  60 {
  61     CBDATA_CLASS(netdbExchangeState);
  62
  63 public:
  64     netdbExchangeState(CachePeer *aPeer, const HttpRequestPointer &theReq) :
  65         p(aPeer),
  66         r(theReq)
  67     {
  68         assert(r);
  69         // TODO: check if we actually need to do this. should be implicit
  70         r->http_ver = Http::ProtocolVersion();
  71     }
  72
  73     ~netdbExchangeState() {
  74         debugs(38, 3, e->url());
  75         storeUnregister(sc, e, this);
  76         e->unlock("netdbExchangeDone");
  77     }
  78
  79     CbcPointer<CachePeer> p;
  80     StoreEntry *e = nullptr;
  81     store_client *sc = nullptr;
  82     HttpRequestPointer r;
  83
  84     /// for receiving a NetDB reply body from Store and interpreting it
  85     Store::ParsingBuffer parsingBuffer;
  86
  87     netdb_conn_state_t connstate = STATE_HEADER;
  88 };
  89
  90 CBDATA_CLASS_INIT(netdbExchangeState);
  91
  92 static hash_table *addr_table = nullptr;
  93 static hash_table *host_table = nullptr;
  94
  95 static Ip::Address networkFromInaddr(const Ip::Address &a);
  96 static void netdbRelease(netdbEntry * n);
  97
  98 static void netdbHashInsert(netdbEntry * n, Ip::Address &addr);
  99 static void netdbHashDelete(const char *key);
 100 static void netdbHostInsert(netdbEntry * n, const char *hostname);
 101 static void netdbHostDelete(const net_db_name * x);
 102 static void netdbPurgeLRU(void);
 103 static netdbEntry *netdbLookupHost(const char *key);
 104 static net_db_peer *netdbPeerByName(const netdbEntry * n, const char *);
 105 static net_db_peer *netdbPeerAdd(netdbEntry * n, CachePeer * e);
 106 static const char *netdbPeerName(const char *name);
 107 static IPH netdbSendPing;
 108 static STCB netdbExchangeHandleReply;
 109
 110 /* We have to keep a local list of CachePeer names.  The Peers structure
 111  * gets freed during a reconfigure.  We want this database to
 112  * remain persisitent, so _net_db_peer->peername points into this
 113  * linked list */
 114 static wordlist *peer_names = nullptr;
 115
 116 static void
 117 netdbHashInsert(netdbEntry * n, Ip::Address &addr)
 118 {
 119     networkFromInaddr(addr).toStr(n->network, MAX_IPSTRLEN);
 120     n->key = n->network;
 121     assert(hash_lookup(addr_table, n->network) == nullptr);
 122     hash_join(addr_table, n);
 123 }
 124
 125 static void
 126 netdbHashDelete(const char *key)
 127 {
 128     hash_link *hptr = (hash_link *)hash_lookup(addr_table, key);
 129
 130     if (hptr == nullptr) {
 131         debug_trap("netdbHashDelete: key not found");
 132         return;
 133     }
 134
 135     hash_remove_link(addr_table, hptr);
 136 }
 137
 138 net_db_name::net_db_name(const char *hostname, netdbEntry *e) :
 139     next(e ? e->hosts : nullptr),
 140     net_db_entry(e)
 141 {
 142     key = xstrdup(hostname);
 143     if (e) {
 144         e->hosts = this;
 145         ++ e->link_count;
 146     }
 147 }
 148
 149 static void
 150 netdbHostInsert(netdbEntry * n, const char *hostname)
 151 {
 152     net_db_name *x = new net_db_name(hostname, n);
 153     assert(hash_lookup(host_table, hostname) == nullptr);
 154     hash_join(host_table, x);
 155 }
 156
 157 static void
 158 netdbHostDelete(const net_db_name * x)
 159 {
 160     assert(x != nullptr);
 161     assert(x->net_db_entry != nullptr);
 162
 163     netdbEntry *n = x->net_db_entry;
 164     -- n->link_count;
 165
 166     for (auto **X = &n->hosts; *X; X = &(*X)->next) {
 167         if (*X == x) {
 168             *X = x->next;
 169             break;
 170         }
 171     }
 172
 173     hash_remove_link(host_table, (hash_link *) x);
 174     delete x;
 175 }
 176
 177 static netdbEntry *
 178 netdbLookupHost(const char *key)
 179 {
 180     net_db_name *x = (net_db_name *) hash_lookup(host_table, key);
 181     return x ? x->net_db_entry : nullptr;
 182 }
 183
 184 static void
 185 netdbRelease(netdbEntry * n)
 186 {
 187     net_db_name *x;
 188     net_db_name *next;
 189
 190     for (x = n->hosts; x; x = next) {
 191         next = x->next;
 192         netdbHostDelete(x);
 193     }
 194
 195     n->hosts = nullptr;
 196     safe_free(n->peers);
 197     n->peers = nullptr;
 198     n->n_peers = 0;
 199     n->n_peers_alloc = 0;
 200
 201     if (n->link_count == 0) {
 202         netdbHashDelete(n->network);
 203         delete n;
 204     }
 205 }
 206
 207 static int
 208 netdbLRU(const void *A, const void *B)
 209 {
 210     const netdbEntry *const *n1 = (const netdbEntry *const *)A;
 211     const netdbEntry *const *n2 = (const netdbEntry *const *)B;
 212
 213     if ((*n1)->last_use_time > (*n2)->last_use_time)
 214         return (1);
 215
 216     if ((*n1)->last_use_time < (*n2)->last_use_time)
 217         return (-1);
 218
 219     return (0);
 220 }
 221
 222 static void
 223 netdbPurgeLRU(void)
 224 {
 225     netdbEntry *n;
 226     netdbEntry **list;
 227     int k = 0;
 228     int list_count = 0;
 229     list = (netdbEntry **)xcalloc(netdbEntry::UseCount(), sizeof(netdbEntry *));
 230     hash_first(addr_table);
 231
 232     while ((n = (netdbEntry *) hash_next(addr_table))) {
 233         assert(list_count < netdbEntry::UseCount());
 234         *(list + list_count) = n;
 235         ++list_count;
 236     }
 237
 238     qsort((char *) list,
 239           list_count,
 240           sizeof(netdbEntry *),
 241           netdbLRU);
 242
 243     for (k = 0; k < list_count; ++k) {
 244         if (netdbEntry::UseCount() < Config.Netdb.low)
 245             break;
 246
 247         netdbRelease(*(list + k));
 248     }
 249
 250     xfree(list);
 251 }
 252
 253 static netdbEntry *
 254 netdbLookupAddr(const Ip::Address &addr)
 255 {
 256     netdbEntry *n;
 257     char *key = new char[MAX_IPSTRLEN];
 258     networkFromInaddr(addr).toStr(key,MAX_IPSTRLEN);
 259     n = (netdbEntry *) hash_lookup(addr_table, key);
 260     delete[] key;
 261     return n;
 262 }
 263
 264 static netdbEntry *
 265 netdbAdd(Ip::Address &addr)
 266 {
 267     netdbEntry *n;
 268
 269     if (netdbEntry::UseCount() > Config.Netdb.high)
 270         netdbPurgeLRU();
 271
 272     if ((n = netdbLookupAddr(addr)) == nullptr) {
 273         n = new netdbEntry;
 274         netdbHashInsert(n, addr);
 275     }
 276
 277     return n;
 278 }
 279
 280 static void
 281 netdbSendPing(const ipcache_addrs *ia, const Dns::LookupDetails &, void *data)
 282 {
 283     Ip::Address addr;
 284     char *hostname = nullptr;
 285     static_cast<generic_cbdata *>(data)->unwrap(&hostname);
 286     netdbEntry *n;
 287     netdbEntry *na;
 288     net_db_name *x;
 289     net_db_name **X;
 290
 291     if (ia == nullptr) {
 292         xfree(hostname);
 293         return;
 294     }
 295
 296     addr = ia->current();
 297
 298     if ((n = netdbLookupHost(hostname)) == nullptr) {
 299         n = netdbAdd(addr);
 300         netdbHostInsert(n, hostname);
 301     } else if ((na = netdbLookupAddr(addr)) != n) {
 302         /*
 303          *hostname moved from 'network n' to 'network na'!
 304          */
 305
 306         if (na == nullptr)
 307             na = netdbAdd(addr);
 308
 309         debugs(38, 3, "netdbSendPing: " << hostname << " moved from " << n->network << " to " << na->network);
 310
 311         x = (net_db_name *) hash_lookup(host_table, hostname);
 312
 313         if (x == nullptr) {
 314             debugs(38, DBG_IMPORTANT, "ERROR: Squid BUG: net_db_name list bug: " << hostname << " not found");
 315             xfree(hostname);
 316             return;
 317         }
 318
 319         /* remove net_db_name from 'network n' linked list */
 320         for (X = &n->hosts; *X; X = &(*X)->next) {
 321             if (*X == x) {
 322                 *X = x->next;
 323                 break;
 324             }
 325         }
 326
 327         -- n->link_count;
 328         /* point to 'network na' from host entry */
 329         x->net_db_entry = na;
 330         /* link net_db_name to 'network na' */
 331         x->next = na->hosts;
 332         na->hosts = x;
 333         ++ na->link_count;
 334         n = na;
 335     }
 336
 337     if (n->next_ping_time <= squid_curtime) {
 338         debugs(38, 3, "netdbSendPing: pinging " << hostname);
 339         icmpEngine.DomainPing(addr, hostname);
 340         ++ n->pings_sent;
 341         n->next_ping_time = squid_curtime + Config.Netdb.period;
 342         n->last_use_time = squid_curtime;
 343     }
 344
 345     xfree(hostname);
 346 }
 347
 348 static Ip::Address
 349 networkFromInaddr(const Ip::Address &in)
 350 {
 351     Ip::Address out;
 352
 353     out = in;
 354
 355     /* in IPv6 the 'network' should be the routing section. */
 356     if ( in.isIPv6() ) {
 357         out.applyMask(64, AF_INET6);
 358         debugs(14, 5, "networkFromInaddr : Masked IPv6 Address to " << in << "/64 routing part.");
 359         return out;
 360     }
 361
 362     debugs(14, 5, "networkFromInaddr : Masked IPv4 Address to " << out << "/24.");
 363
 364     /* use /24 for everything under IPv4 */
 365     out.applyMask(24, AF_INET);
 366     debugs(14, 5, "networkFromInaddr : Masked IPv4 Address to " << in << "/24.");
 367
 368     return out;
 369 }
 370
 371 static int
 372 sortByRtt(const void *A, const void *B)
 373 {
 374     const netdbEntry *const *n1 = (const netdbEntry *const *)A;
 375     const netdbEntry *const *n2 = (const netdbEntry *const *)B;
 376
 377     if ((*n1)->rtt > (*n2)->rtt)
 378         return 1;
 379     else if ((*n1)->rtt < (*n2)->rtt)
 380         return -1;
 381     else
 382         return 0;
 383 }
 384
 385 static net_db_peer *
 386 netdbPeerByName(const netdbEntry * n, const char *peername)
 387 {
 388     int i;
 389     net_db_peer *p = n->peers;
 390
 391     for (i = 0; i < n->n_peers; ++i, ++p) {
 392         if (!strcmp(p->peername, peername))
 393             return p;
 394     }
 395
 396     return nullptr;
 397 }
 398
 399 static net_db_peer *
 400 netdbPeerAdd(netdbEntry * n, CachePeer * e)
 401 {
 402     net_db_peer *p;
 403     net_db_peer *o;
 404     int osize;
 405     int i;
 406
 407     if (n->n_peers == n->n_peers_alloc) {
 408         o = n->peers;
 409         osize = n->n_peers_alloc;
 410
 411         if (n->n_peers_alloc == 0)
 412             n->n_peers_alloc = 2;
 413         else
 414             n->n_peers_alloc <<= 1;
 415
 416         debugs(38, 3, "netdbPeerAdd: Growing peer list for '" << n->network << "' to " << n->n_peers_alloc);
 417
 418         n->peers = (net_db_peer *)xcalloc(n->n_peers_alloc, sizeof(net_db_peer));
 419
 420         for (i = 0; i < osize; ++i)
 421             *(n->peers + i) = *(o + i);
 422
 423         if (osize) {
 424             safe_free(o);
 425         }
 426     }
 427
 428     p = n->peers + n->n_peers;
 429     p->peername = netdbPeerName(e->host);
 430     ++ n->n_peers;
 431     return p;
 432 }
 433
 434 static int
 435 sortPeerByRtt(const void *A, const void *B)
 436 {
 437     const net_db_peer *p1 = (net_db_peer *)A;
 438     const net_db_peer *p2 = (net_db_peer *)B;
 439
 440     if (p1->rtt > p2->rtt)
 441         return 1;
 442     else if (p1->rtt < p2->rtt)
 443         return -1;
 444     else
 445         return 0;
 446 }
 447
 448 static void
 449 netdbSaveState(void *)
 450 {
 451     if (strcmp(Config.netdbFilename, "none") == 0)
 452         return;
 453
 454     Logfile *lf;
 455     netdbEntry *n;
 456     net_db_name *x;
 457
 458     struct timeval start = current_time;
 459     int count = 0;
 460     /*
 461      * This was nicer when we were using stdio, but thanks to
 462      * Solaris bugs, its a bad idea.  fopen can fail if more than
 463      * 256 FDs are open.
 464      */
 465     /*
 466      * unlink() is here because there is currently no way to make
 467      * logfileOpen() use O_TRUNC.
 468      */
 469     unlink(Config.netdbFilename);
 470     lf = logfileOpen(Config.netdbFilename, 4096, 0);
 471
 472     if (!lf) {
 473         int xerrno = errno;
 474         debugs(50, DBG_IMPORTANT, MYNAME << Config.netdbFilename << ": " << xstrerr(xerrno));
 475         return;
 476     }
 477
 478     hash_first(addr_table);
 479
 480     while ((n = (netdbEntry *) hash_next(addr_table))) {
 481         if (n->pings_recv == 0)
 482             continue;
 483
 484         logfilePrintf(lf, "%s %d %d %10.5f %10.5f %d %d",
 485                       n->network,
 486                       n->pings_sent,
 487                       n->pings_recv,
 488                       n->hops,
 489                       n->rtt,
 490                       (int) n->next_ping_time,
 491                       (int) n->last_use_time);
 492
 493         for (x = n->hosts; x; x = x->next)
 494             logfilePrintf(lf, " %s", hashKeyStr(x));
 495
 496         logfilePrintf(lf, "\n");
 497
 498         ++count;
 499
 500 #undef RBUF_SZ
 501
 502     }
 503
 504     logfileClose(lf);
 505     getCurrentTime();
 506     debugs(38, DBG_IMPORTANT, "NETDB state saved; " <<
 507            count << " entries, " <<
 508            tvSubMsec(start, current_time) << " msec" );
 509     eventAddIsh("netdbSaveState", netdbSaveState, nullptr, 3600.0, 1);
 510 }
 511
 512 static void
 513 netdbReloadState(void)
 514 {
 515     if (strcmp(Config.netdbFilename, "none") == 0)
 516         return;
 517
 518     char *s;
 519     int fd;
 520     int l;
 521
 522     struct stat sb;
 523     netdbEntry *n;
 524     netdbEntry N;
 525
 526     Ip::Address addr;
 527     int count = 0;
 528
 529     struct timeval start = current_time;
 530     /*
 531      * This was nicer when we were using stdio, but thanks to
 532      * Solaris bugs, its a bad idea.  fopen can fail if more than
 533      * 256 FDs are open.
 534      */
 535     fd = file_open(Config.netdbFilename, O_RDONLY | O_BINARY);
 536
 537     if (fd < 0)
 538         return;
 539
 540     if (fstat(fd, &sb) < 0) {
 541         file_close(fd);
 542         return;
 543     }
 544
 545     char *t;
 546     char *buf = (char *)xcalloc(1, sb.st_size + 1);
 547     t = buf;
 548     l = FD_READ_METHOD(fd, buf, sb.st_size);
 549     file_close(fd);
 550
 551     if (l <= 0) {
 552         safe_free (buf);
 553         return;
 554     };
 555
 556     while ((s = strchr(t, '\n'))) {
 557         char *q;
 558         assert(s - buf < l);
 559         *s = '\0';
 560         N = netdbEntry();
 561         q = strtok(t, w_space);
 562         t = s + 1;
 563
 564         if (nullptr == q)
 565             continue;
 566
 567         if (! (addr = q) )
 568             continue;
 569
 570         if (netdbLookupAddr(addr) != nullptr)  /* no dups! */
 571             continue;
 572
 573         if ((q = strtok(nullptr, w_space)) == nullptr)
 574             continue;
 575
 576         N.pings_sent = atoi(q);
 577
 578         if ((q = strtok(nullptr, w_space)) == nullptr)
 579             continue;
 580
 581         N.pings_recv = atoi(q);
 582
 583         if (N.pings_recv == 0)
 584             continue;
 585
 586         /* give this measurement low weight */
 587         N.pings_sent = 1;
 588
 589         N.pings_recv = 1;
 590
 591         if ((q = strtok(nullptr, w_space)) == nullptr)
 592             continue;
 593
 594         N.hops = atof(q);
 595
 596         if ((q = strtok(nullptr, w_space)) == nullptr)
 597             continue;
 598
 599         N.rtt = atof(q);
 600
 601         if ((q = strtok(nullptr, w_space)) == nullptr)
 602             continue;
 603
 604         N.next_ping_time = (time_t) atoi(q);
 605
 606         if ((q = strtok(nullptr, w_space)) == nullptr)
 607             continue;
 608
 609         N.last_use_time = (time_t) atoi(q);
 610
 611         n = new netdbEntry;
 612
 613         memcpy(n, &N, sizeof(netdbEntry));
 614
 615         netdbHashInsert(n, addr);
 616
 617         while ((q = strtok(nullptr, w_space)) != nullptr) {
 618             if (netdbLookupHost(q) != nullptr) /* no dups! */
 619                 continue;
 620
 621             netdbHostInsert(n, q);
 622         }
 623
 624         ++count;
 625     }
 626
 627     xfree(buf);
 628     getCurrentTime();
 629     debugs(38, DBG_IMPORTANT, "NETDB state reloaded; " <<
 630            count << " entries, " <<
 631            tvSubMsec(start, current_time) << " msec" );
 632 }
 633
 634 static const char *
 635 netdbPeerName(const char *name)
 636 {
 637     const wordlist *w;
 638
 639     for (w = peer_names; w; w = w->next) {
 640         if (!strcmp(w->key, name))
 641             return w->key;
 642     }
 643
 644     return wordlistAdd(&peer_names, name);
 645 }
 646
 647 static void
 648 netdbExchangeHandleReply(void *data, StoreIOBuffer receivedData)
 649 {
 650     Ip::Address addr;
 651
 652     netdbExchangeState *ex = (netdbExchangeState *)data;
 653
 654     struct in_addr line_addr;
 655     double rtt;
 656     double hops;
 657     int j;
 658     int nused = 0;
 659
 660     size_t rec_sz = 0; // received record size (TODO: make const)
 661     rec_sz += 1 + sizeof(struct in_addr);
 662     rec_sz += 1 + sizeof(int);
 663     rec_sz += 1 + sizeof(int);
 664     // to make progress without growing buffer space, we must parse at least one record per call
 665     Assure(rec_sz <= ex->parsingBuffer.capacity());
 666     debugs(38, 3, "netdbExchangeHandleReply: " << receivedData.length << " read bytes");
 667
 668     if (!ex->p.valid()) {
 669         debugs(38, 3, "netdbExchangeHandleReply: Peer became invalid");
 670         delete ex;
 671         return;
 672     }
 673
 674     debugs(38, 3, "for " << *ex->p);
 675
 676     if (receivedData.flags.error) {
 677         delete ex;
 678         return;
 679     }
 680
 681     if (ex->connstate == STATE_HEADER) {
 682         const auto scode = ex->e->mem().baseReply().sline.status();
 683         assert(scode != Http::scNone);
 684         debugs(38, 3, "reply status " << scode);
 685         if (scode != Http::scOkay) {
 686             delete ex;
 687             return;
 688         }
 689         ex->connstate = STATE_BODY;
 690     }
 691
 692     assert(ex->connstate == STATE_BODY);
 693
 694     ex->parsingBuffer.appended(receivedData.data, receivedData.length);
 695     auto p = ex->parsingBuffer.c_str(); // current parsing position
 696     auto size = ex->parsingBuffer.contentSize(); // bytes we still need to parse
 697
 698     /* If we get here, we have some body to parse .. */
 699     debugs(38, 5, "netdbExchangeHandleReply: start parsing loop, size = " << size);
 700
 701     while (size >= rec_sz) {
 702         debugs(38, 5, "netdbExchangeHandleReply: in parsing loop, size = " << size);
 703         addr.setAnyAddr();
 704         hops = rtt = 0.0;
 705
 706         size_t o; // current record parsing offset
 707         for (o = 0; o < rec_sz;) {
 708             switch ((int) *(p + o)) {
 709
 710             case NETDB_EX_NETWORK:
 711                 ++o;
 712                 // XXX: NetDB can still only send IPv4
 713                 memcpy(&line_addr, p + o, sizeof(struct in_addr));
 714                 addr = line_addr;
 715                 o += sizeof(struct in_addr);
 716                 break;
 717
 718             case NETDB_EX_RTT:
 719                 ++o;
 720                 memcpy(&j, p + o, sizeof(int));
 721                 o += sizeof(int);
 722                 rtt = (double) ntohl(j) / 1000.0;
 723                 break;
 724
 725             case NETDB_EX_HOPS:
 726                 ++o;
 727                 memcpy(&j, p + o, sizeof(int));
 728                 o += sizeof(int);
 729                 hops = (double) ntohl(j) / 1000.0;
 730                 break;
 731
 732             default:
 733                 debugs(38, DBG_IMPORTANT, "ERROR: netdbExchangeHandleReply: corrupt data, aborting");
 734                 delete ex;
 735                 return;
 736             }
 737         }
 738
 739         if (!addr.isAnyAddr() && rtt > 0)
 740             netdbExchangeUpdatePeer(addr, ex->p.get(), rtt, hops);
 741
 742         assert(o == rec_sz);
 743
 744         size -= rec_sz;
 745
 746         p += rec_sz;
 747
 748         ++nused;
 749     }
 750
 751     const auto parsedSize = ex->parsingBuffer.contentSize() - size;
 752     ex->parsingBuffer.consume(parsedSize);
 753
 754     debugs(38, 3, "netdbExchangeHandleReply: size left over in this buffer: " << size << " bytes");
 755
 756     debugs(38, 3, "netdbExchangeHandleReply: used " << nused <<
 757            " entries, (x " << rec_sz << " bytes) == " << nused * rec_sz <<
 758            " bytes total");
 759
 760     if (EBIT_TEST(ex->e->flags, ENTRY_ABORTED)) {
 761         debugs(38, 3, "netdbExchangeHandleReply: ENTRY_ABORTED");
 762         delete ex;
 763         return;
 764     }
 765
 766     if (ex->sc->atEof()) {
 767         if (const auto leftoverBytes = ex->parsingBuffer.contentSize())
 768             debugs(38, 2, "discarding a partially received record due to Store EOF: " << leftoverBytes);
 769         delete ex;
 770         return;
 771     }
 772
 773     // TODO: To protect us from a broken peer sending an "infinite" stream of
 774     // new addresses, limit the cumulative number of received bytes or records?
 775
 776     const auto remainingSpace = ex->parsingBuffer.space().positionAt(receivedData.offset + receivedData.length);
 777     // rec_sz is at most buffer capacity, and we consume all fully loaded records
 778     Assure(remainingSpace.length);
 779     storeClientCopy(ex->sc, ex->e, remainingSpace, netdbExchangeHandleReply, ex);
 780 }
 781
 782 #endif /* USE_ICMP */
 783
 784 /* PUBLIC FUNCTIONS */
 785
 786 void
 787 netdbInit(void)
 788 {
 789 #if USE_ICMP
 790     Mgr::RegisterAction("netdb", "Network Measurement Database", netdbDump, 0, 1);
 791
 792     if (addr_table)
 793         return;
 794
 795     int n = hashPrime(Config.Netdb.high / 4);
 796
 797     addr_table = hash_create((HASHCMP *) strcmp, n, hash_string);
 798
 799     n = hashPrime(3 * Config.Netdb.high / 4);
 800
 801     host_table = hash_create((HASHCMP *) strcmp, n, hash_string);
 802
 803     eventAddIsh("netdbSaveState", netdbSaveState, nullptr, 3600.0, 1);
 804
 805     netdbReloadState();
 806
 807 #endif
 808 }
 809
 810 void
 811 netdbPingSite(const char *hostname)
 812 {
 813 #if USE_ICMP
 814     netdbEntry *n;
 815
 816     if ((n = netdbLookupHost(hostname)) != nullptr)
 817         if (n->next_ping_time > squid_curtime)
 818             return;
 819
 820     ipcache_nbgethostbyname(hostname, netdbSendPing,
 821                             new generic_cbdata(xstrdup(hostname)));
 822 #else
 823     (void)hostname;
 824 #endif
 825 }
 826
 827 void
 828 netdbHandlePingReply(const Ip::Address &from, int hops, int rtt)
 829 {
 830 #if USE_ICMP
 831     netdbEntry *n;
 832     int N;
 833     debugs(38, 3, "netdbHandlePingReply: from " << from);
 834
 835     if ((n = netdbLookupAddr(from)) == nullptr)
 836         return;
 837
 838     N = ++n->pings_recv;
 839
 840     if (N > 5)
 841         N = 5;
 842
 843     if (rtt < 1)
 844         rtt = 1;
 845
 846     n->hops = ((n->hops * (N - 1)) + hops) / N;
 847
 848     n->rtt = ((n->rtt * (N - 1)) + rtt) / N;
 849
 850     debugs(38, 3, "netdbHandlePingReply: " << n->network  << "; rtt="<<
 851            std::setw(5)<< std::setprecision(2) << n->rtt << "  hops="<<
 852            std::setw(4) << n->hops);
 853 #else
 854     (void)from;
 855     (void)hops;
 856     (void)rtt;
 857 #endif
 858 }
 859
 860 void
 861 netdbDump(StoreEntry * sentry)
 862 {
 863 #if USE_ICMP
 864     netdbEntry *n;
 865     netdbEntry **list;
 866     net_db_name *x;
 867     int k;
 868     int i;
 869     int j;
 870     net_db_peer *p;
 871     storeAppendPrintf(sentry, "Network DB Statistics:\n");
 872     storeAppendPrintf(sentry, "%-46.46s %9s %7s %5s %s\n",  /* Max between 16 (IPv4) or 46 (IPv6)   */
 873                       "Network",
 874                       "recv/sent",
 875                       "RTT",
 876                       "Hops",
 877                       "Hostnames");
 878     list = (netdbEntry **)xcalloc(netdbEntry::UseCount(), sizeof(netdbEntry *));
 879     i = 0;
 880     hash_first(addr_table);
 881
 882     while ((n = (netdbEntry *) hash_next(addr_table))) {
 883         *(list + i) = n;
 884         ++i;
 885     }
 886
 887     if (i != netdbEntry::UseCount())
 888         debugs(38, DBG_CRITICAL, "WARNING: netdb_addrs count off, found " << i <<
 889                ", expected " << netdbEntry::UseCount());
 890
 891     qsort((char *) list,
 892           i,
 893           sizeof(netdbEntry *),
 894           sortByRtt);
 895
 896     for (k = 0; k < i; ++k) {
 897         n = *(list + k);
 898         storeAppendPrintf(sentry, "%-46.46s %4d/%4d %7.1f %5.1f", /* Max between 16 (IPv4) or 46 (IPv6)   */
 899                           n->network,
 900                           n->pings_recv,
 901                           n->pings_sent,
 902                           n->rtt,
 903                           n->hops);
 904
 905         for (x = n->hosts; x; x = x->next)
 906             storeAppendPrintf(sentry, " %s", hashKeyStr(x));
 907
 908         storeAppendPrintf(sentry, "\n");
 909
 910         p = n->peers;
 911
 912         for (j = 0; j < n->n_peers; ++j, ++p) {
 913             storeAppendPrintf(sentry, "    %-22.22s %7.1f %5.1f\n",
 914                               p->peername,
 915                               p->rtt,
 916                               p->hops);
 917         }
 918     }
 919
 920     xfree(list);
 921 #else
 922
 923     storeAppendPrintf(sentry,"NETDB support not compiled into this Squid cache.\n");
 924 #endif
 925 }
 926
 927 int
 928 netdbHostHops(const char *host)
 929 {
 930 #if USE_ICMP
 931     netdbEntry *n = netdbLookupHost(host);
 932
 933     if (n) {
 934         n->last_use_time = squid_curtime;
 935         return (int) (n->hops + 0.5);
 936     }
 937 #else
 938     (void)host;
 939 #endif
 940     return 0;
 941 }
 942
 943 int
 944 netdbHostRtt(const char *host)
 945 {
 946 #if USE_ICMP
 947     netdbEntry *n = netdbLookupHost(host);
 948
 949     if (n) {
 950         n->last_use_time = squid_curtime;
 951         return (int) (n->rtt + 0.5);
 952     }
 953
 954 #else
 955     (void)host;
 956 #endif
 957     return 0;
 958 }
 959
 960 void
 961 netdbHostData(const char *host, int *samp, int *rtt, int *hops)
 962 {
 963 #if USE_ICMP
 964     netdbEntry *n = netdbLookupHost(host);
 965
 966     if (n == nullptr)
 967         return;
 968
 969     *samp = n->pings_recv;
 970
 971     *rtt = (int) (n->rtt + 0.5);
 972
 973     *hops = (int) (n->hops + 0.5);
 974
 975     n->last_use_time = squid_curtime;
 976
 977 #else
 978     (void)hops;
 979     (void)rtt;
 980     (void)samp;
 981     (void)host;
 982 #endif
 983 }
 984
 985 void
 986 netdbUpdatePeer(const AnyP::Uri &url, CachePeer *e, int irtt, int ihops)
 987 {
 988 #if USE_ICMP
 989     netdbEntry *n;
 990     double rtt = (double) irtt;
 991     double hops = (double) ihops;
 992     net_db_peer *p;
 993     debugs(38, 3, url.host() << ", " << ihops << " hops, " << irtt << " rtt");
 994     n = netdbLookupHost(url.host());
 995
 996     if (n == nullptr) {
 997         debugs(38, 3, "host " << url.host() << " not found");
 998         return;
 999     }
1000
1001     if ((p = netdbPeerByName(n, e->host)) == nullptr)
1002         p = netdbPeerAdd(n, e);
1003
1004     p->rtt = rtt;
1005
1006     p->hops = hops;
1007
1008     p->expires = squid_curtime + 3600;
1009
1010     if (n->n_peers < 2)
1011         return;
1012
1013     qsort((char *) n->peers,
1014           n->n_peers,
1015           sizeof(net_db_peer),
1016           sortPeerByRtt);
1017
1018 #else
1019     (void)ihops;
1020     (void)irtt;
1021     (void)e;
1022     (void)url;
1023 #endif
1024 }
1025
1026 void
1027 netdbExchangeUpdatePeer(Ip::Address &addr, CachePeer * e, double rtt, double hops)
1028 {
1029 #if USE_ICMP
1030     netdbEntry *n;
1031     net_db_peer *p;
1032     debugs(38, 5, "netdbExchangeUpdatePeer: '" << addr << "', "<<
1033            std::setfill('0')<< std::setprecision(2) << hops << " hops, " <<
1034            rtt << " rtt");
1035
1036     if ( !addr.isIPv4() ) {
1037         debugs(38, 5, "netdbExchangeUpdatePeer: Aborting peer update for '" << addr << "', NetDB cannot handle IPv6.");
1038         return;
1039     }
1040
1041     n = netdbLookupAddr(addr);
1042
1043     if (n == nullptr)
1044         n = netdbAdd(addr);
1045
1046     assert(nullptr != n);
1047
1048     if ((p = netdbPeerByName(n, e->host)) == nullptr)
1049         p = netdbPeerAdd(n, e);
1050
1051     p->rtt = rtt;
1052
1053     p->hops = hops;
1054
1055     p->expires = squid_curtime + 3600;  /* XXX ? */
1056
1057     if (n->n_peers < 2)
1058         return;
1059
1060     qsort((char *) n->peers,
1061           n->n_peers,
1062           sizeof(net_db_peer),
1063           sortPeerByRtt);
1064
1065 #else
1066     (void)hops;
1067     (void)rtt;
1068     (void)e;
1069     (void)addr;
1070 #endif
1071 }
1072
1073 void
1074 netdbDeleteAddrNetwork(Ip::Address &addr)
1075 {
1076 #if USE_ICMP
1077     netdbEntry *n = netdbLookupAddr(addr);
1078
1079     if (n == nullptr)
1080         return;
1081
1082     debugs(38, 3, "netdbDeleteAddrNetwork: " << n->network);
1083
1084     netdbRelease(n);
1085 #else
1086     (void)addr;
1087 #endif
1088 }
1089
1090 void
1091 netdbBinaryExchange(StoreEntry * s)
1092 {
1093     HttpReply *reply = new HttpReply;
1094 #if USE_ICMP
1095
1096     Ip::Address addr;
1097
1098     netdbEntry *n;
1099     int i;
1100     int j;
1101     int rec_sz;
1102     char *buf;
1103
1104     struct in_addr line_addr;
1105     s->buffer();
1106     reply->setHeaders(Http::scOkay, "OK", nullptr, -1, squid_curtime, -2);
1107     s->replaceHttpReply(reply);
1108     rec_sz = 0;
1109     rec_sz += 1 + sizeof(struct in_addr);
1110     rec_sz += 1 + sizeof(int);
1111     rec_sz += 1 + sizeof(int);
1112     buf = (char *)memAllocate(MEM_4K_BUF);
1113     i = 0;
1114     hash_first(addr_table);
1115
1116     while ((n = (netdbEntry *) hash_next(addr_table))) {
1117         if (0.0 == n->rtt)
1118             continue;
1119
1120         if (n->rtt > 60000) /* RTT > 1 MIN probably bogus */
1121             continue;
1122
1123         if (! (addr = n->network) )
1124             continue;
1125
1126         // XXX: NetDB cannot yet handle IPv6 addresses
1127         if ( !addr.isIPv4() )
1128             continue;
1129
1130         buf[i] = (char) NETDB_EX_NETWORK;
1131         ++i;
1132
1133         addr.getInAddr(line_addr);
1134         memcpy(&buf[i], &line_addr, sizeof(struct in_addr));
1135
1136         i += sizeof(struct in_addr);
1137
1138         buf[i] = (char) NETDB_EX_RTT;
1139         ++i;
1140
1141         j = htonl((int) (n->rtt * 1000));
1142
1143         memcpy(&buf[i], &j, sizeof(int));
1144
1145         i += sizeof(int);
1146
1147         buf[i] = (char) NETDB_EX_HOPS;
1148         ++i;
1149
1150         j = htonl((int) (n->hops * 1000));
1151
1152         memcpy(&buf[i], &j, sizeof(int));
1153
1154         i += sizeof(int);
1155
1156         if (i + rec_sz > 4096) {
1157             s->append(buf, i);
1158             i = 0;
1159         }
1160     }
1161
1162     if (i > 0) {
1163         s->append(buf, i);
1164         i = 0;
1165     }
1166
1167     assert(0 == i);
1168     s->flush();
1169     memFree(buf, MEM_4K_BUF);
1170 #else
1171
1172     reply->setHeaders(Http::scBadRequest, "Bad Request", nullptr, -1, squid_curtime, -2);
1173     s->replaceHttpReply(reply);
1174     storeAppendPrintf(s, "NETDB support not compiled into this Squid cache.\n");
1175 #endif
1176
1177     s->complete();
1178 }
1179
1180 void
1181 netdbExchangeStart(void *data)
1182 {
1183 #if USE_ICMP
1184     CachePeer *p = (CachePeer *)data;
1185     static const SBuf netDB("netdb");
1186     char *uri = internalRemoteUri(p->secure.encryptTransport, p->host, p->http_port, "/squid-internal-dynamic/", netDB);
1187     debugs(38, 3, "Requesting '" << uri << "'");
1188     const auto mx = MasterXaction::MakePortless<XactionInitiator::initIcmp>();
1189     HttpRequestPointer req(HttpRequest::FromUrlXXX(uri, mx));
1190
1191     if (!req) {
1192         debugs(38, DBG_IMPORTANT, "ERROR: " << MYNAME << ": Bad URI " << uri);
1193         return;
1194     }
1195
1196     netdbExchangeState *ex = new netdbExchangeState(p, req);
1197     ex->e = storeCreateEntry(uri, uri, RequestFlags(), Http::METHOD_GET);
1198     assert(nullptr != ex->e);
1199
1200     ex->sc = storeClientListAdd(ex->e, ex);
1201     storeClientCopy(ex->sc, ex->e, ex->parsingBuffer.makeInitialSpace(), netdbExchangeHandleReply, ex);
1202
1203     ex->r->flags.loopDetected = true;   /* cheat! -- force direct */
1204
1205     // XXX: send as Proxy-Authenticate instead
1206     if (p->login)
1207         ex->r->url.userInfo(SBuf(p->login));
1208
1209     FwdState::fwdStart(Comm::ConnectionPointer(), ex->e, ex->r.getRaw());
1210 #else
1211     (void)data;
1212 #endif
1213 }
1214
1215 #if USE_ICMP
1216 /// a netdbClosestParent() helper to find the first usable parent CachePeer
1217 /// responsible for the given hostname
1218 static CachePeer *
1219 findUsableParentAtHostname(PeerSelector *ps, const char * const hostname, const HttpRequest &request)
1220 {
1221     for (const auto &peer: CurrentCachePeers()) {
1222         const auto p = peer.get();
1223         // Both fields should be lowercase, but no code ensures that invariant.
1224         // TODO: net_db_peer should point to CachePeer instead of its hostname!
1225         if (strcasecmp(p->host, hostname) != 0)
1226             continue;
1227
1228         if (neighborType(p, request.url) != PEER_PARENT)
1229             continue;
1230
1231         if (!peerHTTPOkay(p, ps))
1232             continue;
1233
1234         return p;
1235     }
1236
1237     return nullptr;
1238 }
1239 #endif
1240
1241 CachePeer *
1242 netdbClosestParent(PeerSelector *ps)
1243 {
1244 #if USE_ICMP
1245     assert(ps);
1246     HttpRequest *request = ps->request;
1247
1248     netdbEntry *n;
1249     const ipcache_addrs *ia;
1250     net_db_peer *h;
1251     int i;
1252     n = netdbLookupHost(request->url.host());
1253
1254     if (nullptr == n) {
1255         /* try IP addr */
1256         ia = ipcache_gethostbyname(request->url.host(), 0);
1257
1258         if (nullptr != ia)
1259             n = netdbLookupAddr(ia->current());
1260     }
1261
1262     if (nullptr == n)
1263         return nullptr;
1264
1265     if (0 == n->n_peers)
1266         return nullptr;
1267
1268     n->last_use_time = squid_curtime;
1269
1270     /*
1271      * Find the parent with the least RTT to the origin server.
1272      * Make sure we don't return a parent who is farther away than
1273      * we are.  Note, the n->peers list is pre-sorted by RTT.
1274      */
1275     for (i = 0; i < n->n_peers; ++i) {
1276         h = &n->peers[i];
1277
1278         if (n->rtt > 0)
1279             if (n->rtt < h->rtt)
1280                 break;
1281
1282         if (const auto p = findUsableParentAtHostname(ps, h->peername, *request))
1283             return p;
1284     }
1285
1286 #else
1287     (void)ps;
1288 #endif
1289     return nullptr;
1290 }
1291