src/icmp/net_db.cc

   1 /*
   2  * Copyright (C) 1996-2022 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 38    Network Measurement Database */
  10
  11 /*
  12  * XXX XXX XXX
  13  *
  14  * This code may be slightly broken now. If you're getting consistent
  15  * (sometimes working) corrupt data exchanges, please contact adrian
  16  * (adrian@squid-cache.org) to sort them out.
  17  */
  18
  19 #include "squid.h"
  20 #include "CachePeer.h"
  21 #include "cbdata.h"
  22 #include "event.h"
  23 #include "fde.h"
  24 #include "fs_io.h"
  25 #include "FwdState.h"
  26 #include "HttpReply.h"
  27 #include "icmp/net_db.h"
  28 #include "internal.h"
  29 #include "ip/Address.h"
  30 #include "log/File.h"
  31 #include "MemObject.h"
  32 #include "mgr/Registration.h"
  33 #include "mime_header.h"
  34 #include "neighbors.h"
  35 #include "PeerSelectState.h"
  36 #include "SquidConfig.h"
  37 #include "Store.h"
  38 #include "StoreClient.h"
  39 #include "tools.h"
  40 #include "wordlist.h"
  41
  42 #if HAVE_SYS_STAT_H
  43 #include <sys/stat.h>
  44 #endif
  45
  46 #if USE_ICMP
  47 #include "icmp/IcmpSquid.h"
  48 #include "ipcache.h"
  49 #include "StoreClient.h"
  50
  51 #define NETDB_REQBUF_SZ 4096
  52
  53 typedef enum {
  54     STATE_NONE,
  55     STATE_HEADER,
  56     STATE_BODY
  57 } netdb_conn_state_t;
  58
  59 class netdbExchangeState
  60 {
  61     CBDATA_CLASS(netdbExchangeState);
  62
  63 public:
  64     netdbExchangeState(CachePeer *aPeer, const HttpRequestPointer &theReq) :
  65         p(aPeer),
  66         r(theReq)
  67     {
  68         *buf = 0;
  69         assert(r);
  70         // TODO: check if we actually need to do this. should be implicit
  71         r->http_ver = Http::ProtocolVersion();
  72     }
  73
  74     ~netdbExchangeState() {
  75         debugs(38, 3, e->url());
  76         storeUnregister(sc, e, this);
  77         e->unlock("netdbExchangeDone");
  78     }
  79
  80     CbcPointer<CachePeer> p;
  81     StoreEntry *e = nullptr;
  82     store_client *sc = nullptr;
  83     HttpRequestPointer r;
  84     int64_t used = 0;
  85     size_t buf_sz = NETDB_REQBUF_SZ;
  86     char buf[NETDB_REQBUF_SZ];
  87     int buf_ofs = 0;
  88     netdb_conn_state_t connstate = STATE_HEADER;
  89 };
  90
  91 CBDATA_CLASS_INIT(netdbExchangeState);
  92
  93 static hash_table *addr_table = nullptr;
  94 static hash_table *host_table = nullptr;
  95
  96 Ip::Address networkFromInaddr(const Ip::Address &a);
  97 static void netdbRelease(netdbEntry * n);
  98
  99 static void netdbHashInsert(netdbEntry * n, Ip::Address &addr);
 100 static void netdbHashDelete(const char *key);
 101 static void netdbHostInsert(netdbEntry * n, const char *hostname);
 102 static void netdbHostDelete(const net_db_name * x);
 103 static void netdbPurgeLRU(void);
 104 static netdbEntry *netdbLookupHost(const char *key);
 105 static net_db_peer *netdbPeerByName(const netdbEntry * n, const char *);
 106 static net_db_peer *netdbPeerAdd(netdbEntry * n, CachePeer * e);
 107 static const char *netdbPeerName(const char *name);
 108 static IPH netdbSendPing;
 109 static FREE netdbFreeNameEntry;
 110 static FREE netdbFreeNetdbEntry;
 111 static STCB netdbExchangeHandleReply;
 112
 113 /* We have to keep a local list of CachePeer names.  The Peers structure
 114  * gets freed during a reconfigure.  We want this database to
 115  * remain persisitent, so _net_db_peer->peername points into this
 116  * linked list */
 117 static wordlist *peer_names = nullptr;
 118
 119 static void
 120 netdbHashInsert(netdbEntry * n, Ip::Address &addr)
 121 {
 122     networkFromInaddr(addr).toStr(n->network, MAX_IPSTRLEN);
 123     n->key = n->network;
 124     assert(hash_lookup(addr_table, n->network) == nullptr);
 125     hash_join(addr_table, n);
 126 }
 127
 128 static void
 129 netdbHashDelete(const char *key)
 130 {
 131     hash_link *hptr = (hash_link *)hash_lookup(addr_table, key);
 132
 133     if (hptr == nullptr) {
 134         debug_trap("netdbHashDelete: key not found");
 135         return;
 136     }
 137
 138     hash_remove_link(addr_table, hptr);
 139 }
 140
 141 net_db_name::net_db_name(const char *hostname, netdbEntry *e) :
 142     next(e ? e->hosts : nullptr),
 143     net_db_entry(e)
 144 {
 145     key = xstrdup(hostname);
 146     if (e) {
 147         e->hosts = this;
 148         ++ e->link_count;
 149     }
 150 }
 151
 152 static void
 153 netdbHostInsert(netdbEntry * n, const char *hostname)
 154 {
 155     net_db_name *x = new net_db_name(hostname, n);
 156     assert(hash_lookup(host_table, hostname) == nullptr);
 157     hash_join(host_table, x);
 158 }
 159
 160 static void
 161 netdbHostDelete(const net_db_name * x)
 162 {
 163     assert(x != nullptr);
 164     assert(x->net_db_entry != nullptr);
 165
 166     netdbEntry *n = x->net_db_entry;
 167     -- n->link_count;
 168
 169     for (auto **X = &n->hosts; *X; X = &(*X)->next) {
 170         if (*X == x) {
 171             *X = x->next;
 172             break;
 173         }
 174     }
 175
 176     hash_remove_link(host_table, (hash_link *) x);
 177     delete x;
 178 }
 179
 180 static netdbEntry *
 181 netdbLookupHost(const char *key)
 182 {
 183     net_db_name *x = (net_db_name *) hash_lookup(host_table, key);
 184     return x ? x->net_db_entry : nullptr;
 185 }
 186
 187 static void
 188 netdbRelease(netdbEntry * n)
 189 {
 190     net_db_name *x;
 191     net_db_name *next;
 192
 193     for (x = n->hosts; x; x = next) {
 194         next = x->next;
 195         netdbHostDelete(x);
 196     }
 197
 198     n->hosts = nullptr;
 199     safe_free(n->peers);
 200     n->peers = nullptr;
 201     n->n_peers = 0;
 202     n->n_peers_alloc = 0;
 203
 204     if (n->link_count == 0) {
 205         netdbHashDelete(n->network);
 206         delete n;
 207     }
 208 }
 209
 210 static int
 211 netdbLRU(const void *A, const void *B)
 212 {
 213     const netdbEntry *const *n1 = (const netdbEntry *const *)A;
 214     const netdbEntry *const *n2 = (const netdbEntry *const *)B;
 215
 216     if ((*n1)->last_use_time > (*n2)->last_use_time)
 217         return (1);
 218
 219     if ((*n1)->last_use_time < (*n2)->last_use_time)
 220         return (-1);
 221
 222     return (0);
 223 }
 224
 225 static void
 226 netdbPurgeLRU(void)
 227 {
 228     netdbEntry *n;
 229     netdbEntry **list;
 230     int k = 0;
 231     int list_count = 0;
 232     int removed = 0;
 233     list = (netdbEntry **)xcalloc(netdbEntry::UseCount(), sizeof(netdbEntry *));
 234     hash_first(addr_table);
 235
 236     while ((n = (netdbEntry *) hash_next(addr_table))) {
 237         assert(list_count < netdbEntry::UseCount());
 238         *(list + list_count) = n;
 239         ++list_count;
 240     }
 241
 242     qsort((char *) list,
 243           list_count,
 244           sizeof(netdbEntry *),
 245           netdbLRU);
 246
 247     for (k = 0; k < list_count; ++k) {
 248         if (netdbEntry::UseCount() < Config.Netdb.low)
 249             break;
 250
 251         netdbRelease(*(list + k));
 252
 253         ++removed;
 254     }
 255
 256     xfree(list);
 257 }
 258
 259 static netdbEntry *
 260 netdbLookupAddr(const Ip::Address &addr)
 261 {
 262     netdbEntry *n;
 263     char *key = new char[MAX_IPSTRLEN];
 264     networkFromInaddr(addr).toStr(key,MAX_IPSTRLEN);
 265     n = (netdbEntry *) hash_lookup(addr_table, key);
 266     delete[] key;
 267     return n;
 268 }
 269
 270 static netdbEntry *
 271 netdbAdd(Ip::Address &addr)
 272 {
 273     netdbEntry *n;
 274
 275     if (netdbEntry::UseCount() > Config.Netdb.high)
 276         netdbPurgeLRU();
 277
 278     if ((n = netdbLookupAddr(addr)) == nullptr) {
 279         n = new netdbEntry;
 280         netdbHashInsert(n, addr);
 281     }
 282
 283     return n;
 284 }
 285
 286 static void
 287 netdbSendPing(const ipcache_addrs *ia, const Dns::LookupDetails &, void *data)
 288 {
 289     Ip::Address addr;
 290     char *hostname = nullptr;
 291     static_cast<generic_cbdata *>(data)->unwrap(&hostname);
 292     netdbEntry *n;
 293     netdbEntry *na;
 294     net_db_name *x;
 295     net_db_name **X;
 296
 297     if (ia == nullptr) {
 298         xfree(hostname);
 299         return;
 300     }
 301
 302     addr = ia->current();
 303
 304     if ((n = netdbLookupHost(hostname)) == nullptr) {
 305         n = netdbAdd(addr);
 306         netdbHostInsert(n, hostname);
 307     } else if ((na = netdbLookupAddr(addr)) != n) {
 308         /*
 309          *hostname moved from 'network n' to 'network na'!
 310          */
 311
 312         if (na == nullptr)
 313             na = netdbAdd(addr);
 314
 315         debugs(38, 3, "netdbSendPing: " << hostname << " moved from " << n->network << " to " << na->network);
 316
 317         x = (net_db_name *) hash_lookup(host_table, hostname);
 318
 319         if (x == nullptr) {
 320             debugs(38, DBG_IMPORTANT, "ERROR: Squid BUG: net_db_name list bug: " << hostname << " not found");
 321             xfree(hostname);
 322             return;
 323         }
 324
 325         /* remove net_db_name from 'network n' linked list */
 326         for (X = &n->hosts; *X; X = &(*X)->next) {
 327             if (*X == x) {
 328                 *X = x->next;
 329                 break;
 330             }
 331         }
 332
 333         -- n->link_count;
 334         /* point to 'network na' from host entry */
 335         x->net_db_entry = na;
 336         /* link net_db_name to 'network na' */
 337         x->next = na->hosts;
 338         na->hosts = x;
 339         ++ na->link_count;
 340         n = na;
 341     }
 342
 343     if (n->next_ping_time <= squid_curtime) {
 344         debugs(38, 3, "netdbSendPing: pinging " << hostname);
 345         icmpEngine.DomainPing(addr, hostname);
 346         ++ n->pings_sent;
 347         n->next_ping_time = squid_curtime + Config.Netdb.period;
 348         n->last_use_time = squid_curtime;
 349     }
 350
 351     xfree(hostname);
 352 }
 353
 354 Ip::Address
 355 networkFromInaddr(const Ip::Address &in)
 356 {
 357     Ip::Address out;
 358
 359     out = in;
 360
 361     /* in IPv6 the 'network' should be the routing section. */
 362     if ( in.isIPv6() ) {
 363         out.applyMask(64, AF_INET6);
 364         debugs(14, 5, "networkFromInaddr : Masked IPv6 Address to " << in << "/64 routing part.");
 365         return out;
 366     }
 367
 368 #if USE_CLASSFUL
 369     struct in_addr b;
 370
 371     in.getInAddr(b);
 372
 373     if (IN_CLASSC(b.s_addr))
 374         b.s_addr &= IN_CLASSC_NET;
 375     else if (IN_CLASSB(b.s_addr))
 376         b.s_addr &= IN_CLASSB_NET;
 377     else if (IN_CLASSA(b.s_addr))
 378         b.s_addr &= IN_CLASSA_NET;
 379
 380     out = b;
 381
 382 #endif
 383
 384     debugs(14, 5, "networkFromInaddr : Masked IPv4 Address to " << out << "/24.");
 385
 386     /* use /24 for everything under IPv4 */
 387     out.applyMask(24, AF_INET);
 388     debugs(14, 5, "networkFromInaddr : Masked IPv4 Address to " << in << "/24.");
 389
 390     return out;
 391 }
 392
 393 static int
 394 sortByRtt(const void *A, const void *B)
 395 {
 396     const netdbEntry *const *n1 = (const netdbEntry *const *)A;
 397     const netdbEntry *const *n2 = (const netdbEntry *const *)B;
 398
 399     if ((*n1)->rtt > (*n2)->rtt)
 400         return 1;
 401     else if ((*n1)->rtt < (*n2)->rtt)
 402         return -1;
 403     else
 404         return 0;
 405 }
 406
 407 static net_db_peer *
 408 netdbPeerByName(const netdbEntry * n, const char *peername)
 409 {
 410     int i;
 411     net_db_peer *p = n->peers;
 412
 413     for (i = 0; i < n->n_peers; ++i, ++p) {
 414         if (!strcmp(p->peername, peername))
 415             return p;
 416     }
 417
 418     return nullptr;
 419 }
 420
 421 static net_db_peer *
 422 netdbPeerAdd(netdbEntry * n, CachePeer * e)
 423 {
 424     net_db_peer *p;
 425     net_db_peer *o;
 426     int osize;
 427     int i;
 428
 429     if (n->n_peers == n->n_peers_alloc) {
 430         o = n->peers;
 431         osize = n->n_peers_alloc;
 432
 433         if (n->n_peers_alloc == 0)
 434             n->n_peers_alloc = 2;
 435         else
 436             n->n_peers_alloc <<= 1;
 437
 438         debugs(38, 3, "netdbPeerAdd: Growing peer list for '" << n->network << "' to " << n->n_peers_alloc);
 439
 440         n->peers = (net_db_peer *)xcalloc(n->n_peers_alloc, sizeof(net_db_peer));
 441
 442         for (i = 0; i < osize; ++i)
 443             *(n->peers + i) = *(o + i);
 444
 445         if (osize) {
 446             safe_free(o);
 447         }
 448     }
 449
 450     p = n->peers + n->n_peers;
 451     p->peername = netdbPeerName(e->host);
 452     ++ n->n_peers;
 453     return p;
 454 }
 455
 456 static int
 457 sortPeerByRtt(const void *A, const void *B)
 458 {
 459     const net_db_peer *p1 = (net_db_peer *)A;
 460     const net_db_peer *p2 = (net_db_peer *)B;
 461
 462     if (p1->rtt > p2->rtt)
 463         return 1;
 464     else if (p1->rtt < p2->rtt)
 465         return -1;
 466     else
 467         return 0;
 468 }
 469
 470 static void
 471 netdbSaveState(void *)
 472 {
 473     if (strcmp(Config.netdbFilename, "none") == 0)
 474         return;
 475
 476     Logfile *lf;
 477     netdbEntry *n;
 478     net_db_name *x;
 479
 480     struct timeval start = current_time;
 481     int count = 0;
 482     /*
 483      * This was nicer when we were using stdio, but thanks to
 484      * Solaris bugs, its a bad idea.  fopen can fail if more than
 485      * 256 FDs are open.
 486      */
 487     /*
 488      * unlink() is here because there is currently no way to make
 489      * logfileOpen() use O_TRUNC.
 490      */
 491     unlink(Config.netdbFilename);
 492     lf = logfileOpen(Config.netdbFilename, 4096, 0);
 493
 494     if (!lf) {
 495         int xerrno = errno;
 496         debugs(50, DBG_IMPORTANT, MYNAME << Config.netdbFilename << ": " << xstrerr(xerrno));
 497         return;
 498     }
 499
 500     hash_first(addr_table);
 501
 502     while ((n = (netdbEntry *) hash_next(addr_table))) {
 503         if (n->pings_recv == 0)
 504             continue;
 505
 506         logfilePrintf(lf, "%s %d %d %10.5f %10.5f %d %d",
 507                       n->network,
 508                       n->pings_sent,
 509                       n->pings_recv,
 510                       n->hops,
 511                       n->rtt,
 512                       (int) n->next_ping_time,
 513                       (int) n->last_use_time);
 514
 515         for (x = n->hosts; x; x = x->next)
 516             logfilePrintf(lf, " %s", hashKeyStr(x));
 517
 518         logfilePrintf(lf, "\n");
 519
 520         ++count;
 521
 522 #undef RBUF_SZ
 523
 524     }
 525
 526     logfileClose(lf);
 527     getCurrentTime();
 528     debugs(38, DBG_IMPORTANT, "NETDB state saved; " <<
 529            count << " entries, " <<
 530            tvSubMsec(start, current_time) << " msec" );
 531     eventAddIsh("netdbSaveState", netdbSaveState, nullptr, 3600.0, 1);
 532 }
 533
 534 static void
 535 netdbReloadState(void)
 536 {
 537     if (strcmp(Config.netdbFilename, "none") == 0)
 538         return;
 539
 540     char *s;
 541     int fd;
 542     int l;
 543
 544     struct stat sb;
 545     netdbEntry *n;
 546     netdbEntry N;
 547
 548     Ip::Address addr;
 549     int count = 0;
 550
 551     struct timeval start = current_time;
 552     /*
 553      * This was nicer when we were using stdio, but thanks to
 554      * Solaris bugs, its a bad idea.  fopen can fail if more than
 555      * 256 FDs are open.
 556      */
 557     fd = file_open(Config.netdbFilename, O_RDONLY | O_BINARY);
 558
 559     if (fd < 0)
 560         return;
 561
 562     if (fstat(fd, &sb) < 0) {
 563         file_close(fd);
 564         return;
 565     }
 566
 567     char *t;
 568     char *buf = (char *)xcalloc(1, sb.st_size + 1);
 569     t = buf;
 570     l = FD_READ_METHOD(fd, buf, sb.st_size);
 571     file_close(fd);
 572
 573     if (l <= 0) {
 574         safe_free (buf);
 575         return;
 576     };
 577
 578     while ((s = strchr(t, '\n'))) {
 579         char *q;
 580         assert(s - buf < l);
 581         *s = '\0';
 582         N = netdbEntry();
 583         q = strtok(t, w_space);
 584         t = s + 1;
 585
 586         if (nullptr == q)
 587             continue;
 588
 589         if (! (addr = q) )
 590             continue;
 591
 592         if (netdbLookupAddr(addr) != nullptr)  /* no dups! */
 593             continue;
 594
 595         if ((q = strtok(nullptr, w_space)) == nullptr)
 596             continue;
 597
 598         N.pings_sent = atoi(q);
 599
 600         if ((q = strtok(nullptr, w_space)) == nullptr)
 601             continue;
 602
 603         N.pings_recv = atoi(q);
 604
 605         if (N.pings_recv == 0)
 606             continue;
 607
 608         /* give this measurement low weight */
 609         N.pings_sent = 1;
 610
 611         N.pings_recv = 1;
 612
 613         if ((q = strtok(nullptr, w_space)) == nullptr)
 614             continue;
 615
 616         N.hops = atof(q);
 617
 618         if ((q = strtok(nullptr, w_space)) == nullptr)
 619             continue;
 620
 621         N.rtt = atof(q);
 622
 623         if ((q = strtok(nullptr, w_space)) == nullptr)
 624             continue;
 625
 626         N.next_ping_time = (time_t) atoi(q);
 627
 628         if ((q = strtok(nullptr, w_space)) == nullptr)
 629             continue;
 630
 631         N.last_use_time = (time_t) atoi(q);
 632
 633         n = new netdbEntry;
 634
 635         memcpy(n, &N, sizeof(netdbEntry));
 636
 637         netdbHashInsert(n, addr);
 638
 639         while ((q = strtok(nullptr, w_space)) != nullptr) {
 640             if (netdbLookupHost(q) != nullptr) /* no dups! */
 641                 continue;
 642
 643             netdbHostInsert(n, q);
 644         }
 645
 646         ++count;
 647     }
 648
 649     xfree(buf);
 650     getCurrentTime();
 651     debugs(38, DBG_IMPORTANT, "NETDB state reloaded; " <<
 652            count << " entries, " <<
 653            tvSubMsec(start, current_time) << " msec" );
 654 }
 655
 656 static const char *
 657 netdbPeerName(const char *name)
 658 {
 659     const wordlist *w;
 660
 661     for (w = peer_names; w; w = w->next) {
 662         if (!strcmp(w->key, name))
 663             return w->key;
 664     }
 665
 666     return wordlistAdd(&peer_names, name);
 667 }
 668
 669 static void
 670 netdbFreeNetdbEntry(void *data)
 671 {
 672     netdbEntry *n = (netdbEntry *)data;
 673     safe_free(n->peers);
 674     delete n;
 675 }
 676
 677 static void
 678 netdbFreeNameEntry(void *data)
 679 {
 680     net_db_name *x = (net_db_name *)data;
 681     delete x;
 682 }
 683
 684 static void
 685 netdbExchangeHandleReply(void *data, StoreIOBuffer receivedData)
 686 {
 687     Ip::Address addr;
 688
 689     netdbExchangeState *ex = (netdbExchangeState *)data;
 690     int rec_sz = 0;
 691     int o;
 692
 693     struct in_addr line_addr;
 694     double rtt;
 695     double hops;
 696     char *p;
 697     int j;
 698     size_t hdr_sz;
 699     int nused = 0;
 700     int size;
 701     int oldbufofs = ex->buf_ofs;
 702
 703     rec_sz = 0;
 704     rec_sz += 1 + sizeof(struct in_addr);
 705     rec_sz += 1 + sizeof(int);
 706     rec_sz += 1 + sizeof(int);
 707     debugs(38, 3, "netdbExchangeHandleReply: " << receivedData.length << " read bytes");
 708
 709     if (!ex->p.valid()) {
 710         debugs(38, 3, "netdbExchangeHandleReply: Peer became invalid");
 711         delete ex;
 712         return;
 713     }
 714
 715     debugs(38, 3, "netdbExchangeHandleReply: for '" << ex->p->host << ":" << ex->p->http_port << "'");
 716
 717     if (receivedData.length == 0 && !receivedData.flags.error) {
 718         debugs(38, 3, "netdbExchangeHandleReply: Done");
 719         delete ex;
 720         return;
 721     }
 722
 723     p = ex->buf;
 724
 725     /* Get the size of the buffer now */
 726     size = ex->buf_ofs + receivedData.length;
 727     debugs(38, 3, "netdbExchangeHandleReply: " << size << " bytes buf");
 728
 729     /* Check if we're still doing headers */
 730
 731     if (ex->connstate == STATE_HEADER) {
 732
 733         ex->buf_ofs += receivedData.length;
 734
 735         /* skip reply headers */
 736
 737         if ((hdr_sz = headersEnd(p, ex->buf_ofs))) {
 738             debugs(38, 5, "netdbExchangeHandleReply: hdr_sz = " << hdr_sz);
 739             const auto scode = ex->e->mem().baseReply().sline.status();
 740             assert(scode != Http::scNone);
 741             debugs(38, 3, "netdbExchangeHandleReply: reply status " << scode);
 742
 743             if (scode != Http::scOkay) {
 744                 delete ex;
 745                 return;
 746             }
 747
 748             assert((size_t)ex->buf_ofs >= hdr_sz);
 749
 750             /*
 751              * Now, point p to the part of the buffer where the data
 752              * starts, and update the size accordingly
 753              */
 754             assert(ex->used == 0);
 755             ex->used = hdr_sz;
 756             size = ex->buf_ofs - hdr_sz;
 757             p += hdr_sz;
 758
 759             /* Finally, set the conn state mode to STATE_BODY */
 760             ex->connstate = STATE_BODY;
 761         } else {
 762             StoreIOBuffer tempBuffer;
 763             tempBuffer.offset = ex->buf_ofs;
 764             tempBuffer.length = ex->buf_sz - ex->buf_ofs;
 765             tempBuffer.data = ex->buf + ex->buf_ofs;
 766             /* Have more headers .. */
 767             storeClientCopy(ex->sc, ex->e, tempBuffer,
 768                             netdbExchangeHandleReply, ex);
 769             return;
 770         }
 771     }
 772
 773     assert(ex->connstate == STATE_BODY);
 774
 775     /* If we get here, we have some body to parse .. */
 776     debugs(38, 5, "netdbExchangeHandleReply: start parsing loop, size = " << size);
 777
 778     while (size >= rec_sz) {
 779         debugs(38, 5, "netdbExchangeHandleReply: in parsing loop, size = " << size);
 780         addr.setAnyAddr();
 781         hops = rtt = 0.0;
 782
 783         for (o = 0; o < rec_sz;) {
 784             switch ((int) *(p + o)) {
 785
 786             case NETDB_EX_NETWORK:
 787                 ++o;
 788                 // XXX: NetDB can still only send IPv4
 789                 memcpy(&line_addr, p + o, sizeof(struct in_addr));
 790                 addr = line_addr;
 791                 o += sizeof(struct in_addr);
 792                 break;
 793
 794             case NETDB_EX_RTT:
 795                 ++o;
 796                 memcpy(&j, p + o, sizeof(int));
 797                 o += sizeof(int);
 798                 rtt = (double) ntohl(j) / 1000.0;
 799                 break;
 800
 801             case NETDB_EX_HOPS:
 802                 ++o;
 803                 memcpy(&j, p + o, sizeof(int));
 804                 o += sizeof(int);
 805                 hops = (double) ntohl(j) / 1000.0;
 806                 break;
 807
 808             default:
 809                 debugs(38, DBG_IMPORTANT, "ERROR: netdbExchangeHandleReply: corrupt data, aborting");
 810                 delete ex;
 811                 return;
 812             }
 813         }
 814
 815         if (!addr.isAnyAddr() && rtt > 0)
 816             netdbExchangeUpdatePeer(addr, ex->p.get(), rtt, hops);
 817
 818         assert(o == rec_sz);
 819
 820         ex->used += rec_sz;
 821
 822         size -= rec_sz;
 823
 824         p += rec_sz;
 825
 826         ++nused;
 827     }
 828
 829     /*
 830      * Copy anything that is left over to the beginning of the buffer,
 831      * and adjust buf_ofs accordingly
 832      */
 833
 834     /*
 835      * Evilly, size refers to the buf size left now,
 836      * ex->buf_ofs is the original buffer size, so just copy that
 837      * much data over
 838      */
 839     memmove(ex->buf, ex->buf + (ex->buf_ofs - size), size);
 840
 841     ex->buf_ofs = size;
 842
 843     /*
 844      * And don't re-copy the remaining data ..
 845      */
 846     ex->used += size;
 847
 848     /*
 849      * Now the tricky bit - size _included_ the leftover bit from the _last_
 850      * storeClientCopy. We don't want to include that, or our offset will be wrong.
 851      * So, don't count the size of the leftover buffer we began with.
 852      * This can _disappear_ when we're not tracking offsets ..
 853      */
 854     ex->used -= oldbufofs;
 855
 856     debugs(38, 3, "netdbExchangeHandleReply: size left over in this buffer: " << size << " bytes");
 857
 858     debugs(38, 3, "netdbExchangeHandleReply: used " << nused <<
 859            " entries, (x " << rec_sz << " bytes) == " << nused * rec_sz <<
 860            " bytes total");
 861
 862     debugs(38, 3, "netdbExchangeHandleReply: used " << ex->used);
 863
 864     if (EBIT_TEST(ex->e->flags, ENTRY_ABORTED)) {
 865         debugs(38, 3, "netdbExchangeHandleReply: ENTRY_ABORTED");
 866         delete ex;
 867     } else if (ex->e->store_status == STORE_PENDING) {
 868         StoreIOBuffer tempBuffer;
 869         tempBuffer.offset = ex->used;
 870         tempBuffer.length = ex->buf_sz - ex->buf_ofs;
 871         tempBuffer.data = ex->buf + ex->buf_ofs;
 872         debugs(38, 3, "netdbExchangeHandleReply: EOF not received");
 873         storeClientCopy(ex->sc, ex->e, tempBuffer,
 874                         netdbExchangeHandleReply, ex);
 875     }
 876 }
 877
 878 #endif /* USE_ICMP */
 879
 880 /* PUBLIC FUNCTIONS */
 881
 882 void
 883 netdbInit(void)
 884 {
 885 #if USE_ICMP
 886     Mgr::RegisterAction("netdb", "Network Measurement Database", netdbDump, 0, 1);
 887
 888     if (addr_table)
 889         return;
 890
 891     int n = hashPrime(Config.Netdb.high / 4);
 892
 893     addr_table = hash_create((HASHCMP *) strcmp, n, hash_string);
 894
 895     n = hashPrime(3 * Config.Netdb.high / 4);
 896
 897     host_table = hash_create((HASHCMP *) strcmp, n, hash_string);
 898
 899     eventAddIsh("netdbSaveState", netdbSaveState, nullptr, 3600.0, 1);
 900
 901     netdbReloadState();
 902
 903 #endif
 904 }
 905
 906 void
 907 netdbPingSite(const char *hostname)
 908 {
 909 #if USE_ICMP
 910     netdbEntry *n;
 911
 912     if ((n = netdbLookupHost(hostname)) != nullptr)
 913         if (n->next_ping_time > squid_curtime)
 914             return;
 915
 916     ipcache_nbgethostbyname(hostname, netdbSendPing,
 917                             new generic_cbdata(xstrdup(hostname)));
 918 #else
 919     (void)hostname;
 920 #endif
 921 }
 922
 923 void
 924 netdbHandlePingReply(const Ip::Address &from, int hops, int rtt)
 925 {
 926 #if USE_ICMP
 927     netdbEntry *n;
 928     int N;
 929     debugs(38, 3, "netdbHandlePingReply: from " << from);
 930
 931     if ((n = netdbLookupAddr(from)) == nullptr)
 932         return;
 933
 934     N = ++n->pings_recv;
 935
 936     if (N > 5)
 937         N = 5;
 938
 939     if (rtt < 1)
 940         rtt = 1;
 941
 942     n->hops = ((n->hops * (N - 1)) + hops) / N;
 943
 944     n->rtt = ((n->rtt * (N - 1)) + rtt) / N;
 945
 946     debugs(38, 3, "netdbHandlePingReply: " << n->network  << "; rtt="<<
 947            std::setw(5)<< std::setprecision(2) << n->rtt << "  hops="<<
 948            std::setw(4) << n->hops);
 949 #else
 950     (void)from;
 951     (void)hops;
 952     (void)rtt;
 953 #endif
 954 }
 955
 956 void
 957 netdbFreeMemory(void)
 958 {
 959 #if USE_ICMP
 960     hashFreeItems(addr_table, netdbFreeNetdbEntry);
 961     hashFreeMemory(addr_table);
 962     addr_table = nullptr;
 963     hashFreeItems(host_table, netdbFreeNameEntry);
 964     hashFreeMemory(host_table);
 965     host_table = nullptr;
 966     wordlistDestroy(&peer_names);
 967     peer_names = nullptr;
 968 #endif
 969 }
 970
 971 void
 972 netdbDump(StoreEntry * sentry)
 973 {
 974 #if USE_ICMP
 975     netdbEntry *n;
 976     netdbEntry **list;
 977     net_db_name *x;
 978     int k;
 979     int i;
 980     int j;
 981     net_db_peer *p;
 982     storeAppendPrintf(sentry, "Network DB Statistics:\n");
 983     storeAppendPrintf(sentry, "%-46.46s %9s %7s %5s %s\n",  /* Max between 16 (IPv4) or 46 (IPv6)   */
 984                       "Network",
 985                       "recv/sent",
 986                       "RTT",
 987                       "Hops",
 988                       "Hostnames");
 989     list = (netdbEntry **)xcalloc(netdbEntry::UseCount(), sizeof(netdbEntry *));
 990     i = 0;
 991     hash_first(addr_table);
 992
 993     while ((n = (netdbEntry *) hash_next(addr_table))) {
 994         *(list + i) = n;
 995         ++i;
 996     }
 997
 998     if (i != netdbEntry::UseCount())
 999         debugs(38, DBG_CRITICAL, "WARNING: netdb_addrs count off, found " << i <<
1000                ", expected " << netdbEntry::UseCount());
1001
1002     qsort((char *) list,
1003           i,
1004           sizeof(netdbEntry *),
1005           sortByRtt);
1006
1007     for (k = 0; k < i; ++k) {
1008         n = *(list + k);
1009         storeAppendPrintf(sentry, "%-46.46s %4d/%4d %7.1f %5.1f", /* Max between 16 (IPv4) or 46 (IPv6)   */
1010                           n->network,
1011                           n->pings_recv,
1012                           n->pings_sent,
1013                           n->rtt,
1014                           n->hops);
1015
1016         for (x = n->hosts; x; x = x->next)
1017             storeAppendPrintf(sentry, " %s", hashKeyStr(x));
1018
1019         storeAppendPrintf(sentry, "\n");
1020
1021         p = n->peers;
1022
1023         for (j = 0; j < n->n_peers; ++j, ++p) {
1024             storeAppendPrintf(sentry, "    %-22.22s %7.1f %5.1f\n",
1025                               p->peername,
1026                               p->rtt,
1027                               p->hops);
1028         }
1029     }
1030
1031     xfree(list);
1032 #else
1033
1034     storeAppendPrintf(sentry,"NETDB support not compiled into this Squid cache.\n");
1035 #endif
1036 }
1037
1038 int
1039 netdbHostHops(const char *host)
1040 {
1041 #if USE_ICMP
1042     netdbEntry *n = netdbLookupHost(host);
1043
1044     if (n) {
1045         n->last_use_time = squid_curtime;
1046         return (int) (n->hops + 0.5);
1047     }
1048 #else
1049     (void)host;
1050 #endif
1051     return 0;
1052 }
1053
1054 int
1055 netdbHostRtt(const char *host)
1056 {
1057 #if USE_ICMP
1058     netdbEntry *n = netdbLookupHost(host);
1059
1060     if (n) {
1061         n->last_use_time = squid_curtime;
1062         return (int) (n->rtt + 0.5);
1063     }
1064
1065 #else
1066     (void)host;
1067 #endif
1068     return 0;
1069 }
1070
1071 void
1072 netdbHostData(const char *host, int *samp, int *rtt, int *hops)
1073 {
1074 #if USE_ICMP
1075     netdbEntry *n = netdbLookupHost(host);
1076
1077     if (n == nullptr)
1078         return;
1079
1080     *samp = n->pings_recv;
1081
1082     *rtt = (int) (n->rtt + 0.5);
1083
1084     *hops = (int) (n->hops + 0.5);
1085
1086     n->last_use_time = squid_curtime;
1087
1088 #else
1089     (void)hops;
1090     (void)rtt;
1091     (void)samp;
1092     (void)host;
1093 #endif
1094 }
1095
1096 void
1097 netdbUpdatePeer(const AnyP::Uri &url, CachePeer *e, int irtt, int ihops)
1098 {
1099 #if USE_ICMP
1100     netdbEntry *n;
1101     double rtt = (double) irtt;
1102     double hops = (double) ihops;
1103     net_db_peer *p;
1104     debugs(38, 3, url.host() << ", " << ihops << " hops, " << irtt << " rtt");
1105     n = netdbLookupHost(url.host());
1106
1107     if (n == nullptr) {
1108         debugs(38, 3, "host " << url.host() << " not found");
1109         return;
1110     }
1111
1112     if ((p = netdbPeerByName(n, e->host)) == nullptr)
1113         p = netdbPeerAdd(n, e);
1114
1115     p->rtt = rtt;
1116
1117     p->hops = hops;
1118
1119     p->expires = squid_curtime + 3600;
1120
1121     if (n->n_peers < 2)
1122         return;
1123
1124     qsort((char *) n->peers,
1125           n->n_peers,
1126           sizeof(net_db_peer),
1127           sortPeerByRtt);
1128
1129 #else
1130     (void)ihops;
1131     (void)irtt;
1132     (void)e;
1133     (void)url;
1134 #endif
1135 }
1136
1137 void
1138 netdbExchangeUpdatePeer(Ip::Address &addr, CachePeer * e, double rtt, double hops)
1139 {
1140 #if USE_ICMP
1141     netdbEntry *n;
1142     net_db_peer *p;
1143     debugs(38, 5, "netdbExchangeUpdatePeer: '" << addr << "', "<<
1144            std::setfill('0')<< std::setprecision(2) << hops << " hops, " <<
1145            rtt << " rtt");
1146
1147     if ( !addr.isIPv4() ) {
1148         debugs(38, 5, "netdbExchangeUpdatePeer: Aborting peer update for '" << addr << "', NetDB cannot handle IPv6.");
1149         return;
1150     }
1151
1152     n = netdbLookupAddr(addr);
1153
1154     if (n == nullptr)
1155         n = netdbAdd(addr);
1156
1157     assert(nullptr != n);
1158
1159     if ((p = netdbPeerByName(n, e->host)) == nullptr)
1160         p = netdbPeerAdd(n, e);
1161
1162     p->rtt = rtt;
1163
1164     p->hops = hops;
1165
1166     p->expires = squid_curtime + 3600;  /* XXX ? */
1167
1168     if (n->n_peers < 2)
1169         return;
1170
1171     qsort((char *) n->peers,
1172           n->n_peers,
1173           sizeof(net_db_peer),
1174           sortPeerByRtt);
1175
1176 #else
1177     (void)hops;
1178     (void)rtt;
1179     (void)e;
1180     (void)addr;
1181 #endif
1182 }
1183
1184 void
1185 netdbDeleteAddrNetwork(Ip::Address &addr)
1186 {
1187 #if USE_ICMP
1188     netdbEntry *n = netdbLookupAddr(addr);
1189
1190     if (n == nullptr)
1191         return;
1192
1193     debugs(38, 3, "netdbDeleteAddrNetwork: " << n->network);
1194
1195     netdbRelease(n);
1196 #else
1197     (void)addr;
1198 #endif
1199 }
1200
1201 void
1202 netdbBinaryExchange(StoreEntry * s)
1203 {
1204     HttpReply *reply = new HttpReply;
1205 #if USE_ICMP
1206
1207     Ip::Address addr;
1208
1209     netdbEntry *n;
1210     int i;
1211     int j;
1212     int rec_sz;
1213     char *buf;
1214
1215     struct in_addr line_addr;
1216     s->buffer();
1217     reply->setHeaders(Http::scOkay, "OK", nullptr, -1, squid_curtime, -2);
1218     s->replaceHttpReply(reply);
1219     rec_sz = 0;
1220     rec_sz += 1 + sizeof(struct in_addr);
1221     rec_sz += 1 + sizeof(int);
1222     rec_sz += 1 + sizeof(int);
1223     buf = (char *)memAllocate(MEM_4K_BUF);
1224     i = 0;
1225     hash_first(addr_table);
1226
1227     while ((n = (netdbEntry *) hash_next(addr_table))) {
1228         if (0.0 == n->rtt)
1229             continue;
1230
1231         if (n->rtt > 60000) /* RTT > 1 MIN probably bogus */
1232             continue;
1233
1234         if (! (addr = n->network) )
1235             continue;
1236
1237         // XXX: NetDB cannot yet handle IPv6 addresses
1238         if ( !addr.isIPv4() )
1239             continue;
1240
1241         buf[i] = (char) NETDB_EX_NETWORK;
1242         ++i;
1243
1244         addr.getInAddr(line_addr);
1245         memcpy(&buf[i], &line_addr, sizeof(struct in_addr));
1246
1247         i += sizeof(struct in_addr);
1248
1249         buf[i] = (char) NETDB_EX_RTT;
1250         ++i;
1251
1252         j = htonl((int) (n->rtt * 1000));
1253
1254         memcpy(&buf[i], &j, sizeof(int));
1255
1256         i += sizeof(int);
1257
1258         buf[i] = (char) NETDB_EX_HOPS;
1259         ++i;
1260
1261         j = htonl((int) (n->hops * 1000));
1262
1263         memcpy(&buf[i], &j, sizeof(int));
1264
1265         i += sizeof(int);
1266
1267         if (i + rec_sz > 4096) {
1268             s->append(buf, i);
1269             i = 0;
1270         }
1271     }
1272
1273     if (i > 0) {
1274         s->append(buf, i);
1275         i = 0;
1276     }
1277
1278     assert(0 == i);
1279     s->flush();
1280     memFree(buf, MEM_4K_BUF);
1281 #else
1282
1283     reply->setHeaders(Http::scBadRequest, "Bad Request", NULL, -1, squid_curtime, -2);
1284     s->replaceHttpReply(reply);
1285     storeAppendPrintf(s, "NETDB support not compiled into this Squid cache.\n");
1286 #endif
1287
1288     s->complete();
1289 }
1290
1291 void
1292 netdbExchangeStart(void *data)
1293 {
1294 #if USE_ICMP
1295     CachePeer *p = (CachePeer *)data;
1296     static const SBuf netDB("netdb");
1297     char *uri = internalRemoteUri(p->secure.encryptTransport, p->host, p->http_port, "/squid-internal-dynamic/", netDB);
1298     debugs(38, 3, "Requesting '" << uri << "'");
1299     const auto mx = MasterXaction::MakePortless<XactionInitiator::initIcmp>();
1300     HttpRequestPointer req(HttpRequest::FromUrlXXX(uri, mx));
1301
1302     if (!req) {
1303         debugs(38, DBG_IMPORTANT, "ERROR: " << MYNAME << ": Bad URI " << uri);
1304         return;
1305     }
1306
1307     netdbExchangeState *ex = new netdbExchangeState(p, req);
1308     ex->e = storeCreateEntry(uri, uri, RequestFlags(), Http::METHOD_GET);
1309     assert(nullptr != ex->e);
1310
1311     StoreIOBuffer tempBuffer;
1312     tempBuffer.length = ex->buf_sz;
1313     tempBuffer.data = ex->buf;
1314
1315     ex->sc = storeClientListAdd(ex->e, ex);
1316
1317     storeClientCopy(ex->sc, ex->e, tempBuffer,
1318                     netdbExchangeHandleReply, ex);
1319     ex->r->flags.loopDetected = true;   /* cheat! -- force direct */
1320
1321     // XXX: send as Proxy-Authenticate instead
1322     if (p->login)
1323         ex->r->url.userInfo(SBuf(p->login));
1324
1325     FwdState::fwdStart(Comm::ConnectionPointer(), ex->e, ex->r.getRaw());
1326 #else
1327     (void)data;
1328 #endif
1329 }
1330
1331 CachePeer *
1332 netdbClosestParent(PeerSelector *ps)
1333 {
1334 #if USE_ICMP
1335     assert(ps);
1336     HttpRequest *request = ps->request;
1337
1338     CachePeer *p = nullptr;
1339     netdbEntry *n;
1340     const ipcache_addrs *ia;
1341     net_db_peer *h;
1342     int i;
1343     n = netdbLookupHost(request->url.host());
1344
1345     if (nullptr == n) {
1346         /* try IP addr */
1347         ia = ipcache_gethostbyname(request->url.host(), 0);
1348
1349         if (nullptr != ia)
1350             n = netdbLookupAddr(ia->current());
1351     }
1352
1353     if (nullptr == n)
1354         return nullptr;
1355
1356     if (0 == n->n_peers)
1357         return nullptr;
1358
1359     n->last_use_time = squid_curtime;
1360
1361     /*
1362      * Find the parent with the least RTT to the origin server.
1363      * Make sure we don't return a parent who is farther away than
1364      * we are.  Note, the n->peers list is pre-sorted by RTT.
1365      */
1366     for (i = 0; i < n->n_peers; ++i) {
1367         h = &n->peers[i];
1368
1369         if (n->rtt > 0)
1370             if (n->rtt < h->rtt)
1371                 break;
1372
1373         p = peerFindByName(h->peername);
1374
1375         if (nullptr == p)      /* not found */
1376             continue;
1377
1378         if (neighborType(p, request->url) != PEER_PARENT)
1379             continue;
1380
1381         if (!peerHTTPOkay(p, ps))  /* not allowed */
1382             continue;
1383
1384         return p;
1385     }
1386
1387 #else
1388     (void)ps;
1389 #endif
1390     return nullptr;
1391 }
1392