src/icmp/net_db.cc

   1 /*
   2  * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 38    Network Measurement Database */
  10
  11 /*
  12  * XXX XXX XXX
  13  *
  14  * This code may be slightly broken now. If you're getting consistent
  15  * (sometimes working) corrupt data exchanges, please contact adrian
  16  * (adrian@squid-cache.org) to sort them out.
  17  */
  18
  19 #include "squid.h"
  20 #include "CachePeer.h"
  21 #include "cbdata.h"
  22 #include "disk.h"
  23 #include "event.h"
  24 #include "fde.h"
  25 #include "FwdState.h"
  26 #include "HttpReply.h"
  27 #include "HttpRequest.h"
  28 #include "icmp/net_db.h"
  29 #include "internal.h"
  30 #include "ip/Address.h"
  31 #include "log/File.h"
  32 #include "MemObject.h"
  33 #include "mgr/Registration.h"
  34 #include "mime_header.h"
  35 #include "neighbors.h"
  36 #include "SquidTime.h"
  37 #include "Store.h"
  38 #include "StoreClient.h"
  39 #include "SwapDir.h"
  40 #include "tools.h"
  41 #include "URL.h"
  42 #include "wordlist.h"
  43
  44 #if HAVE_SYS_STAT_H
  45 #include <sys/stat.h>
  46 #endif
  47
  48 #if USE_ICMP
  49 #include "icmp/IcmpSquid.h"
  50 #include "ipcache.h"
  51 #include "StoreClient.h"
  52
  53 #define NETDB_REQBUF_SZ 4096
  54
  55 typedef enum {
  56     STATE_NONE,
  57     STATE_HEADER,
  58     STATE_BODY
  59 } netdb_conn_state_t;
  60
  61 class netdbExchangeState
  62 {
  63     CBDATA_CLASS(netdbExchangeState);
  64
  65 public:
  66     netdbExchangeState(CachePeer *aPeer, HttpRequest *theReq) :
  67         p(cbdataReference(aPeer)),
  68         e(NULL),
  69         sc(NULL),
  70         r(theReq),
  71         used(0),
  72         buf_sz(NETDB_REQBUF_SZ),
  73         buf_ofs(0),
  74         connstate(STATE_HEADER)
  75     {
  76         *buf = 0;
  77
  78         assert(NULL != r);
  79         HTTPMSGLOCK(r);
  80         // TODO: check if we actually need to do this. should be implicit
  81         r->http_ver = Http::ProtocolVersion();
  82     }
  83
  84     ~netdbExchangeState() {
  85         debugs(38, 3, e->url());
  86         storeUnregister(sc, e, this);
  87         e->unlock("netdbExchangeDone");
  88         HTTPMSGUNLOCK(r);
  89         cbdataReferenceDone(p);
  90     }
  91
  92     CachePeer *p;
  93     StoreEntry *e;
  94     store_client *sc;
  95     HttpRequest *r;
  96     int64_t used;
  97     size_t buf_sz;
  98     char buf[NETDB_REQBUF_SZ];
  99     int buf_ofs;
 100     netdb_conn_state_t connstate;
 101 };
 102
 103 CBDATA_CLASS_INIT(netdbExchangeState);
 104
 105 static hash_table *addr_table = NULL;
 106 static hash_table *host_table = NULL;
 107
 108 Ip::Address networkFromInaddr(const Ip::Address &a);
 109 static void netdbRelease(netdbEntry * n);
 110
 111 static void netdbHashInsert(netdbEntry * n, Ip::Address &addr);
 112 static void netdbHashDelete(const char *key);
 113 static void netdbHostInsert(netdbEntry * n, const char *hostname);
 114 static void netdbHostDelete(const net_db_name * x);
 115 static void netdbPurgeLRU(void);
 116 static netdbEntry *netdbLookupHost(const char *key);
 117 static net_db_peer *netdbPeerByName(const netdbEntry * n, const char *);
 118 static net_db_peer *netdbPeerAdd(netdbEntry * n, CachePeer * e);
 119 static const char *netdbPeerName(const char *name);
 120 static IPH netdbSendPing;
 121 static QS sortPeerByRtt;
 122 static QS sortByRtt;
 123 static QS netdbLRU;
 124 static FREE netdbFreeNameEntry;
 125 static FREE netdbFreeNetdbEntry;
 126 static STCB netdbExchangeHandleReply;
 127
 128 /* We have to keep a local list of CachePeer names.  The Peers structure
 129  * gets freed during a reconfigure.  We want this database to
 130  * remain persisitent, so _net_db_peer->peername points into this
 131  * linked list */
 132 static wordlist *peer_names = NULL;
 133
 134 static void
 135 netdbHashInsert(netdbEntry * n, Ip::Address &addr)
 136 {
 137     networkFromInaddr(addr).toStr(n->network, MAX_IPSTRLEN);
 138     n->hash.key = n->network;
 139     assert(hash_lookup(addr_table, n->network) == NULL);
 140     hash_join(addr_table, &n->hash);
 141 }
 142
 143 static void
 144 netdbHashDelete(const char *key)
 145 {
 146     hash_link *hptr = (hash_link *)hash_lookup(addr_table, key);
 147
 148     if (hptr == NULL) {
 149         debug_trap("netdbHashDelete: key not found");
 150         return;
 151     }
 152
 153     hash_remove_link(addr_table, hptr);
 154 }
 155
 156 static void
 157 netdbHostInsert(netdbEntry * n, const char *hostname)
 158 {
 159     net_db_name *x = (net_db_name *)memAllocate(MEM_NET_DB_NAME);
 160     x->hash.key = xstrdup(hostname);
 161     x->next = n->hosts;
 162     n->hosts = x;
 163     x->net_db_entry = n;
 164     assert(hash_lookup(host_table, hostname) == NULL);
 165     hash_join(host_table, &x->hash);
 166     ++ n->link_count;
 167 }
 168
 169 static void
 170 netdbHostDelete(const net_db_name * x)
 171 {
 172     netdbEntry *n;
 173     net_db_name **X;
 174     assert(x != NULL);
 175     assert(x->net_db_entry != NULL);
 176     n = x->net_db_entry;
 177     -- n->link_count;
 178
 179     for (X = &n->hosts; *X; X = &(*X)->next) {
 180         if (*X == x) {
 181             *X = x->next;
 182             break;
 183         }
 184     }
 185
 186     hash_remove_link(host_table, (hash_link *) x);
 187     xfree(x->hash.key);
 188     memFree((void *) x, MEM_NET_DB_NAME);
 189 }
 190
 191 static netdbEntry *
 192 netdbLookupHost(const char *key)
 193 {
 194     net_db_name *x = (net_db_name *) hash_lookup(host_table, key);
 195     return x ? x->net_db_entry : NULL;
 196 }
 197
 198 static void
 199 netdbRelease(netdbEntry * n)
 200 {
 201     net_db_name *x;
 202     net_db_name *next;
 203
 204     for (x = n->hosts; x; x = next) {
 205         next = x->next;
 206         netdbHostDelete(x);
 207     }
 208
 209     n->hosts = NULL;
 210     safe_free(n->peers);
 211     n->peers = NULL;
 212     n->n_peers = 0;
 213     n->n_peers_alloc = 0;
 214
 215     if (n->link_count == 0) {
 216         netdbHashDelete(n->network);
 217         memFree(n, MEM_NETDBENTRY);
 218     }
 219 }
 220
 221 static int
 222 netdbLRU(const void *A, const void *B)
 223 {
 224     const netdbEntry *const *n1 = (const netdbEntry *const *)A;
 225     const netdbEntry *const *n2 = (const netdbEntry *const *)B;
 226
 227     if ((*n1)->last_use_time > (*n2)->last_use_time)
 228         return (1);
 229
 230     if ((*n1)->last_use_time < (*n2)->last_use_time)
 231         return (-1);
 232
 233     return (0);
 234 }
 235
 236 static void
 237 netdbPurgeLRU(void)
 238 {
 239     netdbEntry *n;
 240     netdbEntry **list;
 241     int k = 0;
 242     int list_count = 0;
 243     int removed = 0;
 244     list = (netdbEntry **)xcalloc(memInUse(MEM_NETDBENTRY), sizeof(netdbEntry *));
 245     hash_first(addr_table);
 246
 247     while ((n = (netdbEntry *) hash_next(addr_table))) {
 248         assert(list_count < memInUse(MEM_NETDBENTRY));
 249         *(list + list_count) = n;
 250         ++list_count;
 251     }
 252
 253     qsort((char *) list,
 254           list_count,
 255           sizeof(netdbEntry *),
 256           netdbLRU);
 257
 258     for (k = 0; k < list_count; ++k) {
 259         if (memInUse(MEM_NETDBENTRY) < Config.Netdb.low)
 260             break;
 261
 262         netdbRelease(*(list + k));
 263
 264         ++removed;
 265     }
 266
 267     xfree(list);
 268 }
 269
 270 static netdbEntry *
 271 netdbLookupAddr(const Ip::Address &addr)
 272 {
 273     netdbEntry *n;
 274     char *key = new char[MAX_IPSTRLEN];
 275     networkFromInaddr(addr).toStr(key,MAX_IPSTRLEN);
 276     n = (netdbEntry *) hash_lookup(addr_table, key);
 277     delete[] key;
 278     return n;
 279 }
 280
 281 static netdbEntry *
 282 netdbAdd(Ip::Address &addr)
 283 {
 284     netdbEntry *n;
 285
 286     if (memInUse(MEM_NETDBENTRY) > Config.Netdb.high)
 287         netdbPurgeLRU();
 288
 289     if ((n = netdbLookupAddr(addr)) == NULL) {
 290         n = (netdbEntry *)memAllocate(MEM_NETDBENTRY);
 291         netdbHashInsert(n, addr);
 292     }
 293
 294     return n;
 295 }
 296
 297 static void
 298 netdbSendPing(const ipcache_addrs *ia, const Dns::LookupDetails &, void *data)
 299 {
 300     Ip::Address addr;
 301     char *hostname = NULL;
 302     static_cast<generic_cbdata *>(data)->unwrap(&hostname);
 303     netdbEntry *n;
 304     netdbEntry *na;
 305     net_db_name *x;
 306     net_db_name **X;
 307
 308     if (ia == NULL) {
 309         xfree(hostname);
 310         return;
 311     }
 312
 313     addr = ia->in_addrs[ia->cur];
 314
 315     if ((n = netdbLookupHost(hostname)) == NULL) {
 316         n = netdbAdd(addr);
 317         netdbHostInsert(n, hostname);
 318     } else if ((na = netdbLookupAddr(addr)) != n) {
 319         /*
 320          *hostname moved from 'network n' to 'network na'!
 321          */
 322
 323         if (na == NULL)
 324             na = netdbAdd(addr);
 325
 326         debugs(38, 3, "netdbSendPing: " << hostname << " moved from " << n->network << " to " << na->network);
 327
 328         x = (net_db_name *) hash_lookup(host_table, hostname);
 329
 330         if (x == NULL) {
 331             debugs(38, DBG_IMPORTANT, "netdbSendPing: net_db_name list bug: " << hostname << " not found");
 332             xfree(hostname);
 333             return;
 334         }
 335
 336         /* remove net_db_name from 'network n' linked list */
 337         for (X = &n->hosts; *X; X = &(*X)->next) {
 338             if (*X == x) {
 339                 *X = x->next;
 340                 break;
 341             }
 342         }
 343
 344         -- n->link_count;
 345         /* point to 'network na' from host entry */
 346         x->net_db_entry = na;
 347         /* link net_db_name to 'network na' */
 348         x->next = na->hosts;
 349         na->hosts = x;
 350         ++ na->link_count;
 351         n = na;
 352     }
 353
 354     if (n->next_ping_time <= squid_curtime) {
 355         debugs(38, 3, "netdbSendPing: pinging " << hostname);
 356         icmpEngine.DomainPing(addr, hostname);
 357         ++ n->pings_sent;
 358         n->next_ping_time = squid_curtime + Config.Netdb.period;
 359         n->last_use_time = squid_curtime;
 360     }
 361
 362     xfree(hostname);
 363 }
 364
 365 Ip::Address
 366 networkFromInaddr(const Ip::Address &in)
 367 {
 368     Ip::Address out;
 369
 370     out = in;
 371
 372     /* in IPv6 the 'network' should be the routing section. */
 373     if ( in.isIPv6() ) {
 374         out.applyMask(64, AF_INET6);
 375         debugs(14, 5, "networkFromInaddr : Masked IPv6 Address to " << in << "/64 routing part.");
 376         return out;
 377     }
 378
 379 #if USE_CLASSFUL
 380     struct in_addr b;
 381
 382     in.getInAddr(b);
 383
 384     if (IN_CLASSC(b.s_addr))
 385         b.s_addr &= IN_CLASSC_NET;
 386     else if (IN_CLASSB(b.s_addr))
 387         b.s_addr &= IN_CLASSB_NET;
 388     else if (IN_CLASSA(b.s_addr))
 389         b.s_addr &= IN_CLASSA_NET;
 390
 391     out = b;
 392
 393 #endif
 394
 395     debugs(14, 5, "networkFromInaddr : Masked IPv4 Address to " << out << "/24.");
 396
 397     /* use /24 for everything under IPv4 */
 398     out.applyMask(24, AF_INET);
 399     debugs(14, 5, "networkFromInaddr : Masked IPv4 Address to " << in << "/24.");
 400
 401     return out;
 402 }
 403
 404 static int
 405 sortByRtt(const void *A, const void *B)
 406 {
 407     const netdbEntry *const *n1 = (const netdbEntry *const *)A;
 408     const netdbEntry *const *n2 = (const netdbEntry *const *)B;
 409
 410     if ((*n1)->rtt > (*n2)->rtt)
 411         return 1;
 412     else if ((*n1)->rtt < (*n2)->rtt)
 413         return -1;
 414     else
 415         return 0;
 416 }
 417
 418 static net_db_peer *
 419 netdbPeerByName(const netdbEntry * n, const char *peername)
 420 {
 421     int i;
 422     net_db_peer *p = n->peers;
 423
 424     for (i = 0; i < n->n_peers; ++i, ++p) {
 425         if (!strcmp(p->peername, peername))
 426             return p;
 427     }
 428
 429     return NULL;
 430 }
 431
 432 static net_db_peer *
 433 netdbPeerAdd(netdbEntry * n, CachePeer * e)
 434 {
 435     net_db_peer *p;
 436     net_db_peer *o;
 437     int osize;
 438     int i;
 439
 440     if (n->n_peers == n->n_peers_alloc) {
 441         o = n->peers;
 442         osize = n->n_peers_alloc;
 443
 444         if (n->n_peers_alloc == 0)
 445             n->n_peers_alloc = 2;
 446         else
 447             n->n_peers_alloc <<= 1;
 448
 449         debugs(38, 3, "netdbPeerAdd: Growing peer list for '" << n->network << "' to " << n->n_peers_alloc);
 450
 451         n->peers = (net_db_peer *)xcalloc(n->n_peers_alloc, sizeof(net_db_peer));
 452
 453         for (i = 0; i < osize; ++i)
 454             *(n->peers + i) = *(o + i);
 455
 456         if (osize) {
 457             safe_free(o);
 458         }
 459     }
 460
 461     p = n->peers + n->n_peers;
 462     p->peername = netdbPeerName(e->host);
 463     ++ n->n_peers;
 464     return p;
 465 }
 466
 467 static int
 468 sortPeerByRtt(const void *A, const void *B)
 469 {
 470     const net_db_peer *p1 = (net_db_peer *)A;
 471     const net_db_peer *p2 = (net_db_peer *)B;
 472
 473     if (p1->rtt > p2->rtt)
 474         return 1;
 475     else if (p1->rtt < p2->rtt)
 476         return -1;
 477     else
 478         return 0;
 479 }
 480
 481 static void
 482 netdbSaveState(void *foo)
 483 {
 484     if (strcmp(Config.netdbFilename, "none") == 0)
 485         return;
 486
 487     Logfile *lf;
 488     netdbEntry *n;
 489     net_db_name *x;
 490
 491     struct timeval start = current_time;
 492     int count = 0;
 493     /*
 494      * This was nicer when we were using stdio, but thanks to
 495      * Solaris bugs, its a bad idea.  fopen can fail if more than
 496      * 256 FDs are open.
 497      */
 498     /*
 499      * unlink() is here because there is currently no way to make
 500      * logfileOpen() use O_TRUNC.
 501      */
 502     unlink(Config.netdbFilename);
 503     lf = logfileOpen(Config.netdbFilename, 4096, 0);
 504
 505     if (NULL == lf) {
 506         debugs(50, DBG_IMPORTANT, "netdbSaveState: " << Config.netdbFilename << ": " << xstrerror());
 507         return;
 508     }
 509
 510     hash_first(addr_table);
 511
 512     while ((n = (netdbEntry *) hash_next(addr_table))) {
 513         if (n->pings_recv == 0)
 514             continue;
 515
 516         logfilePrintf(lf, "%s %d %d %10.5f %10.5f %d %d",
 517                       n->network,
 518                       n->pings_sent,
 519                       n->pings_recv,
 520                       n->hops,
 521                       n->rtt,
 522                       (int) n->next_ping_time,
 523                       (int) n->last_use_time);
 524
 525         for (x = n->hosts; x; x = x->next)
 526             logfilePrintf(lf, " %s", hashKeyStr(&x->hash));
 527
 528         logfilePrintf(lf, "\n");
 529
 530         ++count;
 531
 532 #undef RBUF_SZ
 533
 534     }
 535
 536     logfileClose(lf);
 537     getCurrentTime();
 538     debugs(38, DBG_IMPORTANT, "NETDB state saved; " <<
 539            count << " entries, " <<
 540            tvSubMsec(start, current_time) << " msec" );
 541     eventAddIsh("netdbSaveState", netdbSaveState, NULL, 3600.0, 1);
 542 }
 543
 544 static void
 545 netdbReloadState(void)
 546 {
 547     if (strcmp(Config.netdbFilename, "none") == 0)
 548         return;
 549
 550     char *s;
 551     int fd;
 552     int l;
 553
 554     struct stat sb;
 555     netdbEntry *n;
 556     netdbEntry N;
 557
 558     Ip::Address addr;
 559     int count = 0;
 560
 561     struct timeval start = current_time;
 562     /*
 563      * This was nicer when we were using stdio, but thanks to
 564      * Solaris bugs, its a bad idea.  fopen can fail if more than
 565      * 256 FDs are open.
 566      */
 567     fd = file_open(Config.netdbFilename, O_RDONLY | O_BINARY);
 568
 569     if (fd < 0)
 570         return;
 571
 572     if (fstat(fd, &sb) < 0) {
 573         file_close(fd);
 574         return;
 575     }
 576
 577     char *t;
 578     char *buf = (char *)xcalloc(1, sb.st_size + 1);
 579     t = buf;
 580     l = FD_READ_METHOD(fd, buf, sb.st_size);
 581     file_close(fd);
 582
 583     if (l <= 0) {
 584         safe_free (buf);
 585         return;
 586     };
 587
 588     while ((s = strchr(t, '\n'))) {
 589         char *q;
 590         assert(s - buf < l);
 591         *s = '\0';
 592         memset(&N, '\0', sizeof(netdbEntry));
 593         q = strtok(t, w_space);
 594         t = s + 1;
 595
 596         if (NULL == q)
 597             continue;
 598
 599         if (! (addr = q) )
 600             continue;
 601
 602         if (netdbLookupAddr(addr) != NULL)  /* no dups! */
 603             continue;
 604
 605         if ((q = strtok(NULL, w_space)) == NULL)
 606             continue;
 607
 608         N.pings_sent = atoi(q);
 609
 610         if ((q = strtok(NULL, w_space)) == NULL)
 611             continue;
 612
 613         N.pings_recv = atoi(q);
 614
 615         if (N.pings_recv == 0)
 616             continue;
 617
 618         /* give this measurement low weight */
 619         N.pings_sent = 1;
 620
 621         N.pings_recv = 1;
 622
 623         if ((q = strtok(NULL, w_space)) == NULL)
 624             continue;
 625
 626         N.hops = atof(q);
 627
 628         if ((q = strtok(NULL, w_space)) == NULL)
 629             continue;
 630
 631         N.rtt = atof(q);
 632
 633         if ((q = strtok(NULL, w_space)) == NULL)
 634             continue;
 635
 636         N.next_ping_time = (time_t) atoi(q);
 637
 638         if ((q = strtok(NULL, w_space)) == NULL)
 639             continue;
 640
 641         N.last_use_time = (time_t) atoi(q);
 642
 643         n = (netdbEntry *)memAllocate(MEM_NETDBENTRY);
 644
 645         memcpy(n, &N, sizeof(netdbEntry));
 646
 647         netdbHashInsert(n, addr);
 648
 649         while ((q = strtok(NULL, w_space)) != NULL) {
 650             if (netdbLookupHost(q) != NULL) /* no dups! */
 651                 continue;
 652
 653             netdbHostInsert(n, q);
 654         }
 655
 656         ++count;
 657     }
 658
 659     xfree(buf);
 660     getCurrentTime();
 661     debugs(38, DBG_IMPORTANT, "NETDB state reloaded; " <<
 662            count << " entries, " <<
 663            tvSubMsec(start, current_time) << " msec" );
 664 }
 665
 666 static const char *
 667 netdbPeerName(const char *name)
 668 {
 669     const wordlist *w;
 670
 671     for (w = peer_names; w; w = w->next) {
 672         if (!strcmp(w->key, name))
 673             return w->key;
 674     }
 675
 676     return wordlistAdd(&peer_names, name);
 677 }
 678
 679 static void
 680 netdbFreeNetdbEntry(void *data)
 681 {
 682     netdbEntry *n = (netdbEntry *)data;
 683     safe_free(n->peers);
 684     memFree(n, MEM_NETDBENTRY);
 685 }
 686
 687 static void
 688 netdbFreeNameEntry(void *data)
 689 {
 690     net_db_name *x = (net_db_name *)data;
 691     xfree(x->hash.key);
 692     memFree(x, MEM_NET_DB_NAME);
 693 }
 694
 695 static void
 696 netdbExchangeHandleReply(void *data, StoreIOBuffer receivedData)
 697 {
 698     Ip::Address addr;
 699
 700     netdbExchangeState *ex = (netdbExchangeState *)data;
 701     int rec_sz = 0;
 702     int o;
 703
 704     struct in_addr line_addr;
 705     double rtt;
 706     double hops;
 707     char *p;
 708     int j;
 709     HttpReply const *rep;
 710     size_t hdr_sz;
 711     int nused = 0;
 712     int size;
 713     int oldbufofs = ex->buf_ofs;
 714
 715     rec_sz = 0;
 716     rec_sz += 1 + sizeof(struct in_addr);
 717     rec_sz += 1 + sizeof(int);
 718     rec_sz += 1 + sizeof(int);
 719     debugs(38, 3, "netdbExchangeHandleReply: " << receivedData.length << " read bytes");
 720
 721     if (!cbdataReferenceValid(ex->p)) {
 722         debugs(38, 3, "netdbExchangeHandleReply: Peer became invalid");
 723         delete ex;
 724         return;
 725     }
 726
 727     debugs(38, 3, "netdbExchangeHandleReply: for '" << ex->p->host << ":" << ex->p->http_port << "'");
 728
 729     if (receivedData.length == 0 && !receivedData.flags.error) {
 730         debugs(38, 3, "netdbExchangeHandleReply: Done");
 731         delete ex;
 732         return;
 733     }
 734
 735     p = ex->buf;
 736
 737     /* Get the size of the buffer now */
 738     size = ex->buf_ofs + receivedData.length;
 739     debugs(38, 3, "netdbExchangeHandleReply: " << size << " bytes buf");
 740
 741     /* Check if we're still doing headers */
 742
 743     if (ex->connstate == STATE_HEADER) {
 744
 745         ex->buf_ofs += receivedData.length;
 746
 747         /* skip reply headers */
 748
 749         if ((hdr_sz = headersEnd(p, ex->buf_ofs))) {
 750             debugs(38, 5, "netdbExchangeHandleReply: hdr_sz = " << hdr_sz);
 751             rep = ex->e->getReply();
 752             assert(rep->sline.status() != Http::scNone);
 753             debugs(38, 3, "netdbExchangeHandleReply: reply status " << rep->sline.status());
 754
 755             if (rep->sline.status() != Http::scOkay) {
 756                 delete ex;
 757                 return;
 758             }
 759
 760             assert((size_t)ex->buf_ofs >= hdr_sz);
 761
 762             /*
 763              * Now, point p to the part of the buffer where the data
 764              * starts, and update the size accordingly
 765              */
 766             assert(ex->used == 0);
 767             ex->used = hdr_sz;
 768             size = ex->buf_ofs - hdr_sz;
 769             p += hdr_sz;
 770
 771             /* Finally, set the conn state mode to STATE_BODY */
 772             ex->connstate = STATE_BODY;
 773         } else {
 774             StoreIOBuffer tempBuffer;
 775             tempBuffer.offset = ex->buf_ofs;
 776             tempBuffer.length = ex->buf_sz - ex->buf_ofs;
 777             tempBuffer.data = ex->buf + ex->buf_ofs;
 778             /* Have more headers .. */
 779             storeClientCopy(ex->sc, ex->e, tempBuffer,
 780                             netdbExchangeHandleReply, ex);
 781             return;
 782         }
 783     }
 784
 785     assert(ex->connstate == STATE_BODY);
 786
 787     /* If we get here, we have some body to parse .. */
 788     debugs(38, 5, "netdbExchangeHandleReply: start parsing loop, size = " << size);
 789
 790     while (size >= rec_sz) {
 791         debugs(38, 5, "netdbExchangeHandleReply: in parsing loop, size = " << size);
 792         addr.setAnyAddr();
 793         hops = rtt = 0.0;
 794
 795         for (o = 0; o < rec_sz;) {
 796             switch ((int) *(p + o)) {
 797
 798             case NETDB_EX_NETWORK:
 799                 ++o;
 800                 /* FIXME INET6 : NetDB can still ony send IPv4 */
 801                 memcpy(&line_addr, p + o, sizeof(struct in_addr));
 802                 addr = line_addr;
 803                 o += sizeof(struct in_addr);
 804                 break;
 805
 806             case NETDB_EX_RTT:
 807                 ++o;
 808                 memcpy(&j, p + o, sizeof(int));
 809                 o += sizeof(int);
 810                 rtt = (double) ntohl(j) / 1000.0;
 811                 break;
 812
 813             case NETDB_EX_HOPS:
 814                 ++o;
 815                 memcpy(&j, p + o, sizeof(int));
 816                 o += sizeof(int);
 817                 hops = (double) ntohl(j) / 1000.0;
 818                 break;
 819
 820             default:
 821                 debugs(38, DBG_IMPORTANT, "netdbExchangeHandleReply: corrupt data, aborting");
 822                 delete ex;
 823                 return;
 824             }
 825         }
 826
 827         if (!addr.isAnyAddr() && rtt > 0)
 828             netdbExchangeUpdatePeer(addr, ex->p, rtt, hops);
 829
 830         assert(o == rec_sz);
 831
 832         ex->used += rec_sz;
 833
 834         size -= rec_sz;
 835
 836         p += rec_sz;
 837
 838         ++nused;
 839     }
 840
 841     /*
 842      * Copy anything that is left over to the beginning of the buffer,
 843      * and adjust buf_ofs accordingly
 844      */
 845
 846     /*
 847      * Evilly, size refers to the buf size left now,
 848      * ex->buf_ofs is the original buffer size, so just copy that
 849      * much data over
 850      */
 851     memmove(ex->buf, ex->buf + (ex->buf_ofs - size), size);
 852
 853     ex->buf_ofs = size;
 854
 855     /*
 856      * And don't re-copy the remaining data ..
 857      */
 858     ex->used += size;
 859
 860     /*
 861      * Now the tricky bit - size _included_ the leftover bit from the _last_
 862      * storeClientCopy. We don't want to include that, or our offset will be wrong.
 863      * So, don't count the size of the leftover buffer we began with.
 864      * This can _disappear_ when we're not tracking offsets ..
 865      */
 866     ex->used -= oldbufofs;
 867
 868     debugs(38, 3, "netdbExchangeHandleReply: size left over in this buffer: " << size << " bytes");
 869
 870     debugs(38, 3, "netdbExchangeHandleReply: used " << nused <<
 871            " entries, (x " << rec_sz << " bytes) == " << nused * rec_sz <<
 872            " bytes total");
 873
 874     debugs(38, 3, "netdbExchangeHandleReply: used " << ex->used);
 875
 876     if (EBIT_TEST(ex->e->flags, ENTRY_ABORTED)) {
 877         debugs(38, 3, "netdbExchangeHandleReply: ENTRY_ABORTED");
 878         delete ex;
 879     } else if (ex->e->store_status == STORE_PENDING) {
 880         StoreIOBuffer tempBuffer;
 881         tempBuffer.offset = ex->used;
 882         tempBuffer.length = ex->buf_sz - ex->buf_ofs;
 883         tempBuffer.data = ex->buf + ex->buf_ofs;
 884         debugs(38, 3, "netdbExchangeHandleReply: EOF not received");
 885         storeClientCopy(ex->sc, ex->e, tempBuffer,
 886                         netdbExchangeHandleReply, ex);
 887     }
 888 }
 889
 890 static void
 891 netdbRegisterWithCacheManager(void)
 892 {
 893     Mgr::RegisterAction("netdb", "Network Measurement Database", netdbDump, 0, 1);
 894 }
 895
 896 #endif /* USE_ICMP */
 897
 898 /* PUBLIC FUNCTIONS */
 899
 900 void
 901 netdbInit(void)
 902 {
 903 #if USE_ICMP
 904     int n;
 905
 906     netdbRegisterWithCacheManager();
 907
 908     if (addr_table)
 909         return;
 910
 911     n = hashPrime(Config.Netdb.high / 4);
 912
 913     addr_table = hash_create((HASHCMP *) strcmp, n, hash_string);
 914
 915     n = hashPrime(3 * Config.Netdb.high / 4);
 916
 917     host_table = hash_create((HASHCMP *) strcmp, n, hash_string);
 918
 919     eventAddIsh("netdbSaveState", netdbSaveState, NULL, 3600.0, 1);
 920
 921     netdbReloadState();
 922
 923 #endif
 924 }
 925
 926 void
 927 netdbPingSite(const char *hostname)
 928 {
 929 #if USE_ICMP
 930     netdbEntry *n;
 931
 932     if ((n = netdbLookupHost(hostname)) != NULL)
 933         if (n->next_ping_time > squid_curtime)
 934             return;
 935
 936     ipcache_nbgethostbyname(hostname, netdbSendPing,
 937                             new generic_cbdata(xstrdup(hostname)));
 938
 939 #endif
 940 }
 941
 942 void
 943 netdbHandlePingReply(const Ip::Address &from, int hops, int rtt)
 944 {
 945 #if USE_ICMP
 946     netdbEntry *n;
 947     int N;
 948     debugs(38, 3, "netdbHandlePingReply: from " << from);
 949
 950     if ((n = netdbLookupAddr(from)) == NULL)
 951         return;
 952
 953     N = ++n->pings_recv;
 954
 955     if (N > 5)
 956         N = 5;
 957
 958     if (rtt < 1)
 959         rtt = 1;
 960
 961     n->hops = ((n->hops * (N - 1)) + hops) / N;
 962
 963     n->rtt = ((n->rtt * (N - 1)) + rtt) / N;
 964
 965     debugs(38, 3, "netdbHandlePingReply: " << n->network  << "; rtt="<<
 966            std::setw(5)<< std::setprecision(2) << n->rtt << "  hops="<<
 967            std::setw(4) << n->hops);
 968
 969 #endif
 970 }
 971
 972 void
 973 netdbFreeMemory(void)
 974 {
 975 #if USE_ICMP
 976     hashFreeItems(addr_table, netdbFreeNetdbEntry);
 977     hashFreeMemory(addr_table);
 978     addr_table = NULL;
 979     hashFreeItems(host_table, netdbFreeNameEntry);
 980     hashFreeMemory(host_table);
 981     host_table = NULL;
 982     wordlistDestroy(&peer_names);
 983     peer_names = NULL;
 984 #endif
 985 }
 986
 987 void
 988 netdbDump(StoreEntry * sentry)
 989 {
 990 #if USE_ICMP
 991     netdbEntry *n;
 992     netdbEntry **list;
 993     net_db_name *x;
 994     int k;
 995     int i;
 996     int j;
 997     net_db_peer *p;
 998     storeAppendPrintf(sentry, "Network DB Statistics:\n");
 999     storeAppendPrintf(sentry, "%-46.46s %9s %7s %5s %s\n",  /* Max between 16 (IPv4) or 46 (IPv6)   */
1000                       "Network",
1001                       "recv/sent",
1002                       "RTT",
1003                       "Hops",
1004                       "Hostnames");
1005     list = (netdbEntry **)xcalloc(memInUse(MEM_NETDBENTRY), sizeof(netdbEntry *));
1006     i = 0;
1007     hash_first(addr_table);
1008
1009     while ((n = (netdbEntry *) hash_next(addr_table))) {
1010         *(list + i) = n;
1011         ++i;
1012     }
1013
1014     if (i != memInUse(MEM_NETDBENTRY))
1015         debugs(38, DBG_CRITICAL, "WARNING: netdb_addrs count off, found " << i <<
1016                ", expected " << memInUse(MEM_NETDBENTRY));
1017
1018     qsort((char *) list,
1019           i,
1020           sizeof(netdbEntry *),
1021           sortByRtt);
1022
1023     for (k = 0; k < i; ++k) {
1024         n = *(list + k);
1025         storeAppendPrintf(sentry, "%-46.46s %4d/%4d %7.1f %5.1f", /* Max between 16 (IPv4) or 46 (IPv6)   */
1026                           n->network,
1027                           n->pings_recv,
1028                           n->pings_sent,
1029                           n->rtt,
1030                           n->hops);
1031
1032         for (x = n->hosts; x; x = x->next)
1033             storeAppendPrintf(sentry, " %s", hashKeyStr(&x->hash));
1034
1035         storeAppendPrintf(sentry, "\n");
1036
1037         p = n->peers;
1038
1039         for (j = 0; j < n->n_peers; ++j, ++p) {
1040             storeAppendPrintf(sentry, "    %-22.22s %7.1f %5.1f\n",
1041                               p->peername,
1042                               p->rtt,
1043                               p->hops);
1044         }
1045     }
1046
1047     xfree(list);
1048 #else
1049
1050     storeAppendPrintf(sentry,"NETDB support not compiled into this Squid cache.\n");
1051 #endif
1052 }
1053
1054 int
1055 netdbHostHops(const char *host)
1056 {
1057 #if USE_ICMP
1058     netdbEntry *n = netdbLookupHost(host);
1059
1060     if (n) {
1061         n->last_use_time = squid_curtime;
1062         return (int) (n->hops + 0.5);
1063     }
1064
1065 #endif
1066     return 0;
1067 }
1068
1069 int
1070 netdbHostRtt(const char *host)
1071 {
1072 #if USE_ICMP
1073     netdbEntry *n = netdbLookupHost(host);
1074
1075     if (n) {
1076         n->last_use_time = squid_curtime;
1077         return (int) (n->rtt + 0.5);
1078     }
1079
1080 #endif
1081     return 0;
1082 }
1083
1084 void
1085 netdbHostData(const char *host, int *samp, int *rtt, int *hops)
1086 {
1087 #if USE_ICMP
1088     netdbEntry *n = netdbLookupHost(host);
1089
1090     if (n == NULL)
1091         return;
1092
1093     *samp = n->pings_recv;
1094
1095     *rtt = (int) (n->rtt + 0.5);
1096
1097     *hops = (int) (n->hops + 0.5);
1098
1099     n->last_use_time = squid_curtime;
1100
1101 #endif
1102 }
1103
1104 void
1105 netdbUpdatePeer(const URL &url, CachePeer * e, int irtt, int ihops)
1106 {
1107 #if USE_ICMP
1108     netdbEntry *n;
1109     double rtt = (double) irtt;
1110     double hops = (double) ihops;
1111     net_db_peer *p;
1112     debugs(38, 3, url.host() << ", " << ihops << " hops, " << irtt << " rtt");
1113     n = netdbLookupHost(url.host());
1114
1115     if (n == NULL) {
1116         debugs(38, 3, "host " << url.host() << " not found");
1117         return;
1118     }
1119
1120     if ((p = netdbPeerByName(n, e->host)) == NULL)
1121         p = netdbPeerAdd(n, e);
1122
1123     p->rtt = rtt;
1124
1125     p->hops = hops;
1126
1127     p->expires = squid_curtime + 3600;
1128
1129     if (n->n_peers < 2)
1130         return;
1131
1132     qsort((char *) n->peers,
1133           n->n_peers,
1134           sizeof(net_db_peer),
1135           sortPeerByRtt);
1136
1137 #endif
1138 }
1139
1140 void
1141 netdbExchangeUpdatePeer(Ip::Address &addr, CachePeer * e, double rtt, double hops)
1142 {
1143 #if USE_ICMP
1144     netdbEntry *n;
1145     net_db_peer *p;
1146     debugs(38, 5, "netdbExchangeUpdatePeer: '" << addr << "', "<<
1147            std::setfill('0')<< std::setprecision(2) << hops << " hops, " <<
1148            rtt << " rtt");
1149
1150     if ( !addr.isIPv4() ) {
1151         debugs(38, 5, "netdbExchangeUpdatePeer: Aborting peer update for '" << addr << "', NetDB cannot handle IPv6.");
1152         return;
1153     }
1154
1155     n = netdbLookupAddr(addr);
1156
1157     if (n == NULL)
1158         n = netdbAdd(addr);
1159
1160     assert(NULL != n);
1161
1162     if ((p = netdbPeerByName(n, e->host)) == NULL)
1163         p = netdbPeerAdd(n, e);
1164
1165     p->rtt = rtt;
1166
1167     p->hops = hops;
1168
1169     p->expires = squid_curtime + 3600;  /* XXX ? */
1170
1171     if (n->n_peers < 2)
1172         return;
1173
1174     qsort((char *) n->peers,
1175           n->n_peers,
1176           sizeof(net_db_peer),
1177           sortPeerByRtt);
1178
1179 #endif
1180 }
1181
1182 void
1183 netdbDeleteAddrNetwork(Ip::Address &addr)
1184 {
1185 #if USE_ICMP
1186     netdbEntry *n = netdbLookupAddr(addr);
1187
1188     if (n == NULL)
1189         return;
1190
1191     debugs(38, 3, "netdbDeleteAddrNetwork: " << n->network);
1192
1193     netdbRelease(n);
1194 #endif
1195 }
1196
1197 void
1198 netdbBinaryExchange(StoreEntry * s)
1199 {
1200     HttpReply *reply = new HttpReply;
1201 #if USE_ICMP
1202
1203     Ip::Address addr;
1204
1205     netdbEntry *n;
1206     int i;
1207     int j;
1208     int rec_sz;
1209     char *buf;
1210
1211     struct in_addr line_addr;
1212     s->buffer();
1213     reply->setHeaders(Http::scOkay, "OK", NULL, -1, squid_curtime, -2);
1214     s->replaceHttpReply(reply);
1215     rec_sz = 0;
1216     rec_sz += 1 + sizeof(struct in_addr);
1217     rec_sz += 1 + sizeof(int);
1218     rec_sz += 1 + sizeof(int);
1219     buf = (char *)memAllocate(MEM_4K_BUF);
1220     i = 0;
1221     hash_first(addr_table);
1222
1223     while ((n = (netdbEntry *) hash_next(addr_table))) {
1224         if (0.0 == n->rtt)
1225             continue;
1226
1227         if (n->rtt > 60000) /* RTT > 1 MIN probably bogus */
1228             continue;
1229
1230         if (! (addr = n->network) )
1231             continue;
1232
1233         /* FIXME INET6 : NetDB cannot yet handle IPv6 addresses. Ensure only IPv4 get sent. */
1234         if ( !addr.isIPv4() )
1235             continue;
1236
1237         buf[i] = (char) NETDB_EX_NETWORK;
1238         ++i;
1239
1240         addr.getInAddr(line_addr);
1241         memcpy(&buf[i], &line_addr, sizeof(struct in_addr));
1242
1243         i += sizeof(struct in_addr);
1244
1245         buf[i] = (char) NETDB_EX_RTT;
1246         ++i;
1247
1248         j = htonl((int) (n->rtt * 1000));
1249
1250         memcpy(&buf[i], &j, sizeof(int));
1251
1252         i += sizeof(int);
1253
1254         buf[i] = (char) NETDB_EX_HOPS;
1255         ++i;
1256
1257         j = htonl((int) (n->hops * 1000));
1258
1259         memcpy(&buf[i], &j, sizeof(int));
1260
1261         i += sizeof(int);
1262
1263         if (i + rec_sz > 4096) {
1264             s->append(buf, i);
1265             i = 0;
1266         }
1267     }
1268
1269     if (i > 0) {
1270         s->append(buf, i);
1271         i = 0;
1272     }
1273
1274     assert(0 == i);
1275     s->flush();
1276     memFree(buf, MEM_4K_BUF);
1277 #else
1278
1279     reply->setHeaders(Http::scBadRequest, "Bad Request", NULL, -1, squid_curtime, -2);
1280     s->replaceHttpReply(reply);
1281     storeAppendPrintf(s, "NETDB support not compiled into this Squid cache.\n");
1282 #endif
1283
1284     s->complete();
1285 }
1286
1287 void
1288 netdbExchangeStart(void *data)
1289 {
1290 #if USE_ICMP
1291     CachePeer *p = (CachePeer *)data;
1292     static const SBuf netDB("netdb");
1293     char *uri = internalRemoteUri(p->host, p->http_port, "/squid-internal-dynamic/", netDB);
1294     debugs(38, 3, "netdbExchangeStart: Requesting '" << uri << "'");
1295     assert(NULL != uri);
1296     HttpRequest *req = HttpRequest::CreateFromUrl(uri);
1297
1298     if (req == NULL) {
1299         debugs(38, DBG_IMPORTANT, "netdbExchangeStart: Bad URI " << uri);
1300         return;
1301     }
1302
1303     netdbExchangeState *ex = new netdbExchangeState(p, req);
1304     ex->e = storeCreateEntry(uri, uri, RequestFlags(), Http::METHOD_GET);
1305     assert(NULL != ex->e);
1306
1307     StoreIOBuffer tempBuffer;
1308     tempBuffer.length = ex->buf_sz;
1309     tempBuffer.data = ex->buf;
1310
1311     ex->sc = storeClientListAdd(ex->e, ex);
1312
1313     storeClientCopy(ex->sc, ex->e, tempBuffer,
1314                     netdbExchangeHandleReply, ex);
1315     ex->r->flags.loopDetected = true;   /* cheat! -- force direct */
1316
1317     // XXX: send as Proxy-Authenticate instead
1318     if (p->login)
1319         ex->r->url.userInfo(SBuf(p->login));
1320
1321     FwdState::fwdStart(Comm::ConnectionPointer(), ex->e, ex->r);
1322
1323 #endif
1324 }
1325
1326 CachePeer *
1327 netdbClosestParent(HttpRequest * request)
1328 {
1329 #if USE_ICMP
1330     CachePeer *p = NULL;
1331     netdbEntry *n;
1332     const ipcache_addrs *ia;
1333     net_db_peer *h;
1334     int i;
1335     n = netdbLookupHost(request->url.host());
1336
1337     if (NULL == n) {
1338         /* try IP addr */
1339         ia = ipcache_gethostbyname(request->url.host(), 0);
1340
1341         if (NULL != ia)
1342             n = netdbLookupAddr(ia->in_addrs[ia->cur]);
1343     }
1344
1345     if (NULL == n)
1346         return NULL;
1347
1348     if (0 == n->n_peers)
1349         return NULL;
1350
1351     n->last_use_time = squid_curtime;
1352
1353     /*
1354      * Find the parent with the least RTT to the origin server.
1355      * Make sure we don't return a parent who is farther away than
1356      * we are.  Note, the n->peers list is pre-sorted by RTT.
1357      */
1358     for (i = 0; i < n->n_peers; ++i) {
1359         h = &n->peers[i];
1360
1361         if (n->rtt > 0)
1362             if (n->rtt < h->rtt)
1363                 break;
1364
1365         p = peerFindByName(h->peername);
1366
1367         if (NULL == p)      /* not found */
1368             continue;
1369
1370         if (neighborType(p, request->url) != PEER_PARENT)
1371             continue;
1372
1373         if (!peerHTTPOkay(p, request))  /* not allowed */
1374             continue;
1375
1376         return p;
1377     }
1378
1379 #endif
1380     return NULL;
1381 }
1382