src/icmp/net_db.cc

   1 /*
   2  * DEBUG: section 38    Network Measurement Database
   3  * AUTHOR: Duane Wessels
   4  *
   5  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   6  * ----------------------------------------------------------
   7  *
   8  *  Squid is the result of efforts by numerous individuals from
   9  *  the Internet community; see the CONTRIBUTORS file for full
  10  *  details.   Many organizations have provided support for Squid's
  11  *  development; see the SPONSORS file for full details.  Squid is
  12  *  Copyrighted (C) 2001 by the Regents of the University of
  13  *  California; see the COPYRIGHT file for full details.  Squid
  14  *  incorporates software developed and/or copyrighted by other
  15  *  sources; see the CREDITS file for full details.
  16  *
  17  *  This program is free software; you can redistribute it and/or modify
  18  *  it under the terms of the GNU General Public License as published by
  19  *  the Free Software Foundation; either version 2 of the License, or
  20  *  (at your option) any later version.
  21  *
  22  *  This program is distributed in the hope that it will be useful,
  23  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  24  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  25  *  GNU General Public License for more details.
  26  *
  27  *  You should have received a copy of the GNU General Public License
  28  *  along with this program; if not, write to the Free Software
  29  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  30  *
  31  */
  32
  33 /*
  34  * XXX XXX XXX
  35  *
  36  * This code may be slightly broken now. If you're getting consistent
  37  * (sometimes working) corrupt data exchanges, please contact adrian
  38  * (adrian@squid-cache.org) to sort them out.
  39  */
  40
  41 #include "squid.h"
  42 #include "icmp/net_db.h"
  43 #include "log/File.h"
  44 #include "cbdata.h"
  45 #include "event.h"
  46 #include "CacheManager.h"
  47 #include "Store.h"
  48 #include "SwapDir.h"
  49 #include "HttpRequest.h"
  50 #include "HttpReply.h"
  51 #include "MemObject.h"
  52 #include "fde.h"
  53 #include "forward.h"
  54 #include "SquidTime.h"
  55 #include "wordlist.h"
  56 #include "ip/Address.h"
  57
  58 #if USE_ICMP
  59 #include "icmp/IcmpSquid.h"
  60 #include "StoreClient.h"
  61
  62 #define NETDB_REQBUF_SZ 4096
  63
  64 typedef enum {
  65     STATE_NONE,
  66     STATE_HEADER,
  67     STATE_BODY
  68 } netdb_conn_state_t;
  69
  70 typedef struct {
  71     peer *p;
  72     StoreEntry *e;
  73     store_client *sc;
  74     HttpRequest *r;
  75     int64_t used;
  76     size_t buf_sz;
  77     char buf[NETDB_REQBUF_SZ];
  78     int buf_ofs;
  79     netdb_conn_state_t connstate;
  80 } netdbExchangeState;
  81
  82 static hash_table *addr_table = NULL;
  83 static hash_table *host_table = NULL;
  84
  85 Ip::Address networkFromInaddr(const Ip::Address &a);
  86 static void netdbRelease(netdbEntry * n);
  87
  88 static void netdbHashInsert(netdbEntry * n, Ip::Address &addr);
  89 static void netdbHashDelete(const char *key);
  90 static void netdbHostInsert(netdbEntry * n, const char *hostname);
  91 static void netdbHostDelete(const net_db_name * x);
  92 static void netdbPurgeLRU(void);
  93 static netdbEntry *netdbLookupHost(const char *key);
  94 static net_db_peer *netdbPeerByName(const netdbEntry * n, const char *);
  95 static net_db_peer *netdbPeerAdd(netdbEntry * n, peer * e);
  96 static const char *netdbPeerName(const char *name);
  97 static IPH netdbSendPing;
  98 static QS sortPeerByRtt;
  99 static QS sortByRtt;
 100 static QS netdbLRU;
 101 static FREE netdbFreeNameEntry;
 102 static FREE netdbFreeNetdbEntry;
 103 static STCB netdbExchangeHandleReply;
 104 static void netdbExchangeDone(void *);
 105
 106 /* We have to keep a local list of peer names.  The Peers structure
 107  * gets freed during a reconfigure.  We want this database to
 108  * remain persisitent, so _net_db_peer->peername points into this
 109  * linked list */
 110 static wordlist *peer_names = NULL;
 111
 112 static void
 113 netdbHashInsert(netdbEntry * n, Ip::Address &addr)
 114 {
 115     networkFromInaddr(addr).NtoA(n->network, MAX_IPSTRLEN);
 116     n->hash.key = n->network;
 117     assert(hash_lookup(addr_table, n->network) == NULL);
 118     hash_join(addr_table, &n->hash);
 119 }
 120
 121 static void
 122 netdbHashDelete(const char *key)
 123 {
 124     hash_link *hptr = (hash_link *)hash_lookup(addr_table, key);
 125
 126     if (hptr == NULL) {
 127         debug_trap("netdbHashDelete: key not found");
 128         return;
 129     }
 130
 131     hash_remove_link(addr_table, hptr);
 132 }
 133
 134 static void
 135 netdbHostInsert(netdbEntry * n, const char *hostname)
 136 {
 137     net_db_name *x = (net_db_name *)memAllocate(MEM_NET_DB_NAME);
 138     x->hash.key = xstrdup(hostname);
 139     x->next = n->hosts;
 140     n->hosts = x;
 141     x->net_db_entry = n;
 142     assert(hash_lookup(host_table, hostname) == NULL);
 143     hash_join(host_table, &x->hash);
 144     n->link_count++;
 145 }
 146
 147 static void
 148 netdbHostDelete(const net_db_name * x)
 149 {
 150     netdbEntry *n;
 151     net_db_name **X;
 152     assert(x != NULL);
 153     assert(x->net_db_entry != NULL);
 154     n = x->net_db_entry;
 155     n->link_count--;
 156
 157     for (X = &n->hosts; *X; X = &(*X)->next) {
 158         if (*X == x) {
 159             *X = x->next;
 160             break;
 161         }
 162     }
 163
 164     hash_remove_link(host_table, (hash_link *) x);
 165     xfree(x->hash.key);
 166     memFree((void *) x, MEM_NET_DB_NAME);
 167 }
 168
 169 static netdbEntry *
 170 netdbLookupHost(const char *key)
 171 {
 172     net_db_name *x = (net_db_name *) hash_lookup(host_table, key);
 173     return x ? x->net_db_entry : NULL;
 174 }
 175
 176 static void
 177 netdbRelease(netdbEntry * n)
 178 {
 179     net_db_name *x;
 180     net_db_name *next;
 181
 182     for (x = n->hosts; x; x = next) {
 183         next = x->next;
 184         netdbHostDelete(x);
 185     }
 186
 187     n->hosts = NULL;
 188     safe_free(n->peers);
 189     n->peers = NULL;
 190     n->n_peers = 0;
 191     n->n_peers_alloc = 0;
 192
 193     if (n->link_count == 0) {
 194         netdbHashDelete(n->network);
 195         memFree(n, MEM_NETDBENTRY);
 196     }
 197 }
 198
 199 static int
 200 netdbLRU(const void *A, const void *B)
 201 {
 202     const netdbEntry *const *n1 = (const netdbEntry *const *)A;
 203     const netdbEntry *const *n2 = (const netdbEntry *const *)B;
 204
 205     if ((*n1)->last_use_time > (*n2)->last_use_time)
 206         return (1);
 207
 208     if ((*n1)->last_use_time < (*n2)->last_use_time)
 209         return (-1);
 210
 211     return (0);
 212 }
 213
 214 static void
 215 netdbPurgeLRU(void)
 216 {
 217     netdbEntry *n;
 218     netdbEntry **list;
 219     int k = 0;
 220     int list_count = 0;
 221     int removed = 0;
 222     list = (netdbEntry **)xcalloc(memInUse(MEM_NETDBENTRY), sizeof(netdbEntry *));
 223     hash_first(addr_table);
 224
 225     while ((n = (netdbEntry *) hash_next(addr_table))) {
 226         assert(list_count < memInUse(MEM_NETDBENTRY));
 227         *(list + list_count) = n;
 228         list_count++;
 229     }
 230
 231     qsort((char *) list,
 232           list_count,
 233           sizeof(netdbEntry *),
 234           netdbLRU);
 235
 236     for (k = 0; k < list_count; k++) {
 237         if (memInUse(MEM_NETDBENTRY) < Config.Netdb.low)
 238             break;
 239
 240         netdbRelease(*(list + k));
 241
 242         removed++;
 243     }
 244
 245     xfree(list);
 246 }
 247
 248 static netdbEntry *
 249 netdbLookupAddr(const Ip::Address &addr)
 250 {
 251     netdbEntry *n;
 252     char *key = new char[MAX_IPSTRLEN];
 253     networkFromInaddr(addr).NtoA(key,MAX_IPSTRLEN);
 254     n = (netdbEntry *) hash_lookup(addr_table, key);
 255     delete[] key;
 256     return n;
 257 }
 258
 259 static netdbEntry *
 260 netdbAdd(Ip::Address &addr)
 261 {
 262     netdbEntry *n;
 263
 264     if (memInUse(MEM_NETDBENTRY) > Config.Netdb.high)
 265         netdbPurgeLRU();
 266
 267     if ((n = netdbLookupAddr(addr)) == NULL) {
 268         n = (netdbEntry *)memAllocate(MEM_NETDBENTRY);
 269         netdbHashInsert(n, addr);
 270     }
 271
 272     return n;
 273 }
 274
 275 static void
 276 netdbSendPing(const ipcache_addrs *ia, const DnsLookupDetails &, void *data)
 277 {
 278     Ip::Address addr;
 279     char *hostname = NULL;
 280     static_cast<generic_cbdata *>(data)->unwrap(&hostname);
 281     netdbEntry *n;
 282     netdbEntry *na;
 283     net_db_name *x;
 284     net_db_name **X;
 285
 286     if (ia == NULL) {
 287         xfree(hostname);
 288         return;
 289     }
 290
 291     addr = ia->in_addrs[ia->cur];
 292
 293     if ((n = netdbLookupHost(hostname)) == NULL) {
 294         n = netdbAdd(addr);
 295         netdbHostInsert(n, hostname);
 296     } else if ((na = netdbLookupAddr(addr)) != n) {
 297         /*
 298          *hostname moved from 'network n' to 'network na'!
 299          */
 300
 301         if (na == NULL)
 302             na = netdbAdd(addr);
 303
 304         debugs(38, 3, "netdbSendPing: " << hostname << " moved from " << n->network << " to " << na->network);
 305
 306         x = (net_db_name *) hash_lookup(host_table, hostname);
 307
 308         if (x == NULL) {
 309             debugs(38, 1, "netdbSendPing: net_db_name list bug: " << hostname << " not found");
 310             xfree(hostname);
 311             return;
 312         }
 313
 314         /* remove net_db_name from 'network n' linked list */
 315         for (X = &n->hosts; *X; X = &(*X)->next) {
 316             if (*X == x) {
 317                 *X = x->next;
 318                 break;
 319             }
 320         }
 321
 322         n->link_count--;
 323         /* point to 'network na' from host entry */
 324         x->net_db_entry = na;
 325         /* link net_db_name to 'network na' */
 326         x->next = na->hosts;
 327         na->hosts = x;
 328         na->link_count++;
 329         n = na;
 330     }
 331
 332     if (n->next_ping_time <= squid_curtime) {
 333         debugs(38, 3, "netdbSendPing: pinging " << hostname);
 334         icmpEngine.DomainPing(addr, hostname);
 335         n->pings_sent++;
 336         n->next_ping_time = squid_curtime + Config.Netdb.period;
 337         n->last_use_time = squid_curtime;
 338     }
 339
 340     xfree(hostname);
 341 }
 342
 343 Ip::Address
 344 networkFromInaddr(const Ip::Address &in)
 345 {
 346     Ip::Address out;
 347
 348     out = in;
 349
 350     /* in IPv6 the 'network' should be the routing section. */
 351     if ( in.IsIPv6() ) {
 352         out.ApplyMask(64, AF_INET6);
 353         debugs(14, 5, "networkFromInaddr : Masked IPv6 Address to " << in << "/64 routing part.");
 354         return out;
 355     }
 356
 357 #if USE_CLASSFUL
 358     struct in_addr b;
 359
 360     in.GetInAddr(b);
 361
 362     if (IN_CLASSC(b.s_addr))
 363         b.s_addr &= IN_CLASSC_NET;
 364     else if (IN_CLASSB(b.s_addr))
 365         b.s_addr &= IN_CLASSB_NET;
 366     else if (IN_CLASSA(b.s_addr))
 367         b.s_addr &= IN_CLASSA_NET;
 368
 369     out = b;
 370
 371 #endif
 372
 373     debugs(14, 5, "networkFromInaddr : Masked IPv4 Address to " << out << "/24.");
 374
 375     /* use /24 for everything under IPv4 */
 376     out.ApplyMask(24, AF_INET);
 377     debugs(14, 5, "networkFromInaddr : Masked IPv4 Address to " << in << "/24.");
 378
 379     return out;
 380 }
 381
 382 static int
 383 sortByRtt(const void *A, const void *B)
 384 {
 385     const netdbEntry *const *n1 = (const netdbEntry *const *)A;
 386     const netdbEntry *const *n2 = (const netdbEntry *const *)B;
 387
 388     if ((*n1)->rtt > (*n2)->rtt)
 389         return 1;
 390     else if ((*n1)->rtt < (*n2)->rtt)
 391         return -1;
 392     else
 393         return 0;
 394 }
 395
 396 static net_db_peer *
 397 netdbPeerByName(const netdbEntry * n, const char *peername)
 398 {
 399     int i;
 400     net_db_peer *p = n->peers;
 401
 402     for (i = 0; i < n->n_peers; i++, p++) {
 403         if (!strcmp(p->peername, peername))
 404             return p;
 405     }
 406
 407     return NULL;
 408 }
 409
 410 static net_db_peer *
 411 netdbPeerAdd(netdbEntry * n, peer * e)
 412 {
 413     net_db_peer *p;
 414     net_db_peer *o;
 415     int osize;
 416     int i;
 417
 418     if (n->n_peers == n->n_peers_alloc) {
 419         o = n->peers;
 420         osize = n->n_peers_alloc;
 421
 422         if (n->n_peers_alloc == 0)
 423             n->n_peers_alloc = 2;
 424         else
 425             n->n_peers_alloc <<= 1;
 426
 427         debugs(38, 3, "netdbPeerAdd: Growing peer list for '" << n->network << "' to " << n->n_peers_alloc);
 428
 429         n->peers = (net_db_peer *)xcalloc(n->n_peers_alloc, sizeof(net_db_peer));
 430
 431         for (i = 0; i < osize; i++)
 432             *(n->peers + i) = *(o + i);
 433
 434         if (osize) {
 435             safe_free(o);
 436         }
 437     }
 438
 439     p = n->peers + n->n_peers;
 440     p->peername = netdbPeerName(e->host);
 441     n->n_peers++;
 442     return p;
 443 }
 444
 445 static int
 446 sortPeerByRtt(const void *A, const void *B)
 447 {
 448     const net_db_peer *p1 = (net_db_peer *)A;
 449     const net_db_peer *p2 = (net_db_peer *)B;
 450
 451     if (p1->rtt > p2->rtt)
 452         return 1;
 453     else if (p1->rtt < p2->rtt)
 454         return -1;
 455     else
 456         return 0;
 457 }
 458
 459 static void
 460 netdbSaveState(void *foo)
 461 {
 462     if (strcmp(Config.netdbFilename, "none") == 0)
 463         return;
 464
 465     Logfile *lf;
 466     netdbEntry *n;
 467     net_db_name *x;
 468
 469     struct timeval start = current_time;
 470     int count = 0;
 471     /*
 472      * This was nicer when we were using stdio, but thanks to
 473      * Solaris bugs, its a bad idea.  fopen can fail if more than
 474      * 256 FDs are open.
 475      */
 476     /*
 477      * unlink() is here because there is currently no way to make
 478      * logfileOpen() use O_TRUNC.
 479      */
 480     unlink(Config.netdbFilename);
 481     lf = logfileOpen(Config.netdbFilename, 4096, 0);
 482
 483     if (NULL == lf) {
 484         debugs(50, 1, "netdbSaveState: " << Config.netdbFilename << ": " << xstrerror());
 485         return;
 486     }
 487
 488     hash_first(addr_table);
 489
 490     while ((n = (netdbEntry *) hash_next(addr_table))) {
 491         if (n->pings_recv == 0)
 492             continue;
 493
 494         logfilePrintf(lf, "%s %d %d %10.5f %10.5f %d %d",
 495                       n->network,
 496                       n->pings_sent,
 497                       n->pings_recv,
 498                       n->hops,
 499                       n->rtt,
 500                       (int) n->next_ping_time,
 501                       (int) n->last_use_time);
 502
 503         for (x = n->hosts; x; x = x->next)
 504             logfilePrintf(lf, " %s", hashKeyStr(&x->hash));
 505
 506         logfilePrintf(lf, "\n");
 507
 508         count++;
 509
 510 #undef RBUF_SZ
 511
 512     }
 513
 514     logfileClose(lf);
 515     getCurrentTime();
 516     debugs(38, 1, "NETDB state saved; " <<
 517            count << " entries, " <<
 518            tvSubMsec(start, current_time) << " msec" );
 519     eventAddIsh("netdbSaveState", netdbSaveState, NULL, 3600.0, 1);
 520 }
 521
 522 static void
 523 netdbReloadState(void)
 524 {
 525     if (strcmp(Config.netdbFilename, "none") == 0)
 526         return;
 527
 528     char *s;
 529     int fd;
 530     int l;
 531
 532     struct stat sb;
 533     netdbEntry *n;
 534     netdbEntry N;
 535
 536     Ip::Address addr;
 537     int count = 0;
 538
 539     struct timeval start = current_time;
 540     /*
 541      * This was nicer when we were using stdio, but thanks to
 542      * Solaris bugs, its a bad idea.  fopen can fail if more than
 543      * 256 FDs are open.
 544      */
 545     fd = file_open(Config.netdbFilename, O_RDONLY | O_BINARY);
 546
 547     if (fd < 0)
 548         return;
 549
 550     if (fstat(fd, &sb) < 0) {
 551         file_close(fd);
 552         return;
 553     }
 554
 555     char *t;
 556     char *buf = (char *)xcalloc(1, sb.st_size + 1);
 557     t = buf;
 558     l = FD_READ_METHOD(fd, buf, sb.st_size);
 559     file_close(fd);
 560
 561     if (l <= 0) {
 562         safe_free (buf);
 563         return;
 564     };
 565
 566     while ((s = strchr(t, '\n'))) {
 567         char *q;
 568         assert(s - buf < l);
 569         *s = '\0';
 570         memset(&N, '\0', sizeof(netdbEntry));
 571         q = strtok(t, w_space);
 572         t = s + 1;
 573
 574         if (NULL == q)
 575             continue;
 576
 577         if (! (addr = q) )
 578             continue;
 579
 580         if (netdbLookupAddr(addr) != NULL)      /* no dups! */
 581             continue;
 582
 583         if ((q = strtok(NULL, w_space)) == NULL)
 584             continue;
 585
 586         N.pings_sent = atoi(q);
 587
 588         if ((q = strtok(NULL, w_space)) == NULL)
 589             continue;
 590
 591         N.pings_recv = atoi(q);
 592
 593         if (N.pings_recv == 0)
 594             continue;
 595
 596         /* give this measurement low weight */
 597         N.pings_sent = 1;
 598
 599         N.pings_recv = 1;
 600
 601         if ((q = strtok(NULL, w_space)) == NULL)
 602             continue;
 603
 604         N.hops = atof(q);
 605
 606         if ((q = strtok(NULL, w_space)) == NULL)
 607             continue;
 608
 609         N.rtt = atof(q);
 610
 611         if ((q = strtok(NULL, w_space)) == NULL)
 612             continue;
 613
 614         N.next_ping_time = (time_t) atoi(q);
 615
 616         if ((q = strtok(NULL, w_space)) == NULL)
 617             continue;
 618
 619         N.last_use_time = (time_t) atoi(q);
 620
 621         n = (netdbEntry *)memAllocate(MEM_NETDBENTRY);
 622
 623         xmemcpy(n, &N, sizeof(netdbEntry));
 624
 625         netdbHashInsert(n, addr);
 626
 627         while ((q = strtok(NULL, w_space)) != NULL) {
 628             if (netdbLookupHost(q) != NULL)     /* no dups! */
 629                 continue;
 630
 631             netdbHostInsert(n, q);
 632         }
 633
 634         count++;
 635     }
 636
 637     xfree(buf);
 638     getCurrentTime();
 639     debugs(38, 1, "NETDB state reloaded; " <<
 640            count << " entries, " <<
 641            tvSubMsec(start, current_time) << " msec" );
 642 }
 643
 644 static const char *
 645 netdbPeerName(const char *name)
 646 {
 647     const wordlist *w;
 648
 649     for (w = peer_names; w; w = w->next) {
 650         if (!strcmp(w->key, name))
 651             return w->key;
 652     }
 653
 654     return wordlistAdd(&peer_names, name);
 655 }
 656
 657 static void
 658 netdbFreeNetdbEntry(void *data)
 659 {
 660     netdbEntry *n = (netdbEntry *)data;
 661     safe_free(n->peers);
 662     memFree(n, MEM_NETDBENTRY);
 663 }
 664
 665 static void
 666 netdbFreeNameEntry(void *data)
 667 {
 668     net_db_name *x = (net_db_name *)data;
 669     xfree(x->hash.key);
 670     memFree(x, MEM_NET_DB_NAME);
 671 }
 672
 673
 674 static void
 675 netdbExchangeHandleReply(void *data, StoreIOBuffer receivedData)
 676 {
 677     Ip::Address addr;
 678
 679     netdbExchangeState *ex = (netdbExchangeState *)data;
 680     int rec_sz = 0;
 681     int o;
 682
 683     struct in_addr line_addr;
 684     double rtt;
 685     double hops;
 686     char *p;
 687     int j;
 688     HttpReply const *rep;
 689     size_t hdr_sz;
 690     int nused = 0;
 691     int size;
 692     int oldbufofs = ex->buf_ofs;
 693
 694     rec_sz = 0;
 695     rec_sz += 1 + sizeof(struct in_addr);
 696     rec_sz += 1 + sizeof(int);
 697     rec_sz += 1 + sizeof(int);
 698     debugs(38, 3, "netdbExchangeHandleReply: " << receivedData.length << " read bytes");
 699
 700     if (!cbdataReferenceValid(ex->p)) {
 701         debugs(38, 3, "netdbExchangeHandleReply: Peer became invalid");
 702         netdbExchangeDone(ex);
 703         return;
 704     }
 705
 706     debugs(38, 3, "netdbExchangeHandleReply: for '" << ex->p->host << ":" << ex->p->http_port << "'");
 707
 708     if (receivedData.length == 0 &&
 709             !receivedData.flags.error) {
 710         debugs(38, 3, "netdbExchangeHandleReply: Done");
 711         netdbExchangeDone(ex);
 712         return;
 713     }
 714
 715     p = ex->buf;
 716
 717     /* Get the size of the buffer now */
 718     size = ex->buf_ofs + receivedData.length;
 719     debugs(38, 3, "netdbExchangeHandleReply: " << size << " bytes buf");
 720
 721     /* Check if we're still doing headers */
 722
 723     if (ex->connstate == STATE_HEADER) {
 724
 725         ex->buf_ofs += receivedData.length;
 726
 727         /* skip reply headers */
 728
 729         if ((hdr_sz = headersEnd(p, ex->buf_ofs))) {
 730             debugs(38, 5, "netdbExchangeHandleReply: hdr_sz = " << hdr_sz);
 731             rep = ex->e->getReply();
 732             assert (0 != rep->sline.status);
 733             debugs(38, 3, "netdbExchangeHandleReply: reply status " << rep->sline.status);
 734
 735             if (HTTP_OK != rep->sline.status) {
 736                 netdbExchangeDone(ex);
 737                 return;
 738             }
 739
 740             assert((size_t)ex->buf_ofs >= hdr_sz);
 741
 742             /*
 743              * Now, point p to the part of the buffer where the data
 744              * starts, and update the size accordingly
 745              */
 746             assert(ex->used == 0);
 747             ex->used = hdr_sz;
 748             size = ex->buf_ofs - hdr_sz;
 749             p += hdr_sz;
 750
 751             /* Finally, set the conn state mode to STATE_BODY */
 752             ex->connstate = STATE_BODY;
 753         } else {
 754             StoreIOBuffer tempBuffer;
 755             tempBuffer.offset = ex->buf_ofs;
 756             tempBuffer.length = ex->buf_sz - ex->buf_ofs;
 757             tempBuffer.data = ex->buf + ex->buf_ofs;
 758             /* Have more headers .. */
 759             storeClientCopy(ex->sc, ex->e, tempBuffer,
 760                             netdbExchangeHandleReply, ex);
 761             return;
 762         }
 763     }
 764
 765     assert(ex->connstate == STATE_BODY);
 766
 767     /* If we get here, we have some body to parse .. */
 768     debugs(38, 5, "netdbExchangeHandleReply: start parsing loop, size = " << size);
 769
 770     while (size >= rec_sz) {
 771         debugs(38, 5, "netdbExchangeHandleReply: in parsing loop, size = " << size);
 772         addr.SetAnyAddr();
 773         hops = rtt = 0.0;
 774
 775         for (o = 0; o < rec_sz;) {
 776             switch ((int) *(p + o)) {
 777
 778             case NETDB_EX_NETWORK:
 779                 o++;
 780                 /* FIXME INET6 : NetDB can still ony send IPv4 */
 781                 xmemcpy(&line_addr, p + o, sizeof(struct in_addr));
 782                 addr = line_addr;
 783                 o += sizeof(struct in_addr);
 784                 break;
 785
 786             case NETDB_EX_RTT:
 787                 o++;
 788                 xmemcpy(&j, p + o, sizeof(int));
 789                 o += sizeof(int);
 790                 rtt = (double) ntohl(j) / 1000.0;
 791                 break;
 792
 793             case NETDB_EX_HOPS:
 794                 o++;
 795                 xmemcpy(&j, p + o, sizeof(int));
 796                 o += sizeof(int);
 797                 hops = (double) ntohl(j) / 1000.0;
 798                 break;
 799
 800             default:
 801                 debugs(38, 1, "netdbExchangeHandleReply: corrupt data, aborting");
 802                 netdbExchangeDone(ex);
 803                 return;
 804             }
 805         }
 806
 807         if (!addr.IsAnyAddr() && rtt > 0)
 808             netdbExchangeUpdatePeer(addr, ex->p, rtt, hops);
 809
 810         assert(o == rec_sz);
 811
 812         ex->used += rec_sz;
 813
 814         size -= rec_sz;
 815
 816         p += rec_sz;
 817
 818         nused++;
 819     }
 820
 821     /*
 822      * Copy anything that is left over to the beginning of the buffer,
 823      * and adjust buf_ofs accordingly
 824      */
 825
 826     /*
 827      * Evilly, size refers to the buf size left now,
 828      * ex->buf_ofs is the original buffer size, so just copy that
 829      * much data over
 830      */
 831     memmove(ex->buf, ex->buf + (ex->buf_ofs - size), size);
 832
 833     ex->buf_ofs = size;
 834
 835     /*
 836      * And don't re-copy the remaining data ..
 837      */
 838     ex->used += size;
 839
 840     /*
 841      * Now the tricky bit - size _included_ the leftover bit from the _last_
 842      * storeClientCopy. We don't want to include that, or our offset will be wrong.
 843      * So, don't count the size of the leftover buffer we began with.
 844      * This can _disappear_ when we're not tracking offsets ..
 845      */
 846     ex->used -= oldbufofs;
 847
 848     debugs(38, 3, "netdbExchangeHandleReply: size left over in this buffer: " << size << " bytes");
 849
 850     debugs(38, 3, "netdbExchangeHandleReply: used " << nused <<
 851            " entries, (x " << rec_sz << " bytes) == " << nused * rec_sz <<
 852            " bytes total");
 853
 854     debugs(38, 3, "netdbExchangeHandleReply: used " << ex->used);
 855
 856     if (EBIT_TEST(ex->e->flags, ENTRY_ABORTED)) {
 857         debugs(38, 3, "netdbExchangeHandleReply: ENTRY_ABORTED");
 858         netdbExchangeDone(ex);
 859     } else if (ex->e->store_status == STORE_PENDING) {
 860         StoreIOBuffer tempBuffer;
 861         tempBuffer.offset = ex->used;
 862         tempBuffer.length = ex->buf_sz - ex->buf_ofs;
 863         tempBuffer.data = ex->buf + ex->buf_ofs;
 864         debugs(38, 3, "netdbExchangeHandleReply: EOF not received");
 865         storeClientCopy(ex->sc, ex->e, tempBuffer,
 866                         netdbExchangeHandleReply, ex);
 867     }
 868 }
 869
 870 static void
 871 netdbExchangeDone(void *data)
 872 {
 873     netdbExchangeState *ex = (netdbExchangeState *)data;
 874     debugs(38, 3, "netdbExchangeDone: " << ex->e->url()  );
 875     HTTPMSGUNLOCK(ex->r);
 876     storeUnregister(ex->sc, ex->e, ex);
 877     ex->e->unlock();
 878     cbdataReferenceDone(ex->p);
 879     cbdataFree(ex);
 880 }
 881
 882 static void
 883 netdbRegisterWithCacheManager(void)
 884 {
 885     CacheManager::GetInstance()->
 886     registerAction("netdb", "Network Measurement Database", netdbDump, 0, 1);
 887 }
 888
 889 #endif /* USE_ICMP */
 890
 891
 892 /* PUBLIC FUNCTIONS */
 893
 894 void
 895 netdbInit(void)
 896 {
 897 #if USE_ICMP
 898     int n;
 899
 900     netdbRegisterWithCacheManager();
 901
 902     if (addr_table)
 903         return;
 904
 905     n = hashPrime(Config.Netdb.high / 4);
 906
 907     addr_table = hash_create((HASHCMP *) strcmp, n, hash_string);
 908
 909     n = hashPrime(3 * Config.Netdb.high / 4);
 910
 911     host_table = hash_create((HASHCMP *) strcmp, n, hash_string);
 912
 913     eventAddIsh("netdbSaveState", netdbSaveState, NULL, 3600.0, 1);
 914
 915     netdbReloadState();
 916
 917 #endif
 918 }
 919
 920 void
 921 netdbPingSite(const char *hostname)
 922 {
 923 #if USE_ICMP
 924     netdbEntry *n;
 925
 926     if ((n = netdbLookupHost(hostname)) != NULL)
 927         if (n->next_ping_time > squid_curtime)
 928             return;
 929
 930     ipcache_nbgethostbyname(hostname, netdbSendPing,
 931                             new generic_cbdata(xstrdup(hostname)));
 932
 933 #endif
 934 }
 935
 936 void
 937 netdbHandlePingReply(const Ip::Address &from, int hops, int rtt)
 938 {
 939 #if USE_ICMP
 940     netdbEntry *n;
 941     int N;
 942     debugs(38, 3, "netdbHandlePingReply: from " << from);
 943
 944     if ((n = netdbLookupAddr(from)) == NULL)
 945         return;
 946
 947     N = ++n->pings_recv;
 948
 949     if (N > 5)
 950         N = 5;
 951
 952     if (rtt < 1)
 953         rtt = 1;
 954
 955     n->hops = ((n->hops * (N - 1)) + hops) / N;
 956
 957     n->rtt = ((n->rtt * (N - 1)) + rtt) / N;
 958
 959     debugs(38, 3, "netdbHandlePingReply: " << n->network  << "; rtt="<<
 960            std::setw(5)<< std::setprecision(2) << n->rtt << "  hops="<<
 961            std::setw(4) << n->hops);
 962
 963 #endif
 964 }
 965
 966 void
 967 netdbFreeMemory(void)
 968 {
 969 #if USE_ICMP
 970     hashFreeItems(addr_table, netdbFreeNetdbEntry);
 971     hashFreeMemory(addr_table);
 972     addr_table = NULL;
 973     hashFreeItems(host_table, netdbFreeNameEntry);
 974     hashFreeMemory(host_table);
 975     host_table = NULL;
 976     wordlistDestroy(&peer_names);
 977     peer_names = NULL;
 978 #endif
 979 }
 980
 981
 982 #if 0 // AYJ: Looks to be unused code.
 983 int
 984 netdbHops(Ip::Address &addr)
 985 {
 986 #if USE_ICMP
 987     netdbEntry *n = netdbLookupAddr(addr);
 988
 989     if (n && n->pings_recv) {
 990         n->last_use_time = squid_curtime;
 991         return (int) (n->hops + 0.5);
 992     }
 993
 994 #endif
 995     return 256;
 996 }
 997 #endif
 998
 999 void
1000 netdbDump(StoreEntry * sentry)
1001 {
1002 #if USE_ICMP
1003     netdbEntry *n;
1004     netdbEntry **list;
1005     net_db_name *x;
1006     int k;
1007     int i;
1008     int j;
1009     net_db_peer *p;
1010     storeAppendPrintf(sentry, "Network DB Statistics:\n");
1011     storeAppendPrintf(sentry, "%-46.46s %9s %7s %5s %s\n",  /* Max between 16 (IPv4) or 46 (IPv6)   */
1012                       "Network",
1013                       "recv/sent",
1014                       "RTT",
1015                       "Hops",
1016                       "Hostnames");
1017     list = (netdbEntry **)xcalloc(memInUse(MEM_NETDBENTRY), sizeof(netdbEntry *));
1018     i = 0;
1019     hash_first(addr_table);
1020
1021     while ((n = (netdbEntry *) hash_next(addr_table)))
1022         *(list + i++) = n;
1023
1024     if (i != memInUse(MEM_NETDBENTRY))
1025         debugs(38, 0, "WARNING: netdb_addrs count off, found " << i <<
1026                ", expected " << memInUse(MEM_NETDBENTRY));
1027
1028     qsort((char *) list,
1029           i,
1030           sizeof(netdbEntry *),
1031           sortByRtt);
1032
1033     for (k = 0; k < i; k++) {
1034         n = *(list + k);
1035         storeAppendPrintf(sentry, "%-46.46s %4d/%4d %7.1f %5.1f", /* Max between 16 (IPv4) or 46 (IPv6)   */
1036                           n->network,
1037                           n->pings_recv,
1038                           n->pings_sent,
1039                           n->rtt,
1040                           n->hops);
1041
1042         for (x = n->hosts; x; x = x->next)
1043             storeAppendPrintf(sentry, " %s", hashKeyStr(&x->hash));
1044
1045         storeAppendPrintf(sentry, "\n");
1046
1047         p = n->peers;
1048
1049         for (j = 0; j < n->n_peers; j++, p++) {
1050             storeAppendPrintf(sentry, "    %-22.22s %7.1f %5.1f\n",
1051                               p->peername,
1052                               p->rtt,
1053                               p->hops);
1054         }
1055     }
1056
1057     xfree(list);
1058 #else
1059
1060     storeAppendPrintf(sentry,"NETDB support not compiled into this Squid cache.\n");
1061 #endif
1062 }
1063
1064 int
1065 netdbHostHops(const char *host)
1066 {
1067 #if USE_ICMP
1068     netdbEntry *n = netdbLookupHost(host);
1069
1070     if (n) {
1071         n->last_use_time = squid_curtime;
1072         return (int) (n->hops + 0.5);
1073     }
1074
1075 #endif
1076     return 0;
1077 }
1078
1079 int
1080 netdbHostRtt(const char *host)
1081 {
1082 #if USE_ICMP
1083     netdbEntry *n = netdbLookupHost(host);
1084
1085     if (n) {
1086         n->last_use_time = squid_curtime;
1087         return (int) (n->rtt + 0.5);
1088     }
1089
1090 #endif
1091     return 0;
1092 }
1093
1094 void
1095 netdbHostData(const char *host, int *samp, int *rtt, int *hops)
1096 {
1097 #if USE_ICMP
1098     netdbEntry *n = netdbLookupHost(host);
1099
1100     if (n == NULL)
1101         return;
1102
1103     *samp = n->pings_recv;
1104
1105     *rtt = (int) (n->rtt + 0.5);
1106
1107     *hops = (int) (n->hops + 0.5);
1108
1109     n->last_use_time = squid_curtime;
1110
1111 #endif
1112 }
1113
1114 void
1115 netdbUpdatePeer(HttpRequest * r, peer * e, int irtt, int ihops)
1116 {
1117 #if USE_ICMP
1118     netdbEntry *n;
1119     double rtt = (double) irtt;
1120     double hops = (double) ihops;
1121     net_db_peer *p;
1122     debugs(38, 3, "netdbUpdatePeer: '" << r->GetHost() << "', " << ihops << " hops, " << irtt << " rtt");
1123     n = netdbLookupHost(r->GetHost());
1124
1125     if (n == NULL) {
1126         debugs(38, 3, "netdbUpdatePeer: host '" << r->GetHost() << "' not found");
1127         return;
1128     }
1129
1130     if ((p = netdbPeerByName(n, e->host)) == NULL)
1131         p = netdbPeerAdd(n, e);
1132
1133     p->rtt = rtt;
1134
1135     p->hops = hops;
1136
1137     p->expires = squid_curtime + 3600;
1138
1139     if (n->n_peers < 2)
1140         return;
1141
1142     qsort((char *) n->peers,
1143           n->n_peers,
1144           sizeof(net_db_peer),
1145           sortPeerByRtt);
1146
1147 #endif
1148 }
1149
1150 void
1151 netdbExchangeUpdatePeer(Ip::Address &addr, peer * e, double rtt, double hops)
1152 {
1153 #if USE_ICMP
1154     netdbEntry *n;
1155     net_db_peer *p;
1156     debugs(38, 5, "netdbExchangeUpdatePeer: '" << addr << "', "<<
1157            std::setfill('0')<< std::setprecision(2) << hops << " hops, " <<
1158            rtt << " rtt");
1159
1160     if ( !addr.IsIPv4() ) {
1161         debugs(38, 5, "netdbExchangeUpdatePeer: Aborting peer update for '" << addr << "', NetDB cannot handle IPv6.");
1162         return;
1163     }
1164
1165     n = netdbLookupAddr(addr);
1166
1167     if (n == NULL)
1168         n = netdbAdd(addr);
1169
1170     assert(NULL != n);
1171
1172     if ((p = netdbPeerByName(n, e->host)) == NULL)
1173         p = netdbPeerAdd(n, e);
1174
1175     p->rtt = rtt;
1176
1177     p->hops = hops;
1178
1179     p->expires = squid_curtime + 3600;  /* XXX ? */
1180
1181     if (n->n_peers < 2)
1182         return;
1183
1184     qsort((char *) n->peers,
1185           n->n_peers,
1186           sizeof(net_db_peer),
1187           sortPeerByRtt);
1188
1189 #endif
1190 }
1191
1192 void
1193 netdbDeleteAddrNetwork(Ip::Address &addr)
1194 {
1195 #if USE_ICMP
1196     netdbEntry *n = netdbLookupAddr(addr);
1197
1198     if (n == NULL)
1199         return;
1200
1201     debugs(38, 3, "netdbDeleteAddrNetwork: " << n->network);
1202
1203     netdbRelease(n);
1204 #endif
1205 }
1206
1207
1208 void
1209 netdbBinaryExchange(StoreEntry * s)
1210 {
1211     HttpReply *reply = new HttpReply;
1212 #if USE_ICMP
1213
1214     Ip::Address addr;
1215
1216     netdbEntry *n;
1217     int i;
1218     int j;
1219     int rec_sz;
1220     char *buf;
1221
1222     struct in_addr line_addr;
1223     s->buffer();
1224     reply->setHeaders(HTTP_OK, "OK", NULL, -1, squid_curtime, -2);
1225     s->replaceHttpReply(reply);
1226     rec_sz = 0;
1227     rec_sz += 1 + sizeof(struct in_addr);
1228     rec_sz += 1 + sizeof(int);
1229     rec_sz += 1 + sizeof(int);
1230     buf = (char *)memAllocate(MEM_4K_BUF);
1231     i = 0;
1232     hash_first(addr_table);
1233
1234     while ((n = (netdbEntry *) hash_next(addr_table))) {
1235         if (0.0 == n->rtt)
1236             continue;
1237
1238         if (n->rtt > 60000)     /* RTT > 1 MIN probably bogus */
1239             continue;
1240
1241         if (! (addr = n->network) )
1242             continue;
1243
1244         /* FIXME INET6 : NetDB cannot yet handle IPv6 addresses. Ensure only IPv4 get sent. */
1245         if ( !addr.IsIPv4() )
1246             continue;
1247
1248         buf[i++] = (char) NETDB_EX_NETWORK;
1249
1250         addr.GetInAddr(line_addr);
1251         xmemcpy(&buf[i], &line_addr, sizeof(struct in_addr));
1252
1253         i += sizeof(struct in_addr);
1254
1255         buf[i++] = (char) NETDB_EX_RTT;
1256
1257         j = htonl((int) (n->rtt * 1000));
1258
1259         xmemcpy(&buf[i], &j, sizeof(int));
1260
1261         i += sizeof(int);
1262
1263         buf[i++] = (char) NETDB_EX_HOPS;
1264
1265         j = htonl((int) (n->hops * 1000));
1266
1267         xmemcpy(&buf[i], &j, sizeof(int));
1268
1269         i += sizeof(int);
1270
1271         if (i + rec_sz > 4096) {
1272             s->append(buf, i);
1273             i = 0;
1274         }
1275     }
1276
1277     if (i > 0) {
1278         s->append(buf, i);
1279         i = 0;
1280     }
1281
1282     assert(0 == i);
1283     s->flush();
1284     memFree(buf, MEM_4K_BUF);
1285 #else
1286
1287     reply->setHeaders(HTTP_BAD_REQUEST, "Bad Request", NULL, -1, squid_curtime, -2);
1288     s->replaceHttpReply(reply);
1289     storeAppendPrintf(s, "NETDB support not compiled into this Squid cache.\n");
1290 #endif
1291
1292     s->complete();
1293 }
1294
1295 #if USE_ICMP
1296 CBDATA_TYPE(netdbExchangeState);
1297 #endif
1298
1299 void
1300 netdbExchangeStart(void *data)
1301 {
1302 #if USE_ICMP
1303     peer *p = (peer *)data;
1304     char *uri;
1305     netdbExchangeState *ex;
1306     StoreIOBuffer tempBuffer;
1307     CBDATA_INIT_TYPE(netdbExchangeState);
1308     ex = cbdataAlloc(netdbExchangeState);
1309     ex->p = cbdataReference(p);
1310     uri = internalRemoteUri(p->host, p->http_port, "/squid-internal-dynamic/", "netdb");
1311     debugs(38, 3, "netdbExchangeStart: Requesting '" << uri << "'");
1312     assert(NULL != uri);
1313     ex->r = HttpRequest::CreateFromUrl(uri);
1314
1315     if (NULL == ex->r) {
1316         debugs(38, 1, "netdbExchangeStart: Bad URI " << uri);
1317         return;
1318     }
1319
1320     HTTPMSGLOCK(ex->r);
1321     assert(NULL != ex->r);
1322     ex->r->http_ver = HttpVersion(1,1);
1323     ex->connstate = STATE_HEADER;
1324     ex->e = storeCreateEntry(uri, uri, request_flags(), METHOD_GET);
1325     ex->buf_sz = NETDB_REQBUF_SZ;
1326     assert(NULL != ex->e);
1327     ex->sc = storeClientListAdd(ex->e, ex);
1328     tempBuffer.offset = 0;
1329     tempBuffer.length = ex->buf_sz;
1330     tempBuffer.data = ex->buf;
1331     storeClientCopy(ex->sc, ex->e, tempBuffer,
1332                     netdbExchangeHandleReply, ex);
1333     ex->r->flags.loopdetect = 1;        /* cheat! -- force direct */
1334
1335     if (p->login)
1336         xstrncpy(ex->r->login, p->login, MAX_LOGIN_SZ);
1337
1338     urlCanonical(ex->r);
1339
1340     Comm::ConnectionPointer nul;
1341     FwdState::fwdStart(nul, ex->e, ex->r);
1342
1343 #endif
1344 }
1345
1346 peer *
1347 netdbClosestParent(HttpRequest * request)
1348 {
1349 #if USE_ICMP
1350     peer *p = NULL;
1351     netdbEntry *n;
1352     const ipcache_addrs *ia;
1353     net_db_peer *h;
1354     int i;
1355     n = netdbLookupHost(request->GetHost());
1356
1357     if (NULL == n) {
1358         /* try IP addr */
1359         ia = ipcache_gethostbyname(request->GetHost(), 0);
1360
1361         if (NULL != ia)
1362             n = netdbLookupAddr(ia->in_addrs[ia->cur]);
1363     }
1364
1365     if (NULL == n)
1366         return NULL;
1367
1368     if (0 == n->n_peers)
1369         return NULL;
1370
1371     n->last_use_time = squid_curtime;
1372
1373     /*
1374      * Find the parent with the least RTT to the origin server.
1375      * Make sure we don't return a parent who is farther away than
1376      * we are.  Note, the n->peers list is pre-sorted by RTT.
1377      */
1378     for (i = 0; i < n->n_peers; i++) {
1379         h = &n->peers[i];
1380
1381         if (n->rtt > 0)
1382             if (n->rtt < h->rtt)
1383                 break;
1384
1385         p = peerFindByName(h->peername);
1386
1387         if (NULL == p)          /* not found */
1388             continue;
1389
1390         if (neighborType(p, request) != PEER_PARENT)
1391             continue;
1392
1393         if (!peerHTTPOkay(p, request))  /* not allowed */
1394             continue;
1395
1396         return p;
1397     }
1398
1399 #endif
1400     return NULL;
1401 }