src/icmp/net_db.cc

   1 /*
   2  * $Id: net_db.cc,v 1.202 2008/01/07 17:12:28 hno Exp $
   3  *
   4  * DEBUG: section 38    Network Measurement Database
   5  * AUTHOR: Duane Wessels
   6  *
   7  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   8  * ----------------------------------------------------------
   9  *
  10  *  Squid is the result of efforts by numerous individuals from
  11  *  the Internet community; see the CONTRIBUTORS file for full
  12  *  details.   Many organizations have provided support for Squid's
  13  *  development; see the SPONSORS file for full details.  Squid is
  14  *  Copyrighted (C) 2001 by the Regents of the University of
  15  *  California; see the COPYRIGHT file for full details.  Squid
  16  *  incorporates software developed and/or copyrighted by other
  17  *  sources; see the CREDITS file for full details.
  18  *
  19  *  This program is free software; you can redistribute it and/or modify
  20  *  it under the terms of the GNU General Public License as published by
  21  *  the Free Software Foundation; either version 2 of the License, or
  22  *  (at your option) any later version.
  23  *
  24  *  This program is distributed in the hope that it will be useful,
  25  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  26  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  27  *  GNU General Public License for more details.
  28  *
  29  *  You should have received a copy of the GNU General Public License
  30  *  along with this program; if not, write to the Free Software
  31  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  32  *
  33  */
  34
  35 /*
  36  * XXX XXX XXX
  37  *
  38  * This code may be slightly broken now. If you're getting consistent
  39  * (sometimes working) corrupt data exchanges, please contact adrian
  40  * (adrian@squid-cache.org) to sort them out.
  41  */
  42
  43 #include "squid.h"
  44 #include "icmp/net_db.h"
  45 #include "cbdata.h"
  46 #include "event.h"
  47 #include "CacheManager.h"
  48 #include "Store.h"
  49 #include "SwapDir.h"
  50 #include "HttpRequest.h"
  51 #include "HttpReply.h"
  52 #include "MemObject.h"
  53 #include "fde.h"
  54 #include "forward.h"
  55 #include "SquidTime.h"
  56 #include "wordlist.h"
  57 #include "IPAddress.h"
  58
  59 #if USE_ICMP
  60 #include "icmp/IcmpSquid.h"
  61 #include "StoreClient.h"
  62
  63 #define NETDB_REQBUF_SZ 4096
  64
  65 typedef enum {
  66     STATE_NONE,
  67     STATE_HEADER,
  68     STATE_BODY
  69 } netdb_conn_state_t;
  70
  71 typedef struct {
  72     peer *p;
  73     StoreEntry *e;
  74     store_client *sc;
  75     HttpRequest *r;
  76     int64_t used;
  77     size_t buf_sz;
  78     char buf[NETDB_REQBUF_SZ];
  79     int buf_ofs;
  80     netdb_conn_state_t connstate;
  81 } netdbExchangeState;
  82
  83 static hash_table *addr_table = NULL;
  84 static hash_table *host_table = NULL;
  85
  86 IPAddress networkFromInaddr(const IPAddress &a);
  87 static void netdbRelease(netdbEntry * n);
  88
  89 static void netdbHashInsert(netdbEntry * n, IPAddress &addr);
  90 static void netdbHashDelete(const char *key);
  91 static void netdbHostInsert(netdbEntry * n, const char *hostname);
  92 static void netdbHostDelete(const net_db_name * x);
  93 static void netdbPurgeLRU(void);
  94 static netdbEntry *netdbLookupHost(const char *key);
  95 static net_db_peer *netdbPeerByName(const netdbEntry * n, const char *);
  96 static net_db_peer *netdbPeerAdd(netdbEntry * n, peer * e);
  97 static const char *netdbPeerName(const char *name);
  98 static IPH netdbSendPing;
  99 static QS sortPeerByRtt;
 100 static QS sortByRtt;
 101 static QS netdbLRU;
 102 static FREE netdbFreeNameEntry;
 103 static FREE netdbFreeNetdbEntry;
 104 static STCB netdbExchangeHandleReply;
 105 static void netdbExchangeDone(void *);
 106
 107 /* We have to keep a local list of peer names.  The Peers structure
 108  * gets freed during a reconfigure.  We want this database to
 109  * remain persisitent, so _net_db_peer->peername points into this
 110  * linked list */
 111 static wordlist *peer_names = NULL;
 112
 113 static void
 114 netdbHashInsert(netdbEntry * n, IPAddress &addr)
 115 {
 116     networkFromInaddr(addr).NtoA(n->network, MAX_IPSTRLEN);
 117     n->hash.key = n->network;
 118     assert(hash_lookup(addr_table, n->network) == NULL);
 119     hash_join(addr_table, &n->hash);
 120 }
 121
 122 static void
 123 netdbHashDelete(const char *key)
 124 {
 125     hash_link *hptr = (hash_link *)hash_lookup(addr_table, key);
 126
 127     if (hptr == NULL) {
 128         debug_trap("netdbHashDelete: key not found");
 129         return;
 130     }
 131
 132     hash_remove_link(addr_table, hptr);
 133 }
 134
 135 static void
 136 netdbHostInsert(netdbEntry * n, const char *hostname)
 137 {
 138     net_db_name *x = (net_db_name *)memAllocate(MEM_NET_DB_NAME);
 139     x->hash.key = xstrdup(hostname);
 140     x->next = n->hosts;
 141     n->hosts = x;
 142     x->net_db_entry = n;
 143     assert(hash_lookup(host_table, hostname) == NULL);
 144     hash_join(host_table, &x->hash);
 145     n->link_count++;
 146 }
 147
 148 static void
 149 netdbHostDelete(const net_db_name * x)
 150 {
 151     netdbEntry *n;
 152     net_db_name **X;
 153     assert(x != NULL);
 154     assert(x->net_db_entry != NULL);
 155     n = x->net_db_entry;
 156     n->link_count--;
 157
 158     for (X = &n->hosts; *X; X = &(*X)->next) {
 159         if (*X == x) {
 160             *X = x->next;
 161             break;
 162         }
 163     }
 164
 165     hash_remove_link(host_table, (hash_link *) x);
 166     xfree(x->hash.key);
 167     memFree((void *) x, MEM_NET_DB_NAME);
 168 }
 169
 170 static netdbEntry *
 171 netdbLookupHost(const char *key)
 172 {
 173     net_db_name *x = (net_db_name *) hash_lookup(host_table, key);
 174     return x ? x->net_db_entry : NULL;
 175 }
 176
 177 static void
 178 netdbRelease(netdbEntry * n)
 179 {
 180     net_db_name *x;
 181     net_db_name *next;
 182
 183     for (x = n->hosts; x; x = next) {
 184         next = x->next;
 185         netdbHostDelete(x);
 186     }
 187
 188     n->hosts = NULL;
 189     safe_free(n->peers);
 190     n->peers = NULL;
 191     n->n_peers = 0;
 192     n->n_peers_alloc = 0;
 193
 194     if (n->link_count == 0) {
 195         netdbHashDelete(n->network);
 196         memFree(n, MEM_NETDBENTRY);
 197     }
 198 }
 199
 200 static int
 201 netdbLRU(const void *A, const void *B)
 202 {
 203     const netdbEntry *const *n1 = (const netdbEntry *const *)A;
 204     const netdbEntry *const *n2 = (const netdbEntry *const *)B;
 205
 206     if ((*n1)->last_use_time > (*n2)->last_use_time)
 207         return (1);
 208
 209     if ((*n1)->last_use_time < (*n2)->last_use_time)
 210         return (-1);
 211
 212     return (0);
 213 }
 214
 215 static void
 216 netdbPurgeLRU(void)
 217 {
 218     netdbEntry *n;
 219     netdbEntry **list;
 220     int k = 0;
 221     int list_count = 0;
 222     int removed = 0;
 223     list = (netdbEntry **)xcalloc(memInUse(MEM_NETDBENTRY), sizeof(netdbEntry *));
 224     hash_first(addr_table);
 225
 226     while ((n = (netdbEntry *) hash_next(addr_table))) {
 227         assert(list_count < memInUse(MEM_NETDBENTRY));
 228         *(list + list_count) = n;
 229         list_count++;
 230     }
 231
 232     qsort((char *) list,
 233           list_count,
 234           sizeof(netdbEntry *),
 235           netdbLRU);
 236
 237     for (k = 0; k < list_count; k++) {
 238         if (memInUse(MEM_NETDBENTRY) < Config.Netdb.low)
 239             break;
 240
 241         netdbRelease(*(list + k));
 242
 243         removed++;
 244     }
 245
 246     xfree(list);
 247 }
 248
 249 static netdbEntry *
 250 netdbLookupAddr(const IPAddress &addr)
 251 {
 252     netdbEntry *n;
 253     char *key = new char[MAX_IPSTRLEN];
 254     networkFromInaddr(addr).NtoA(key,MAX_IPSTRLEN);
 255     n = (netdbEntry *) hash_lookup(addr_table, key);
 256     return n;
 257 }
 258
 259 static netdbEntry *
 260 netdbAdd(IPAddress &addr)
 261 {
 262     netdbEntry *n;
 263
 264     if (memInUse(MEM_NETDBENTRY) > Config.Netdb.high)
 265         netdbPurgeLRU();
 266
 267     if ((n = netdbLookupAddr(addr)) == NULL) {
 268         n = (netdbEntry *)memAllocate(MEM_NETDBENTRY);
 269         netdbHashInsert(n, addr);
 270     }
 271
 272     return n;
 273 }
 274
 275 static void
 276 netdbSendPing(const ipcache_addrs * ia, void *data)
 277 {
 278     IPAddress addr;
 279     char *hostname = NULL;
 280     static_cast<generic_cbdata *>(data)->unwrap(&hostname);
 281     netdbEntry *n;
 282     netdbEntry *na;
 283     net_db_name *x;
 284     net_db_name **X;
 285
 286     if (ia == NULL) {
 287         xfree(hostname);
 288         return;
 289     }
 290
 291     addr = ia->in_addrs[ia->cur];
 292
 293     if ((n = netdbLookupHost(hostname)) == NULL) {
 294         n = netdbAdd(addr);
 295         netdbHostInsert(n, hostname);
 296     } else if ((na = netdbLookupAddr(addr)) != n) {
 297         /*
 298          *hostname moved from 'network n' to 'network na'!
 299          */
 300
 301         if (na == NULL)
 302             na = netdbAdd(addr);
 303
 304         debugs(38, 3, "netdbSendPing: " << hostname << " moved from " << n->network << " to " << na->network);
 305
 306         x = (net_db_name *) hash_lookup(host_table, hostname);
 307
 308         if (x == NULL) {
 309             debugs(38, 1, "netdbSendPing: net_db_name list bug: " << hostname << " not found");
 310             xfree(hostname);
 311             return;
 312         }
 313
 314         /* remove net_db_name from 'network n' linked list */
 315         for (X = &n->hosts; *X; X = &(*X)->next) {
 316             if (*X == x) {
 317                 *X = x->next;
 318                 break;
 319             }
 320         }
 321
 322         n->link_count--;
 323         /* point to 'network na' from host entry */
 324         x->net_db_entry = na;
 325         /* link net_db_name to 'network na' */
 326         x->next = na->hosts;
 327         na->hosts = x;
 328         na->link_count++;
 329         n = na;
 330     }
 331
 332     if (n->next_ping_time <= squid_curtime) {
 333         debugs(38, 3, "netdbSendPing: pinging " << hostname);
 334         icmpEngine.DomainPing(addr, hostname);
 335         n->pings_sent++;
 336         n->next_ping_time = squid_curtime + Config.Netdb.period;
 337         n->last_use_time = squid_curtime;
 338     }
 339
 340     xfree(hostname);
 341 }
 342
 343 IPAddress
 344 networkFromInaddr(const IPAddress &in)
 345 {
 346     IPAddress out;
 347
 348     out = in;
 349 #if USE_IPV6
 350
 351     /* in IPv6 the 'network' should be the routing section. */
 352
 353     if ( in.IsIPv6() ) {
 354         out.ApplyMask(64, AF_INET6);
 355         debugs(14, 5, "networkFromInaddr : Masked IPv6 Address to " << in << "/64 routing part.");
 356         return out;
 357     }
 358 #endif
 359
 360 #if USE_CLASSFUL
 361     struct in_addr b;
 362
 363     in.GetInAddr(b);
 364
 365     if (IN_CLASSC(b.s_addr))
 366         b.s_addr &= IN_CLASSC_NET;
 367     else if (IN_CLASSB(b.s_addr))
 368         b.s_addr &= IN_CLASSB_NET;
 369     else if (IN_CLASSA(b.s_addr))
 370         b.s_addr &= IN_CLASSA_NET;
 371
 372     out = b;
 373
 374 #endif
 375
 376     debugs(14, 5, "networkFromInaddr : Masked IPv4 Address to " << out << "/24.");
 377
 378     /* use /24 for everything under IPv4 */
 379     out.ApplyMask(24, AF_INET);
 380     debugs(14, 5, "networkFromInaddr : Masked IPv4 Address to " << in << "/24.");
 381
 382     return out;
 383 }
 384
 385 static int
 386 sortByRtt(const void *A, const void *B)
 387 {
 388     const netdbEntry *const *n1 = (const netdbEntry *const *)A;
 389     const netdbEntry *const *n2 = (const netdbEntry *const *)B;
 390
 391     if ((*n1)->rtt > (*n2)->rtt)
 392         return 1;
 393     else if ((*n1)->rtt < (*n2)->rtt)
 394         return -1;
 395     else
 396         return 0;
 397 }
 398
 399 static net_db_peer *
 400 netdbPeerByName(const netdbEntry * n, const char *peername)
 401 {
 402     int i;
 403     net_db_peer *p = n->peers;
 404
 405     for (i = 0; i < n->n_peers; i++, p++) {
 406         if (!strcmp(p->peername, peername))
 407             return p;
 408     }
 409
 410     return NULL;
 411 }
 412
 413 static net_db_peer *
 414 netdbPeerAdd(netdbEntry * n, peer * e)
 415 {
 416     net_db_peer *p;
 417     net_db_peer *o;
 418     int osize;
 419     int i;
 420
 421     if (n->n_peers == n->n_peers_alloc) {
 422         o = n->peers;
 423         osize = n->n_peers_alloc;
 424
 425         if (n->n_peers_alloc == 0)
 426             n->n_peers_alloc = 2;
 427         else
 428             n->n_peers_alloc <<= 1;
 429
 430         debugs(38, 3, "netdbPeerAdd: Growing peer list for '" << n->network << "' to " << n->n_peers_alloc);
 431
 432         n->peers = (net_db_peer *)xcalloc(n->n_peers_alloc, sizeof(net_db_peer));
 433
 434         for (i = 0; i < osize; i++)
 435             *(n->peers + i) = *(o + i);
 436
 437         if (osize) {
 438             safe_free(o);
 439         }
 440     }
 441
 442     p = n->peers + n->n_peers;
 443     p->peername = netdbPeerName(e->host);
 444     n->n_peers++;
 445     return p;
 446 }
 447
 448 static int
 449 sortPeerByRtt(const void *A, const void *B)
 450 {
 451     const net_db_peer *p1 = (net_db_peer *)A;
 452     const net_db_peer *p2 = (net_db_peer *)B;
 453
 454     if (p1->rtt > p2->rtt)
 455         return 1;
 456     else if (p1->rtt < p2->rtt)
 457         return -1;
 458     else
 459         return 0;
 460 }
 461
 462 static void
 463 netdbSaveState(void *foo)
 464 {
 465     if (strcmp(Config.netdbFilename, "none") == 0)
 466         return;
 467
 468     Logfile *lf;
 469     netdbEntry *n;
 470     net_db_name *x;
 471
 472     struct timeval start = current_time;
 473     int count = 0;
 474     /*
 475      * This was nicer when we were using stdio, but thanks to
 476      * Solaris bugs, its a bad idea.  fopen can fail if more than
 477      * 256 FDs are open.
 478      */
 479     /*
 480      * unlink() is here because there is currently no way to make
 481      * logfileOpen() use O_TRUNC.
 482      */
 483     unlink(Config.netdbFilename);
 484     lf = logfileOpen(Config.netdbFilename, 4096, 0);
 485
 486     if (NULL == lf) {
 487         debugs(50, 1, "netdbSaveState: " << Config.netdbFilename << ": " << xstrerror());
 488         return;
 489     }
 490
 491     hash_first(addr_table);
 492
 493     while ((n = (netdbEntry *) hash_next(addr_table))) {
 494         if (n->pings_recv == 0)
 495             continue;
 496
 497         logfilePrintf(lf, "%s %d %d %10.5f %10.5f %d %d",
 498                       n->network,
 499                       n->pings_sent,
 500                       n->pings_recv,
 501                       n->hops,
 502                       n->rtt,
 503                       (int) n->next_ping_time,
 504                       (int) n->last_use_time);
 505
 506         for (x = n->hosts; x; x = x->next)
 507             logfilePrintf(lf, " %s", hashKeyStr(&x->hash));
 508
 509         logfilePrintf(lf, "\n");
 510
 511         count++;
 512
 513 #undef RBUF_SZ
 514
 515     }
 516
 517     logfileClose(lf);
 518     getCurrentTime();
 519     debugs(38, 1, "NETDB state saved; " <<
 520            count << " entries, " <<
 521            tvSubMsec(start, current_time) << " msec" );
 522     eventAddIsh("netdbSaveState", netdbSaveState, NULL, 3600.0, 1);
 523 }
 524
 525 static void
 526 netdbReloadState(void)
 527 {
 528     if (strcmp(Config.netdbFilename, "none") == 0)
 529         return;
 530
 531     char *s;
 532     int fd;
 533     int l;
 534
 535     struct stat sb;
 536     netdbEntry *n;
 537     netdbEntry N;
 538
 539     IPAddress addr;
 540     int count = 0;
 541
 542     struct timeval start = current_time;
 543     /*
 544      * This was nicer when we were using stdio, but thanks to
 545      * Solaris bugs, its a bad idea.  fopen can fail if more than
 546      * 256 FDs are open.
 547      */
 548     fd = file_open(Config.netdbFilename, O_RDONLY | O_BINARY);
 549
 550     if (fd < 0)
 551         return;
 552
 553     if (fstat(fd, &sb) < 0) {
 554         file_close(fd);
 555         return;
 556     }
 557
 558     char *t;
 559     char *buf = (char *)xcalloc(1, sb.st_size + 1);
 560     t = buf;
 561     l = FD_READ_METHOD(fd, buf, sb.st_size);
 562     file_close(fd);
 563
 564     if (l <= 0) {
 565         safe_free (buf);
 566         return;
 567     };
 568
 569     while ((s = strchr(t, '\n'))) {
 570         char *q;
 571         assert(s - buf < l);
 572         *s = '\0';
 573         memset(&N, '\0', sizeof(netdbEntry));
 574         q = strtok(t, w_space);
 575         t = s + 1;
 576
 577         if (NULL == q)
 578             continue;
 579
 580         if (! (addr = q) )
 581             continue;
 582
 583         if (netdbLookupAddr(addr) != NULL)      /* no dups! */
 584             continue;
 585
 586         if ((q = strtok(NULL, w_space)) == NULL)
 587             continue;
 588
 589         N.pings_sent = atoi(q);
 590
 591         if ((q = strtok(NULL, w_space)) == NULL)
 592             continue;
 593
 594         N.pings_recv = atoi(q);
 595
 596         if (N.pings_recv == 0)
 597             continue;
 598
 599         /* give this measurement low weight */
 600         N.pings_sent = 1;
 601
 602         N.pings_recv = 1;
 603
 604         if ((q = strtok(NULL, w_space)) == NULL)
 605             continue;
 606
 607         N.hops = atof(q);
 608
 609         if ((q = strtok(NULL, w_space)) == NULL)
 610             continue;
 611
 612         N.rtt = atof(q);
 613
 614         if ((q = strtok(NULL, w_space)) == NULL)
 615             continue;
 616
 617         N.next_ping_time = (time_t) atoi(q);
 618
 619         if ((q = strtok(NULL, w_space)) == NULL)
 620             continue;
 621
 622         N.last_use_time = (time_t) atoi(q);
 623
 624         n = (netdbEntry *)memAllocate(MEM_NETDBENTRY);
 625
 626         xmemcpy(n, &N, sizeof(netdbEntry));
 627
 628         netdbHashInsert(n, addr);
 629
 630         while ((q = strtok(NULL, w_space)) != NULL) {
 631             if (netdbLookupHost(q) != NULL)     /* no dups! */
 632                 continue;
 633
 634             netdbHostInsert(n, q);
 635         }
 636
 637         count++;
 638     }
 639
 640     xfree(buf);
 641     getCurrentTime();
 642     debugs(38, 1, "NETDB state reloaded; " <<
 643            count << " entries, " <<
 644            tvSubMsec(start, current_time) << " msec" );
 645 }
 646
 647 static const char *
 648 netdbPeerName(const char *name)
 649 {
 650     const wordlist *w;
 651
 652     for (w = peer_names; w; w = w->next) {
 653         if (!strcmp(w->key, name))
 654             return w->key;
 655     }
 656
 657     return wordlistAdd(&peer_names, name);
 658 }
 659
 660 static void
 661 netdbFreeNetdbEntry(void *data)
 662 {
 663     netdbEntry *n = (netdbEntry *)data;
 664     safe_free(n->peers);
 665     memFree(n, MEM_NETDBENTRY);
 666 }
 667
 668 static void
 669 netdbFreeNameEntry(void *data)
 670 {
 671     net_db_name *x = (net_db_name *)data;
 672     xfree(x->hash.key);
 673     memFree(x, MEM_NET_DB_NAME);
 674 }
 675
 676
 677 static void
 678 netdbExchangeHandleReply(void *data, StoreIOBuffer receivedData)
 679 {
 680     IPAddress addr;
 681
 682     netdbExchangeState *ex = (netdbExchangeState *)data;
 683     int rec_sz = 0;
 684     int o;
 685
 686     struct in_addr line_addr;
 687     double rtt;
 688     double hops;
 689     char *p;
 690     int j;
 691     HttpReply const *rep;
 692     size_t hdr_sz;
 693     int nused = 0;
 694     int size;
 695     int oldbufofs = ex->buf_ofs;
 696
 697     rec_sz = 0;
 698     rec_sz += 1 + sizeof(struct in_addr);
 699     rec_sz += 1 + sizeof(int);
 700     rec_sz += 1 + sizeof(int);
 701     debugs(38, 3, "netdbExchangeHandleReply: " << receivedData.length << " read bytes");
 702
 703     if (!cbdataReferenceValid(ex->p)) {
 704         debugs(38, 3, "netdbExchangeHandleReply: Peer became invalid");
 705         netdbExchangeDone(ex);
 706         return;
 707     }
 708
 709     debugs(38, 3, "netdbExchangeHandleReply: for '" << ex->p->host << ":" << ex->p->http_port << "'");
 710
 711     if (receivedData.length == 0 &&
 712             !receivedData.flags.error) {
 713         debugs(38, 3, "netdbExchangeHandleReply: Done");
 714         netdbExchangeDone(ex);
 715         return;
 716     }
 717
 718     p = ex->buf;
 719
 720     /* Get the size of the buffer now */
 721     size = ex->buf_ofs + receivedData.length;
 722     debugs(38, 3, "netdbExchangeHandleReply: " << size << " bytes buf");
 723
 724     /* Check if we're still doing headers */
 725
 726     if (ex->connstate == STATE_HEADER) {
 727
 728         ex->buf_ofs += receivedData.length;
 729
 730         /* skip reply headers */
 731
 732         if ((hdr_sz = headersEnd(p, ex->buf_ofs))) {
 733             debugs(38, 5, "netdbExchangeHandleReply: hdr_sz = " << hdr_sz);
 734             rep = ex->e->getReply();
 735             assert (0 != rep->sline.status);
 736             debugs(38, 3, "netdbExchangeHandleReply: reply status " << rep->sline.status);
 737
 738             if (HTTP_OK != rep->sline.status) {
 739                 netdbExchangeDone(ex);
 740                 return;
 741             }
 742
 743             assert((size_t)ex->buf_ofs >= hdr_sz);
 744
 745             /*
 746              * Now, point p to the part of the buffer where the data
 747              * starts, and update the size accordingly
 748              */
 749             assert(ex->used == 0);
 750             ex->used = hdr_sz;
 751             size = ex->buf_ofs - hdr_sz;
 752             p += hdr_sz;
 753
 754             /* Finally, set the conn state mode to STATE_BODY */
 755             ex->connstate = STATE_BODY;
 756         } else {
 757             StoreIOBuffer tempBuffer;
 758             tempBuffer.offset = ex->buf_ofs;
 759             tempBuffer.length = ex->buf_sz - ex->buf_ofs;
 760             tempBuffer.data = ex->buf + ex->buf_ofs;
 761             /* Have more headers .. */
 762             storeClientCopy(ex->sc, ex->e, tempBuffer,
 763                             netdbExchangeHandleReply, ex);
 764             return;
 765         }
 766     }
 767
 768     assert(ex->connstate == STATE_BODY);
 769
 770     /* If we get here, we have some body to parse .. */
 771     debugs(38, 5, "netdbExchangeHandleReply: start parsing loop, size = " << size);
 772
 773     while (size >= rec_sz) {
 774         debugs(38, 5, "netdbExchangeHandleReply: in parsing loop, size = " << size);
 775         addr.SetAnyAddr();
 776         hops = rtt = 0.0;
 777
 778         for (o = 0; o < rec_sz;) {
 779             switch ((int) *(p + o)) {
 780
 781             case NETDB_EX_NETWORK:
 782                 o++;
 783                 /* FIXME INET6 : NetDB can still ony send IPv4 */
 784                 xmemcpy(&line_addr, p + o, sizeof(struct in_addr));
 785                 addr = line_addr;
 786                 o += sizeof(struct in_addr);
 787                 break;
 788
 789             case NETDB_EX_RTT:
 790                 o++;
 791                 xmemcpy(&j, p + o, sizeof(int));
 792                 o += sizeof(int);
 793                 rtt = (double) ntohl(j) / 1000.0;
 794                 break;
 795
 796             case NETDB_EX_HOPS:
 797                 o++;
 798                 xmemcpy(&j, p + o, sizeof(int));
 799                 o += sizeof(int);
 800                 hops = (double) ntohl(j) / 1000.0;
 801                 break;
 802
 803             default:
 804                 debugs(38, 1, "netdbExchangeHandleReply: corrupt data, aborting");
 805                 netdbExchangeDone(ex);
 806                 return;
 807             }
 808         }
 809
 810         if (!addr.IsAnyAddr() && rtt > 0)
 811             netdbExchangeUpdatePeer(addr, ex->p, rtt, hops);
 812
 813         assert(o == rec_sz);
 814
 815         ex->used += rec_sz;
 816
 817         size -= rec_sz;
 818
 819         p += rec_sz;
 820
 821         nused++;
 822     }
 823
 824     /*
 825      * Copy anything that is left over to the beginning of the buffer,
 826      * and adjust buf_ofs accordingly
 827      */
 828
 829     /*
 830      * Evilly, size refers to the buf size left now,
 831      * ex->buf_ofs is the original buffer size, so just copy that
 832      * much data over
 833      */
 834     memmove(ex->buf, ex->buf + (ex->buf_ofs - size), size);
 835
 836     ex->buf_ofs = size;
 837
 838     /*
 839      * And don't re-copy the remaining data ..
 840      */
 841     ex->used += size;
 842
 843     /*
 844      * Now the tricky bit - size _included_ the leftover bit from the _last_
 845      * storeClientCopy. We don't want to include that, or our offset will be wrong.
 846      * So, don't count the size of the leftover buffer we began with.
 847      * This can _disappear_ when we're not tracking offsets ..
 848      */
 849     ex->used -= oldbufofs;
 850
 851     debugs(38, 3, "netdbExchangeHandleReply: size left over in this buffer: " << size << " bytes");
 852
 853     debugs(38, 3, "netdbExchangeHandleReply: used " << nused <<
 854            " entries, (x " << rec_sz << " bytes) == " << nused * rec_sz <<
 855            " bytes total");
 856
 857     debugs(38, 3, "netdbExchangeHandleReply: used " << ex->used);
 858
 859     if (EBIT_TEST(ex->e->flags, ENTRY_ABORTED)) {
 860         debugs(38, 3, "netdbExchangeHandleReply: ENTRY_ABORTED");
 861         netdbExchangeDone(ex);
 862     } else if (ex->e->store_status == STORE_PENDING) {
 863         StoreIOBuffer tempBuffer;
 864         tempBuffer.offset = ex->used;
 865         tempBuffer.length = ex->buf_sz - ex->buf_ofs;
 866         tempBuffer.data = ex->buf + ex->buf_ofs;
 867         debugs(38, 3, "netdbExchangeHandleReply: EOF not received");
 868         storeClientCopy(ex->sc, ex->e, tempBuffer,
 869                         netdbExchangeHandleReply, ex);
 870     }
 871 }
 872
 873 static void
 874 netdbExchangeDone(void *data)
 875 {
 876     netdbExchangeState *ex = (netdbExchangeState *)data;
 877     debugs(38, 3, "netdbExchangeDone: " << ex->e->url()  );
 878     HTTPMSGUNLOCK(ex->r);
 879     storeUnregister(ex->sc, ex->e, ex);
 880     ex->e->unlock();
 881     cbdataReferenceDone(ex->p);
 882     cbdataFree(ex);
 883 }
 884
 885 static void
 886 netdbRegisterWithCacheManager(void)
 887 {
 888     CacheManager::GetInstance()->
 889     registerAction("netdb", "Network Measurement Database", netdbDump, 0, 1);
 890 }
 891
 892 #endif /* USE_ICMP */
 893
 894
 895 /* PUBLIC FUNCTIONS */
 896
 897 void
 898 netdbInit(void)
 899 {
 900 #if USE_ICMP
 901     int n;
 902
 903     netdbRegisterWithCacheManager();
 904
 905     if (addr_table)
 906         return;
 907
 908     n = hashPrime(Config.Netdb.high / 4);
 909
 910     addr_table = hash_create((HASHCMP *) strcmp, n, hash_string);
 911
 912     n = hashPrime(3 * Config.Netdb.high / 4);
 913
 914     host_table = hash_create((HASHCMP *) strcmp, n, hash_string);
 915
 916     eventAddIsh("netdbSaveState", netdbSaveState, NULL, 3600.0, 1);
 917
 918     netdbReloadState();
 919
 920 #endif
 921 }
 922
 923 void
 924 netdbPingSite(const char *hostname)
 925 {
 926 #if USE_ICMP
 927     netdbEntry *n;
 928
 929     if ((n = netdbLookupHost(hostname)) != NULL)
 930         if (n->next_ping_time > squid_curtime)
 931             return;
 932
 933     ipcache_nbgethostbyname(hostname, netdbSendPing,
 934                             new generic_cbdata(xstrdup(hostname)));
 935
 936 #endif
 937 }
 938
 939 void
 940 netdbHandlePingReply(const IPAddress &from, int hops, int rtt)
 941 {
 942 #if USE_ICMP
 943     netdbEntry *n;
 944     int N;
 945     debugs(38, 3, "netdbHandlePingReply: from " << from);
 946
 947     if ((n = netdbLookupAddr(from)) == NULL)
 948         return;
 949
 950     N = ++n->pings_recv;
 951
 952     if (N > 5)
 953         N = 5;
 954
 955     if (rtt < 1)
 956         rtt = 1;
 957
 958     n->hops = ((n->hops * (N - 1)) + hops) / N;
 959
 960     n->rtt = ((n->rtt * (N - 1)) + rtt) / N;
 961
 962     debugs(38, 3, "netdbHandlePingReply: " << n->network  << "; rtt="<<
 963            std::setw(5)<< std::setprecision(2) << n->rtt << "  hops="<<
 964            std::setw(4) << n->hops);
 965
 966 #endif
 967 }
 968
 969 void
 970 netdbFreeMemory(void)
 971 {
 972 #if USE_ICMP
 973     hashFreeItems(addr_table, netdbFreeNetdbEntry);
 974     hashFreeMemory(addr_table);
 975     addr_table = NULL;
 976     hashFreeItems(host_table, netdbFreeNameEntry);
 977     hashFreeMemory(host_table);
 978     host_table = NULL;
 979     wordlistDestroy(&peer_names);
 980     peer_names = NULL;
 981 #endif
 982 }
 983
 984
 985 #if 0 // AYJ: Looks to be unused code.
 986 int
 987 netdbHops(IPAddress &addr)
 988 {
 989 #if USE_ICMP
 990     netdbEntry *n = netdbLookupAddr(addr);
 991
 992     if (n && n->pings_recv) {
 993         n->last_use_time = squid_curtime;
 994         return (int) (n->hops + 0.5);
 995     }
 996
 997 #endif
 998     return 256;
 999 }
1000 #endif
1001
1002 void
1003 netdbDump(StoreEntry * sentry)
1004 {
1005 #if USE_ICMP
1006     netdbEntry *n;
1007     netdbEntry **list;
1008     net_db_name *x;
1009     int k;
1010     int i;
1011     int j;
1012     net_db_peer *p;
1013     storeAppendPrintf(sentry, "Network DB Statistics:\n");
1014     storeAppendPrintf(sentry, "%-46.46s %9s %7s %5s %s\n",  /* Max between 16 (IPv4) or 46 (IPv6)   */
1015                       "Network",
1016                       "recv/sent",
1017                       "RTT",
1018                       "Hops",
1019                       "Hostnames");
1020     list = (netdbEntry **)xcalloc(memInUse(MEM_NETDBENTRY), sizeof(netdbEntry *));
1021     i = 0;
1022     hash_first(addr_table);
1023
1024     while ((n = (netdbEntry *) hash_next(addr_table)))
1025         *(list + i++) = n;
1026
1027     if (i != memInUse(MEM_NETDBENTRY))
1028         debugs(38, 0, "WARNING: netdb_addrs count off, found " << i <<
1029                ", expected " << memInUse(MEM_NETDBENTRY));
1030
1031     qsort((char *) list,
1032           i,
1033           sizeof(netdbEntry *),
1034           sortByRtt);
1035
1036     for (k = 0; k < i; k++) {
1037         n = *(list + k);
1038         storeAppendPrintf(sentry, "%-46.46s %4d/%4d %7.1f %5.1f", /* Max between 16 (IPv4) or 46 (IPv6)   */
1039                           n->network,
1040                           n->pings_recv,
1041                           n->pings_sent,
1042                           n->rtt,
1043                           n->hops);
1044
1045         for (x = n->hosts; x; x = x->next)
1046             storeAppendPrintf(sentry, " %s", hashKeyStr(&x->hash));
1047
1048         storeAppendPrintf(sentry, "\n");
1049
1050         p = n->peers;
1051
1052         for (j = 0; j < n->n_peers; j++, p++) {
1053             storeAppendPrintf(sentry, "    %-22.22s %7.1f %5.1f\n",
1054                               p->peername,
1055                               p->rtt,
1056                               p->hops);
1057         }
1058     }
1059
1060     xfree(list);
1061 #else
1062
1063     storeAppendPrintf(sentry,"NETDB support not compiled into this Squid cache.\n");
1064 #endif
1065 }
1066
1067 int
1068 netdbHostHops(const char *host)
1069 {
1070 #if USE_ICMP
1071     netdbEntry *n = netdbLookupHost(host);
1072
1073     if (n) {
1074         n->last_use_time = squid_curtime;
1075         return (int) (n->hops + 0.5);
1076     }
1077
1078 #endif
1079     return 0;
1080 }
1081
1082 int
1083 netdbHostRtt(const char *host)
1084 {
1085 #if USE_ICMP
1086     netdbEntry *n = netdbLookupHost(host);
1087
1088     if (n) {
1089         n->last_use_time = squid_curtime;
1090         return (int) (n->rtt + 0.5);
1091     }
1092
1093 #endif
1094     return 0;
1095 }
1096
1097 void
1098 netdbHostData(const char *host, int *samp, int *rtt, int *hops)
1099 {
1100 #if USE_ICMP
1101     netdbEntry *n = netdbLookupHost(host);
1102
1103     if (n == NULL)
1104         return;
1105
1106     *samp = n->pings_recv;
1107
1108     *rtt = (int) (n->rtt + 0.5);
1109
1110     *hops = (int) (n->hops + 0.5);
1111
1112     n->last_use_time = squid_curtime;
1113
1114 #endif
1115 }
1116
1117 void
1118 netdbUpdatePeer(HttpRequest * r, peer * e, int irtt, int ihops)
1119 {
1120 #if USE_ICMP
1121     netdbEntry *n;
1122     double rtt = (double) irtt;
1123     double hops = (double) ihops;
1124     net_db_peer *p;
1125     debugs(38, 3, "netdbUpdatePeer: '" << r->GetHost() << "', " << ihops << " hops, " << irtt << " rtt");
1126     n = netdbLookupHost(r->GetHost());
1127
1128     if (n == NULL) {
1129         debugs(38, 3, "netdbUpdatePeer: host '" << r->GetHost() << "' not found");
1130         return;
1131     }
1132
1133     if ((p = netdbPeerByName(n, e->host)) == NULL)
1134         p = netdbPeerAdd(n, e);
1135
1136     p->rtt = rtt;
1137
1138     p->hops = hops;
1139
1140     p->expires = squid_curtime + 3600;
1141
1142     if (n->n_peers < 2)
1143         return;
1144
1145     qsort((char *) n->peers,
1146           n->n_peers,
1147           sizeof(net_db_peer),
1148           sortPeerByRtt);
1149
1150 #endif
1151 }
1152
1153 void
1154 netdbExchangeUpdatePeer(IPAddress &addr, peer * e, double rtt, double hops)
1155 {
1156 #if USE_ICMP
1157     netdbEntry *n;
1158     net_db_peer *p;
1159     debugs(38, 5, "netdbExchangeUpdatePeer: '" << addr << "', "<<
1160            std::setfill('0')<< std::setprecision(2) << hops << " hops, " <<
1161            rtt << " rtt");
1162
1163     if ( !addr.IsIPv4() ) {
1164         debugs(38, 5, "netdbExchangeUpdatePeer: Aborting peer update for '" << addr << "', NetDB cannot handle IPv6.");
1165         return;
1166     }
1167
1168     n = netdbLookupAddr(addr);
1169
1170     if (n == NULL)
1171         n = netdbAdd(addr);
1172
1173     assert(NULL != n);
1174
1175     if ((p = netdbPeerByName(n, e->host)) == NULL)
1176         p = netdbPeerAdd(n, e);
1177
1178     p->rtt = rtt;
1179
1180     p->hops = hops;
1181
1182     p->expires = squid_curtime + 3600;  /* XXX ? */
1183
1184     if (n->n_peers < 2)
1185         return;
1186
1187     qsort((char *) n->peers,
1188           n->n_peers,
1189           sizeof(net_db_peer),
1190           sortPeerByRtt);
1191
1192 #endif
1193 }
1194
1195 void
1196 netdbDeleteAddrNetwork(IPAddress &addr)
1197 {
1198 #if USE_ICMP
1199     netdbEntry *n = netdbLookupAddr(addr);
1200
1201     if (n == NULL)
1202         return;
1203
1204     debugs(38, 3, "netdbDeleteAddrNetwork: " << n->network);
1205
1206     netdbRelease(n);
1207 #endif
1208 }
1209
1210
1211 void
1212 netdbBinaryExchange(StoreEntry * s)
1213 {
1214     HttpReply *reply = new HttpReply;
1215 #if USE_ICMP
1216
1217     IPAddress addr;
1218
1219     netdbEntry *n;
1220     int i;
1221     int j;
1222     int rec_sz;
1223     char *buf;
1224
1225     struct in_addr line_addr;
1226     s->buffer();
1227     HttpVersion version(1, 0);
1228     reply->setHeaders(version, HTTP_OK, "OK", NULL, -1, squid_curtime, -2);
1229     s->replaceHttpReply(reply);
1230     rec_sz = 0;
1231     rec_sz += 1 + sizeof(struct in_addr);
1232     rec_sz += 1 + sizeof(int);
1233     rec_sz += 1 + sizeof(int);
1234     buf = (char *)memAllocate(MEM_4K_BUF);
1235     i = 0;
1236     hash_first(addr_table);
1237
1238     while ((n = (netdbEntry *) hash_next(addr_table))) {
1239         if (0.0 == n->rtt)
1240             continue;
1241
1242         if (n->rtt > 60000)     /* RTT > 1 MIN probably bogus */
1243             continue;
1244
1245         if (! (addr = n->network) )
1246             continue;
1247
1248         /* FIXME INET6 : NetDB cannot yet handle IPv6 addresses. Ensure only IPv4 get sent. */
1249         if ( !addr.IsIPv4() )
1250             continue;
1251
1252         buf[i++] = (char) NETDB_EX_NETWORK;
1253
1254         addr.GetInAddr(line_addr);
1255         xmemcpy(&buf[i], &line_addr, sizeof(struct in_addr));
1256
1257         i += sizeof(struct in_addr);
1258
1259         buf[i++] = (char) NETDB_EX_RTT;
1260
1261         j = htonl((int) (n->rtt * 1000));
1262
1263         xmemcpy(&buf[i], &j, sizeof(int));
1264
1265         i += sizeof(int);
1266
1267         buf[i++] = (char) NETDB_EX_HOPS;
1268
1269         j = htonl((int) (n->hops * 1000));
1270
1271         xmemcpy(&buf[i], &j, sizeof(int));
1272
1273         i += sizeof(int);
1274
1275         if (i + rec_sz > 4096) {
1276             s->append(buf, i);
1277             i = 0;
1278         }
1279     }
1280
1281     if (i > 0) {
1282         s->append(buf, i);
1283         i = 0;
1284     }
1285
1286     assert(0 == i);
1287     s->flush();
1288     memFree(buf, MEM_4K_BUF);
1289 #else
1290
1291     HttpVersion version(1,0);
1292     reply->setHeaders(version, HTTP_BAD_REQUEST, "Bad Request",
1293                       NULL, -1, squid_curtime, -2);
1294     s->replaceHttpReply(reply);
1295     storeAppendPrintf(s, "NETDB support not compiled into this Squid cache.\n");
1296 #endif
1297
1298     s->complete();
1299 }
1300
1301 #if USE_ICMP
1302 CBDATA_TYPE(netdbExchangeState);
1303 #endif
1304
1305 void
1306 netdbExchangeStart(void *data)
1307 {
1308 #if USE_ICMP
1309     peer *p = (peer *)data;
1310     char *uri;
1311     netdbExchangeState *ex;
1312     StoreIOBuffer tempBuffer;
1313     CBDATA_INIT_TYPE(netdbExchangeState);
1314     ex = cbdataAlloc(netdbExchangeState);
1315     ex->p = cbdataReference(p);
1316     uri = internalRemoteUri(p->host, p->http_port, "/squid-internal-dynamic/", "netdb");
1317     debugs(38, 3, "netdbExchangeStart: Requesting '" << uri << "'");
1318     assert(NULL != uri);
1319     ex->r = HttpRequest::CreateFromUrl(uri);
1320
1321     if (NULL == ex->r) {
1322         debugs(38, 1, "netdbExchangeStart: Bad URI " << uri);
1323         return;
1324     }
1325
1326     HTTPMSGLOCK(ex->r);
1327     assert(NULL != ex->r);
1328     ex->r->http_ver = HttpVersion(1,0);
1329     ex->connstate = STATE_HEADER;
1330     ex->e = storeCreateEntry(uri, uri, request_flags(), METHOD_GET);
1331     ex->buf_sz = NETDB_REQBUF_SZ;
1332     assert(NULL != ex->e);
1333     ex->sc = storeClientListAdd(ex->e, ex);
1334     tempBuffer.offset = 0;
1335     tempBuffer.length = ex->buf_sz;
1336     tempBuffer.data = ex->buf;
1337     storeClientCopy(ex->sc, ex->e, tempBuffer,
1338                     netdbExchangeHandleReply, ex);
1339     ex->r->flags.loopdetect = 1;        /* cheat! -- force direct */
1340
1341     if (p->login)
1342         xstrncpy(ex->r->login, p->login, MAX_LOGIN_SZ);
1343
1344     FwdState::fwdStart(-1, ex->e, ex->r);
1345
1346 #endif
1347 }
1348
1349 peer *
1350 netdbClosestParent(HttpRequest * request)
1351 {
1352 #if USE_ICMP
1353     peer *p = NULL;
1354     netdbEntry *n;
1355     const ipcache_addrs *ia;
1356     net_db_peer *h;
1357     int i;
1358     n = netdbLookupHost(request->GetHost());
1359
1360     if (NULL == n) {
1361         /* try IP addr */
1362         ia = ipcache_gethostbyname(request->GetHost(), 0);
1363
1364         if (NULL != ia)
1365             n = netdbLookupAddr(ia->in_addrs[ia->cur]);
1366     }
1367
1368     if (NULL == n)
1369         return NULL;
1370
1371     if (0 == n->n_peers)
1372         return NULL;
1373
1374     n->last_use_time = squid_curtime;
1375
1376     /*
1377      * Find the parent with the least RTT to the origin server.
1378      * Make sure we don't return a parent who is farther away than
1379      * we are.  Note, the n->peers list is pre-sorted by RTT.
1380      */
1381     for (i = 0; i < n->n_peers; i++) {
1382         h = &n->peers[i];
1383
1384         if (n->rtt > 0)
1385             if (n->rtt < h->rtt)
1386                 break;
1387
1388         p = peerFindByName(h->peername);
1389
1390         if (NULL == p)          /* not found */
1391             continue;
1392
1393         if (neighborType(p, request) != PEER_PARENT)
1394             continue;
1395
1396         if (!peerHTTPOkay(p, request))  /* not allowed */
1397             continue;
1398
1399         return p;
1400     }
1401
1402 #endif
1403     return NULL;
1404 }