src/net_db.cc

   1 /*
   2  * $Id: net_db.cc,v 1.201 2007/12/29 18:15:45 hno Exp $
   3  *
   4  * DEBUG: section 38    Network Measurement Database
   5  * AUTHOR: Duane Wessels
   6  *
   7  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   8  * ----------------------------------------------------------
   9  *
  10  *  Squid is the result of efforts by numerous individuals from
  11  *  the Internet community; see the CONTRIBUTORS file for full
  12  *  details.   Many organizations have provided support for Squid's
  13  *  development; see the SPONSORS file for full details.  Squid is
  14  *  Copyrighted (C) 2001 by the Regents of the University of
  15  *  California; see the COPYRIGHT file for full details.  Squid
  16  *  incorporates software developed and/or copyrighted by other
  17  *  sources; see the CREDITS file for full details.
  18  *
  19  *  This program is free software; you can redistribute it and/or modify
  20  *  it under the terms of the GNU General Public License as published by
  21  *  the Free Software Foundation; either version 2 of the License, or
  22  *  (at your option) any later version.
  23  *
  24  *  This program is distributed in the hope that it will be useful,
  25  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  26  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  27  *  GNU General Public License for more details.
  28  *
  29  *  You should have received a copy of the GNU General Public License
  30  *  along with this program; if not, write to the Free Software
  31  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  32  *
  33  */
  34
  35 /*
  36  * XXX XXX XXX
  37  *
  38  * This code may be slightly broken now. If you're getting consistent
  39  * (sometimes working) corrupt data exchanges, please contact adrian
  40  * (adrian@squid-cache.org) to sort them out.
  41  */
  42
  43 #include "squid.h"
  44 #include "cbdata.h"
  45 #include "event.h"
  46 #include "CacheManager.h"
  47 #include "Store.h"
  48 #include "SwapDir.h"
  49 #include "HttpRequest.h"
  50 #include "HttpReply.h"
  51 #include "MemObject.h"
  52 #include "fde.h"
  53 #include "forward.h"
  54 #include "SquidTime.h"
  55 #include "wordlist.h"
  56 #include "IPAddress.h"
  57
  58 #if USE_ICMP
  59 #include "ICMPSquid.h"
  60 #include "StoreClient.h"
  61
  62 #define NETDB_REQBUF_SZ 4096
  63
  64 typedef enum {
  65     STATE_NONE,
  66     STATE_HEADER,
  67     STATE_BODY
  68 } netdb_conn_state_t;
  69
  70 typedef struct
  71 {
  72     peer *p;
  73     StoreEntry *e;
  74     store_client *sc;
  75     HttpRequest *r;
  76     off_t used;
  77     size_t buf_sz;
  78     char buf[NETDB_REQBUF_SZ];
  79     int buf_ofs;
  80     netdb_conn_state_t connstate;
  81 }
  82
  83 netdbExchangeState;
  84
  85 static hash_table *addr_table = NULL;
  86 static hash_table *host_table = NULL;
  87
  88 IPAddress networkFromInaddr(const IPAddress &a);
  89 static void netdbRelease(netdbEntry * n);
  90
  91 static void netdbHashInsert(netdbEntry * n, IPAddress &addr);
  92 static void netdbHashDelete(const char *key);
  93 static void netdbHostInsert(netdbEntry * n, const char *hostname);
  94 static void netdbHostDelete(const net_db_name * x);
  95 static void netdbPurgeLRU(void);
  96 static netdbEntry *netdbLookupHost(const char *key);
  97 static net_db_peer *netdbPeerByName(const netdbEntry * n, const char *);
  98 static net_db_peer *netdbPeerAdd(netdbEntry * n, peer * e);
  99 static const char *netdbPeerName(const char *name);
 100 static IPH netdbSendPing;
 101 static QS sortPeerByRtt;
 102 static QS sortByRtt;
 103 static QS netdbLRU;
 104 static FREE netdbFreeNameEntry;
 105 static FREE netdbFreeNetdbEntry;
 106 static STCB netdbExchangeHandleReply;
 107 static void netdbExchangeDone(void *);
 108
 109 /* We have to keep a local list of peer names.  The Peers structure
 110  * gets freed during a reconfigure.  We want this database to
 111  * remain persisitent, so _net_db_peer->peername points into this
 112  * linked list */
 113 static wordlist *peer_names = NULL;
 114
 115 static void
 116 netdbHashInsert(netdbEntry * n, IPAddress &addr)
 117 {
 118     networkFromInaddr(addr).NtoA(n->network, MAX_IPSTRLEN);
 119     n->hash.key = n->network;
 120     assert(hash_lookup(addr_table, n->network) == NULL);
 121     hash_join(addr_table, &n->hash);
 122 }
 123
 124 static void
 125 netdbHashDelete(const char *key)
 126 {
 127     hash_link *hptr = (hash_link *)hash_lookup(addr_table, key);
 128
 129     if (hptr == NULL) {
 130         debug_trap("netdbHashDelete: key not found");
 131         return;
 132     }
 133
 134     hash_remove_link(addr_table, hptr);
 135 }
 136
 137 static void
 138 netdbHostInsert(netdbEntry * n, const char *hostname)
 139 {
 140     net_db_name *x = (net_db_name *)memAllocate(MEM_NET_DB_NAME);
 141     x->hash.key = xstrdup(hostname);
 142     x->next = n->hosts;
 143     n->hosts = x;
 144     x->net_db_entry = n;
 145     assert(hash_lookup(host_table, hostname) == NULL);
 146     hash_join(host_table, &x->hash);
 147     n->link_count++;
 148 }
 149
 150 static void
 151 netdbHostDelete(const net_db_name * x)
 152 {
 153     netdbEntry *n;
 154     net_db_name **X;
 155     assert(x != NULL);
 156     assert(x->net_db_entry != NULL);
 157     n = x->net_db_entry;
 158     n->link_count--;
 159
 160     for (X = &n->hosts; *X; X = &(*X)->next) {
 161         if (*X == x) {
 162             *X = x->next;
 163             break;
 164         }
 165     }
 166
 167     hash_remove_link(host_table, (hash_link *) x);
 168     xfree(x->hash.key);
 169     memFree((void *) x, MEM_NET_DB_NAME);
 170 }
 171
 172 static netdbEntry *
 173 netdbLookupHost(const char *key)
 174 {
 175     net_db_name *x = (net_db_name *) hash_lookup(host_table, key);
 176     return x ? x->net_db_entry : NULL;
 177 }
 178
 179 static void
 180 netdbRelease(netdbEntry * n)
 181 {
 182     net_db_name *x;
 183     net_db_name *next;
 184
 185     for (x = n->hosts; x; x = next) {
 186         next = x->next;
 187         netdbHostDelete(x);
 188     }
 189
 190     n->hosts = NULL;
 191     safe_free(n->peers);
 192     n->peers = NULL;
 193     n->n_peers = 0;
 194     n->n_peers_alloc = 0;
 195
 196     if (n->link_count == 0) {
 197         netdbHashDelete(n->network);
 198         memFree(n, MEM_NETDBENTRY);
 199     }
 200 }
 201
 202 static int
 203 netdbLRU(const void *A, const void *B)
 204 {
 205     const netdbEntry *const *n1 = (const netdbEntry *const *)A;
 206     const netdbEntry *const *n2 = (const netdbEntry *const *)B;
 207
 208     if ((*n1)->last_use_time > (*n2)->last_use_time)
 209         return (1);
 210
 211     if ((*n1)->last_use_time < (*n2)->last_use_time)
 212         return (-1);
 213
 214     return (0);
 215 }
 216
 217 static void
 218 netdbPurgeLRU(void)
 219 {
 220     netdbEntry *n;
 221     netdbEntry **list;
 222     int k = 0;
 223     int list_count = 0;
 224     int removed = 0;
 225     list = (netdbEntry **)xcalloc(memInUse(MEM_NETDBENTRY), sizeof(netdbEntry *));
 226     hash_first(addr_table);
 227
 228     while ((n = (netdbEntry *) hash_next(addr_table))) {
 229         assert(list_count < memInUse(MEM_NETDBENTRY));
 230         *(list + list_count) = n;
 231         list_count++;
 232     }
 233
 234     qsort((char *) list,
 235           list_count,
 236           sizeof(netdbEntry *),
 237           netdbLRU);
 238
 239     for (k = 0; k < list_count; k++) {
 240         if (memInUse(MEM_NETDBENTRY) < Config.Netdb.low)
 241             break;
 242
 243         netdbRelease(*(list + k));
 244
 245         removed++;
 246     }
 247
 248     xfree(list);
 249 }
 250
 251 static netdbEntry *
 252 netdbLookupAddr(const IPAddress &addr)
 253 {
 254     netdbEntry *n;
 255     char *key = new char[MAX_IPSTRLEN];
 256     networkFromInaddr(addr).NtoA(key,MAX_IPSTRLEN);
 257     n = (netdbEntry *) hash_lookup(addr_table, key);
 258     return n;
 259 }
 260
 261 static netdbEntry *
 262 netdbAdd(IPAddress &addr)
 263 {
 264     netdbEntry *n;
 265
 266     if (memInUse(MEM_NETDBENTRY) > Config.Netdb.high)
 267         netdbPurgeLRU();
 268
 269     if ((n = netdbLookupAddr(addr)) == NULL)
 270     {
 271         n = (netdbEntry *)memAllocate(MEM_NETDBENTRY);
 272         netdbHashInsert(n, addr);
 273     }
 274
 275     return n;
 276 }
 277
 278 static void
 279 netdbSendPing(const ipcache_addrs * ia, void *data)
 280 {
 281     IPAddress addr;
 282     char *hostname = NULL;
 283     static_cast<generic_cbdata *>(data)->unwrap(&hostname);
 284     netdbEntry *n;
 285     netdbEntry *na;
 286     net_db_name *x;
 287     net_db_name **X;
 288
 289     if (ia == NULL) {
 290         xfree(hostname);
 291         return;
 292     }
 293
 294     addr = ia->in_addrs[ia->cur];
 295
 296     if ((n = netdbLookupHost(hostname)) == NULL) {
 297         n = netdbAdd(addr);
 298         netdbHostInsert(n, hostname);
 299     } else if ((na = netdbLookupAddr(addr)) != n) {
 300         /*
 301          *hostname moved from 'network n' to 'network na'!
 302          */
 303
 304         if (na == NULL)
 305             na = netdbAdd(addr);
 306
 307         debugs(38, 3, "netdbSendPing: " << hostname << " moved from " << n->network << " to " << na->network);
 308
 309         x = (net_db_name *) hash_lookup(host_table, hostname);
 310
 311         if (x == NULL) {
 312             debugs(38, 1, "netdbSendPing: net_db_name list bug: " << hostname << " not found");
 313             xfree(hostname);
 314             return;
 315         }
 316
 317         /* remove net_db_name from 'network n' linked list */
 318         for (X = &n->hosts; *X; X = &(*X)->next) {
 319             if (*X == x) {
 320                 *X = x->next;
 321                 break;
 322             }
 323         }
 324
 325         n->link_count--;
 326         /* point to 'network na' from host entry */
 327         x->net_db_entry = na;
 328         /* link net_db_name to 'network na' */
 329         x->next = na->hosts;
 330         na->hosts = x;
 331         na->link_count++;
 332         n = na;
 333     }
 334
 335     if (n->next_ping_time <= squid_curtime) {
 336         debugs(38, 3, "netdbSendPing: pinging " << hostname);
 337         icmpEngine.DomainPing(addr, hostname);
 338         n->pings_sent++;
 339         n->next_ping_time = squid_curtime + Config.Netdb.period;
 340         n->last_use_time = squid_curtime;
 341     }
 342
 343     xfree(hostname);
 344 }
 345
 346 IPAddress
 347 networkFromInaddr(const IPAddress &in)
 348 {
 349     IPAddress out;
 350
 351     out = in;
 352 #if USE_IPV6
 353
 354     /* in IPv6 the 'network' should be the routing section. */
 355
 356     if( in.IsIPv6() )
 357     {
 358         out.ApplyMask(64, AF_INET6);
 359         debugs(14, 5, "networkFromInaddr : Masked IPv6 Address to " << in << "/64 routing part.");
 360         return out;
 361     }
 362 #endif
 363
 364 #if USE_CLASSFUL
 365     struct in_addr b;
 366
 367     in.GetInAddr(b);
 368
 369     if (IN_CLASSC(b.s_addr))
 370         b.s_addr &= IN_CLASSC_NET;
 371     else if (IN_CLASSB(b.s_addr))
 372         b.s_addr &= IN_CLASSB_NET;
 373     else if (IN_CLASSA(b.s_addr))
 374         b.s_addr &= IN_CLASSA_NET;
 375
 376     out = b;
 377
 378 #endif
 379
 380     debugs(14, 5, "networkFromInaddr : Masked IPv4 Address to " << out << "/24.");
 381
 382     /* use /24 for everything under IPv4 */
 383     out.ApplyMask(24, AF_INET);
 384     debugs(14, 5, "networkFromInaddr : Masked IPv4 Address to " << in << "/24.");
 385
 386     return out;
 387 }
 388
 389 static int
 390 sortByRtt(const void *A, const void *B)
 391 {
 392     const netdbEntry *const *n1 = (const netdbEntry *const *)A;
 393     const netdbEntry *const *n2 = (const netdbEntry *const *)B;
 394
 395     if ((*n1)->rtt > (*n2)->rtt)
 396         return 1;
 397     else if ((*n1)->rtt < (*n2)->rtt)
 398         return -1;
 399     else
 400         return 0;
 401 }
 402
 403 static net_db_peer *
 404 netdbPeerByName(const netdbEntry * n, const char *peername)
 405 {
 406     int i;
 407     net_db_peer *p = n->peers;
 408
 409     for (i = 0; i < n->n_peers; i++, p++) {
 410         if (!strcmp(p->peername, peername))
 411             return p;
 412     }
 413
 414     return NULL;
 415 }
 416
 417 static net_db_peer *
 418 netdbPeerAdd(netdbEntry * n, peer * e)
 419 {
 420     net_db_peer *p;
 421     net_db_peer *o;
 422     int osize;
 423     int i;
 424
 425     if (n->n_peers == n->n_peers_alloc) {
 426         o = n->peers;
 427         osize = n->n_peers_alloc;
 428
 429         if (n->n_peers_alloc == 0)
 430             n->n_peers_alloc = 2;
 431         else
 432             n->n_peers_alloc <<= 1;
 433
 434         debugs(38, 3, "netdbPeerAdd: Growing peer list for '" << n->network << "' to " << n->n_peers_alloc);
 435
 436         n->peers = (net_db_peer *)xcalloc(n->n_peers_alloc, sizeof(net_db_peer));
 437
 438         for (i = 0; i < osize; i++)
 439             *(n->peers + i) = *(o + i);
 440
 441         if (osize) {
 442             safe_free(o);
 443         }
 444     }
 445
 446     p = n->peers + n->n_peers;
 447     p->peername = netdbPeerName(e->host);
 448     n->n_peers++;
 449     return p;
 450 }
 451
 452 static int
 453 sortPeerByRtt(const void *A, const void *B)
 454 {
 455     const net_db_peer *p1 = (net_db_peer *)A;
 456     const net_db_peer *p2 = (net_db_peer *)B;
 457
 458     if (p1->rtt > p2->rtt)
 459         return 1;
 460     else if (p1->rtt < p2->rtt)
 461         return -1;
 462     else
 463         return 0;
 464 }
 465
 466 static void
 467 netdbSaveState(void *foo)
 468 {
 469     if (strcmp(Config.netdbFilename, "none") == 0)
 470         return;
 471
 472     Logfile *lf;
 473     netdbEntry *n;
 474     net_db_name *x;
 475
 476     struct timeval start = current_time;
 477     int count = 0;
 478     /*
 479      * This was nicer when we were using stdio, but thanks to
 480      * Solaris bugs, its a bad idea.  fopen can fail if more than
 481      * 256 FDs are open.
 482      */
 483     /*
 484      * unlink() is here because there is currently no way to make
 485      * logfileOpen() use O_TRUNC.
 486      */
 487     unlink(Config.netdbFilename);
 488     lf = logfileOpen(Config.netdbFilename, 4096, 0);
 489
 490     if (NULL == lf) {
 491         debugs(50, 1, "netdbSaveState: " << Config.netdbFilename << ": " << xstrerror());
 492         return;
 493     }
 494
 495     hash_first(addr_table);
 496
 497     while ((n = (netdbEntry *) hash_next(addr_table))) {
 498         if (n->pings_recv == 0)
 499             continue;
 500
 501         logfilePrintf(lf, "%s %d %d %10.5f %10.5f %d %d",
 502                       n->network,
 503                       n->pings_sent,
 504                       n->pings_recv,
 505                       n->hops,
 506                       n->rtt,
 507                       (int) n->next_ping_time,
 508                       (int) n->last_use_time);
 509
 510         for (x = n->hosts; x; x = x->next)
 511             logfilePrintf(lf, " %s", hashKeyStr(&x->hash));
 512
 513         logfilePrintf(lf, "\n");
 514
 515         count++;
 516
 517 #undef RBUF_SZ
 518
 519     }
 520
 521     logfileClose(lf);
 522     getCurrentTime();
 523     debugs(38, 1, "NETDB state saved; " <<
 524                  count << " entries, " <<
 525                  tvSubMsec(start, current_time) << " msec" );
 526     eventAddIsh("netdbSaveState", netdbSaveState, NULL, 3600.0, 1);
 527 }
 528
 529 static void
 530 netdbReloadState(void)
 531 {
 532     if (strcmp(Config.netdbFilename, "none") == 0)
 533         return;
 534
 535     char *s;
 536     int fd;
 537     int l;
 538
 539     struct stat sb;
 540     netdbEntry *n;
 541     netdbEntry N;
 542
 543     IPAddress addr;
 544     int count = 0;
 545
 546     struct timeval start = current_time;
 547     /*
 548      * This was nicer when we were using stdio, but thanks to
 549      * Solaris bugs, its a bad idea.  fopen can fail if more than
 550      * 256 FDs are open.
 551      */
 552     fd = file_open(Config.netdbFilename, O_RDONLY | O_BINARY);
 553
 554     if (fd < 0)
 555         return;
 556
 557     if (fstat(fd, &sb) < 0) {
 558         file_close(fd);
 559         return;
 560     }
 561
 562     char *t;
 563     char *buf = (char *)xcalloc(1, sb.st_size + 1);
 564     t = buf;
 565     l = FD_READ_METHOD(fd, buf, sb.st_size);
 566     file_close(fd);
 567
 568     if (l <= 0) {
 569         safe_free (buf);
 570         return;
 571     };
 572
 573     while ((s = strchr(t, '\n'))) {
 574         char *q;
 575         assert(s - buf < l);
 576         *s = '\0';
 577         memset(&N, '\0', sizeof(netdbEntry));
 578         q = strtok(t, w_space);
 579         t = s + 1;
 580
 581         if (NULL == q)
 582             continue;
 583
 584         if (! (addr = q) )
 585             continue;
 586
 587         if (netdbLookupAddr(addr) != NULL)      /* no dups! */
 588             continue;
 589
 590         if ((q = strtok(NULL, w_space)) == NULL)
 591             continue;
 592
 593         N.pings_sent = atoi(q);
 594
 595         if ((q = strtok(NULL, w_space)) == NULL)
 596             continue;
 597
 598         N.pings_recv = atoi(q);
 599
 600         if (N.pings_recv == 0)
 601             continue;
 602
 603         /* give this measurement low weight */
 604         N.pings_sent = 1;
 605
 606         N.pings_recv = 1;
 607
 608         if ((q = strtok(NULL, w_space)) == NULL)
 609             continue;
 610
 611         N.hops = atof(q);
 612
 613         if ((q = strtok(NULL, w_space)) == NULL)
 614             continue;
 615
 616         N.rtt = atof(q);
 617
 618         if ((q = strtok(NULL, w_space)) == NULL)
 619             continue;
 620
 621         N.next_ping_time = (time_t) atoi(q);
 622
 623         if ((q = strtok(NULL, w_space)) == NULL)
 624             continue;
 625
 626         N.last_use_time = (time_t) atoi(q);
 627
 628         n = (netdbEntry *)memAllocate(MEM_NETDBENTRY);
 629
 630         xmemcpy(n, &N, sizeof(netdbEntry));
 631
 632         netdbHashInsert(n, addr);
 633
 634         while ((q = strtok(NULL, w_space)) != NULL) {
 635             if (netdbLookupHost(q) != NULL)     /* no dups! */
 636                 continue;
 637
 638             netdbHostInsert(n, q);
 639         }
 640
 641         count++;
 642     }
 643
 644     xfree(buf);
 645     getCurrentTime();
 646     debugs(38, 1, "NETDB state reloaded; " <<
 647                   count << " entries, " <<
 648                   tvSubMsec(start, current_time) << " msec" );
 649 }
 650
 651 static const char *
 652 netdbPeerName(const char *name)
 653 {
 654     const wordlist *w;
 655
 656     for (w = peer_names; w; w = w->next) {
 657         if (!strcmp(w->key, name))
 658             return w->key;
 659     }
 660
 661     return wordlistAdd(&peer_names, name);
 662 }
 663
 664 static void
 665 netdbFreeNetdbEntry(void *data)
 666 {
 667     netdbEntry *n = (netdbEntry *)data;
 668     safe_free(n->peers);
 669     memFree(n, MEM_NETDBENTRY);
 670 }
 671
 672 static void
 673 netdbFreeNameEntry(void *data)
 674 {
 675     net_db_name *x = (net_db_name *)data;
 676     xfree(x->hash.key);
 677     memFree(x, MEM_NET_DB_NAME);
 678 }
 679
 680
 681 static void
 682 netdbExchangeHandleReply(void *data, StoreIOBuffer receivedData)
 683 {
 684     IPAddress addr;
 685
 686     netdbExchangeState *ex = (netdbExchangeState *)data;
 687     int rec_sz = 0;
 688     off_t o;
 689
 690     struct in_addr line_addr;
 691     double rtt;
 692     double hops;
 693     char *p;
 694     int j;
 695     HttpReply const *rep;
 696     size_t hdr_sz;
 697     int nused = 0;
 698     int size;
 699     int oldbufofs = ex->buf_ofs;
 700
 701     rec_sz = 0;
 702     rec_sz += 1 + sizeof(struct in_addr);
 703     rec_sz += 1 + sizeof(int);
 704     rec_sz += 1 + sizeof(int);
 705     debugs(38, 3, "netdbExchangeHandleReply: " << receivedData.length << " read bytes");
 706
 707     if (!cbdataReferenceValid(ex->p)) {
 708         debugs(38, 3, "netdbExchangeHandleReply: Peer became invalid");
 709         netdbExchangeDone(ex);
 710         return;
 711     }
 712
 713     debugs(38, 3, "netdbExchangeHandleReply: for '" << ex->p->host << ":" << ex->p->http_port << "'");
 714
 715     if (receivedData.length == 0 &&
 716             !receivedData.flags.error) {
 717         debugs(38, 3, "netdbExchangeHandleReply: Done");
 718         netdbExchangeDone(ex);
 719         return;
 720     }
 721
 722     p = ex->buf;
 723
 724     /* Get the size of the buffer now */
 725     size = ex->buf_ofs + receivedData.length;
 726     debugs(38, 3, "netdbExchangeHandleReply: " << size << " bytes buf");
 727
 728     /* Check if we're still doing headers */
 729
 730     if (ex->connstate == STATE_HEADER) {
 731
 732         ex->buf_ofs += receivedData.length;
 733
 734         /* skip reply headers */
 735
 736         if ((hdr_sz = headersEnd(p, ex->buf_ofs))) {
 737             debugs(38, 5, "netdbExchangeHandleReply: hdr_sz = " << hdr_sz);
 738             rep = ex->e->getReply();
 739             assert (0 != rep->sline.status);
 740             debugs(38, 3, "netdbExchangeHandleReply: reply status " << rep->sline.status);
 741
 742             if (HTTP_OK != rep->sline.status) {
 743                 netdbExchangeDone(ex);
 744                 return;
 745             }
 746
 747             assert((size_t)ex->buf_ofs >= hdr_sz);
 748
 749             /*
 750              * Now, point p to the part of the buffer where the data
 751              * starts, and update the size accordingly
 752              */
 753             assert(ex->used == 0);
 754             ex->used = hdr_sz;
 755             size = ex->buf_ofs - hdr_sz;
 756             p += hdr_sz;
 757
 758             /* Finally, set the conn state mode to STATE_BODY */
 759             ex->connstate = STATE_BODY;
 760         } else {
 761             StoreIOBuffer tempBuffer;
 762             tempBuffer.offset = ex->buf_ofs;
 763             tempBuffer.length = ex->buf_sz - ex->buf_ofs;
 764             tempBuffer.data = ex->buf + ex->buf_ofs;
 765             /* Have more headers .. */
 766             storeClientCopy(ex->sc, ex->e, tempBuffer,
 767                             netdbExchangeHandleReply, ex);
 768             return;
 769         }
 770     }
 771
 772     assert(ex->connstate == STATE_BODY);
 773
 774     /* If we get here, we have some body to parse .. */
 775     debugs(38, 5, "netdbExchangeHandleReply: start parsing loop, size = " << size);
 776
 777     while (size >= rec_sz) {
 778         debugs(38, 5, "netdbExchangeHandleReply: in parsing loop, size = " << size);
 779         addr.SetAnyAddr();
 780         hops = rtt = 0.0;
 781
 782         for (o = 0; o < rec_sz;) {
 783             switch ((int) *(p + o)) {
 784
 785             case NETDB_EX_NETWORK:
 786                 o++;
 787                 /* FIXME INET6 : NetDB can still ony send IPv4 */
 788                 xmemcpy(&line_addr, p + o, sizeof(struct in_addr));
 789                 addr = line_addr;
 790                 o += sizeof(struct in_addr);
 791                 break;
 792
 793             case NETDB_EX_RTT:
 794                 o++;
 795                 xmemcpy(&j, p + o, sizeof(int));
 796                 o += sizeof(int);
 797                 rtt = (double) ntohl(j) / 1000.0;
 798                 break;
 799
 800             case NETDB_EX_HOPS:
 801                 o++;
 802                 xmemcpy(&j, p + o, sizeof(int));
 803                 o += sizeof(int);
 804                 hops = (double) ntohl(j) / 1000.0;
 805                 break;
 806
 807             default:
 808                 debugs(38, 1, "netdbExchangeHandleReply: corrupt data, aborting");
 809                 netdbExchangeDone(ex);
 810                 return;
 811             }
 812         }
 813
 814         if (!addr.IsAnyAddr() && rtt > 0)
 815             netdbExchangeUpdatePeer(addr, ex->p, rtt, hops);
 816
 817         assert(o == rec_sz);
 818
 819         ex->used += rec_sz;
 820
 821         size -= rec_sz;
 822
 823         p += rec_sz;
 824
 825         nused++;
 826     }
 827
 828     /*
 829      * Copy anything that is left over to the beginning of the buffer,
 830      * and adjust buf_ofs accordingly
 831      */
 832
 833     /*
 834      * Evilly, size refers to the buf size left now,
 835      * ex->buf_ofs is the original buffer size, so just copy that
 836      * much data over
 837      */
 838     memmove(ex->buf, ex->buf + (ex->buf_ofs - size), size);
 839
 840     ex->buf_ofs = size;
 841
 842     /*
 843      * And don't re-copy the remaining data ..
 844      */
 845     ex->used += size;
 846
 847     /*
 848      * Now the tricky bit - size _included_ the leftover bit from the _last_
 849      * storeClientCopy. We don't want to include that, or our offset will be wrong.
 850      * So, don't count the size of the leftover buffer we began with.
 851      * This can _disappear_ when we're not tracking offsets ..
 852      */
 853     ex->used -= oldbufofs;
 854
 855     debugs(38, 3, "netdbExchangeHandleReply: size left over in this buffer: " << size << " bytes");
 856
 857     debugs(38, 3, "netdbExchangeHandleReply: used " << nused <<
 858            " entries, (x " << rec_sz << " bytes) == " << nused * rec_sz <<
 859            " bytes total");
 860
 861     debugs(38, 3, "netdbExchangeHandleReply: used " << ex->used);
 862
 863     if (EBIT_TEST(ex->e->flags, ENTRY_ABORTED)) {
 864         debugs(38, 3, "netdbExchangeHandleReply: ENTRY_ABORTED");
 865         netdbExchangeDone(ex);
 866     } else if (ex->e->store_status == STORE_PENDING) {
 867         StoreIOBuffer tempBuffer;
 868         tempBuffer.offset = ex->used;
 869         tempBuffer.length = ex->buf_sz - ex->buf_ofs;
 870         tempBuffer.data = ex->buf + ex->buf_ofs;
 871         debugs(38, 3, "netdbExchangeHandleReply: EOF not received");
 872         storeClientCopy(ex->sc, ex->e, tempBuffer,
 873                         netdbExchangeHandleReply, ex);
 874     }
 875 }
 876
 877 static void
 878 netdbExchangeDone(void *data)
 879 {
 880     netdbExchangeState *ex = (netdbExchangeState *)data;
 881     debugs(38, 3, "netdbExchangeDone: " << ex->e->url()  );
 882     HTTPMSGUNLOCK(ex->r);
 883     storeUnregister(ex->sc, ex->e, ex);
 884     ex->e->unlock();
 885     cbdataReferenceDone(ex->p);
 886     cbdataFree(ex);
 887 }
 888
 889 #endif /* USE_ICMP */
 890
 891 /* PUBLIC FUNCTIONS */
 892
 893 void
 894 netdbInit(void)
 895 {
 896 #if USE_ICMP
 897     int n;
 898
 899     if (addr_table)
 900         return;
 901
 902     n = hashPrime(Config.Netdb.high / 4);
 903
 904     addr_table = hash_create((HASHCMP *) strcmp, n, hash_string);
 905
 906     n = hashPrime(3 * Config.Netdb.high / 4);
 907
 908     host_table = hash_create((HASHCMP *) strcmp, n, hash_string);
 909
 910     eventAddIsh("netdbSaveState", netdbSaveState, NULL, 3600.0, 1);
 911
 912     netdbReloadState();
 913
 914 #endif
 915 }
 916
 917 void
 918 netdbRegisterWitHCacheManager(CacheManager & manager)
 919 {
 920 #if USE_ICMP
 921     manager.registerAction("netdb",
 922                            "Network Measurement Database",
 923                            netdbDump, 0, 1);
 924
 925 #endif
 926 }
 927
 928 void
 929 netdbPingSite(const char *hostname)
 930 {
 931 #if USE_ICMP
 932     netdbEntry *n;
 933
 934     if ((n = netdbLookupHost(hostname)) != NULL)
 935         if (n->next_ping_time > squid_curtime)
 936             return;
 937
 938     ipcache_nbgethostbyname(hostname, netdbSendPing,
 939                             new generic_cbdata(xstrdup(hostname)));
 940
 941 #endif
 942 }
 943
 944 void
 945 netdbHandlePingReply(const IPAddress &from, int hops, int rtt)
 946 {
 947 #if USE_ICMP
 948     netdbEntry *n;
 949     int N;
 950     debugs(38, 3, "netdbHandlePingReply: from " << from);
 951
 952     if ((n = netdbLookupAddr(from)) == NULL)
 953         return;
 954
 955     N = ++n->pings_recv;
 956
 957     if (N > 5)
 958         N = 5;
 959
 960     if (rtt < 1)
 961         rtt = 1;
 962
 963     n->hops = ((n->hops * (N - 1)) + hops) / N;
 964
 965     n->rtt = ((n->rtt * (N - 1)) + rtt) / N;
 966
 967     debugs(38, 3, "netdbHandlePingReply: " << n->network  << "; rtt="<<
 968            std::setw(5)<< std::setprecision(2) << n->rtt << "  hops="<<
 969            std::setw(4) << n->hops);
 970
 971 #endif
 972 }
 973
 974 void
 975 netdbFreeMemory(void)
 976 {
 977 #if USE_ICMP
 978     hashFreeItems(addr_table, netdbFreeNetdbEntry);
 979     hashFreeMemory(addr_table);
 980     addr_table = NULL;
 981     hashFreeItems(host_table, netdbFreeNameEntry);
 982     hashFreeMemory(host_table);
 983     host_table = NULL;
 984     wordlistDestroy(&peer_names);
 985     peer_names = NULL;
 986 #endif
 987 }
 988
 989
 990 #if 0 // AYJ: Looks to be unused code.
 991 int
 992 netdbHops(IPAddress &addr)
 993 {
 994 #if USE_ICMP
 995     netdbEntry *n = netdbLookupAddr(addr);
 996
 997     if (n && n->pings_recv)
 998     {
 999         n->last_use_time = squid_curtime;
1000         return (int) (n->hops + 0.5);
1001     }
1002
1003 #endif
1004     return 256;
1005 }
1006 #endif
1007
1008 void
1009 netdbDump(StoreEntry * sentry)
1010 {
1011 #if USE_ICMP
1012     netdbEntry *n;
1013     netdbEntry **list;
1014     net_db_name *x;
1015     int k;
1016     int i;
1017     int j;
1018     net_db_peer *p;
1019     storeAppendPrintf(sentry, "Network DB Statistics:\n");
1020     storeAppendPrintf(sentry, "%-46.46s %9s %7s %5s %s\n",  /* Max between 16 (IPv4) or 46 (IPv6)   */
1021                       "Network",
1022                       "recv/sent",
1023                       "RTT",
1024                       "Hops",
1025                       "Hostnames");
1026     list = (netdbEntry **)xcalloc(memInUse(MEM_NETDBENTRY), sizeof(netdbEntry *));
1027     i = 0;
1028     hash_first(addr_table);
1029
1030     while ((n = (netdbEntry *) hash_next(addr_table)))
1031         *(list + i++) = n;
1032
1033     if (i != memInUse(MEM_NETDBENTRY))
1034         debugs(38, 0, "WARNING: netdb_addrs count off, found " << i <<
1035                ", expected " << memInUse(MEM_NETDBENTRY));
1036
1037     qsort((char *) list,
1038           i,
1039           sizeof(netdbEntry *),
1040           sortByRtt);
1041
1042     for (k = 0; k < i; k++) {
1043         n = *(list + k);
1044         storeAppendPrintf(sentry, "%-46.46s %4d/%4d %7.1f %5.1f", /* Max between 16 (IPv4) or 46 (IPv6)   */
1045                           n->network,
1046                           n->pings_recv,
1047                           n->pings_sent,
1048                           n->rtt,
1049                           n->hops);
1050
1051         for (x = n->hosts; x; x = x->next)
1052             storeAppendPrintf(sentry, " %s", hashKeyStr(&x->hash));
1053
1054         storeAppendPrintf(sentry, "\n");
1055
1056         p = n->peers;
1057
1058         for (j = 0; j < n->n_peers; j++, p++) {
1059             storeAppendPrintf(sentry, "    %-22.22s %7.1f %5.1f\n",
1060                               p->peername,
1061                               p->rtt,
1062                               p->hops);
1063         }
1064     }
1065
1066     xfree(list);
1067 #else
1068
1069     storeAppendPrintf(sentry,
1070                       "NETDB support not compiled into this Squid cache.\n");
1071 #endif
1072 }
1073
1074 int
1075 netdbHostHops(const char *host)
1076 {
1077 #if USE_ICMP
1078     netdbEntry *n = netdbLookupHost(host);
1079
1080     if (n) {
1081         n->last_use_time = squid_curtime;
1082         return (int) (n->hops + 0.5);
1083     }
1084
1085 #endif
1086     return 0;
1087 }
1088
1089 int
1090 netdbHostRtt(const char *host)
1091 {
1092 #if USE_ICMP
1093     netdbEntry *n = netdbLookupHost(host);
1094
1095     if (n) {
1096         n->last_use_time = squid_curtime;
1097         return (int) (n->rtt + 0.5);
1098     }
1099
1100 #endif
1101     return 0;
1102 }
1103
1104 void
1105 netdbHostData(const char *host, int *samp, int *rtt, int *hops)
1106 {
1107 #if USE_ICMP
1108     netdbEntry *n = netdbLookupHost(host);
1109
1110     if (n == NULL)
1111         return;
1112
1113     *samp = n->pings_recv;
1114
1115     *rtt = (int) (n->rtt + 0.5);
1116
1117     *hops = (int) (n->hops + 0.5);
1118
1119     n->last_use_time = squid_curtime;
1120
1121 #endif
1122 }
1123
1124 void
1125 netdbUpdatePeer(HttpRequest * r, peer * e, int irtt, int ihops)
1126 {
1127 #if USE_ICMP
1128     netdbEntry *n;
1129     double rtt = (double) irtt;
1130     double hops = (double) ihops;
1131     net_db_peer *p;
1132     debugs(38, 3, "netdbUpdatePeer: '" << r->GetHost() << "', " << ihops << " hops, " << irtt << " rtt");
1133     n = netdbLookupHost(r->GetHost());
1134
1135     if (n == NULL) {
1136         debugs(38, 3, "netdbUpdatePeer: host '" << r->GetHost() << "' not found");
1137         return;
1138     }
1139
1140     if ((p = netdbPeerByName(n, e->host)) == NULL)
1141         p = netdbPeerAdd(n, e);
1142
1143     p->rtt = rtt;
1144
1145     p->hops = hops;
1146
1147     p->expires = squid_curtime + 3600;
1148
1149     if (n->n_peers < 2)
1150         return;
1151
1152     qsort((char *) n->peers,
1153           n->n_peers,
1154           sizeof(net_db_peer),
1155           sortPeerByRtt);
1156
1157 #endif
1158 }
1159
1160 void
1161 netdbExchangeUpdatePeer(IPAddress &addr, peer * e, double rtt, double hops)
1162 {
1163 #if USE_ICMP
1164     netdbEntry *n;
1165     net_db_peer *p;
1166     debugs(38, 5, "netdbExchangeUpdatePeer: '" << addr << "', "<<
1167            std::setfill('0')<< std::setprecision(2) << hops << " hops, " <<
1168            rtt << " rtt");
1169
1170     if( !addr.IsIPv4() ) {
1171         debugs(38, 5, "netdbExchangeUpdatePeer: Aborting peer update for '" << addr << "', NetDB cannot handle IPv6.");
1172         return;
1173     }
1174
1175     n = netdbLookupAddr(addr);
1176
1177     if (n == NULL)
1178         n = netdbAdd(addr);
1179
1180     assert(NULL != n);
1181
1182     if ((p = netdbPeerByName(n, e->host)) == NULL)
1183         p = netdbPeerAdd(n, e);
1184
1185     p->rtt = rtt;
1186
1187     p->hops = hops;
1188
1189     p->expires = squid_curtime + 3600;  /* XXX ? */
1190
1191     if (n->n_peers < 2)
1192         return;
1193
1194     qsort((char *) n->peers,
1195           n->n_peers,
1196           sizeof(net_db_peer),
1197           sortPeerByRtt);
1198
1199 #endif
1200 }
1201
1202 void
1203
1204 netdbDeleteAddrNetwork(IPAddress &addr)
1205 {
1206 #if USE_ICMP
1207     netdbEntry *n = netdbLookupAddr(addr);
1208
1209     if (n == NULL)
1210         return;
1211
1212     debugs(38, 3, "netdbDeleteAddrNetwork: " << n->network);
1213
1214     netdbRelease(n);
1215
1216 #endif
1217 }
1218
1219 void
1220 netdbBinaryExchange(StoreEntry * s)
1221 {
1222     HttpReply *reply = new HttpReply;
1223 #if USE_ICMP
1224
1225     IPAddress addr;
1226
1227     netdbEntry *n;
1228     int i;
1229     int j;
1230     int rec_sz;
1231     char *buf;
1232
1233     struct in_addr line_addr;
1234     s->buffer();
1235     HttpVersion version(1, 0);
1236     reply->setHeaders(version, HTTP_OK, "OK", NULL, -1, squid_curtime, -2);
1237     s->replaceHttpReply(reply);
1238     rec_sz = 0;
1239     rec_sz += 1 + sizeof(struct in_addr);
1240     rec_sz += 1 + sizeof(int);
1241     rec_sz += 1 + sizeof(int);
1242     buf = (char *)memAllocate(MEM_4K_BUF);
1243     i = 0;
1244     hash_first(addr_table);
1245
1246     while ((n = (netdbEntry *) hash_next(addr_table))) {
1247         if (0.0 == n->rtt)
1248             continue;
1249
1250         if (n->rtt > 60000)     /* RTT > 1 MIN probably bogus */
1251             continue;
1252
1253         if (! (addr = n->network) )
1254             continue;
1255
1256         /* FIXME INET6 : NetDB cannot yet handle IPv6 addresses. Ensure only IPv4 get sent. */
1257         if( !addr.IsIPv4() )
1258             continue;
1259
1260         buf[i++] = (char) NETDB_EX_NETWORK;
1261
1262         addr.GetInAddr(line_addr);
1263         xmemcpy(&buf[i], &line_addr, sizeof(struct in_addr));
1264
1265         i += sizeof(struct in_addr);
1266
1267         buf[i++] = (char) NETDB_EX_RTT;
1268
1269         j = htonl((int) (n->rtt * 1000));
1270
1271         xmemcpy(&buf[i], &j, sizeof(int));
1272
1273         i += sizeof(int);
1274
1275         buf[i++] = (char) NETDB_EX_HOPS;
1276
1277         j = htonl((int) (n->hops * 1000));
1278
1279         xmemcpy(&buf[i], &j, sizeof(int));
1280
1281         i += sizeof(int);
1282
1283         if (i + rec_sz > 4096) {
1284             s->append(buf, i);
1285             i = 0;
1286         }
1287     }
1288
1289     if (i > 0) {
1290         s->append(buf, i);
1291         i = 0;
1292     }
1293
1294     assert(0 == i);
1295     s->flush();
1296     memFree(buf, MEM_4K_BUF);
1297 #else
1298
1299     HttpVersion version(1,0);
1300     reply->setHeaders(version, HTTP_BAD_REQUEST, "Bad Request",
1301                       NULL, -1, squid_curtime, -2);
1302     s->replaceHttpReply(reply);
1303     storeAppendPrintf(s, "NETDB support not compiled into this Squid cache.\n");
1304 #endif
1305
1306     s->complete();
1307 }
1308
1309 #if USE_ICMP
1310 CBDATA_TYPE(netdbExchangeState);
1311 #endif
1312
1313 void
1314 netdbExchangeStart(void *data)
1315 {
1316 #if USE_ICMP
1317     peer *p = (peer *)data;
1318     char *uri;
1319     netdbExchangeState *ex;
1320     StoreIOBuffer tempBuffer;
1321     CBDATA_INIT_TYPE(netdbExchangeState);
1322     ex = cbdataAlloc(netdbExchangeState);
1323     ex->p = cbdataReference(p);
1324     uri = internalRemoteUri(p->host, p->http_port, "/squid-internal-dynamic/", "netdb");
1325     debugs(38, 3, "netdbExchangeStart: Requesting '" << uri << "'");
1326     assert(NULL != uri);
1327     ex->r = HttpRequest::CreateFromUrl(uri);
1328
1329     if (NULL == ex->r) {
1330         debugs(38, 1, "netdbExchangeStart: Bad URI " << uri);
1331         return;
1332     }
1333
1334     HTTPMSGLOCK(ex->r);
1335     assert(NULL != ex->r);
1336     ex->r->http_ver = HttpVersion(1,0);
1337     ex->connstate = STATE_HEADER;
1338     ex->e = storeCreateEntry(uri, uri, request_flags(), METHOD_GET);
1339     ex->buf_sz = NETDB_REQBUF_SZ;
1340     assert(NULL != ex->e);
1341     ex->sc = storeClientListAdd(ex->e, ex);
1342     tempBuffer.offset = 0;
1343     tempBuffer.length = ex->buf_sz;
1344     tempBuffer.data = ex->buf;
1345     storeClientCopy(ex->sc, ex->e, tempBuffer,
1346                     netdbExchangeHandleReply, ex);
1347     ex->r->flags.loopdetect = 1;        /* cheat! -- force direct */
1348
1349     if (p->login)
1350         xstrncpy(ex->r->login, p->login, MAX_LOGIN_SZ);
1351
1352     FwdState::fwdStart(-1, ex->e, ex->r);
1353
1354 #endif
1355 }
1356
1357 peer *
1358 netdbClosestParent(HttpRequest * request)
1359 {
1360 #if USE_ICMP
1361     peer *p = NULL;
1362     netdbEntry *n;
1363     const ipcache_addrs *ia;
1364     net_db_peer *h;
1365     int i;
1366     n = netdbLookupHost(request->GetHost());
1367
1368     if (NULL == n) {
1369         /* try IP addr */
1370         ia = ipcache_gethostbyname(request->GetHost(), 0);
1371
1372         if (NULL != ia)
1373             n = netdbLookupAddr(ia->in_addrs[ia->cur]);
1374     }
1375
1376     if (NULL == n)
1377         return NULL;
1378
1379     if (0 == n->n_peers)
1380         return NULL;
1381
1382     n->last_use_time = squid_curtime;
1383
1384     /*
1385      * Find the parent with the least RTT to the origin server.
1386      * Make sure we don't return a parent who is farther away than
1387      * we are.  Note, the n->peers list is pre-sorted by RTT.
1388      */
1389     for (i = 0; i < n->n_peers; i++) {
1390         h = &n->peers[i];
1391
1392         if (n->rtt > 0)
1393             if (n->rtt < h->rtt)
1394                 break;
1395
1396         p = peerFindByName(h->peername);
1397
1398         if (NULL == p)          /* not found */
1399             continue;
1400
1401         if (neighborType(p, request) != PEER_PARENT)
1402             continue;
1403
1404         if (!peerHTTPOkay(p, request))  /* not allowed */
1405             continue;
1406
1407         return p;
1408     }
1409
1410 #endif
1411     return NULL;
1412 }