src/icmp/net_db.cc

   1 /*
   2  * DEBUG: section 38    Network Measurement Database
   3  * AUTHOR: Duane Wessels
   4  *
   5  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   6  * ----------------------------------------------------------
   7  *
   8  *  Squid is the result of efforts by numerous individuals from
   9  *  the Internet community; see the CONTRIBUTORS file for full
  10  *  details.   Many organizations have provided support for Squid's
  11  *  development; see the SPONSORS file for full details.  Squid is
  12  *  Copyrighted (C) 2001 by the Regents of the University of
  13  *  California; see the COPYRIGHT file for full details.  Squid
  14  *  incorporates software developed and/or copyrighted by other
  15  *  sources; see the CREDITS file for full details.
  16  *
  17  *  This program is free software; you can redistribute it and/or modify
  18  *  it under the terms of the GNU General Public License as published by
  19  *  the Free Software Foundation; either version 2 of the License, or
  20  *  (at your option) any later version.
  21  *
  22  *  This program is distributed in the hope that it will be useful,
  23  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  24  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  25  *  GNU General Public License for more details.
  26  *
  27  *  You should have received a copy of the GNU General Public License
  28  *  along with this program; if not, write to the Free Software
  29  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  30  *
  31  */
  32
  33 /*
  34  * XXX XXX XXX
  35  *
  36  * This code may be slightly broken now. If you're getting consistent
  37  * (sometimes working) corrupt data exchanges, please contact adrian
  38  * (adrian@squid-cache.org) to sort them out.
  39  */
  40
  41 #include "squid.h"
  42 #include "cbdata.h"
  43 #include "disk.h"
  44 #include "event.h"
  45 #include "fde.h"
  46 #include "forward.h"
  47 #include "HttpReply.h"
  48 #include "HttpRequest.h"
  49 #include "icmp/net_db.h"
  50 #include "internal.h"
  51 #include "ip/Address.h"
  52 #include "log/File.h"
  53 #include "Mem.h"
  54 #include "MemObject.h"
  55 #include "mgr/Registration.h"
  56 #include "mime_header.h"
  57 #include "neighbors.h"
  58 #include "SquidTime.h"
  59 #include "Store.h"
  60 #include "StoreClient.h"
  61 #include "SwapDir.h"
  62 #include "tools.h"
  63 #include "URL.h"
  64 #include "wordlist.h"
  65
  66 #if HAVE_SYS_STAT_H
  67 #include <sys/stat.h>
  68 #endif
  69
  70 #if USE_ICMP
  71 #include "icmp/IcmpSquid.h"
  72 #include "ipcache.h"
  73 #include "StoreClient.h"
  74
  75 #define NETDB_REQBUF_SZ 4096
  76
  77 typedef enum {
  78     STATE_NONE,
  79     STATE_HEADER,
  80     STATE_BODY
  81 } netdb_conn_state_t;
  82
  83 typedef struct {
  84     peer *p;
  85     StoreEntry *e;
  86     store_client *sc;
  87     HttpRequest *r;
  88     int64_t used;
  89     size_t buf_sz;
  90     char buf[NETDB_REQBUF_SZ];
  91     int buf_ofs;
  92     netdb_conn_state_t connstate;
  93 } netdbExchangeState;
  94
  95 static hash_table *addr_table = NULL;
  96 static hash_table *host_table = NULL;
  97
  98 Ip::Address networkFromInaddr(const Ip::Address &a);
  99 static void netdbRelease(netdbEntry * n);
 100
 101 static void netdbHashInsert(netdbEntry * n, Ip::Address &addr);
 102 static void netdbHashDelete(const char *key);
 103 static void netdbHostInsert(netdbEntry * n, const char *hostname);
 104 static void netdbHostDelete(const net_db_name * x);
 105 static void netdbPurgeLRU(void);
 106 static netdbEntry *netdbLookupHost(const char *key);
 107 static net_db_peer *netdbPeerByName(const netdbEntry * n, const char *);
 108 static net_db_peer *netdbPeerAdd(netdbEntry * n, peer * e);
 109 static const char *netdbPeerName(const char *name);
 110 static IPH netdbSendPing;
 111 static QS sortPeerByRtt;
 112 static QS sortByRtt;
 113 static QS netdbLRU;
 114 static FREE netdbFreeNameEntry;
 115 static FREE netdbFreeNetdbEntry;
 116 static STCB netdbExchangeHandleReply;
 117 static void netdbExchangeDone(void *);
 118
 119 /* We have to keep a local list of peer names.  The Peers structure
 120  * gets freed during a reconfigure.  We want this database to
 121  * remain persisitent, so _net_db_peer->peername points into this
 122  * linked list */
 123 static wordlist *peer_names = NULL;
 124
 125 static void
 126 netdbHashInsert(netdbEntry * n, Ip::Address &addr)
 127 {
 128     networkFromInaddr(addr).NtoA(n->network, MAX_IPSTRLEN);
 129     n->hash.key = n->network;
 130     assert(hash_lookup(addr_table, n->network) == NULL);
 131     hash_join(addr_table, &n->hash);
 132 }
 133
 134 static void
 135 netdbHashDelete(const char *key)
 136 {
 137     hash_link *hptr = (hash_link *)hash_lookup(addr_table, key);
 138
 139     if (hptr == NULL) {
 140         debug_trap("netdbHashDelete: key not found");
 141         return;
 142     }
 143
 144     hash_remove_link(addr_table, hptr);
 145 }
 146
 147 static void
 148 netdbHostInsert(netdbEntry * n, const char *hostname)
 149 {
 150     net_db_name *x = (net_db_name *)memAllocate(MEM_NET_DB_NAME);
 151     x->hash.key = xstrdup(hostname);
 152     x->next = n->hosts;
 153     n->hosts = x;
 154     x->net_db_entry = n;
 155     assert(hash_lookup(host_table, hostname) == NULL);
 156     hash_join(host_table, &x->hash);
 157     ++ n->link_count;
 158 }
 159
 160 static void
 161 netdbHostDelete(const net_db_name * x)
 162 {
 163     netdbEntry *n;
 164     net_db_name **X;
 165     assert(x != NULL);
 166     assert(x->net_db_entry != NULL);
 167     n = x->net_db_entry;
 168     -- n->link_count;
 169
 170     for (X = &n->hosts; *X; X = &(*X)->next) {
 171         if (*X == x) {
 172             *X = x->next;
 173             break;
 174         }
 175     }
 176
 177     hash_remove_link(host_table, (hash_link *) x);
 178     xfree(x->hash.key);
 179     memFree((void *) x, MEM_NET_DB_NAME);
 180 }
 181
 182 static netdbEntry *
 183 netdbLookupHost(const char *key)
 184 {
 185     net_db_name *x = (net_db_name *) hash_lookup(host_table, key);
 186     return x ? x->net_db_entry : NULL;
 187 }
 188
 189 static void
 190 netdbRelease(netdbEntry * n)
 191 {
 192     net_db_name *x;
 193     net_db_name *next;
 194
 195     for (x = n->hosts; x; x = next) {
 196         next = x->next;
 197         netdbHostDelete(x);
 198     }
 199
 200     n->hosts = NULL;
 201     safe_free(n->peers);
 202     n->peers = NULL;
 203     n->n_peers = 0;
 204     n->n_peers_alloc = 0;
 205
 206     if (n->link_count == 0) {
 207         netdbHashDelete(n->network);
 208         memFree(n, MEM_NETDBENTRY);
 209     }
 210 }
 211
 212 static int
 213 netdbLRU(const void *A, const void *B)
 214 {
 215     const netdbEntry *const *n1 = (const netdbEntry *const *)A;
 216     const netdbEntry *const *n2 = (const netdbEntry *const *)B;
 217
 218     if ((*n1)->last_use_time > (*n2)->last_use_time)
 219         return (1);
 220
 221     if ((*n1)->last_use_time < (*n2)->last_use_time)
 222         return (-1);
 223
 224     return (0);
 225 }
 226
 227 static void
 228 netdbPurgeLRU(void)
 229 {
 230     netdbEntry *n;
 231     netdbEntry **list;
 232     int k = 0;
 233     int list_count = 0;
 234     int removed = 0;
 235     list = (netdbEntry **)xcalloc(memInUse(MEM_NETDBENTRY), sizeof(netdbEntry *));
 236     hash_first(addr_table);
 237
 238     while ((n = (netdbEntry *) hash_next(addr_table))) {
 239         assert(list_count < memInUse(MEM_NETDBENTRY));
 240         *(list + list_count) = n;
 241         ++list_count;
 242     }
 243
 244     qsort((char *) list,
 245           list_count,
 246           sizeof(netdbEntry *),
 247           netdbLRU);
 248
 249     for (k = 0; k < list_count; ++k) {
 250         if (memInUse(MEM_NETDBENTRY) < Config.Netdb.low)
 251             break;
 252
 253         netdbRelease(*(list + k));
 254
 255         ++removed;
 256     }
 257
 258     xfree(list);
 259 }
 260
 261 static netdbEntry *
 262 netdbLookupAddr(const Ip::Address &addr)
 263 {
 264     netdbEntry *n;
 265     char *key = new char[MAX_IPSTRLEN];
 266     networkFromInaddr(addr).NtoA(key,MAX_IPSTRLEN);
 267     n = (netdbEntry *) hash_lookup(addr_table, key);
 268     delete[] key;
 269     return n;
 270 }
 271
 272 static netdbEntry *
 273 netdbAdd(Ip::Address &addr)
 274 {
 275     netdbEntry *n;
 276
 277     if (memInUse(MEM_NETDBENTRY) > Config.Netdb.high)
 278         netdbPurgeLRU();
 279
 280     if ((n = netdbLookupAddr(addr)) == NULL) {
 281         n = (netdbEntry *)memAllocate(MEM_NETDBENTRY);
 282         netdbHashInsert(n, addr);
 283     }
 284
 285     return n;
 286 }
 287
 288 static void
 289 netdbSendPing(const ipcache_addrs *ia, const DnsLookupDetails &, void *data)
 290 {
 291     Ip::Address addr;
 292     char *hostname = NULL;
 293     static_cast<generic_cbdata *>(data)->unwrap(&hostname);
 294     netdbEntry *n;
 295     netdbEntry *na;
 296     net_db_name *x;
 297     net_db_name **X;
 298
 299     if (ia == NULL) {
 300         xfree(hostname);
 301         return;
 302     }
 303
 304     addr = ia->in_addrs[ia->cur];
 305
 306     if ((n = netdbLookupHost(hostname)) == NULL) {
 307         n = netdbAdd(addr);
 308         netdbHostInsert(n, hostname);
 309     } else if ((na = netdbLookupAddr(addr)) != n) {
 310         /*
 311          *hostname moved from 'network n' to 'network na'!
 312          */
 313
 314         if (na == NULL)
 315             na = netdbAdd(addr);
 316
 317         debugs(38, 3, "netdbSendPing: " << hostname << " moved from " << n->network << " to " << na->network);
 318
 319         x = (net_db_name *) hash_lookup(host_table, hostname);
 320
 321         if (x == NULL) {
 322             debugs(38, DBG_IMPORTANT, "netdbSendPing: net_db_name list bug: " << hostname << " not found");
 323             xfree(hostname);
 324             return;
 325         }
 326
 327         /* remove net_db_name from 'network n' linked list */
 328         for (X = &n->hosts; *X; X = &(*X)->next) {
 329             if (*X == x) {
 330                 *X = x->next;
 331                 break;
 332             }
 333         }
 334
 335         -- n->link_count;
 336         /* point to 'network na' from host entry */
 337         x->net_db_entry = na;
 338         /* link net_db_name to 'network na' */
 339         x->next = na->hosts;
 340         na->hosts = x;
 341         ++ na->link_count;
 342         n = na;
 343     }
 344
 345     if (n->next_ping_time <= squid_curtime) {
 346         debugs(38, 3, "netdbSendPing: pinging " << hostname);
 347         icmpEngine.DomainPing(addr, hostname);
 348         ++ n->pings_sent;
 349         n->next_ping_time = squid_curtime + Config.Netdb.period;
 350         n->last_use_time = squid_curtime;
 351     }
 352
 353     xfree(hostname);
 354 }
 355
 356 Ip::Address
 357 networkFromInaddr(const Ip::Address &in)
 358 {
 359     Ip::Address out;
 360
 361     out = in;
 362
 363     /* in IPv6 the 'network' should be the routing section. */
 364     if ( in.IsIPv6() ) {
 365         out.ApplyMask(64, AF_INET6);
 366         debugs(14, 5, "networkFromInaddr : Masked IPv6 Address to " << in << "/64 routing part.");
 367         return out;
 368     }
 369
 370 #if USE_CLASSFUL
 371     struct in_addr b;
 372
 373     in.GetInAddr(b);
 374
 375     if (IN_CLASSC(b.s_addr))
 376         b.s_addr &= IN_CLASSC_NET;
 377     else if (IN_CLASSB(b.s_addr))
 378         b.s_addr &= IN_CLASSB_NET;
 379     else if (IN_CLASSA(b.s_addr))
 380         b.s_addr &= IN_CLASSA_NET;
 381
 382     out = b;
 383
 384 #endif
 385
 386     debugs(14, 5, "networkFromInaddr : Masked IPv4 Address to " << out << "/24.");
 387
 388     /* use /24 for everything under IPv4 */
 389     out.ApplyMask(24, AF_INET);
 390     debugs(14, 5, "networkFromInaddr : Masked IPv4 Address to " << in << "/24.");
 391
 392     return out;
 393 }
 394
 395 static int
 396 sortByRtt(const void *A, const void *B)
 397 {
 398     const netdbEntry *const *n1 = (const netdbEntry *const *)A;
 399     const netdbEntry *const *n2 = (const netdbEntry *const *)B;
 400
 401     if ((*n1)->rtt > (*n2)->rtt)
 402         return 1;
 403     else if ((*n1)->rtt < (*n2)->rtt)
 404         return -1;
 405     else
 406         return 0;
 407 }
 408
 409 static net_db_peer *
 410 netdbPeerByName(const netdbEntry * n, const char *peername)
 411 {
 412     int i;
 413     net_db_peer *p = n->peers;
 414
 415     for (i = 0; i < n->n_peers; ++i, ++p) {
 416         if (!strcmp(p->peername, peername))
 417             return p;
 418     }
 419
 420     return NULL;
 421 }
 422
 423 static net_db_peer *
 424 netdbPeerAdd(netdbEntry * n, peer * e)
 425 {
 426     net_db_peer *p;
 427     net_db_peer *o;
 428     int osize;
 429     int i;
 430
 431     if (n->n_peers == n->n_peers_alloc) {
 432         o = n->peers;
 433         osize = n->n_peers_alloc;
 434
 435         if (n->n_peers_alloc == 0)
 436             n->n_peers_alloc = 2;
 437         else
 438             n->n_peers_alloc <<= 1;
 439
 440         debugs(38, 3, "netdbPeerAdd: Growing peer list for '" << n->network << "' to " << n->n_peers_alloc);
 441
 442         n->peers = (net_db_peer *)xcalloc(n->n_peers_alloc, sizeof(net_db_peer));
 443
 444         for (i = 0; i < osize; ++i)
 445             *(n->peers + i) = *(o + i);
 446
 447         if (osize) {
 448             safe_free(o);
 449         }
 450     }
 451
 452     p = n->peers + n->n_peers;
 453     p->peername = netdbPeerName(e->host);
 454     ++ n->n_peers;
 455     return p;
 456 }
 457
 458 static int
 459 sortPeerByRtt(const void *A, const void *B)
 460 {
 461     const net_db_peer *p1 = (net_db_peer *)A;
 462     const net_db_peer *p2 = (net_db_peer *)B;
 463
 464     if (p1->rtt > p2->rtt)
 465         return 1;
 466     else if (p1->rtt < p2->rtt)
 467         return -1;
 468     else
 469         return 0;
 470 }
 471
 472 static void
 473 netdbSaveState(void *foo)
 474 {
 475     if (strcmp(Config.netdbFilename, "none") == 0)
 476         return;
 477
 478     Logfile *lf;
 479     netdbEntry *n;
 480     net_db_name *x;
 481
 482     struct timeval start = current_time;
 483     int count = 0;
 484     /*
 485      * This was nicer when we were using stdio, but thanks to
 486      * Solaris bugs, its a bad idea.  fopen can fail if more than
 487      * 256 FDs are open.
 488      */
 489     /*
 490      * unlink() is here because there is currently no way to make
 491      * logfileOpen() use O_TRUNC.
 492      */
 493     unlink(Config.netdbFilename);
 494     lf = logfileOpen(Config.netdbFilename, 4096, 0);
 495
 496     if (NULL == lf) {
 497         debugs(50, DBG_IMPORTANT, "netdbSaveState: " << Config.netdbFilename << ": " << xstrerror());
 498         return;
 499     }
 500
 501     hash_first(addr_table);
 502
 503     while ((n = (netdbEntry *) hash_next(addr_table))) {
 504         if (n->pings_recv == 0)
 505             continue;
 506
 507         logfilePrintf(lf, "%s %d %d %10.5f %10.5f %d %d",
 508                       n->network,
 509                       n->pings_sent,
 510                       n->pings_recv,
 511                       n->hops,
 512                       n->rtt,
 513                       (int) n->next_ping_time,
 514                       (int) n->last_use_time);
 515
 516         for (x = n->hosts; x; x = x->next)
 517             logfilePrintf(lf, " %s", hashKeyStr(&x->hash));
 518
 519         logfilePrintf(lf, "\n");
 520
 521         ++count;
 522
 523 #undef RBUF_SZ
 524
 525     }
 526
 527     logfileClose(lf);
 528     getCurrentTime();
 529     debugs(38, DBG_IMPORTANT, "NETDB state saved; " <<
 530            count << " entries, " <<
 531            tvSubMsec(start, current_time) << " msec" );
 532     eventAddIsh("netdbSaveState", netdbSaveState, NULL, 3600.0, 1);
 533 }
 534
 535 static void
 536 netdbReloadState(void)
 537 {
 538     if (strcmp(Config.netdbFilename, "none") == 0)
 539         return;
 540
 541     char *s;
 542     int fd;
 543     int l;
 544
 545     struct stat sb;
 546     netdbEntry *n;
 547     netdbEntry N;
 548
 549     Ip::Address addr;
 550     int count = 0;
 551
 552     struct timeval start = current_time;
 553     /*
 554      * This was nicer when we were using stdio, but thanks to
 555      * Solaris bugs, its a bad idea.  fopen can fail if more than
 556      * 256 FDs are open.
 557      */
 558     fd = file_open(Config.netdbFilename, O_RDONLY | O_BINARY);
 559
 560     if (fd < 0)
 561         return;
 562
 563     if (fstat(fd, &sb) < 0) {
 564         file_close(fd);
 565         return;
 566     }
 567
 568     char *t;
 569     char *buf = (char *)xcalloc(1, sb.st_size + 1);
 570     t = buf;
 571     l = FD_READ_METHOD(fd, buf, sb.st_size);
 572     file_close(fd);
 573
 574     if (l <= 0) {
 575         safe_free (buf);
 576         return;
 577     };
 578
 579     while ((s = strchr(t, '\n'))) {
 580         char *q;
 581         assert(s - buf < l);
 582         *s = '\0';
 583         memset(&N, '\0', sizeof(netdbEntry));
 584         q = strtok(t, w_space);
 585         t = s + 1;
 586
 587         if (NULL == q)
 588             continue;
 589
 590         if (! (addr = q) )
 591             continue;
 592
 593         if (netdbLookupAddr(addr) != NULL)      /* no dups! */
 594             continue;
 595
 596         if ((q = strtok(NULL, w_space)) == NULL)
 597             continue;
 598
 599         N.pings_sent = atoi(q);
 600
 601         if ((q = strtok(NULL, w_space)) == NULL)
 602             continue;
 603
 604         N.pings_recv = atoi(q);
 605
 606         if (N.pings_recv == 0)
 607             continue;
 608
 609         /* give this measurement low weight */
 610         N.pings_sent = 1;
 611
 612         N.pings_recv = 1;
 613
 614         if ((q = strtok(NULL, w_space)) == NULL)
 615             continue;
 616
 617         N.hops = atof(q);
 618
 619         if ((q = strtok(NULL, w_space)) == NULL)
 620             continue;
 621
 622         N.rtt = atof(q);
 623
 624         if ((q = strtok(NULL, w_space)) == NULL)
 625             continue;
 626
 627         N.next_ping_time = (time_t) atoi(q);
 628
 629         if ((q = strtok(NULL, w_space)) == NULL)
 630             continue;
 631
 632         N.last_use_time = (time_t) atoi(q);
 633
 634         n = (netdbEntry *)memAllocate(MEM_NETDBENTRY);
 635
 636         memcpy(n, &N, sizeof(netdbEntry));
 637
 638         netdbHashInsert(n, addr);
 639
 640         while ((q = strtok(NULL, w_space)) != NULL) {
 641             if (netdbLookupHost(q) != NULL)     /* no dups! */
 642                 continue;
 643
 644             netdbHostInsert(n, q);
 645         }
 646
 647         ++count;
 648     }
 649
 650     xfree(buf);
 651     getCurrentTime();
 652     debugs(38, DBG_IMPORTANT, "NETDB state reloaded; " <<
 653            count << " entries, " <<
 654            tvSubMsec(start, current_time) << " msec" );
 655 }
 656
 657 static const char *
 658 netdbPeerName(const char *name)
 659 {
 660     const wordlist *w;
 661
 662     for (w = peer_names; w; w = w->next) {
 663         if (!strcmp(w->key, name))
 664             return w->key;
 665     }
 666
 667     return wordlistAdd(&peer_names, name);
 668 }
 669
 670 static void
 671 netdbFreeNetdbEntry(void *data)
 672 {
 673     netdbEntry *n = (netdbEntry *)data;
 674     safe_free(n->peers);
 675     memFree(n, MEM_NETDBENTRY);
 676 }
 677
 678 static void
 679 netdbFreeNameEntry(void *data)
 680 {
 681     net_db_name *x = (net_db_name *)data;
 682     xfree(x->hash.key);
 683     memFree(x, MEM_NET_DB_NAME);
 684 }
 685
 686 static void
 687 netdbExchangeHandleReply(void *data, StoreIOBuffer receivedData)
 688 {
 689     Ip::Address addr;
 690
 691     netdbExchangeState *ex = (netdbExchangeState *)data;
 692     int rec_sz = 0;
 693     int o;
 694
 695     struct in_addr line_addr;
 696     double rtt;
 697     double hops;
 698     char *p;
 699     int j;
 700     HttpReply const *rep;
 701     size_t hdr_sz;
 702     int nused = 0;
 703     int size;
 704     int oldbufofs = ex->buf_ofs;
 705
 706     rec_sz = 0;
 707     rec_sz += 1 + sizeof(struct in_addr);
 708     rec_sz += 1 + sizeof(int);
 709     rec_sz += 1 + sizeof(int);
 710     debugs(38, 3, "netdbExchangeHandleReply: " << receivedData.length << " read bytes");
 711
 712     if (!cbdataReferenceValid(ex->p)) {
 713         debugs(38, 3, "netdbExchangeHandleReply: Peer became invalid");
 714         netdbExchangeDone(ex);
 715         return;
 716     }
 717
 718     debugs(38, 3, "netdbExchangeHandleReply: for '" << ex->p->host << ":" << ex->p->http_port << "'");
 719
 720     if (receivedData.length == 0 &&
 721             !receivedData.flags.error) {
 722         debugs(38, 3, "netdbExchangeHandleReply: Done");
 723         netdbExchangeDone(ex);
 724         return;
 725     }
 726
 727     p = ex->buf;
 728
 729     /* Get the size of the buffer now */
 730     size = ex->buf_ofs + receivedData.length;
 731     debugs(38, 3, "netdbExchangeHandleReply: " << size << " bytes buf");
 732
 733     /* Check if we're still doing headers */
 734
 735     if (ex->connstate == STATE_HEADER) {
 736
 737         ex->buf_ofs += receivedData.length;
 738
 739         /* skip reply headers */
 740
 741         if ((hdr_sz = headersEnd(p, ex->buf_ofs))) {
 742             debugs(38, 5, "netdbExchangeHandleReply: hdr_sz = " << hdr_sz);
 743             rep = ex->e->getReply();
 744             assert (0 != rep->sline.status);
 745             debugs(38, 3, "netdbExchangeHandleReply: reply status " << rep->sline.status);
 746
 747             if (HTTP_OK != rep->sline.status) {
 748                 netdbExchangeDone(ex);
 749                 return;
 750             }
 751
 752             assert((size_t)ex->buf_ofs >= hdr_sz);
 753
 754             /*
 755              * Now, point p to the part of the buffer where the data
 756              * starts, and update the size accordingly
 757              */
 758             assert(ex->used == 0);
 759             ex->used = hdr_sz;
 760             size = ex->buf_ofs - hdr_sz;
 761             p += hdr_sz;
 762
 763             /* Finally, set the conn state mode to STATE_BODY */
 764             ex->connstate = STATE_BODY;
 765         } else {
 766             StoreIOBuffer tempBuffer;
 767             tempBuffer.offset = ex->buf_ofs;
 768             tempBuffer.length = ex->buf_sz - ex->buf_ofs;
 769             tempBuffer.data = ex->buf + ex->buf_ofs;
 770             /* Have more headers .. */
 771             storeClientCopy(ex->sc, ex->e, tempBuffer,
 772                             netdbExchangeHandleReply, ex);
 773             return;
 774         }
 775     }
 776
 777     assert(ex->connstate == STATE_BODY);
 778
 779     /* If we get here, we have some body to parse .. */
 780     debugs(38, 5, "netdbExchangeHandleReply: start parsing loop, size = " << size);
 781
 782     while (size >= rec_sz) {
 783         debugs(38, 5, "netdbExchangeHandleReply: in parsing loop, size = " << size);
 784         addr.SetAnyAddr();
 785         hops = rtt = 0.0;
 786
 787         for (o = 0; o < rec_sz;) {
 788             switch ((int) *(p + o)) {
 789
 790             case NETDB_EX_NETWORK:
 791                 ++o;
 792                 /* FIXME INET6 : NetDB can still ony send IPv4 */
 793                 memcpy(&line_addr, p + o, sizeof(struct in_addr));
 794                 addr = line_addr;
 795                 o += sizeof(struct in_addr);
 796                 break;
 797
 798             case NETDB_EX_RTT:
 799                 ++o;
 800                 memcpy(&j, p + o, sizeof(int));
 801                 o += sizeof(int);
 802                 rtt = (double) ntohl(j) / 1000.0;
 803                 break;
 804
 805             case NETDB_EX_HOPS:
 806                 ++o;
 807                 memcpy(&j, p + o, sizeof(int));
 808                 o += sizeof(int);
 809                 hops = (double) ntohl(j) / 1000.0;
 810                 break;
 811
 812             default:
 813                 debugs(38, DBG_IMPORTANT, "netdbExchangeHandleReply: corrupt data, aborting");
 814                 netdbExchangeDone(ex);
 815                 return;
 816             }
 817         }
 818
 819         if (!addr.IsAnyAddr() && rtt > 0)
 820             netdbExchangeUpdatePeer(addr, ex->p, rtt, hops);
 821
 822         assert(o == rec_sz);
 823
 824         ex->used += rec_sz;
 825
 826         size -= rec_sz;
 827
 828         p += rec_sz;
 829
 830         ++nused;
 831     }
 832
 833     /*
 834      * Copy anything that is left over to the beginning of the buffer,
 835      * and adjust buf_ofs accordingly
 836      */
 837
 838     /*
 839      * Evilly, size refers to the buf size left now,
 840      * ex->buf_ofs is the original buffer size, so just copy that
 841      * much data over
 842      */
 843     memmove(ex->buf, ex->buf + (ex->buf_ofs - size), size);
 844
 845     ex->buf_ofs = size;
 846
 847     /*
 848      * And don't re-copy the remaining data ..
 849      */
 850     ex->used += size;
 851
 852     /*
 853      * Now the tricky bit - size _included_ the leftover bit from the _last_
 854      * storeClientCopy. We don't want to include that, or our offset will be wrong.
 855      * So, don't count the size of the leftover buffer we began with.
 856      * This can _disappear_ when we're not tracking offsets ..
 857      */
 858     ex->used -= oldbufofs;
 859
 860     debugs(38, 3, "netdbExchangeHandleReply: size left over in this buffer: " << size << " bytes");
 861
 862     debugs(38, 3, "netdbExchangeHandleReply: used " << nused <<
 863            " entries, (x " << rec_sz << " bytes) == " << nused * rec_sz <<
 864            " bytes total");
 865
 866     debugs(38, 3, "netdbExchangeHandleReply: used " << ex->used);
 867
 868     if (EBIT_TEST(ex->e->flags, ENTRY_ABORTED)) {
 869         debugs(38, 3, "netdbExchangeHandleReply: ENTRY_ABORTED");
 870         netdbExchangeDone(ex);
 871     } else if (ex->e->store_status == STORE_PENDING) {
 872         StoreIOBuffer tempBuffer;
 873         tempBuffer.offset = ex->used;
 874         tempBuffer.length = ex->buf_sz - ex->buf_ofs;
 875         tempBuffer.data = ex->buf + ex->buf_ofs;
 876         debugs(38, 3, "netdbExchangeHandleReply: EOF not received");
 877         storeClientCopy(ex->sc, ex->e, tempBuffer,
 878                         netdbExchangeHandleReply, ex);
 879     }
 880 }
 881
 882 static void
 883 netdbExchangeDone(void *data)
 884 {
 885     netdbExchangeState *ex = (netdbExchangeState *)data;
 886     debugs(38, 3, "netdbExchangeDone: " << ex->e->url()  );
 887     HTTPMSGUNLOCK(ex->r);
 888     storeUnregister(ex->sc, ex->e, ex);
 889     ex->e->unlock();
 890     cbdataReferenceDone(ex->p);
 891     cbdataFree(ex);
 892 }
 893
 894 static void
 895 netdbRegisterWithCacheManager(void)
 896 {
 897     Mgr::RegisterAction("netdb", "Network Measurement Database", netdbDump, 0, 1);
 898 }
 899
 900 #endif /* USE_ICMP */
 901
 902 /* PUBLIC FUNCTIONS */
 903
 904 void
 905 netdbInit(void)
 906 {
 907 #if USE_ICMP
 908     int n;
 909
 910     netdbRegisterWithCacheManager();
 911
 912     if (addr_table)
 913         return;
 914
 915     n = hashPrime(Config.Netdb.high / 4);
 916
 917     addr_table = hash_create((HASHCMP *) strcmp, n, hash_string);
 918
 919     n = hashPrime(3 * Config.Netdb.high / 4);
 920
 921     host_table = hash_create((HASHCMP *) strcmp, n, hash_string);
 922
 923     eventAddIsh("netdbSaveState", netdbSaveState, NULL, 3600.0, 1);
 924
 925     netdbReloadState();
 926
 927 #endif
 928 }
 929
 930 void
 931 netdbPingSite(const char *hostname)
 932 {
 933 #if USE_ICMP
 934     netdbEntry *n;
 935
 936     if ((n = netdbLookupHost(hostname)) != NULL)
 937         if (n->next_ping_time > squid_curtime)
 938             return;
 939
 940     ipcache_nbgethostbyname(hostname, netdbSendPing,
 941                             new generic_cbdata(xstrdup(hostname)));
 942
 943 #endif
 944 }
 945
 946 void
 947 netdbHandlePingReply(const Ip::Address &from, int hops, int rtt)
 948 {
 949 #if USE_ICMP
 950     netdbEntry *n;
 951     int N;
 952     debugs(38, 3, "netdbHandlePingReply: from " << from);
 953
 954     if ((n = netdbLookupAddr(from)) == NULL)
 955         return;
 956
 957     N = ++n->pings_recv;
 958
 959     if (N > 5)
 960         N = 5;
 961
 962     if (rtt < 1)
 963         rtt = 1;
 964
 965     n->hops = ((n->hops * (N - 1)) + hops) / N;
 966
 967     n->rtt = ((n->rtt * (N - 1)) + rtt) / N;
 968
 969     debugs(38, 3, "netdbHandlePingReply: " << n->network  << "; rtt="<<
 970            std::setw(5)<< std::setprecision(2) << n->rtt << "  hops="<<
 971            std::setw(4) << n->hops);
 972
 973 #endif
 974 }
 975
 976 void
 977 netdbFreeMemory(void)
 978 {
 979 #if USE_ICMP
 980     hashFreeItems(addr_table, netdbFreeNetdbEntry);
 981     hashFreeMemory(addr_table);
 982     addr_table = NULL;
 983     hashFreeItems(host_table, netdbFreeNameEntry);
 984     hashFreeMemory(host_table);
 985     host_table = NULL;
 986     wordlistDestroy(&peer_names);
 987     peer_names = NULL;
 988 #endif
 989 }
 990
 991 #if 0 // AYJ: Looks to be unused code.
 992 int
 993 netdbHops(Ip::Address &addr)
 994 {
 995 #if USE_ICMP
 996     netdbEntry *n = netdbLookupAddr(addr);
 997
 998     if (n && n->pings_recv) {
 999         n->last_use_time = squid_curtime;
1000         return (int) (n->hops + 0.5);
1001     }
1002
1003 #endif
1004     return 256;
1005 }
1006 #endif
1007
1008 void
1009 netdbDump(StoreEntry * sentry)
1010 {
1011 #if USE_ICMP
1012     netdbEntry *n;
1013     netdbEntry **list;
1014     net_db_name *x;
1015     int k;
1016     int i;
1017     int j;
1018     net_db_peer *p;
1019     storeAppendPrintf(sentry, "Network DB Statistics:\n");
1020     storeAppendPrintf(sentry, "%-46.46s %9s %7s %5s %s\n",  /* Max between 16 (IPv4) or 46 (IPv6)   */
1021                       "Network",
1022                       "recv/sent",
1023                       "RTT",
1024                       "Hops",
1025                       "Hostnames");
1026     list = (netdbEntry **)xcalloc(memInUse(MEM_NETDBENTRY), sizeof(netdbEntry *));
1027     i = 0;
1028     hash_first(addr_table);
1029
1030     while ((n = (netdbEntry *) hash_next(addr_table))) {
1031         *(list + i) = n;
1032         ++i;
1033     }
1034
1035     if (i != memInUse(MEM_NETDBENTRY))
1036         debugs(38, DBG_CRITICAL, "WARNING: netdb_addrs count off, found " << i <<
1037                ", expected " << memInUse(MEM_NETDBENTRY));
1038
1039     qsort((char *) list,
1040           i,
1041           sizeof(netdbEntry *),
1042           sortByRtt);
1043
1044     for (k = 0; k < i; ++k) {
1045         n = *(list + k);
1046         storeAppendPrintf(sentry, "%-46.46s %4d/%4d %7.1f %5.1f", /* Max between 16 (IPv4) or 46 (IPv6)   */
1047                           n->network,
1048                           n->pings_recv,
1049                           n->pings_sent,
1050                           n->rtt,
1051                           n->hops);
1052
1053         for (x = n->hosts; x; x = x->next)
1054             storeAppendPrintf(sentry, " %s", hashKeyStr(&x->hash));
1055
1056         storeAppendPrintf(sentry, "\n");
1057
1058         p = n->peers;
1059
1060         for (j = 0; j < n->n_peers; ++j, ++p) {
1061             storeAppendPrintf(sentry, "    %-22.22s %7.1f %5.1f\n",
1062                               p->peername,
1063                               p->rtt,
1064                               p->hops);
1065         }
1066     }
1067
1068     xfree(list);
1069 #else
1070
1071     storeAppendPrintf(sentry,"NETDB support not compiled into this Squid cache.\n");
1072 #endif
1073 }
1074
1075 int
1076 netdbHostHops(const char *host)
1077 {
1078 #if USE_ICMP
1079     netdbEntry *n = netdbLookupHost(host);
1080
1081     if (n) {
1082         n->last_use_time = squid_curtime;
1083         return (int) (n->hops + 0.5);
1084     }
1085
1086 #endif
1087     return 0;
1088 }
1089
1090 int
1091 netdbHostRtt(const char *host)
1092 {
1093 #if USE_ICMP
1094     netdbEntry *n = netdbLookupHost(host);
1095
1096     if (n) {
1097         n->last_use_time = squid_curtime;
1098         return (int) (n->rtt + 0.5);
1099     }
1100
1101 #endif
1102     return 0;
1103 }
1104
1105 void
1106 netdbHostData(const char *host, int *samp, int *rtt, int *hops)
1107 {
1108 #if USE_ICMP
1109     netdbEntry *n = netdbLookupHost(host);
1110
1111     if (n == NULL)
1112         return;
1113
1114     *samp = n->pings_recv;
1115
1116     *rtt = (int) (n->rtt + 0.5);
1117
1118     *hops = (int) (n->hops + 0.5);
1119
1120     n->last_use_time = squid_curtime;
1121
1122 #endif
1123 }
1124
1125 void
1126 netdbUpdatePeer(HttpRequest * r, peer * e, int irtt, int ihops)
1127 {
1128 #if USE_ICMP
1129     netdbEntry *n;
1130     double rtt = (double) irtt;
1131     double hops = (double) ihops;
1132     net_db_peer *p;
1133     debugs(38, 3, "netdbUpdatePeer: '" << r->GetHost() << "', " << ihops << " hops, " << irtt << " rtt");
1134     n = netdbLookupHost(r->GetHost());
1135
1136     if (n == NULL) {
1137         debugs(38, 3, "netdbUpdatePeer: host '" << r->GetHost() << "' not found");
1138         return;
1139     }
1140
1141     if ((p = netdbPeerByName(n, e->host)) == NULL)
1142         p = netdbPeerAdd(n, e);
1143
1144     p->rtt = rtt;
1145
1146     p->hops = hops;
1147
1148     p->expires = squid_curtime + 3600;
1149
1150     if (n->n_peers < 2)
1151         return;
1152
1153     qsort((char *) n->peers,
1154           n->n_peers,
1155           sizeof(net_db_peer),
1156           sortPeerByRtt);
1157
1158 #endif
1159 }
1160
1161 void
1162 netdbExchangeUpdatePeer(Ip::Address &addr, peer * e, double rtt, double hops)
1163 {
1164 #if USE_ICMP
1165     netdbEntry *n;
1166     net_db_peer *p;
1167     debugs(38, 5, "netdbExchangeUpdatePeer: '" << addr << "', "<<
1168            std::setfill('0')<< std::setprecision(2) << hops << " hops, " <<
1169            rtt << " rtt");
1170
1171     if ( !addr.IsIPv4() ) {
1172         debugs(38, 5, "netdbExchangeUpdatePeer: Aborting peer update for '" << addr << "', NetDB cannot handle IPv6.");
1173         return;
1174     }
1175
1176     n = netdbLookupAddr(addr);
1177
1178     if (n == NULL)
1179         n = netdbAdd(addr);
1180
1181     assert(NULL != n);
1182
1183     if ((p = netdbPeerByName(n, e->host)) == NULL)
1184         p = netdbPeerAdd(n, e);
1185
1186     p->rtt = rtt;
1187
1188     p->hops = hops;
1189
1190     p->expires = squid_curtime + 3600;  /* XXX ? */
1191
1192     if (n->n_peers < 2)
1193         return;
1194
1195     qsort((char *) n->peers,
1196           n->n_peers,
1197           sizeof(net_db_peer),
1198           sortPeerByRtt);
1199
1200 #endif
1201 }
1202
1203 void
1204 netdbDeleteAddrNetwork(Ip::Address &addr)
1205 {
1206 #if USE_ICMP
1207     netdbEntry *n = netdbLookupAddr(addr);
1208
1209     if (n == NULL)
1210         return;
1211
1212     debugs(38, 3, "netdbDeleteAddrNetwork: " << n->network);
1213
1214     netdbRelease(n);
1215 #endif
1216 }
1217
1218 void
1219 netdbBinaryExchange(StoreEntry * s)
1220 {
1221     HttpReply *reply = new HttpReply;
1222 #if USE_ICMP
1223
1224     Ip::Address addr;
1225
1226     netdbEntry *n;
1227     int i;
1228     int j;
1229     int rec_sz;
1230     char *buf;
1231
1232     struct in_addr line_addr;
1233     s->buffer();
1234     reply->setHeaders(HTTP_OK, "OK", NULL, -1, squid_curtime, -2);
1235     s->replaceHttpReply(reply);
1236     rec_sz = 0;
1237     rec_sz += 1 + sizeof(struct in_addr);
1238     rec_sz += 1 + sizeof(int);
1239     rec_sz += 1 + sizeof(int);
1240     buf = (char *)memAllocate(MEM_4K_BUF);
1241     i = 0;
1242     hash_first(addr_table);
1243
1244     while ((n = (netdbEntry *) hash_next(addr_table))) {
1245         if (0.0 == n->rtt)
1246             continue;
1247
1248         if (n->rtt > 60000)     /* RTT > 1 MIN probably bogus */
1249             continue;
1250
1251         if (! (addr = n->network) )
1252             continue;
1253
1254         /* FIXME INET6 : NetDB cannot yet handle IPv6 addresses. Ensure only IPv4 get sent. */
1255         if ( !addr.IsIPv4() )
1256             continue;
1257
1258         buf[i] = (char) NETDB_EX_NETWORK;
1259         ++i;
1260
1261         addr.GetInAddr(line_addr);
1262         memcpy(&buf[i], &line_addr, sizeof(struct in_addr));
1263
1264         i += sizeof(struct in_addr);
1265
1266         buf[i] = (char) NETDB_EX_RTT;
1267         ++i;
1268
1269         j = htonl((int) (n->rtt * 1000));
1270
1271         memcpy(&buf[i], &j, sizeof(int));
1272
1273         i += sizeof(int);
1274
1275         buf[i] = (char) NETDB_EX_HOPS;
1276         ++i;
1277
1278         j = htonl((int) (n->hops * 1000));
1279
1280         memcpy(&buf[i], &j, sizeof(int));
1281
1282         i += sizeof(int);
1283
1284         if (i + rec_sz > 4096) {
1285             s->append(buf, i);
1286             i = 0;
1287         }
1288     }
1289
1290     if (i > 0) {
1291         s->append(buf, i);
1292         i = 0;
1293     }
1294
1295     assert(0 == i);
1296     s->flush();
1297     memFree(buf, MEM_4K_BUF);
1298 #else
1299
1300     reply->setHeaders(HTTP_BAD_REQUEST, "Bad Request", NULL, -1, squid_curtime, -2);
1301     s->replaceHttpReply(reply);
1302     storeAppendPrintf(s, "NETDB support not compiled into this Squid cache.\n");
1303 #endif
1304
1305     s->complete();
1306 }
1307
1308 #if USE_ICMP
1309 CBDATA_TYPE(netdbExchangeState);
1310 #endif
1311
1312 void
1313 netdbExchangeStart(void *data)
1314 {
1315 #if USE_ICMP
1316     peer *p = (peer *)data;
1317     char *uri;
1318     netdbExchangeState *ex;
1319     StoreIOBuffer tempBuffer;
1320     CBDATA_INIT_TYPE(netdbExchangeState);
1321     ex = cbdataAlloc(netdbExchangeState);
1322     ex->p = cbdataReference(p);
1323     uri = internalRemoteUri(p->host, p->http_port, "/squid-internal-dynamic/", "netdb");
1324     debugs(38, 3, "netdbExchangeStart: Requesting '" << uri << "'");
1325     assert(NULL != uri);
1326     ex->r = HttpRequest::CreateFromUrl(uri);
1327
1328     if (NULL == ex->r) {
1329         debugs(38, DBG_IMPORTANT, "netdbExchangeStart: Bad URI " << uri);
1330         return;
1331     }
1332
1333     HTTPMSGLOCK(ex->r);
1334     assert(NULL != ex->r);
1335     ex->r->http_ver = HttpVersion(1,1);
1336     ex->connstate = STATE_HEADER;
1337     ex->e = storeCreateEntry(uri, uri, request_flags(), METHOD_GET);
1338     ex->buf_sz = NETDB_REQBUF_SZ;
1339     assert(NULL != ex->e);
1340     ex->sc = storeClientListAdd(ex->e, ex);
1341     tempBuffer.offset = 0;
1342     tempBuffer.length = ex->buf_sz;
1343     tempBuffer.data = ex->buf;
1344     storeClientCopy(ex->sc, ex->e, tempBuffer,
1345                     netdbExchangeHandleReply, ex);
1346     ex->r->flags.loopdetect = 1;        /* cheat! -- force direct */
1347
1348     if (p->login)
1349         xstrncpy(ex->r->login, p->login, MAX_LOGIN_SZ);
1350
1351     urlCanonical(ex->r);
1352
1353     FwdState::fwdStart(Comm::ConnectionPointer(), ex->e, ex->r);
1354
1355 #endif
1356 }
1357
1358 peer *
1359 netdbClosestParent(HttpRequest * request)
1360 {
1361 #if USE_ICMP
1362     peer *p = NULL;
1363     netdbEntry *n;
1364     const ipcache_addrs *ia;
1365     net_db_peer *h;
1366     int i;
1367     n = netdbLookupHost(request->GetHost());
1368
1369     if (NULL == n) {
1370         /* try IP addr */
1371         ia = ipcache_gethostbyname(request->GetHost(), 0);
1372
1373         if (NULL != ia)
1374             n = netdbLookupAddr(ia->in_addrs[ia->cur]);
1375     }
1376
1377     if (NULL == n)
1378         return NULL;
1379
1380     if (0 == n->n_peers)
1381         return NULL;
1382
1383     n->last_use_time = squid_curtime;
1384
1385     /*
1386      * Find the parent with the least RTT to the origin server.
1387      * Make sure we don't return a parent who is farther away than
1388      * we are.  Note, the n->peers list is pre-sorted by RTT.
1389      */
1390     for (i = 0; i < n->n_peers; ++i) {
1391         h = &n->peers[i];
1392
1393         if (n->rtt > 0)
1394             if (n->rtt < h->rtt)
1395                 break;
1396
1397         p = peerFindByName(h->peername);
1398
1399         if (NULL == p)          /* not found */
1400             continue;
1401
1402         if (neighborType(p, request) != PEER_PARENT)
1403             continue;
1404
1405         if (!peerHTTPOkay(p, request))  /* not allowed */
1406             continue;
1407
1408         return p;
1409     }
1410
1411 #endif
1412     return NULL;
1413 }