src/icmp/net_db.cc

   1 /*
   2  * DEBUG: section 38    Network Measurement Database
   3  * AUTHOR: Duane Wessels
   4  *
   5  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   6  * ----------------------------------------------------------
   7  *
   8  *  Squid is the result of efforts by numerous individuals from
   9  *  the Internet community; see the CONTRIBUTORS file for full
  10  *  details.   Many organizations have provided support for Squid's
  11  *  development; see the SPONSORS file for full details.  Squid is
  12  *  Copyrighted (C) 2001 by the Regents of the University of
  13  *  California; see the COPYRIGHT file for full details.  Squid
  14  *  incorporates software developed and/or copyrighted by other
  15  *  sources; see the CREDITS file for full details.
  16  *
  17  *  This program is free software; you can redistribute it and/or modify
  18  *  it under the terms of the GNU General Public License as published by
  19  *  the Free Software Foundation; either version 2 of the License, or
  20  *  (at your option) any later version.
  21  *
  22  *  This program is distributed in the hope that it will be useful,
  23  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  24  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  25  *  GNU General Public License for more details.
  26  *
  27  *  You should have received a copy of the GNU General Public License
  28  *  along with this program; if not, write to the Free Software
  29  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  30  *
  31  */
  32
  33 /*
  34  * XXX XXX XXX
  35  *
  36  * This code may be slightly broken now. If you're getting consistent
  37  * (sometimes working) corrupt data exchanges, please contact adrian
  38  * (adrian@squid-cache.org) to sort them out.
  39  */
  40
  41 #include "squid.h"
  42 #include "icmp/net_db.h"
  43 #include "cbdata.h"
  44 #include "event.h"
  45 #include "CacheManager.h"
  46 #include "Store.h"
  47 #include "SwapDir.h"
  48 #include "HttpRequest.h"
  49 #include "HttpReply.h"
  50 #include "MemObject.h"
  51 #include "fde.h"
  52 #include "forward.h"
  53 #include "SquidTime.h"
  54 #include "wordlist.h"
  55 #include "ip/IpAddress.h"
  56
  57 #if USE_ICMP
  58 #include "icmp/IcmpSquid.h"
  59 #include "StoreClient.h"
  60
  61 #define NETDB_REQBUF_SZ 4096
  62
  63 typedef enum {
  64     STATE_NONE,
  65     STATE_HEADER,
  66     STATE_BODY
  67 } netdb_conn_state_t;
  68
  69 typedef struct {
  70     peer *p;
  71     StoreEntry *e;
  72     store_client *sc;
  73     HttpRequest *r;
  74     int64_t used;
  75     size_t buf_sz;
  76     char buf[NETDB_REQBUF_SZ];
  77     int buf_ofs;
  78     netdb_conn_state_t connstate;
  79 } netdbExchangeState;
  80
  81 static hash_table *addr_table = NULL;
  82 static hash_table *host_table = NULL;
  83
  84 IpAddress networkFromInaddr(const IpAddress &a);
  85 static void netdbRelease(netdbEntry * n);
  86
  87 static void netdbHashInsert(netdbEntry * n, IpAddress &addr);
  88 static void netdbHashDelete(const char *key);
  89 static void netdbHostInsert(netdbEntry * n, const char *hostname);
  90 static void netdbHostDelete(const net_db_name * x);
  91 static void netdbPurgeLRU(void);
  92 static netdbEntry *netdbLookupHost(const char *key);
  93 static net_db_peer *netdbPeerByName(const netdbEntry * n, const char *);
  94 static net_db_peer *netdbPeerAdd(netdbEntry * n, peer * e);
  95 static const char *netdbPeerName(const char *name);
  96 static IPH netdbSendPing;
  97 static QS sortPeerByRtt;
  98 static QS sortByRtt;
  99 static QS netdbLRU;
 100 static FREE netdbFreeNameEntry;
 101 static FREE netdbFreeNetdbEntry;
 102 static STCB netdbExchangeHandleReply;
 103 static void netdbExchangeDone(void *);
 104
 105 /* We have to keep a local list of peer names.  The Peers structure
 106  * gets freed during a reconfigure.  We want this database to
 107  * remain persisitent, so _net_db_peer->peername points into this
 108  * linked list */
 109 static wordlist *peer_names = NULL;
 110
 111 static void
 112 netdbHashInsert(netdbEntry * n, IpAddress &addr)
 113 {
 114     networkFromInaddr(addr).NtoA(n->network, MAX_IPSTRLEN);
 115     n->hash.key = n->network;
 116     assert(hash_lookup(addr_table, n->network) == NULL);
 117     hash_join(addr_table, &n->hash);
 118 }
 119
 120 static void
 121 netdbHashDelete(const char *key)
 122 {
 123     hash_link *hptr = (hash_link *)hash_lookup(addr_table, key);
 124
 125     if (hptr == NULL) {
 126         debug_trap("netdbHashDelete: key not found");
 127         return;
 128     }
 129
 130     hash_remove_link(addr_table, hptr);
 131 }
 132
 133 static void
 134 netdbHostInsert(netdbEntry * n, const char *hostname)
 135 {
 136     net_db_name *x = (net_db_name *)memAllocate(MEM_NET_DB_NAME);
 137     x->hash.key = xstrdup(hostname);
 138     x->next = n->hosts;
 139     n->hosts = x;
 140     x->net_db_entry = n;
 141     assert(hash_lookup(host_table, hostname) == NULL);
 142     hash_join(host_table, &x->hash);
 143     n->link_count++;
 144 }
 145
 146 static void
 147 netdbHostDelete(const net_db_name * x)
 148 {
 149     netdbEntry *n;
 150     net_db_name **X;
 151     assert(x != NULL);
 152     assert(x->net_db_entry != NULL);
 153     n = x->net_db_entry;
 154     n->link_count--;
 155
 156     for (X = &n->hosts; *X; X = &(*X)->next) {
 157         if (*X == x) {
 158             *X = x->next;
 159             break;
 160         }
 161     }
 162
 163     hash_remove_link(host_table, (hash_link *) x);
 164     xfree(x->hash.key);
 165     memFree((void *) x, MEM_NET_DB_NAME);
 166 }
 167
 168 static netdbEntry *
 169 netdbLookupHost(const char *key)
 170 {
 171     net_db_name *x = (net_db_name *) hash_lookup(host_table, key);
 172     return x ? x->net_db_entry : NULL;
 173 }
 174
 175 static void
 176 netdbRelease(netdbEntry * n)
 177 {
 178     net_db_name *x;
 179     net_db_name *next;
 180
 181     for (x = n->hosts; x; x = next) {
 182         next = x->next;
 183         netdbHostDelete(x);
 184     }
 185
 186     n->hosts = NULL;
 187     safe_free(n->peers);
 188     n->peers = NULL;
 189     n->n_peers = 0;
 190     n->n_peers_alloc = 0;
 191
 192     if (n->link_count == 0) {
 193         netdbHashDelete(n->network);
 194         memFree(n, MEM_NETDBENTRY);
 195     }
 196 }
 197
 198 static int
 199 netdbLRU(const void *A, const void *B)
 200 {
 201     const netdbEntry *const *n1 = (const netdbEntry *const *)A;
 202     const netdbEntry *const *n2 = (const netdbEntry *const *)B;
 203
 204     if ((*n1)->last_use_time > (*n2)->last_use_time)
 205         return (1);
 206
 207     if ((*n1)->last_use_time < (*n2)->last_use_time)
 208         return (-1);
 209
 210     return (0);
 211 }
 212
 213 static void
 214 netdbPurgeLRU(void)
 215 {
 216     netdbEntry *n;
 217     netdbEntry **list;
 218     int k = 0;
 219     int list_count = 0;
 220     int removed = 0;
 221     list = (netdbEntry **)xcalloc(memInUse(MEM_NETDBENTRY), sizeof(netdbEntry *));
 222     hash_first(addr_table);
 223
 224     while ((n = (netdbEntry *) hash_next(addr_table))) {
 225         assert(list_count < memInUse(MEM_NETDBENTRY));
 226         *(list + list_count) = n;
 227         list_count++;
 228     }
 229
 230     qsort((char *) list,
 231           list_count,
 232           sizeof(netdbEntry *),
 233           netdbLRU);
 234
 235     for (k = 0; k < list_count; k++) {
 236         if (memInUse(MEM_NETDBENTRY) < Config.Netdb.low)
 237             break;
 238
 239         netdbRelease(*(list + k));
 240
 241         removed++;
 242     }
 243
 244     xfree(list);
 245 }
 246
 247 static netdbEntry *
 248 netdbLookupAddr(const IpAddress &addr)
 249 {
 250     netdbEntry *n;
 251     char *key = new char[MAX_IPSTRLEN];
 252     networkFromInaddr(addr).NtoA(key,MAX_IPSTRLEN);
 253     n = (netdbEntry *) hash_lookup(addr_table, key);
 254     return n;
 255 }
 256
 257 static netdbEntry *
 258 netdbAdd(IpAddress &addr)
 259 {
 260     netdbEntry *n;
 261
 262     if (memInUse(MEM_NETDBENTRY) > Config.Netdb.high)
 263         netdbPurgeLRU();
 264
 265     if ((n = netdbLookupAddr(addr)) == NULL) {
 266         n = (netdbEntry *)memAllocate(MEM_NETDBENTRY);
 267         netdbHashInsert(n, addr);
 268     }
 269
 270     return n;
 271 }
 272
 273 static void
 274 netdbSendPing(const ipcache_addrs *ia, const DnsLookupDetails &, void *data)
 275 {
 276     IpAddress addr;
 277     char *hostname = NULL;
 278     static_cast<generic_cbdata *>(data)->unwrap(&hostname);
 279     netdbEntry *n;
 280     netdbEntry *na;
 281     net_db_name *x;
 282     net_db_name **X;
 283
 284     if (ia == NULL) {
 285         xfree(hostname);
 286         return;
 287     }
 288
 289     addr = ia->in_addrs[ia->cur];
 290
 291     if ((n = netdbLookupHost(hostname)) == NULL) {
 292         n = netdbAdd(addr);
 293         netdbHostInsert(n, hostname);
 294     } else if ((na = netdbLookupAddr(addr)) != n) {
 295         /*
 296          *hostname moved from 'network n' to 'network na'!
 297          */
 298
 299         if (na == NULL)
 300             na = netdbAdd(addr);
 301
 302         debugs(38, 3, "netdbSendPing: " << hostname << " moved from " << n->network << " to " << na->network);
 303
 304         x = (net_db_name *) hash_lookup(host_table, hostname);
 305
 306         if (x == NULL) {
 307             debugs(38, 1, "netdbSendPing: net_db_name list bug: " << hostname << " not found");
 308             xfree(hostname);
 309             return;
 310         }
 311
 312         /* remove net_db_name from 'network n' linked list */
 313         for (X = &n->hosts; *X; X = &(*X)->next) {
 314             if (*X == x) {
 315                 *X = x->next;
 316                 break;
 317             }
 318         }
 319
 320         n->link_count--;
 321         /* point to 'network na' from host entry */
 322         x->net_db_entry = na;
 323         /* link net_db_name to 'network na' */
 324         x->next = na->hosts;
 325         na->hosts = x;
 326         na->link_count++;
 327         n = na;
 328     }
 329
 330     if (n->next_ping_time <= squid_curtime) {
 331         debugs(38, 3, "netdbSendPing: pinging " << hostname);
 332         icmpEngine.DomainPing(addr, hostname);
 333         n->pings_sent++;
 334         n->next_ping_time = squid_curtime + Config.Netdb.period;
 335         n->last_use_time = squid_curtime;
 336     }
 337
 338     xfree(hostname);
 339 }
 340
 341 IpAddress
 342 networkFromInaddr(const IpAddress &in)
 343 {
 344     IpAddress out;
 345
 346     out = in;
 347 #if USE_IPV6
 348
 349     /* in IPv6 the 'network' should be the routing section. */
 350
 351     if ( in.IsIPv6() ) {
 352         out.ApplyMask(64, AF_INET6);
 353         debugs(14, 5, "networkFromInaddr : Masked IPv6 Address to " << in << "/64 routing part.");
 354         return out;
 355     }
 356 #endif
 357
 358 #if USE_CLASSFUL
 359     struct in_addr b;
 360
 361     in.GetInAddr(b);
 362
 363     if (IN_CLASSC(b.s_addr))
 364         b.s_addr &= IN_CLASSC_NET;
 365     else if (IN_CLASSB(b.s_addr))
 366         b.s_addr &= IN_CLASSB_NET;
 367     else if (IN_CLASSA(b.s_addr))
 368         b.s_addr &= IN_CLASSA_NET;
 369
 370     out = b;
 371
 372 #endif
 373
 374     debugs(14, 5, "networkFromInaddr : Masked IPv4 Address to " << out << "/24.");
 375
 376     /* use /24 for everything under IPv4 */
 377     out.ApplyMask(24, AF_INET);
 378     debugs(14, 5, "networkFromInaddr : Masked IPv4 Address to " << in << "/24.");
 379
 380     return out;
 381 }
 382
 383 static int
 384 sortByRtt(const void *A, const void *B)
 385 {
 386     const netdbEntry *const *n1 = (const netdbEntry *const *)A;
 387     const netdbEntry *const *n2 = (const netdbEntry *const *)B;
 388
 389     if ((*n1)->rtt > (*n2)->rtt)
 390         return 1;
 391     else if ((*n1)->rtt < (*n2)->rtt)
 392         return -1;
 393     else
 394         return 0;
 395 }
 396
 397 static net_db_peer *
 398 netdbPeerByName(const netdbEntry * n, const char *peername)
 399 {
 400     int i;
 401     net_db_peer *p = n->peers;
 402
 403     for (i = 0; i < n->n_peers; i++, p++) {
 404         if (!strcmp(p->peername, peername))
 405             return p;
 406     }
 407
 408     return NULL;
 409 }
 410
 411 static net_db_peer *
 412 netdbPeerAdd(netdbEntry * n, peer * e)
 413 {
 414     net_db_peer *p;
 415     net_db_peer *o;
 416     int osize;
 417     int i;
 418
 419     if (n->n_peers == n->n_peers_alloc) {
 420         o = n->peers;
 421         osize = n->n_peers_alloc;
 422
 423         if (n->n_peers_alloc == 0)
 424             n->n_peers_alloc = 2;
 425         else
 426             n->n_peers_alloc <<= 1;
 427
 428         debugs(38, 3, "netdbPeerAdd: Growing peer list for '" << n->network << "' to " << n->n_peers_alloc);
 429
 430         n->peers = (net_db_peer *)xcalloc(n->n_peers_alloc, sizeof(net_db_peer));
 431
 432         for (i = 0; i < osize; i++)
 433             *(n->peers + i) = *(o + i);
 434
 435         if (osize) {
 436             safe_free(o);
 437         }
 438     }
 439
 440     p = n->peers + n->n_peers;
 441     p->peername = netdbPeerName(e->host);
 442     n->n_peers++;
 443     return p;
 444 }
 445
 446 static int
 447 sortPeerByRtt(const void *A, const void *B)
 448 {
 449     const net_db_peer *p1 = (net_db_peer *)A;
 450     const net_db_peer *p2 = (net_db_peer *)B;
 451
 452     if (p1->rtt > p2->rtt)
 453         return 1;
 454     else if (p1->rtt < p2->rtt)
 455         return -1;
 456     else
 457         return 0;
 458 }
 459
 460 static void
 461 netdbSaveState(void *foo)
 462 {
 463     if (strcmp(Config.netdbFilename, "none") == 0)
 464         return;
 465
 466     Logfile *lf;
 467     netdbEntry *n;
 468     net_db_name *x;
 469
 470     struct timeval start = current_time;
 471     int count = 0;
 472     /*
 473      * This was nicer when we were using stdio, but thanks to
 474      * Solaris bugs, its a bad idea.  fopen can fail if more than
 475      * 256 FDs are open.
 476      */
 477     /*
 478      * unlink() is here because there is currently no way to make
 479      * logfileOpen() use O_TRUNC.
 480      */
 481     unlink(Config.netdbFilename);
 482     lf = logfileOpen(Config.netdbFilename, 4096, 0);
 483
 484     if (NULL == lf) {
 485         debugs(50, 1, "netdbSaveState: " << Config.netdbFilename << ": " << xstrerror());
 486         return;
 487     }
 488
 489     hash_first(addr_table);
 490
 491     while ((n = (netdbEntry *) hash_next(addr_table))) {
 492         if (n->pings_recv == 0)
 493             continue;
 494
 495         logfilePrintf(lf, "%s %d %d %10.5f %10.5f %d %d",
 496                       n->network,
 497                       n->pings_sent,
 498                       n->pings_recv,
 499                       n->hops,
 500                       n->rtt,
 501                       (int) n->next_ping_time,
 502                       (int) n->last_use_time);
 503
 504         for (x = n->hosts; x; x = x->next)
 505             logfilePrintf(lf, " %s", hashKeyStr(&x->hash));
 506
 507         logfilePrintf(lf, "\n");
 508
 509         count++;
 510
 511 #undef RBUF_SZ
 512
 513     }
 514
 515     logfileClose(lf);
 516     getCurrentTime();
 517     debugs(38, 1, "NETDB state saved; " <<
 518            count << " entries, " <<
 519            tvSubMsec(start, current_time) << " msec" );
 520     eventAddIsh("netdbSaveState", netdbSaveState, NULL, 3600.0, 1);
 521 }
 522
 523 static void
 524 netdbReloadState(void)
 525 {
 526     if (strcmp(Config.netdbFilename, "none") == 0)
 527         return;
 528
 529     char *s;
 530     int fd;
 531     int l;
 532
 533     struct stat sb;
 534     netdbEntry *n;
 535     netdbEntry N;
 536
 537     IpAddress addr;
 538     int count = 0;
 539
 540     struct timeval start = current_time;
 541     /*
 542      * This was nicer when we were using stdio, but thanks to
 543      * Solaris bugs, its a bad idea.  fopen can fail if more than
 544      * 256 FDs are open.
 545      */
 546     fd = file_open(Config.netdbFilename, O_RDONLY | O_BINARY);
 547
 548     if (fd < 0)
 549         return;
 550
 551     if (fstat(fd, &sb) < 0) {
 552         file_close(fd);
 553         return;
 554     }
 555
 556     char *t;
 557     char *buf = (char *)xcalloc(1, sb.st_size + 1);
 558     t = buf;
 559     l = FD_READ_METHOD(fd, buf, sb.st_size);
 560     file_close(fd);
 561
 562     if (l <= 0) {
 563         safe_free (buf);
 564         return;
 565     };
 566
 567     while ((s = strchr(t, '\n'))) {
 568         char *q;
 569         assert(s - buf < l);
 570         *s = '\0';
 571         memset(&N, '\0', sizeof(netdbEntry));
 572         q = strtok(t, w_space);
 573         t = s + 1;
 574
 575         if (NULL == q)
 576             continue;
 577
 578         if (! (addr = q) )
 579             continue;
 580
 581         if (netdbLookupAddr(addr) != NULL)      /* no dups! */
 582             continue;
 583
 584         if ((q = strtok(NULL, w_space)) == NULL)
 585             continue;
 586
 587         N.pings_sent = atoi(q);
 588
 589         if ((q = strtok(NULL, w_space)) == NULL)
 590             continue;
 591
 592         N.pings_recv = atoi(q);
 593
 594         if (N.pings_recv == 0)
 595             continue;
 596
 597         /* give this measurement low weight */
 598         N.pings_sent = 1;
 599
 600         N.pings_recv = 1;
 601
 602         if ((q = strtok(NULL, w_space)) == NULL)
 603             continue;
 604
 605         N.hops = atof(q);
 606
 607         if ((q = strtok(NULL, w_space)) == NULL)
 608             continue;
 609
 610         N.rtt = atof(q);
 611
 612         if ((q = strtok(NULL, w_space)) == NULL)
 613             continue;
 614
 615         N.next_ping_time = (time_t) atoi(q);
 616
 617         if ((q = strtok(NULL, w_space)) == NULL)
 618             continue;
 619
 620         N.last_use_time = (time_t) atoi(q);
 621
 622         n = (netdbEntry *)memAllocate(MEM_NETDBENTRY);
 623
 624         xmemcpy(n, &N, sizeof(netdbEntry));
 625
 626         netdbHashInsert(n, addr);
 627
 628         while ((q = strtok(NULL, w_space)) != NULL) {
 629             if (netdbLookupHost(q) != NULL)     /* no dups! */
 630                 continue;
 631
 632             netdbHostInsert(n, q);
 633         }
 634
 635         count++;
 636     }
 637
 638     xfree(buf);
 639     getCurrentTime();
 640     debugs(38, 1, "NETDB state reloaded; " <<
 641            count << " entries, " <<
 642            tvSubMsec(start, current_time) << " msec" );
 643 }
 644
 645 static const char *
 646 netdbPeerName(const char *name)
 647 {
 648     const wordlist *w;
 649
 650     for (w = peer_names; w; w = w->next) {
 651         if (!strcmp(w->key, name))
 652             return w->key;
 653     }
 654
 655     return wordlistAdd(&peer_names, name);
 656 }
 657
 658 static void
 659 netdbFreeNetdbEntry(void *data)
 660 {
 661     netdbEntry *n = (netdbEntry *)data;
 662     safe_free(n->peers);
 663     memFree(n, MEM_NETDBENTRY);
 664 }
 665
 666 static void
 667 netdbFreeNameEntry(void *data)
 668 {
 669     net_db_name *x = (net_db_name *)data;
 670     xfree(x->hash.key);
 671     memFree(x, MEM_NET_DB_NAME);
 672 }
 673
 674
 675 static void
 676 netdbExchangeHandleReply(void *data, StoreIOBuffer receivedData)
 677 {
 678     IpAddress addr;
 679
 680     netdbExchangeState *ex = (netdbExchangeState *)data;
 681     int rec_sz = 0;
 682     int o;
 683
 684     struct in_addr line_addr;
 685     double rtt;
 686     double hops;
 687     char *p;
 688     int j;
 689     HttpReply const *rep;
 690     size_t hdr_sz;
 691     int nused = 0;
 692     int size;
 693     int oldbufofs = ex->buf_ofs;
 694
 695     rec_sz = 0;
 696     rec_sz += 1 + sizeof(struct in_addr);
 697     rec_sz += 1 + sizeof(int);
 698     rec_sz += 1 + sizeof(int);
 699     debugs(38, 3, "netdbExchangeHandleReply: " << receivedData.length << " read bytes");
 700
 701     if (!cbdataReferenceValid(ex->p)) {
 702         debugs(38, 3, "netdbExchangeHandleReply: Peer became invalid");
 703         netdbExchangeDone(ex);
 704         return;
 705     }
 706
 707     debugs(38, 3, "netdbExchangeHandleReply: for '" << ex->p->host << ":" << ex->p->http_port << "'");
 708
 709     if (receivedData.length == 0 &&
 710             !receivedData.flags.error) {
 711         debugs(38, 3, "netdbExchangeHandleReply: Done");
 712         netdbExchangeDone(ex);
 713         return;
 714     }
 715
 716     p = ex->buf;
 717
 718     /* Get the size of the buffer now */
 719     size = ex->buf_ofs + receivedData.length;
 720     debugs(38, 3, "netdbExchangeHandleReply: " << size << " bytes buf");
 721
 722     /* Check if we're still doing headers */
 723
 724     if (ex->connstate == STATE_HEADER) {
 725
 726         ex->buf_ofs += receivedData.length;
 727
 728         /* skip reply headers */
 729
 730         if ((hdr_sz = headersEnd(p, ex->buf_ofs))) {
 731             debugs(38, 5, "netdbExchangeHandleReply: hdr_sz = " << hdr_sz);
 732             rep = ex->e->getReply();
 733             assert (0 != rep->sline.status);
 734             debugs(38, 3, "netdbExchangeHandleReply: reply status " << rep->sline.status);
 735
 736             if (HTTP_OK != rep->sline.status) {
 737                 netdbExchangeDone(ex);
 738                 return;
 739             }
 740
 741             assert((size_t)ex->buf_ofs >= hdr_sz);
 742
 743             /*
 744              * Now, point p to the part of the buffer where the data
 745              * starts, and update the size accordingly
 746              */
 747             assert(ex->used == 0);
 748             ex->used = hdr_sz;
 749             size = ex->buf_ofs - hdr_sz;
 750             p += hdr_sz;
 751
 752             /* Finally, set the conn state mode to STATE_BODY */
 753             ex->connstate = STATE_BODY;
 754         } else {
 755             StoreIOBuffer tempBuffer;
 756             tempBuffer.offset = ex->buf_ofs;
 757             tempBuffer.length = ex->buf_sz - ex->buf_ofs;
 758             tempBuffer.data = ex->buf + ex->buf_ofs;
 759             /* Have more headers .. */
 760             storeClientCopy(ex->sc, ex->e, tempBuffer,
 761                             netdbExchangeHandleReply, ex);
 762             return;
 763         }
 764     }
 765
 766     assert(ex->connstate == STATE_BODY);
 767
 768     /* If we get here, we have some body to parse .. */
 769     debugs(38, 5, "netdbExchangeHandleReply: start parsing loop, size = " << size);
 770
 771     while (size >= rec_sz) {
 772         debugs(38, 5, "netdbExchangeHandleReply: in parsing loop, size = " << size);
 773         addr.SetAnyAddr();
 774         hops = rtt = 0.0;
 775
 776         for (o = 0; o < rec_sz;) {
 777             switch ((int) *(p + o)) {
 778
 779             case NETDB_EX_NETWORK:
 780                 o++;
 781                 /* FIXME INET6 : NetDB can still ony send IPv4 */
 782                 xmemcpy(&line_addr, p + o, sizeof(struct in_addr));
 783                 addr = line_addr;
 784                 o += sizeof(struct in_addr);
 785                 break;
 786
 787             case NETDB_EX_RTT:
 788                 o++;
 789                 xmemcpy(&j, p + o, sizeof(int));
 790                 o += sizeof(int);
 791                 rtt = (double) ntohl(j) / 1000.0;
 792                 break;
 793
 794             case NETDB_EX_HOPS:
 795                 o++;
 796                 xmemcpy(&j, p + o, sizeof(int));
 797                 o += sizeof(int);
 798                 hops = (double) ntohl(j) / 1000.0;
 799                 break;
 800
 801             default:
 802                 debugs(38, 1, "netdbExchangeHandleReply: corrupt data, aborting");
 803                 netdbExchangeDone(ex);
 804                 return;
 805             }
 806         }
 807
 808         if (!addr.IsAnyAddr() && rtt > 0)
 809             netdbExchangeUpdatePeer(addr, ex->p, rtt, hops);
 810
 811         assert(o == rec_sz);
 812
 813         ex->used += rec_sz;
 814
 815         size -= rec_sz;
 816
 817         p += rec_sz;
 818
 819         nused++;
 820     }
 821
 822     /*
 823      * Copy anything that is left over to the beginning of the buffer,
 824      * and adjust buf_ofs accordingly
 825      */
 826
 827     /*
 828      * Evilly, size refers to the buf size left now,
 829      * ex->buf_ofs is the original buffer size, so just copy that
 830      * much data over
 831      */
 832     memmove(ex->buf, ex->buf + (ex->buf_ofs - size), size);
 833
 834     ex->buf_ofs = size;
 835
 836     /*
 837      * And don't re-copy the remaining data ..
 838      */
 839     ex->used += size;
 840
 841     /*
 842      * Now the tricky bit - size _included_ the leftover bit from the _last_
 843      * storeClientCopy. We don't want to include that, or our offset will be wrong.
 844      * So, don't count the size of the leftover buffer we began with.
 845      * This can _disappear_ when we're not tracking offsets ..
 846      */
 847     ex->used -= oldbufofs;
 848
 849     debugs(38, 3, "netdbExchangeHandleReply: size left over in this buffer: " << size << " bytes");
 850
 851     debugs(38, 3, "netdbExchangeHandleReply: used " << nused <<
 852            " entries, (x " << rec_sz << " bytes) == " << nused * rec_sz <<
 853            " bytes total");
 854
 855     debugs(38, 3, "netdbExchangeHandleReply: used " << ex->used);
 856
 857     if (EBIT_TEST(ex->e->flags, ENTRY_ABORTED)) {
 858         debugs(38, 3, "netdbExchangeHandleReply: ENTRY_ABORTED");
 859         netdbExchangeDone(ex);
 860     } else if (ex->e->store_status == STORE_PENDING) {
 861         StoreIOBuffer tempBuffer;
 862         tempBuffer.offset = ex->used;
 863         tempBuffer.length = ex->buf_sz - ex->buf_ofs;
 864         tempBuffer.data = ex->buf + ex->buf_ofs;
 865         debugs(38, 3, "netdbExchangeHandleReply: EOF not received");
 866         storeClientCopy(ex->sc, ex->e, tempBuffer,
 867                         netdbExchangeHandleReply, ex);
 868     }
 869 }
 870
 871 static void
 872 netdbExchangeDone(void *data)
 873 {
 874     netdbExchangeState *ex = (netdbExchangeState *)data;
 875     debugs(38, 3, "netdbExchangeDone: " << ex->e->url()  );
 876     HTTPMSGUNLOCK(ex->r);
 877     storeUnregister(ex->sc, ex->e, ex);
 878     ex->e->unlock();
 879     cbdataReferenceDone(ex->p);
 880     cbdataFree(ex);
 881 }
 882
 883 static void
 884 netdbRegisterWithCacheManager(void)
 885 {
 886     CacheManager::GetInstance()->
 887     registerAction("netdb", "Network Measurement Database", netdbDump, 0, 1);
 888 }
 889
 890 #endif /* USE_ICMP */
 891
 892
 893 /* PUBLIC FUNCTIONS */
 894
 895 void
 896 netdbInit(void)
 897 {
 898 #if USE_ICMP
 899     int n;
 900
 901     netdbRegisterWithCacheManager();
 902
 903     if (addr_table)
 904         return;
 905
 906     n = hashPrime(Config.Netdb.high / 4);
 907
 908     addr_table = hash_create((HASHCMP *) strcmp, n, hash_string);
 909
 910     n = hashPrime(3 * Config.Netdb.high / 4);
 911
 912     host_table = hash_create((HASHCMP *) strcmp, n, hash_string);
 913
 914     eventAddIsh("netdbSaveState", netdbSaveState, NULL, 3600.0, 1);
 915
 916     netdbReloadState();
 917
 918 #endif
 919 }
 920
 921 void
 922 netdbPingSite(const char *hostname)
 923 {
 924 #if USE_ICMP
 925     netdbEntry *n;
 926
 927     if ((n = netdbLookupHost(hostname)) != NULL)
 928         if (n->next_ping_time > squid_curtime)
 929             return;
 930
 931     ipcache_nbgethostbyname(hostname, netdbSendPing,
 932                             new generic_cbdata(xstrdup(hostname)));
 933
 934 #endif
 935 }
 936
 937 void
 938 netdbHandlePingReply(const IpAddress &from, int hops, int rtt)
 939 {
 940 #if USE_ICMP
 941     netdbEntry *n;
 942     int N;
 943     debugs(38, 3, "netdbHandlePingReply: from " << from);
 944
 945     if ((n = netdbLookupAddr(from)) == NULL)
 946         return;
 947
 948     N = ++n->pings_recv;
 949
 950     if (N > 5)
 951         N = 5;
 952
 953     if (rtt < 1)
 954         rtt = 1;
 955
 956     n->hops = ((n->hops * (N - 1)) + hops) / N;
 957
 958     n->rtt = ((n->rtt * (N - 1)) + rtt) / N;
 959
 960     debugs(38, 3, "netdbHandlePingReply: " << n->network  << "; rtt="<<
 961            std::setw(5)<< std::setprecision(2) << n->rtt << "  hops="<<
 962            std::setw(4) << n->hops);
 963
 964 #endif
 965 }
 966
 967 void
 968 netdbFreeMemory(void)
 969 {
 970 #if USE_ICMP
 971     hashFreeItems(addr_table, netdbFreeNetdbEntry);
 972     hashFreeMemory(addr_table);
 973     addr_table = NULL;
 974     hashFreeItems(host_table, netdbFreeNameEntry);
 975     hashFreeMemory(host_table);
 976     host_table = NULL;
 977     wordlistDestroy(&peer_names);
 978     peer_names = NULL;
 979 #endif
 980 }
 981
 982
 983 #if 0 // AYJ: Looks to be unused code.
 984 int
 985 netdbHops(IpAddress &addr)
 986 {
 987 #if USE_ICMP
 988     netdbEntry *n = netdbLookupAddr(addr);
 989
 990     if (n && n->pings_recv) {
 991         n->last_use_time = squid_curtime;
 992         return (int) (n->hops + 0.5);
 993     }
 994
 995 #endif
 996     return 256;
 997 }
 998 #endif
 999
1000 void
1001 netdbDump(StoreEntry * sentry)
1002 {
1003 #if USE_ICMP
1004     netdbEntry *n;
1005     netdbEntry **list;
1006     net_db_name *x;
1007     int k;
1008     int i;
1009     int j;
1010     net_db_peer *p;
1011     storeAppendPrintf(sentry, "Network DB Statistics:\n");
1012     storeAppendPrintf(sentry, "%-46.46s %9s %7s %5s %s\n",  /* Max between 16 (IPv4) or 46 (IPv6)   */
1013                       "Network",
1014                       "recv/sent",
1015                       "RTT",
1016                       "Hops",
1017                       "Hostnames");
1018     list = (netdbEntry **)xcalloc(memInUse(MEM_NETDBENTRY), sizeof(netdbEntry *));
1019     i = 0;
1020     hash_first(addr_table);
1021
1022     while ((n = (netdbEntry *) hash_next(addr_table)))
1023         *(list + i++) = n;
1024
1025     if (i != memInUse(MEM_NETDBENTRY))
1026         debugs(38, 0, "WARNING: netdb_addrs count off, found " << i <<
1027                ", expected " << memInUse(MEM_NETDBENTRY));
1028
1029     qsort((char *) list,
1030           i,
1031           sizeof(netdbEntry *),
1032           sortByRtt);
1033
1034     for (k = 0; k < i; k++) {
1035         n = *(list + k);
1036         storeAppendPrintf(sentry, "%-46.46s %4d/%4d %7.1f %5.1f", /* Max between 16 (IPv4) or 46 (IPv6)   */
1037                           n->network,
1038                           n->pings_recv,
1039                           n->pings_sent,
1040                           n->rtt,
1041                           n->hops);
1042
1043         for (x = n->hosts; x; x = x->next)
1044             storeAppendPrintf(sentry, " %s", hashKeyStr(&x->hash));
1045
1046         storeAppendPrintf(sentry, "\n");
1047
1048         p = n->peers;
1049
1050         for (j = 0; j < n->n_peers; j++, p++) {
1051             storeAppendPrintf(sentry, "    %-22.22s %7.1f %5.1f\n",
1052                               p->peername,
1053                               p->rtt,
1054                               p->hops);
1055         }
1056     }
1057
1058     xfree(list);
1059 #else
1060
1061     storeAppendPrintf(sentry,"NETDB support not compiled into this Squid cache.\n");
1062 #endif
1063 }
1064
1065 int
1066 netdbHostHops(const char *host)
1067 {
1068 #if USE_ICMP
1069     netdbEntry *n = netdbLookupHost(host);
1070
1071     if (n) {
1072         n->last_use_time = squid_curtime;
1073         return (int) (n->hops + 0.5);
1074     }
1075
1076 #endif
1077     return 0;
1078 }
1079
1080 int
1081 netdbHostRtt(const char *host)
1082 {
1083 #if USE_ICMP
1084     netdbEntry *n = netdbLookupHost(host);
1085
1086     if (n) {
1087         n->last_use_time = squid_curtime;
1088         return (int) (n->rtt + 0.5);
1089     }
1090
1091 #endif
1092     return 0;
1093 }
1094
1095 void
1096 netdbHostData(const char *host, int *samp, int *rtt, int *hops)
1097 {
1098 #if USE_ICMP
1099     netdbEntry *n = netdbLookupHost(host);
1100
1101     if (n == NULL)
1102         return;
1103
1104     *samp = n->pings_recv;
1105
1106     *rtt = (int) (n->rtt + 0.5);
1107
1108     *hops = (int) (n->hops + 0.5);
1109
1110     n->last_use_time = squid_curtime;
1111
1112 #endif
1113 }
1114
1115 void
1116 netdbUpdatePeer(HttpRequest * r, peer * e, int irtt, int ihops)
1117 {
1118 #if USE_ICMP
1119     netdbEntry *n;
1120     double rtt = (double) irtt;
1121     double hops = (double) ihops;
1122     net_db_peer *p;
1123     debugs(38, 3, "netdbUpdatePeer: '" << r->GetHost() << "', " << ihops << " hops, " << irtt << " rtt");
1124     n = netdbLookupHost(r->GetHost());
1125
1126     if (n == NULL) {
1127         debugs(38, 3, "netdbUpdatePeer: host '" << r->GetHost() << "' not found");
1128         return;
1129     }
1130
1131     if ((p = netdbPeerByName(n, e->host)) == NULL)
1132         p = netdbPeerAdd(n, e);
1133
1134     p->rtt = rtt;
1135
1136     p->hops = hops;
1137
1138     p->expires = squid_curtime + 3600;
1139
1140     if (n->n_peers < 2)
1141         return;
1142
1143     qsort((char *) n->peers,
1144           n->n_peers,
1145           sizeof(net_db_peer),
1146           sortPeerByRtt);
1147
1148 #endif
1149 }
1150
1151 void
1152 netdbExchangeUpdatePeer(IpAddress &addr, peer * e, double rtt, double hops)
1153 {
1154 #if USE_ICMP
1155     netdbEntry *n;
1156     net_db_peer *p;
1157     debugs(38, 5, "netdbExchangeUpdatePeer: '" << addr << "', "<<
1158            std::setfill('0')<< std::setprecision(2) << hops << " hops, " <<
1159            rtt << " rtt");
1160
1161     if ( !addr.IsIPv4() ) {
1162         debugs(38, 5, "netdbExchangeUpdatePeer: Aborting peer update for '" << addr << "', NetDB cannot handle IPv6.");
1163         return;
1164     }
1165
1166     n = netdbLookupAddr(addr);
1167
1168     if (n == NULL)
1169         n = netdbAdd(addr);
1170
1171     assert(NULL != n);
1172
1173     if ((p = netdbPeerByName(n, e->host)) == NULL)
1174         p = netdbPeerAdd(n, e);
1175
1176     p->rtt = rtt;
1177
1178     p->hops = hops;
1179
1180     p->expires = squid_curtime + 3600;  /* XXX ? */
1181
1182     if (n->n_peers < 2)
1183         return;
1184
1185     qsort((char *) n->peers,
1186           n->n_peers,
1187           sizeof(net_db_peer),
1188           sortPeerByRtt);
1189
1190 #endif
1191 }
1192
1193 void
1194 netdbDeleteAddrNetwork(IpAddress &addr)
1195 {
1196 #if USE_ICMP
1197     netdbEntry *n = netdbLookupAddr(addr);
1198
1199     if (n == NULL)
1200         return;
1201
1202     debugs(38, 3, "netdbDeleteAddrNetwork: " << n->network);
1203
1204     netdbRelease(n);
1205 #endif
1206 }
1207
1208
1209 void
1210 netdbBinaryExchange(StoreEntry * s)
1211 {
1212     HttpReply *reply = new HttpReply;
1213 #if USE_ICMP
1214
1215     IpAddress addr;
1216
1217     netdbEntry *n;
1218     int i;
1219     int j;
1220     int rec_sz;
1221     char *buf;
1222
1223     struct in_addr line_addr;
1224     s->buffer();
1225     HttpVersion version(1, 0);
1226     reply->setHeaders(version, HTTP_OK, "OK", NULL, -1, squid_curtime, -2);
1227     s->replaceHttpReply(reply);
1228     rec_sz = 0;
1229     rec_sz += 1 + sizeof(struct in_addr);
1230     rec_sz += 1 + sizeof(int);
1231     rec_sz += 1 + sizeof(int);
1232     buf = (char *)memAllocate(MEM_4K_BUF);
1233     i = 0;
1234     hash_first(addr_table);
1235
1236     while ((n = (netdbEntry *) hash_next(addr_table))) {
1237         if (0.0 == n->rtt)
1238             continue;
1239
1240         if (n->rtt > 60000)     /* RTT > 1 MIN probably bogus */
1241             continue;
1242
1243         if (! (addr = n->network) )
1244             continue;
1245
1246         /* FIXME INET6 : NetDB cannot yet handle IPv6 addresses. Ensure only IPv4 get sent. */
1247         if ( !addr.IsIPv4() )
1248             continue;
1249
1250         buf[i++] = (char) NETDB_EX_NETWORK;
1251
1252         addr.GetInAddr(line_addr);
1253         xmemcpy(&buf[i], &line_addr, sizeof(struct in_addr));
1254
1255         i += sizeof(struct in_addr);
1256
1257         buf[i++] = (char) NETDB_EX_RTT;
1258
1259         j = htonl((int) (n->rtt * 1000));
1260
1261         xmemcpy(&buf[i], &j, sizeof(int));
1262
1263         i += sizeof(int);
1264
1265         buf[i++] = (char) NETDB_EX_HOPS;
1266
1267         j = htonl((int) (n->hops * 1000));
1268
1269         xmemcpy(&buf[i], &j, sizeof(int));
1270
1271         i += sizeof(int);
1272
1273         if (i + rec_sz > 4096) {
1274             s->append(buf, i);
1275             i = 0;
1276         }
1277     }
1278
1279     if (i > 0) {
1280         s->append(buf, i);
1281         i = 0;
1282     }
1283
1284     assert(0 == i);
1285     s->flush();
1286     memFree(buf, MEM_4K_BUF);
1287 #else
1288
1289     HttpVersion version(1,0);
1290     reply->setHeaders(version, HTTP_BAD_REQUEST, "Bad Request",
1291                       NULL, -1, squid_curtime, -2);
1292     s->replaceHttpReply(reply);
1293     storeAppendPrintf(s, "NETDB support not compiled into this Squid cache.\n");
1294 #endif
1295
1296     s->complete();
1297 }
1298
1299 #if USE_ICMP
1300 CBDATA_TYPE(netdbExchangeState);
1301 #endif
1302
1303 void
1304 netdbExchangeStart(void *data)
1305 {
1306 #if USE_ICMP
1307     peer *p = (peer *)data;
1308     char *uri;
1309     netdbExchangeState *ex;
1310     StoreIOBuffer tempBuffer;
1311     CBDATA_INIT_TYPE(netdbExchangeState);
1312     ex = cbdataAlloc(netdbExchangeState);
1313     ex->p = cbdataReference(p);
1314     uri = internalRemoteUri(p->host, p->http_port, "/squid-internal-dynamic/", "netdb");
1315     debugs(38, 3, "netdbExchangeStart: Requesting '" << uri << "'");
1316     assert(NULL != uri);
1317     ex->r = HttpRequest::CreateFromUrl(uri);
1318
1319     if (NULL == ex->r) {
1320         debugs(38, 1, "netdbExchangeStart: Bad URI " << uri);
1321         return;
1322     }
1323
1324     HTTPMSGLOCK(ex->r);
1325     assert(NULL != ex->r);
1326     ex->r->http_ver = HttpVersion(1,0);
1327     ex->connstate = STATE_HEADER;
1328     ex->e = storeCreateEntry(uri, uri, request_flags(), METHOD_GET);
1329     ex->buf_sz = NETDB_REQBUF_SZ;
1330     assert(NULL != ex->e);
1331     ex->sc = storeClientListAdd(ex->e, ex);
1332     tempBuffer.offset = 0;
1333     tempBuffer.length = ex->buf_sz;
1334     tempBuffer.data = ex->buf;
1335     storeClientCopy(ex->sc, ex->e, tempBuffer,
1336                     netdbExchangeHandleReply, ex);
1337     ex->r->flags.loopdetect = 1;        /* cheat! -- force direct */
1338
1339     if (p->login)
1340         xstrncpy(ex->r->login, p->login, MAX_LOGIN_SZ);
1341
1342     urlCanonical(ex->r);
1343
1344     FwdState::fwdStart(-1, ex->e, ex->r);
1345
1346 #endif
1347 }
1348
1349 peer *
1350 netdbClosestParent(HttpRequest * request)
1351 {
1352 #if USE_ICMP
1353     peer *p = NULL;
1354     netdbEntry *n;
1355     const ipcache_addrs *ia;
1356     net_db_peer *h;
1357     int i;
1358     n = netdbLookupHost(request->GetHost());
1359
1360     if (NULL == n) {
1361         /* try IP addr */
1362         ia = ipcache_gethostbyname(request->GetHost(), 0);
1363
1364         if (NULL != ia)
1365             n = netdbLookupAddr(ia->in_addrs[ia->cur]);
1366     }
1367
1368     if (NULL == n)
1369         return NULL;
1370
1371     if (0 == n->n_peers)
1372         return NULL;
1373
1374     n->last_use_time = squid_curtime;
1375
1376     /*
1377      * Find the parent with the least RTT to the origin server.
1378      * Make sure we don't return a parent who is farther away than
1379      * we are.  Note, the n->peers list is pre-sorted by RTT.
1380      */
1381     for (i = 0; i < n->n_peers; i++) {
1382         h = &n->peers[i];
1383
1384         if (n->rtt > 0)
1385             if (n->rtt < h->rtt)
1386                 break;
1387
1388         p = peerFindByName(h->peername);
1389
1390         if (NULL == p)          /* not found */
1391             continue;
1392
1393         if (neighborType(p, request) != PEER_PARENT)
1394             continue;
1395
1396         if (!peerHTTPOkay(p, request))  /* not allowed */
1397             continue;
1398
1399         return p;
1400     }
1401
1402 #endif
1403     return NULL;
1404 }