src/icmp/net_db.cc

   1 /*
   2  * DEBUG: section 38    Network Measurement Database
   3  * AUTHOR: Duane Wessels
   4  *
   5  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   6  * ----------------------------------------------------------
   7  *
   8  *  Squid is the result of efforts by numerous individuals from
   9  *  the Internet community; see the CONTRIBUTORS file for full
  10  *  details.   Many organizations have provided support for Squid's
  11  *  development; see the SPONSORS file for full details.  Squid is
  12  *  Copyrighted (C) 2001 by the Regents of the University of
  13  *  California; see the COPYRIGHT file for full details.  Squid
  14  *  incorporates software developed and/or copyrighted by other
  15  *  sources; see the CREDITS file for full details.
  16  *
  17  *  This program is free software; you can redistribute it and/or modify
  18  *  it under the terms of the GNU General Public License as published by
  19  *  the Free Software Foundation; either version 2 of the License, or
  20  *  (at your option) any later version.
  21  *
  22  *  This program is distributed in the hope that it will be useful,
  23  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  24  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  25  *  GNU General Public License for more details.
  26  *
  27  *  You should have received a copy of the GNU General Public License
  28  *  along with this program; if not, write to the Free Software
  29  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  30  *
  31  */
  32
  33 /*
  34  * XXX XXX XXX
  35  *
  36  * This code may be slightly broken now. If you're getting consistent
  37  * (sometimes working) corrupt data exchanges, please contact adrian
  38  * (adrian@squid-cache.org) to sort them out.
  39  */
  40
  41 #include "squid.h"
  42 #include "icmp/net_db.h"
  43 #include "log/File.h"
  44 #include "cbdata.h"
  45 #include "event.h"
  46 #include "CacheManager.h"
  47 #include "Store.h"
  48 #include "SwapDir.h"
  49 #include "HttpRequest.h"
  50 #include "HttpReply.h"
  51 #include "MemObject.h"
  52 #include "fde.h"
  53 #include "forward.h"
  54 #include "SquidTime.h"
  55 #include "wordlist.h"
  56 #include "ip/Address.h"
  57
  58 #if USE_ICMP
  59 #include "icmp/IcmpSquid.h"
  60 #include "StoreClient.h"
  61
  62 #define NETDB_REQBUF_SZ 4096
  63
  64 typedef enum {
  65     STATE_NONE,
  66     STATE_HEADER,
  67     STATE_BODY
  68 } netdb_conn_state_t;
  69
  70 typedef struct {
  71     peer *p;
  72     StoreEntry *e;
  73     store_client *sc;
  74     HttpRequest *r;
  75     int64_t used;
  76     size_t buf_sz;
  77     char buf[NETDB_REQBUF_SZ];
  78     int buf_ofs;
  79     netdb_conn_state_t connstate;
  80 } netdbExchangeState;
  81
  82 static hash_table *addr_table = NULL;
  83 static hash_table *host_table = NULL;
  84
  85 Ip::Address networkFromInaddr(const Ip::Address &a);
  86 static void netdbRelease(netdbEntry * n);
  87
  88 static void netdbHashInsert(netdbEntry * n, Ip::Address &addr);
  89 static void netdbHashDelete(const char *key);
  90 static void netdbHostInsert(netdbEntry * n, const char *hostname);
  91 static void netdbHostDelete(const net_db_name * x);
  92 static void netdbPurgeLRU(void);
  93 static netdbEntry *netdbLookupHost(const char *key);
  94 static net_db_peer *netdbPeerByName(const netdbEntry * n, const char *);
  95 static net_db_peer *netdbPeerAdd(netdbEntry * n, peer * e);
  96 static const char *netdbPeerName(const char *name);
  97 static IPH netdbSendPing;
  98 static QS sortPeerByRtt;
  99 static QS sortByRtt;
 100 static QS netdbLRU;
 101 static FREE netdbFreeNameEntry;
 102 static FREE netdbFreeNetdbEntry;
 103 static STCB netdbExchangeHandleReply;
 104 static void netdbExchangeDone(void *);
 105
 106 /* We have to keep a local list of peer names.  The Peers structure
 107  * gets freed during a reconfigure.  We want this database to
 108  * remain persisitent, so _net_db_peer->peername points into this
 109  * linked list */
 110 static wordlist *peer_names = NULL;
 111
 112 static void
 113 netdbHashInsert(netdbEntry * n, Ip::Address &addr)
 114 {
 115     networkFromInaddr(addr).NtoA(n->network, MAX_IPSTRLEN);
 116     n->hash.key = n->network;
 117     assert(hash_lookup(addr_table, n->network) == NULL);
 118     hash_join(addr_table, &n->hash);
 119 }
 120
 121 static void
 122 netdbHashDelete(const char *key)
 123 {
 124     hash_link *hptr = (hash_link *)hash_lookup(addr_table, key);
 125
 126     if (hptr == NULL) {
 127         debug_trap("netdbHashDelete: key not found");
 128         return;
 129     }
 130
 131     hash_remove_link(addr_table, hptr);
 132 }
 133
 134 static void
 135 netdbHostInsert(netdbEntry * n, const char *hostname)
 136 {
 137     net_db_name *x = (net_db_name *)memAllocate(MEM_NET_DB_NAME);
 138     x->hash.key = xstrdup(hostname);
 139     x->next = n->hosts;
 140     n->hosts = x;
 141     x->net_db_entry = n;
 142     assert(hash_lookup(host_table, hostname) == NULL);
 143     hash_join(host_table, &x->hash);
 144     n->link_count++;
 145 }
 146
 147 static void
 148 netdbHostDelete(const net_db_name * x)
 149 {
 150     netdbEntry *n;
 151     net_db_name **X;
 152     assert(x != NULL);
 153     assert(x->net_db_entry != NULL);
 154     n = x->net_db_entry;
 155     n->link_count--;
 156
 157     for (X = &n->hosts; *X; X = &(*X)->next) {
 158         if (*X == x) {
 159             *X = x->next;
 160             break;
 161         }
 162     }
 163
 164     hash_remove_link(host_table, (hash_link *) x);
 165     xfree(x->hash.key);
 166     memFree((void *) x, MEM_NET_DB_NAME);
 167 }
 168
 169 static netdbEntry *
 170 netdbLookupHost(const char *key)
 171 {
 172     net_db_name *x = (net_db_name *) hash_lookup(host_table, key);
 173     return x ? x->net_db_entry : NULL;
 174 }
 175
 176 static void
 177 netdbRelease(netdbEntry * n)
 178 {
 179     net_db_name *x;
 180     net_db_name *next;
 181
 182     for (x = n->hosts; x; x = next) {
 183         next = x->next;
 184         netdbHostDelete(x);
 185     }
 186
 187     n->hosts = NULL;
 188     safe_free(n->peers);
 189     n->peers = NULL;
 190     n->n_peers = 0;
 191     n->n_peers_alloc = 0;
 192
 193     if (n->link_count == 0) {
 194         netdbHashDelete(n->network);
 195         memFree(n, MEM_NETDBENTRY);
 196     }
 197 }
 198
 199 static int
 200 netdbLRU(const void *A, const void *B)
 201 {
 202     const netdbEntry *const *n1 = (const netdbEntry *const *)A;
 203     const netdbEntry *const *n2 = (const netdbEntry *const *)B;
 204
 205     if ((*n1)->last_use_time > (*n2)->last_use_time)
 206         return (1);
 207
 208     if ((*n1)->last_use_time < (*n2)->last_use_time)
 209         return (-1);
 210
 211     return (0);
 212 }
 213
 214 static void
 215 netdbPurgeLRU(void)
 216 {
 217     netdbEntry *n;
 218     netdbEntry **list;
 219     int k = 0;
 220     int list_count = 0;
 221     int removed = 0;
 222     list = (netdbEntry **)xcalloc(memInUse(MEM_NETDBENTRY), sizeof(netdbEntry *));
 223     hash_first(addr_table);
 224
 225     while ((n = (netdbEntry *) hash_next(addr_table))) {
 226         assert(list_count < memInUse(MEM_NETDBENTRY));
 227         *(list + list_count) = n;
 228         list_count++;
 229     }
 230
 231     qsort((char *) list,
 232           list_count,
 233           sizeof(netdbEntry *),
 234           netdbLRU);
 235
 236     for (k = 0; k < list_count; k++) {
 237         if (memInUse(MEM_NETDBENTRY) < Config.Netdb.low)
 238             break;
 239
 240         netdbRelease(*(list + k));
 241
 242         removed++;
 243     }
 244
 245     xfree(list);
 246 }
 247
 248 static netdbEntry *
 249 netdbLookupAddr(const Ip::Address &addr)
 250 {
 251     netdbEntry *n;
 252     char *key = new char[MAX_IPSTRLEN];
 253     networkFromInaddr(addr).NtoA(key,MAX_IPSTRLEN);
 254     n = (netdbEntry *) hash_lookup(addr_table, key);
 255     return n;
 256 }
 257
 258 static netdbEntry *
 259 netdbAdd(Ip::Address &addr)
 260 {
 261     netdbEntry *n;
 262
 263     if (memInUse(MEM_NETDBENTRY) > Config.Netdb.high)
 264         netdbPurgeLRU();
 265
 266     if ((n = netdbLookupAddr(addr)) == NULL) {
 267         n = (netdbEntry *)memAllocate(MEM_NETDBENTRY);
 268         netdbHashInsert(n, addr);
 269     }
 270
 271     return n;
 272 }
 273
 274 static void
 275 netdbSendPing(const ipcache_addrs *ia, const DnsLookupDetails &, void *data)
 276 {
 277     Ip::Address addr;
 278     char *hostname = NULL;
 279     static_cast<generic_cbdata *>(data)->unwrap(&hostname);
 280     netdbEntry *n;
 281     netdbEntry *na;
 282     net_db_name *x;
 283     net_db_name **X;
 284
 285     if (ia == NULL) {
 286         xfree(hostname);
 287         return;
 288     }
 289
 290     addr = ia->in_addrs[ia->cur];
 291
 292     if ((n = netdbLookupHost(hostname)) == NULL) {
 293         n = netdbAdd(addr);
 294         netdbHostInsert(n, hostname);
 295     } else if ((na = netdbLookupAddr(addr)) != n) {
 296         /*
 297          *hostname moved from 'network n' to 'network na'!
 298          */
 299
 300         if (na == NULL)
 301             na = netdbAdd(addr);
 302
 303         debugs(38, 3, "netdbSendPing: " << hostname << " moved from " << n->network << " to " << na->network);
 304
 305         x = (net_db_name *) hash_lookup(host_table, hostname);
 306
 307         if (x == NULL) {
 308             debugs(38, 1, "netdbSendPing: net_db_name list bug: " << hostname << " not found");
 309             xfree(hostname);
 310             return;
 311         }
 312
 313         /* remove net_db_name from 'network n' linked list */
 314         for (X = &n->hosts; *X; X = &(*X)->next) {
 315             if (*X == x) {
 316                 *X = x->next;
 317                 break;
 318             }
 319         }
 320
 321         n->link_count--;
 322         /* point to 'network na' from host entry */
 323         x->net_db_entry = na;
 324         /* link net_db_name to 'network na' */
 325         x->next = na->hosts;
 326         na->hosts = x;
 327         na->link_count++;
 328         n = na;
 329     }
 330
 331     if (n->next_ping_time <= squid_curtime) {
 332         debugs(38, 3, "netdbSendPing: pinging " << hostname);
 333         icmpEngine.DomainPing(addr, hostname);
 334         n->pings_sent++;
 335         n->next_ping_time = squid_curtime + Config.Netdb.period;
 336         n->last_use_time = squid_curtime;
 337     }
 338
 339     xfree(hostname);
 340 }
 341
 342 Ip::Address
 343 networkFromInaddr(const Ip::Address &in)
 344 {
 345     Ip::Address out;
 346
 347     out = in;
 348
 349     /* in IPv6 the 'network' should be the routing section. */
 350     if ( in.IsIPv6() ) {
 351         out.ApplyMask(64, AF_INET6);
 352         debugs(14, 5, "networkFromInaddr : Masked IPv6 Address to " << in << "/64 routing part.");
 353         return out;
 354     }
 355
 356 #if USE_CLASSFUL
 357     struct in_addr b;
 358
 359     in.GetInAddr(b);
 360
 361     if (IN_CLASSC(b.s_addr))
 362         b.s_addr &= IN_CLASSC_NET;
 363     else if (IN_CLASSB(b.s_addr))
 364         b.s_addr &= IN_CLASSB_NET;
 365     else if (IN_CLASSA(b.s_addr))
 366         b.s_addr &= IN_CLASSA_NET;
 367
 368     out = b;
 369
 370 #endif
 371
 372     debugs(14, 5, "networkFromInaddr : Masked IPv4 Address to " << out << "/24.");
 373
 374     /* use /24 for everything under IPv4 */
 375     out.ApplyMask(24, AF_INET);
 376     debugs(14, 5, "networkFromInaddr : Masked IPv4 Address to " << in << "/24.");
 377
 378     return out;
 379 }
 380
 381 static int
 382 sortByRtt(const void *A, const void *B)
 383 {
 384     const netdbEntry *const *n1 = (const netdbEntry *const *)A;
 385     const netdbEntry *const *n2 = (const netdbEntry *const *)B;
 386
 387     if ((*n1)->rtt > (*n2)->rtt)
 388         return 1;
 389     else if ((*n1)->rtt < (*n2)->rtt)
 390         return -1;
 391     else
 392         return 0;
 393 }
 394
 395 static net_db_peer *
 396 netdbPeerByName(const netdbEntry * n, const char *peername)
 397 {
 398     int i;
 399     net_db_peer *p = n->peers;
 400
 401     for (i = 0; i < n->n_peers; i++, p++) {
 402         if (!strcmp(p->peername, peername))
 403             return p;
 404     }
 405
 406     return NULL;
 407 }
 408
 409 static net_db_peer *
 410 netdbPeerAdd(netdbEntry * n, peer * e)
 411 {
 412     net_db_peer *p;
 413     net_db_peer *o;
 414     int osize;
 415     int i;
 416
 417     if (n->n_peers == n->n_peers_alloc) {
 418         o = n->peers;
 419         osize = n->n_peers_alloc;
 420
 421         if (n->n_peers_alloc == 0)
 422             n->n_peers_alloc = 2;
 423         else
 424             n->n_peers_alloc <<= 1;
 425
 426         debugs(38, 3, "netdbPeerAdd: Growing peer list for '" << n->network << "' to " << n->n_peers_alloc);
 427
 428         n->peers = (net_db_peer *)xcalloc(n->n_peers_alloc, sizeof(net_db_peer));
 429
 430         for (i = 0; i < osize; i++)
 431             *(n->peers + i) = *(o + i);
 432
 433         if (osize) {
 434             safe_free(o);
 435         }
 436     }
 437
 438     p = n->peers + n->n_peers;
 439     p->peername = netdbPeerName(e->host);
 440     n->n_peers++;
 441     return p;
 442 }
 443
 444 static int
 445 sortPeerByRtt(const void *A, const void *B)
 446 {
 447     const net_db_peer *p1 = (net_db_peer *)A;
 448     const net_db_peer *p2 = (net_db_peer *)B;
 449
 450     if (p1->rtt > p2->rtt)
 451         return 1;
 452     else if (p1->rtt < p2->rtt)
 453         return -1;
 454     else
 455         return 0;
 456 }
 457
 458 static void
 459 netdbSaveState(void *foo)
 460 {
 461     if (strcmp(Config.netdbFilename, "none") == 0)
 462         return;
 463
 464     Logfile *lf;
 465     netdbEntry *n;
 466     net_db_name *x;
 467
 468     struct timeval start = current_time;
 469     int count = 0;
 470     /*
 471      * This was nicer when we were using stdio, but thanks to
 472      * Solaris bugs, its a bad idea.  fopen can fail if more than
 473      * 256 FDs are open.
 474      */
 475     /*
 476      * unlink() is here because there is currently no way to make
 477      * logfileOpen() use O_TRUNC.
 478      */
 479     unlink(Config.netdbFilename);
 480     lf = logfileOpen(Config.netdbFilename, 4096, 0);
 481
 482     if (NULL == lf) {
 483         debugs(50, 1, "netdbSaveState: " << Config.netdbFilename << ": " << xstrerror());
 484         return;
 485     }
 486
 487     hash_first(addr_table);
 488
 489     while ((n = (netdbEntry *) hash_next(addr_table))) {
 490         if (n->pings_recv == 0)
 491             continue;
 492
 493         logfilePrintf(lf, "%s %d %d %10.5f %10.5f %d %d",
 494                       n->network,
 495                       n->pings_sent,
 496                       n->pings_recv,
 497                       n->hops,
 498                       n->rtt,
 499                       (int) n->next_ping_time,
 500                       (int) n->last_use_time);
 501
 502         for (x = n->hosts; x; x = x->next)
 503             logfilePrintf(lf, " %s", hashKeyStr(&x->hash));
 504
 505         logfilePrintf(lf, "\n");
 506
 507         count++;
 508
 509 #undef RBUF_SZ
 510
 511     }
 512
 513     logfileClose(lf);
 514     getCurrentTime();
 515     debugs(38, 1, "NETDB state saved; " <<
 516            count << " entries, " <<
 517            tvSubMsec(start, current_time) << " msec" );
 518     eventAddIsh("netdbSaveState", netdbSaveState, NULL, 3600.0, 1);
 519 }
 520
 521 static void
 522 netdbReloadState(void)
 523 {
 524     if (strcmp(Config.netdbFilename, "none") == 0)
 525         return;
 526
 527     char *s;
 528     int fd;
 529     int l;
 530
 531     struct stat sb;
 532     netdbEntry *n;
 533     netdbEntry N;
 534
 535     Ip::Address addr;
 536     int count = 0;
 537
 538     struct timeval start = current_time;
 539     /*
 540      * This was nicer when we were using stdio, but thanks to
 541      * Solaris bugs, its a bad idea.  fopen can fail if more than
 542      * 256 FDs are open.
 543      */
 544     fd = file_open(Config.netdbFilename, O_RDONLY | O_BINARY);
 545
 546     if (fd < 0)
 547         return;
 548
 549     if (fstat(fd, &sb) < 0) {
 550         file_close(fd);
 551         return;
 552     }
 553
 554     char *t;
 555     char *buf = (char *)xcalloc(1, sb.st_size + 1);
 556     t = buf;
 557     l = FD_READ_METHOD(fd, buf, sb.st_size);
 558     file_close(fd);
 559
 560     if (l <= 0) {
 561         safe_free (buf);
 562         return;
 563     };
 564
 565     while ((s = strchr(t, '\n'))) {
 566         char *q;
 567         assert(s - buf < l);
 568         *s = '\0';
 569         memset(&N, '\0', sizeof(netdbEntry));
 570         q = strtok(t, w_space);
 571         t = s + 1;
 572
 573         if (NULL == q)
 574             continue;
 575
 576         if (! (addr = q) )
 577             continue;
 578
 579         if (netdbLookupAddr(addr) != NULL)      /* no dups! */
 580             continue;
 581
 582         if ((q = strtok(NULL, w_space)) == NULL)
 583             continue;
 584
 585         N.pings_sent = atoi(q);
 586
 587         if ((q = strtok(NULL, w_space)) == NULL)
 588             continue;
 589
 590         N.pings_recv = atoi(q);
 591
 592         if (N.pings_recv == 0)
 593             continue;
 594
 595         /* give this measurement low weight */
 596         N.pings_sent = 1;
 597
 598         N.pings_recv = 1;
 599
 600         if ((q = strtok(NULL, w_space)) == NULL)
 601             continue;
 602
 603         N.hops = atof(q);
 604
 605         if ((q = strtok(NULL, w_space)) == NULL)
 606             continue;
 607
 608         N.rtt = atof(q);
 609
 610         if ((q = strtok(NULL, w_space)) == NULL)
 611             continue;
 612
 613         N.next_ping_time = (time_t) atoi(q);
 614
 615         if ((q = strtok(NULL, w_space)) == NULL)
 616             continue;
 617
 618         N.last_use_time = (time_t) atoi(q);
 619
 620         n = (netdbEntry *)memAllocate(MEM_NETDBENTRY);
 621
 622         xmemcpy(n, &N, sizeof(netdbEntry));
 623
 624         netdbHashInsert(n, addr);
 625
 626         while ((q = strtok(NULL, w_space)) != NULL) {
 627             if (netdbLookupHost(q) != NULL)     /* no dups! */
 628                 continue;
 629
 630             netdbHostInsert(n, q);
 631         }
 632
 633         count++;
 634     }
 635
 636     xfree(buf);
 637     getCurrentTime();
 638     debugs(38, 1, "NETDB state reloaded; " <<
 639            count << " entries, " <<
 640            tvSubMsec(start, current_time) << " msec" );
 641 }
 642
 643 static const char *
 644 netdbPeerName(const char *name)
 645 {
 646     const wordlist *w;
 647
 648     for (w = peer_names; w; w = w->next) {
 649         if (!strcmp(w->key, name))
 650             return w->key;
 651     }
 652
 653     return wordlistAdd(&peer_names, name);
 654 }
 655
 656 static void
 657 netdbFreeNetdbEntry(void *data)
 658 {
 659     netdbEntry *n = (netdbEntry *)data;
 660     safe_free(n->peers);
 661     memFree(n, MEM_NETDBENTRY);
 662 }
 663
 664 static void
 665 netdbFreeNameEntry(void *data)
 666 {
 667     net_db_name *x = (net_db_name *)data;
 668     xfree(x->hash.key);
 669     memFree(x, MEM_NET_DB_NAME);
 670 }
 671
 672
 673 static void
 674 netdbExchangeHandleReply(void *data, StoreIOBuffer receivedData)
 675 {
 676     Ip::Address addr;
 677
 678     netdbExchangeState *ex = (netdbExchangeState *)data;
 679     int rec_sz = 0;
 680     int o;
 681
 682     struct in_addr line_addr;
 683     double rtt;
 684     double hops;
 685     char *p;
 686     int j;
 687     HttpReply const *rep;
 688     size_t hdr_sz;
 689     int nused = 0;
 690     int size;
 691     int oldbufofs = ex->buf_ofs;
 692
 693     rec_sz = 0;
 694     rec_sz += 1 + sizeof(struct in_addr);
 695     rec_sz += 1 + sizeof(int);
 696     rec_sz += 1 + sizeof(int);
 697     debugs(38, 3, "netdbExchangeHandleReply: " << receivedData.length << " read bytes");
 698
 699     if (!cbdataReferenceValid(ex->p)) {
 700         debugs(38, 3, "netdbExchangeHandleReply: Peer became invalid");
 701         netdbExchangeDone(ex);
 702         return;
 703     }
 704
 705     debugs(38, 3, "netdbExchangeHandleReply: for '" << ex->p->host << ":" << ex->p->http_port << "'");
 706
 707     if (receivedData.length == 0 &&
 708             !receivedData.flags.error) {
 709         debugs(38, 3, "netdbExchangeHandleReply: Done");
 710         netdbExchangeDone(ex);
 711         return;
 712     }
 713
 714     p = ex->buf;
 715
 716     /* Get the size of the buffer now */
 717     size = ex->buf_ofs + receivedData.length;
 718     debugs(38, 3, "netdbExchangeHandleReply: " << size << " bytes buf");
 719
 720     /* Check if we're still doing headers */
 721
 722     if (ex->connstate == STATE_HEADER) {
 723
 724         ex->buf_ofs += receivedData.length;
 725
 726         /* skip reply headers */
 727
 728         if ((hdr_sz = headersEnd(p, ex->buf_ofs))) {
 729             debugs(38, 5, "netdbExchangeHandleReply: hdr_sz = " << hdr_sz);
 730             rep = ex->e->getReply();
 731             assert (0 != rep->sline.status);
 732             debugs(38, 3, "netdbExchangeHandleReply: reply status " << rep->sline.status);
 733
 734             if (HTTP_OK != rep->sline.status) {
 735                 netdbExchangeDone(ex);
 736                 return;
 737             }
 738
 739             assert((size_t)ex->buf_ofs >= hdr_sz);
 740
 741             /*
 742              * Now, point p to the part of the buffer where the data
 743              * starts, and update the size accordingly
 744              */
 745             assert(ex->used == 0);
 746             ex->used = hdr_sz;
 747             size = ex->buf_ofs - hdr_sz;
 748             p += hdr_sz;
 749
 750             /* Finally, set the conn state mode to STATE_BODY */
 751             ex->connstate = STATE_BODY;
 752         } else {
 753             StoreIOBuffer tempBuffer;
 754             tempBuffer.offset = ex->buf_ofs;
 755             tempBuffer.length = ex->buf_sz - ex->buf_ofs;
 756             tempBuffer.data = ex->buf + ex->buf_ofs;
 757             /* Have more headers .. */
 758             storeClientCopy(ex->sc, ex->e, tempBuffer,
 759                             netdbExchangeHandleReply, ex);
 760             return;
 761         }
 762     }
 763
 764     assert(ex->connstate == STATE_BODY);
 765
 766     /* If we get here, we have some body to parse .. */
 767     debugs(38, 5, "netdbExchangeHandleReply: start parsing loop, size = " << size);
 768
 769     while (size >= rec_sz) {
 770         debugs(38, 5, "netdbExchangeHandleReply: in parsing loop, size = " << size);
 771         addr.SetAnyAddr();
 772         hops = rtt = 0.0;
 773
 774         for (o = 0; o < rec_sz;) {
 775             switch ((int) *(p + o)) {
 776
 777             case NETDB_EX_NETWORK:
 778                 o++;
 779                 /* FIXME INET6 : NetDB can still ony send IPv4 */
 780                 xmemcpy(&line_addr, p + o, sizeof(struct in_addr));
 781                 addr = line_addr;
 782                 o += sizeof(struct in_addr);
 783                 break;
 784
 785             case NETDB_EX_RTT:
 786                 o++;
 787                 xmemcpy(&j, p + o, sizeof(int));
 788                 o += sizeof(int);
 789                 rtt = (double) ntohl(j) / 1000.0;
 790                 break;
 791
 792             case NETDB_EX_HOPS:
 793                 o++;
 794                 xmemcpy(&j, p + o, sizeof(int));
 795                 o += sizeof(int);
 796                 hops = (double) ntohl(j) / 1000.0;
 797                 break;
 798
 799             default:
 800                 debugs(38, 1, "netdbExchangeHandleReply: corrupt data, aborting");
 801                 netdbExchangeDone(ex);
 802                 return;
 803             }
 804         }
 805
 806         if (!addr.IsAnyAddr() && rtt > 0)
 807             netdbExchangeUpdatePeer(addr, ex->p, rtt, hops);
 808
 809         assert(o == rec_sz);
 810
 811         ex->used += rec_sz;
 812
 813         size -= rec_sz;
 814
 815         p += rec_sz;
 816
 817         nused++;
 818     }
 819
 820     /*
 821      * Copy anything that is left over to the beginning of the buffer,
 822      * and adjust buf_ofs accordingly
 823      */
 824
 825     /*
 826      * Evilly, size refers to the buf size left now,
 827      * ex->buf_ofs is the original buffer size, so just copy that
 828      * much data over
 829      */
 830     memmove(ex->buf, ex->buf + (ex->buf_ofs - size), size);
 831
 832     ex->buf_ofs = size;
 833
 834     /*
 835      * And don't re-copy the remaining data ..
 836      */
 837     ex->used += size;
 838
 839     /*
 840      * Now the tricky bit - size _included_ the leftover bit from the _last_
 841      * storeClientCopy. We don't want to include that, or our offset will be wrong.
 842      * So, don't count the size of the leftover buffer we began with.
 843      * This can _disappear_ when we're not tracking offsets ..
 844      */
 845     ex->used -= oldbufofs;
 846
 847     debugs(38, 3, "netdbExchangeHandleReply: size left over in this buffer: " << size << " bytes");
 848
 849     debugs(38, 3, "netdbExchangeHandleReply: used " << nused <<
 850            " entries, (x " << rec_sz << " bytes) == " << nused * rec_sz <<
 851            " bytes total");
 852
 853     debugs(38, 3, "netdbExchangeHandleReply: used " << ex->used);
 854
 855     if (EBIT_TEST(ex->e->flags, ENTRY_ABORTED)) {
 856         debugs(38, 3, "netdbExchangeHandleReply: ENTRY_ABORTED");
 857         netdbExchangeDone(ex);
 858     } else if (ex->e->store_status == STORE_PENDING) {
 859         StoreIOBuffer tempBuffer;
 860         tempBuffer.offset = ex->used;
 861         tempBuffer.length = ex->buf_sz - ex->buf_ofs;
 862         tempBuffer.data = ex->buf + ex->buf_ofs;
 863         debugs(38, 3, "netdbExchangeHandleReply: EOF not received");
 864         storeClientCopy(ex->sc, ex->e, tempBuffer,
 865                         netdbExchangeHandleReply, ex);
 866     }
 867 }
 868
 869 static void
 870 netdbExchangeDone(void *data)
 871 {
 872     netdbExchangeState *ex = (netdbExchangeState *)data;
 873     debugs(38, 3, "netdbExchangeDone: " << ex->e->url()  );
 874     HTTPMSGUNLOCK(ex->r);
 875     storeUnregister(ex->sc, ex->e, ex);
 876     ex->e->unlock();
 877     cbdataReferenceDone(ex->p);
 878     cbdataFree(ex);
 879 }
 880
 881 static void
 882 netdbRegisterWithCacheManager(void)
 883 {
 884     CacheManager::GetInstance()->
 885     registerAction("netdb", "Network Measurement Database", netdbDump, 0, 1);
 886 }
 887
 888 #endif /* USE_ICMP */
 889
 890
 891 /* PUBLIC FUNCTIONS */
 892
 893 void
 894 netdbInit(void)
 895 {
 896 #if USE_ICMP
 897     int n;
 898
 899     netdbRegisterWithCacheManager();
 900
 901     if (addr_table)
 902         return;
 903
 904     n = hashPrime(Config.Netdb.high / 4);
 905
 906     addr_table = hash_create((HASHCMP *) strcmp, n, hash_string);
 907
 908     n = hashPrime(3 * Config.Netdb.high / 4);
 909
 910     host_table = hash_create((HASHCMP *) strcmp, n, hash_string);
 911
 912     eventAddIsh("netdbSaveState", netdbSaveState, NULL, 3600.0, 1);
 913
 914     netdbReloadState();
 915
 916 #endif
 917 }
 918
 919 void
 920 netdbPingSite(const char *hostname)
 921 {
 922 #if USE_ICMP
 923     netdbEntry *n;
 924
 925     if ((n = netdbLookupHost(hostname)) != NULL)
 926         if (n->next_ping_time > squid_curtime)
 927             return;
 928
 929     ipcache_nbgethostbyname(hostname, netdbSendPing,
 930                             new generic_cbdata(xstrdup(hostname)));
 931
 932 #endif
 933 }
 934
 935 void
 936 netdbHandlePingReply(const Ip::Address &from, int hops, int rtt)
 937 {
 938 #if USE_ICMP
 939     netdbEntry *n;
 940     int N;
 941     debugs(38, 3, "netdbHandlePingReply: from " << from);
 942
 943     if ((n = netdbLookupAddr(from)) == NULL)
 944         return;
 945
 946     N = ++n->pings_recv;
 947
 948     if (N > 5)
 949         N = 5;
 950
 951     if (rtt < 1)
 952         rtt = 1;
 953
 954     n->hops = ((n->hops * (N - 1)) + hops) / N;
 955
 956     n->rtt = ((n->rtt * (N - 1)) + rtt) / N;
 957
 958     debugs(38, 3, "netdbHandlePingReply: " << n->network  << "; rtt="<<
 959            std::setw(5)<< std::setprecision(2) << n->rtt << "  hops="<<
 960            std::setw(4) << n->hops);
 961
 962 #endif
 963 }
 964
 965 void
 966 netdbFreeMemory(void)
 967 {
 968 #if USE_ICMP
 969     hashFreeItems(addr_table, netdbFreeNetdbEntry);
 970     hashFreeMemory(addr_table);
 971     addr_table = NULL;
 972     hashFreeItems(host_table, netdbFreeNameEntry);
 973     hashFreeMemory(host_table);
 974     host_table = NULL;
 975     wordlistDestroy(&peer_names);
 976     peer_names = NULL;
 977 #endif
 978 }
 979
 980
 981 #if 0 // AYJ: Looks to be unused code.
 982 int
 983 netdbHops(Ip::Address &addr)
 984 {
 985 #if USE_ICMP
 986     netdbEntry *n = netdbLookupAddr(addr);
 987
 988     if (n && n->pings_recv) {
 989         n->last_use_time = squid_curtime;
 990         return (int) (n->hops + 0.5);
 991     }
 992
 993 #endif
 994     return 256;
 995 }
 996 #endif
 997
 998 void
 999 netdbDump(StoreEntry * sentry)
1000 {
1001 #if USE_ICMP
1002     netdbEntry *n;
1003     netdbEntry **list;
1004     net_db_name *x;
1005     int k;
1006     int i;
1007     int j;
1008     net_db_peer *p;
1009     storeAppendPrintf(sentry, "Network DB Statistics:\n");
1010     storeAppendPrintf(sentry, "%-46.46s %9s %7s %5s %s\n",  /* Max between 16 (IPv4) or 46 (IPv6)   */
1011                       "Network",
1012                       "recv/sent",
1013                       "RTT",
1014                       "Hops",
1015                       "Hostnames");
1016     list = (netdbEntry **)xcalloc(memInUse(MEM_NETDBENTRY), sizeof(netdbEntry *));
1017     i = 0;
1018     hash_first(addr_table);
1019
1020     while ((n = (netdbEntry *) hash_next(addr_table)))
1021         *(list + i++) = n;
1022
1023     if (i != memInUse(MEM_NETDBENTRY))
1024         debugs(38, 0, "WARNING: netdb_addrs count off, found " << i <<
1025                ", expected " << memInUse(MEM_NETDBENTRY));
1026
1027     qsort((char *) list,
1028           i,
1029           sizeof(netdbEntry *),
1030           sortByRtt);
1031
1032     for (k = 0; k < i; k++) {
1033         n = *(list + k);
1034         storeAppendPrintf(sentry, "%-46.46s %4d/%4d %7.1f %5.1f", /* Max between 16 (IPv4) or 46 (IPv6)   */
1035                           n->network,
1036                           n->pings_recv,
1037                           n->pings_sent,
1038                           n->rtt,
1039                           n->hops);
1040
1041         for (x = n->hosts; x; x = x->next)
1042             storeAppendPrintf(sentry, " %s", hashKeyStr(&x->hash));
1043
1044         storeAppendPrintf(sentry, "\n");
1045
1046         p = n->peers;
1047
1048         for (j = 0; j < n->n_peers; j++, p++) {
1049             storeAppendPrintf(sentry, "    %-22.22s %7.1f %5.1f\n",
1050                               p->peername,
1051                               p->rtt,
1052                               p->hops);
1053         }
1054     }
1055
1056     xfree(list);
1057 #else
1058
1059     storeAppendPrintf(sentry,"NETDB support not compiled into this Squid cache.\n");
1060 #endif
1061 }
1062
1063 int
1064 netdbHostHops(const char *host)
1065 {
1066 #if USE_ICMP
1067     netdbEntry *n = netdbLookupHost(host);
1068
1069     if (n) {
1070         n->last_use_time = squid_curtime;
1071         return (int) (n->hops + 0.5);
1072     }
1073
1074 #endif
1075     return 0;
1076 }
1077
1078 int
1079 netdbHostRtt(const char *host)
1080 {
1081 #if USE_ICMP
1082     netdbEntry *n = netdbLookupHost(host);
1083
1084     if (n) {
1085         n->last_use_time = squid_curtime;
1086         return (int) (n->rtt + 0.5);
1087     }
1088
1089 #endif
1090     return 0;
1091 }
1092
1093 void
1094 netdbHostData(const char *host, int *samp, int *rtt, int *hops)
1095 {
1096 #if USE_ICMP
1097     netdbEntry *n = netdbLookupHost(host);
1098
1099     if (n == NULL)
1100         return;
1101
1102     *samp = n->pings_recv;
1103
1104     *rtt = (int) (n->rtt + 0.5);
1105
1106     *hops = (int) (n->hops + 0.5);
1107
1108     n->last_use_time = squid_curtime;
1109
1110 #endif
1111 }
1112
1113 void
1114 netdbUpdatePeer(HttpRequest * r, peer * e, int irtt, int ihops)
1115 {
1116 #if USE_ICMP
1117     netdbEntry *n;
1118     double rtt = (double) irtt;
1119     double hops = (double) ihops;
1120     net_db_peer *p;
1121     debugs(38, 3, "netdbUpdatePeer: '" << r->GetHost() << "', " << ihops << " hops, " << irtt << " rtt");
1122     n = netdbLookupHost(r->GetHost());
1123
1124     if (n == NULL) {
1125         debugs(38, 3, "netdbUpdatePeer: host '" << r->GetHost() << "' not found");
1126         return;
1127     }
1128
1129     if ((p = netdbPeerByName(n, e->host)) == NULL)
1130         p = netdbPeerAdd(n, e);
1131
1132     p->rtt = rtt;
1133
1134     p->hops = hops;
1135
1136     p->expires = squid_curtime + 3600;
1137
1138     if (n->n_peers < 2)
1139         return;
1140
1141     qsort((char *) n->peers,
1142           n->n_peers,
1143           sizeof(net_db_peer),
1144           sortPeerByRtt);
1145
1146 #endif
1147 }
1148
1149 void
1150 netdbExchangeUpdatePeer(Ip::Address &addr, peer * e, double rtt, double hops)
1151 {
1152 #if USE_ICMP
1153     netdbEntry *n;
1154     net_db_peer *p;
1155     debugs(38, 5, "netdbExchangeUpdatePeer: '" << addr << "', "<<
1156            std::setfill('0')<< std::setprecision(2) << hops << " hops, " <<
1157            rtt << " rtt");
1158
1159     if ( !addr.IsIPv4() ) {
1160         debugs(38, 5, "netdbExchangeUpdatePeer: Aborting peer update for '" << addr << "', NetDB cannot handle IPv6.");
1161         return;
1162     }
1163
1164     n = netdbLookupAddr(addr);
1165
1166     if (n == NULL)
1167         n = netdbAdd(addr);
1168
1169     assert(NULL != n);
1170
1171     if ((p = netdbPeerByName(n, e->host)) == NULL)
1172         p = netdbPeerAdd(n, e);
1173
1174     p->rtt = rtt;
1175
1176     p->hops = hops;
1177
1178     p->expires = squid_curtime + 3600;  /* XXX ? */
1179
1180     if (n->n_peers < 2)
1181         return;
1182
1183     qsort((char *) n->peers,
1184           n->n_peers,
1185           sizeof(net_db_peer),
1186           sortPeerByRtt);
1187
1188 #endif
1189 }
1190
1191 void
1192 netdbDeleteAddrNetwork(Ip::Address &addr)
1193 {
1194 #if USE_ICMP
1195     netdbEntry *n = netdbLookupAddr(addr);
1196
1197     if (n == NULL)
1198         return;
1199
1200     debugs(38, 3, "netdbDeleteAddrNetwork: " << n->network);
1201
1202     netdbRelease(n);
1203 #endif
1204 }
1205
1206
1207 void
1208 netdbBinaryExchange(StoreEntry * s)
1209 {
1210     HttpReply *reply = new HttpReply;
1211 #if USE_ICMP
1212
1213     Ip::Address addr;
1214
1215     netdbEntry *n;
1216     int i;
1217     int j;
1218     int rec_sz;
1219     char *buf;
1220
1221     struct in_addr line_addr;
1222     s->buffer();
1223     reply->setHeaders(HTTP_OK, "OK", NULL, -1, squid_curtime, -2);
1224     s->replaceHttpReply(reply);
1225     rec_sz = 0;
1226     rec_sz += 1 + sizeof(struct in_addr);
1227     rec_sz += 1 + sizeof(int);
1228     rec_sz += 1 + sizeof(int);
1229     buf = (char *)memAllocate(MEM_4K_BUF);
1230     i = 0;
1231     hash_first(addr_table);
1232
1233     while ((n = (netdbEntry *) hash_next(addr_table))) {
1234         if (0.0 == n->rtt)
1235             continue;
1236
1237         if (n->rtt > 60000)     /* RTT > 1 MIN probably bogus */
1238             continue;
1239
1240         if (! (addr = n->network) )
1241             continue;
1242
1243         /* FIXME INET6 : NetDB cannot yet handle IPv6 addresses. Ensure only IPv4 get sent. */
1244         if ( !addr.IsIPv4() )
1245             continue;
1246
1247         buf[i++] = (char) NETDB_EX_NETWORK;
1248
1249         addr.GetInAddr(line_addr);
1250         xmemcpy(&buf[i], &line_addr, sizeof(struct in_addr));
1251
1252         i += sizeof(struct in_addr);
1253
1254         buf[i++] = (char) NETDB_EX_RTT;
1255
1256         j = htonl((int) (n->rtt * 1000));
1257
1258         xmemcpy(&buf[i], &j, sizeof(int));
1259
1260         i += sizeof(int);
1261
1262         buf[i++] = (char) NETDB_EX_HOPS;
1263
1264         j = htonl((int) (n->hops * 1000));
1265
1266         xmemcpy(&buf[i], &j, sizeof(int));
1267
1268         i += sizeof(int);
1269
1270         if (i + rec_sz > 4096) {
1271             s->append(buf, i);
1272             i = 0;
1273         }
1274     }
1275
1276     if (i > 0) {
1277         s->append(buf, i);
1278         i = 0;
1279     }
1280
1281     assert(0 == i);
1282     s->flush();
1283     memFree(buf, MEM_4K_BUF);
1284 #else
1285
1286     reply->setHeaders(HTTP_BAD_REQUEST, "Bad Request", NULL, -1, squid_curtime, -2);
1287     s->replaceHttpReply(reply);
1288     storeAppendPrintf(s, "NETDB support not compiled into this Squid cache.\n");
1289 #endif
1290
1291     s->complete();
1292 }
1293
1294 #if USE_ICMP
1295 CBDATA_TYPE(netdbExchangeState);
1296 #endif
1297
1298 void
1299 netdbExchangeStart(void *data)
1300 {
1301 #if USE_ICMP
1302     peer *p = (peer *)data;
1303     char *uri;
1304     netdbExchangeState *ex;
1305     StoreIOBuffer tempBuffer;
1306     CBDATA_INIT_TYPE(netdbExchangeState);
1307     ex = cbdataAlloc(netdbExchangeState);
1308     ex->p = cbdataReference(p);
1309     uri = internalRemoteUri(p->host, p->http_port, "/squid-internal-dynamic/", "netdb");
1310     debugs(38, 3, "netdbExchangeStart: Requesting '" << uri << "'");
1311     assert(NULL != uri);
1312     ex->r = HttpRequest::CreateFromUrl(uri);
1313
1314     if (NULL == ex->r) {
1315         debugs(38, 1, "netdbExchangeStart: Bad URI " << uri);
1316         return;
1317     }
1318
1319     HTTPMSGLOCK(ex->r);
1320     assert(NULL != ex->r);
1321     ex->r->http_ver = HttpVersion(1,1);
1322     ex->connstate = STATE_HEADER;
1323     ex->e = storeCreateEntry(uri, uri, request_flags(), METHOD_GET);
1324     ex->buf_sz = NETDB_REQBUF_SZ;
1325     assert(NULL != ex->e);
1326     ex->sc = storeClientListAdd(ex->e, ex);
1327     tempBuffer.offset = 0;
1328     tempBuffer.length = ex->buf_sz;
1329     tempBuffer.data = ex->buf;
1330     storeClientCopy(ex->sc, ex->e, tempBuffer,
1331                     netdbExchangeHandleReply, ex);
1332     ex->r->flags.loopdetect = 1;        /* cheat! -- force direct */
1333
1334     if (p->login)
1335         xstrncpy(ex->r->login, p->login, MAX_LOGIN_SZ);
1336
1337     urlCanonical(ex->r);
1338
1339     FwdState::fwdStart(-1, ex->e, ex->r);
1340
1341 #endif
1342 }
1343
1344 peer *
1345 netdbClosestParent(HttpRequest * request)
1346 {
1347 #if USE_ICMP
1348     peer *p = NULL;
1349     netdbEntry *n;
1350     const ipcache_addrs *ia;
1351     net_db_peer *h;
1352     int i;
1353     n = netdbLookupHost(request->GetHost());
1354
1355     if (NULL == n) {
1356         /* try IP addr */
1357         ia = ipcache_gethostbyname(request->GetHost(), 0);
1358
1359         if (NULL != ia)
1360             n = netdbLookupAddr(ia->in_addrs[ia->cur]);
1361     }
1362
1363     if (NULL == n)
1364         return NULL;
1365
1366     if (0 == n->n_peers)
1367         return NULL;
1368
1369     n->last_use_time = squid_curtime;
1370
1371     /*
1372      * Find the parent with the least RTT to the origin server.
1373      * Make sure we don't return a parent who is farther away than
1374      * we are.  Note, the n->peers list is pre-sorted by RTT.
1375      */
1376     for (i = 0; i < n->n_peers; i++) {
1377         h = &n->peers[i];
1378
1379         if (n->rtt > 0)
1380             if (n->rtt < h->rtt)
1381                 break;
1382
1383         p = peerFindByName(h->peername);
1384
1385         if (NULL == p)          /* not found */
1386             continue;
1387
1388         if (neighborType(p, request) != PEER_PARENT)
1389             continue;
1390
1391         if (!peerHTTPOkay(p, request))  /* not allowed */
1392             continue;
1393
1394         return p;
1395     }
1396
1397 #endif
1398     return NULL;
1399 }