src/icmp/net_db.cc

   1 /*
   2  * DEBUG: section 38    Network Measurement Database
   3  * AUTHOR: Duane Wessels
   4  *
   5  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   6  * ----------------------------------------------------------
   7  *
   8  *  Squid is the result of efforts by numerous individuals from
   9  *  the Internet community; see the CONTRIBUTORS file for full
  10  *  details.   Many organizations have provided support for Squid's
  11  *  development; see the SPONSORS file for full details.  Squid is
  12  *  Copyrighted (C) 2001 by the Regents of the University of
  13  *  California; see the COPYRIGHT file for full details.  Squid
  14  *  incorporates software developed and/or copyrighted by other
  15  *  sources; see the CREDITS file for full details.
  16  *
  17  *  This program is free software; you can redistribute it and/or modify
  18  *  it under the terms of the GNU General Public License as published by
  19  *  the Free Software Foundation; either version 2 of the License, or
  20  *  (at your option) any later version.
  21  *
  22  *  This program is distributed in the hope that it will be useful,
  23  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  24  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  25  *  GNU General Public License for more details.
  26  *
  27  *  You should have received a copy of the GNU General Public License
  28  *  along with this program; if not, write to the Free Software
  29  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  30  *
  31  */
  32
  33 /*
  34  * XXX XXX XXX
  35  *
  36  * This code may be slightly broken now. If you're getting consistent
  37  * (sometimes working) corrupt data exchanges, please contact adrian
  38  * (adrian@squid-cache.org) to sort them out.
  39  */
  40
  41 #include "squid.h"
  42 #include "icmp/net_db.h"
  43 #include "log/File.h"
  44 #include "cbdata.h"
  45 #include "event.h"
  46 #include "CacheManager.h"
  47 #include "Store.h"
  48 #include "SwapDir.h"
  49 #include "HttpRequest.h"
  50 #include "HttpReply.h"
  51 #include "MemObject.h"
  52 #include "fde.h"
  53 #include "forward.h"
  54 #include "SquidTime.h"
  55 #include "wordlist.h"
  56 #include "ip/Address.h"
  57
  58 #if USE_ICMP
  59 #include "icmp/IcmpSquid.h"
  60 #include "StoreClient.h"
  61
  62 #define NETDB_REQBUF_SZ 4096
  63
  64 typedef enum {
  65     STATE_NONE,
  66     STATE_HEADER,
  67     STATE_BODY
  68 } netdb_conn_state_t;
  69
  70 typedef struct {
  71     peer *p;
  72     StoreEntry *e;
  73     store_client *sc;
  74     HttpRequest *r;
  75     int64_t used;
  76     size_t buf_sz;
  77     char buf[NETDB_REQBUF_SZ];
  78     int buf_ofs;
  79     netdb_conn_state_t connstate;
  80 } netdbExchangeState;
  81
  82 static hash_table *addr_table = NULL;
  83 static hash_table *host_table = NULL;
  84
  85 Ip::Address networkFromInaddr(const Ip::Address &a);
  86 static void netdbRelease(netdbEntry * n);
  87
  88 static void netdbHashInsert(netdbEntry * n, Ip::Address &addr);
  89 static void netdbHashDelete(const char *key);
  90 static void netdbHostInsert(netdbEntry * n, const char *hostname);
  91 static void netdbHostDelete(const net_db_name * x);
  92 static void netdbPurgeLRU(void);
  93 static netdbEntry *netdbLookupHost(const char *key);
  94 static net_db_peer *netdbPeerByName(const netdbEntry * n, const char *);
  95 static net_db_peer *netdbPeerAdd(netdbEntry * n, peer * e);
  96 static const char *netdbPeerName(const char *name);
  97 static IPH netdbSendPing;
  98 static QS sortPeerByRtt;
  99 static QS sortByRtt;
 100 static QS netdbLRU;
 101 static FREE netdbFreeNameEntry;
 102 static FREE netdbFreeNetdbEntry;
 103 static STCB netdbExchangeHandleReply;
 104 static void netdbExchangeDone(void *);
 105
 106 /* We have to keep a local list of peer names.  The Peers structure
 107  * gets freed during a reconfigure.  We want this database to
 108  * remain persisitent, so _net_db_peer->peername points into this
 109  * linked list */
 110 static wordlist *peer_names = NULL;
 111
 112 static void
 113 netdbHashInsert(netdbEntry * n, Ip::Address &addr)
 114 {
 115     networkFromInaddr(addr).NtoA(n->network, MAX_IPSTRLEN);
 116     n->hash.key = n->network;
 117     assert(hash_lookup(addr_table, n->network) == NULL);
 118     hash_join(addr_table, &n->hash);
 119 }
 120
 121 static void
 122 netdbHashDelete(const char *key)
 123 {
 124     hash_link *hptr = (hash_link *)hash_lookup(addr_table, key);
 125
 126     if (hptr == NULL) {
 127         debug_trap("netdbHashDelete: key not found");
 128         return;
 129     }
 130
 131     hash_remove_link(addr_table, hptr);
 132 }
 133
 134 static void
 135 netdbHostInsert(netdbEntry * n, const char *hostname)
 136 {
 137     net_db_name *x = (net_db_name *)memAllocate(MEM_NET_DB_NAME);
 138     x->hash.key = xstrdup(hostname);
 139     x->next = n->hosts;
 140     n->hosts = x;
 141     x->net_db_entry = n;
 142     assert(hash_lookup(host_table, hostname) == NULL);
 143     hash_join(host_table, &x->hash);
 144     n->link_count++;
 145 }
 146
 147 static void
 148 netdbHostDelete(const net_db_name * x)
 149 {
 150     netdbEntry *n;
 151     net_db_name **X;
 152     assert(x != NULL);
 153     assert(x->net_db_entry != NULL);
 154     n = x->net_db_entry;
 155     n->link_count--;
 156
 157     for (X = &n->hosts; *X; X = &(*X)->next) {
 158         if (*X == x) {
 159             *X = x->next;
 160             break;
 161         }
 162     }
 163
 164     hash_remove_link(host_table, (hash_link *) x);
 165     xfree(x->hash.key);
 166     memFree((void *) x, MEM_NET_DB_NAME);
 167 }
 168
 169 static netdbEntry *
 170 netdbLookupHost(const char *key)
 171 {
 172     net_db_name *x = (net_db_name *) hash_lookup(host_table, key);
 173     return x ? x->net_db_entry : NULL;
 174 }
 175
 176 static void
 177 netdbRelease(netdbEntry * n)
 178 {
 179     net_db_name *x;
 180     net_db_name *next;
 181
 182     for (x = n->hosts; x; x = next) {
 183         next = x->next;
 184         netdbHostDelete(x);
 185     }
 186
 187     n->hosts = NULL;
 188     safe_free(n->peers);
 189     n->peers = NULL;
 190     n->n_peers = 0;
 191     n->n_peers_alloc = 0;
 192
 193     if (n->link_count == 0) {
 194         netdbHashDelete(n->network);
 195         memFree(n, MEM_NETDBENTRY);
 196     }
 197 }
 198
 199 static int
 200 netdbLRU(const void *A, const void *B)
 201 {
 202     const netdbEntry *const *n1 = (const netdbEntry *const *)A;
 203     const netdbEntry *const *n2 = (const netdbEntry *const *)B;
 204
 205     if ((*n1)->last_use_time > (*n2)->last_use_time)
 206         return (1);
 207
 208     if ((*n1)->last_use_time < (*n2)->last_use_time)
 209         return (-1);
 210
 211     return (0);
 212 }
 213
 214 static void
 215 netdbPurgeLRU(void)
 216 {
 217     netdbEntry *n;
 218     netdbEntry **list;
 219     int k = 0;
 220     int list_count = 0;
 221     int removed = 0;
 222     list = (netdbEntry **)xcalloc(memInUse(MEM_NETDBENTRY), sizeof(netdbEntry *));
 223     hash_first(addr_table);
 224
 225     while ((n = (netdbEntry *) hash_next(addr_table))) {
 226         assert(list_count < memInUse(MEM_NETDBENTRY));
 227         *(list + list_count) = n;
 228         list_count++;
 229     }
 230
 231     qsort((char *) list,
 232           list_count,
 233           sizeof(netdbEntry *),
 234           netdbLRU);
 235
 236     for (k = 0; k < list_count; k++) {
 237         if (memInUse(MEM_NETDBENTRY) < Config.Netdb.low)
 238             break;
 239
 240         netdbRelease(*(list + k));
 241
 242         removed++;
 243     }
 244
 245     xfree(list);
 246 }
 247
 248 static netdbEntry *
 249 netdbLookupAddr(const Ip::Address &addr)
 250 {
 251     netdbEntry *n;
 252     char *key = new char[MAX_IPSTRLEN];
 253     networkFromInaddr(addr).NtoA(key,MAX_IPSTRLEN);
 254     n = (netdbEntry *) hash_lookup(addr_table, key);
 255     return n;
 256 }
 257
 258 static netdbEntry *
 259 netdbAdd(Ip::Address &addr)
 260 {
 261     netdbEntry *n;
 262
 263     if (memInUse(MEM_NETDBENTRY) > Config.Netdb.high)
 264         netdbPurgeLRU();
 265
 266     if ((n = netdbLookupAddr(addr)) == NULL) {
 267         n = (netdbEntry *)memAllocate(MEM_NETDBENTRY);
 268         netdbHashInsert(n, addr);
 269     }
 270
 271     return n;
 272 }
 273
 274 static void
 275 netdbSendPing(const ipcache_addrs *ia, const DnsLookupDetails &, void *data)
 276 {
 277     Ip::Address addr;
 278     char *hostname = NULL;
 279     static_cast<generic_cbdata *>(data)->unwrap(&hostname);
 280     netdbEntry *n;
 281     netdbEntry *na;
 282     net_db_name *x;
 283     net_db_name **X;
 284
 285     if (ia == NULL) {
 286         xfree(hostname);
 287         return;
 288     }
 289
 290     addr = ia->in_addrs[ia->cur];
 291
 292     if ((n = netdbLookupHost(hostname)) == NULL) {
 293         n = netdbAdd(addr);
 294         netdbHostInsert(n, hostname);
 295     } else if ((na = netdbLookupAddr(addr)) != n) {
 296         /*
 297          *hostname moved from 'network n' to 'network na'!
 298          */
 299
 300         if (na == NULL)
 301             na = netdbAdd(addr);
 302
 303         debugs(38, 3, "netdbSendPing: " << hostname << " moved from " << n->network << " to " << na->network);
 304
 305         x = (net_db_name *) hash_lookup(host_table, hostname);
 306
 307         if (x == NULL) {
 308             debugs(38, 1, "netdbSendPing: net_db_name list bug: " << hostname << " not found");
 309             xfree(hostname);
 310             return;
 311         }
 312
 313         /* remove net_db_name from 'network n' linked list */
 314         for (X = &n->hosts; *X; X = &(*X)->next) {
 315             if (*X == x) {
 316                 *X = x->next;
 317                 break;
 318             }
 319         }
 320
 321         n->link_count--;
 322         /* point to 'network na' from host entry */
 323         x->net_db_entry = na;
 324         /* link net_db_name to 'network na' */
 325         x->next = na->hosts;
 326         na->hosts = x;
 327         na->link_count++;
 328         n = na;
 329     }
 330
 331     if (n->next_ping_time <= squid_curtime) {
 332         debugs(38, 3, "netdbSendPing: pinging " << hostname);
 333         icmpEngine.DomainPing(addr, hostname);
 334         n->pings_sent++;
 335         n->next_ping_time = squid_curtime + Config.Netdb.period;
 336         n->last_use_time = squid_curtime;
 337     }
 338
 339     xfree(hostname);
 340 }
 341
 342 Ip::Address
 343 networkFromInaddr(const Ip::Address &in)
 344 {
 345     Ip::Address out;
 346
 347     out = in;
 348 #if USE_IPV6
 349
 350     /* in IPv6 the 'network' should be the routing section. */
 351
 352     if ( in.IsIPv6() ) {
 353         out.ApplyMask(64, AF_INET6);
 354         debugs(14, 5, "networkFromInaddr : Masked IPv6 Address to " << in << "/64 routing part.");
 355         return out;
 356     }
 357 #endif
 358
 359 #if USE_CLASSFUL
 360     struct in_addr b;
 361
 362     in.GetInAddr(b);
 363
 364     if (IN_CLASSC(b.s_addr))
 365         b.s_addr &= IN_CLASSC_NET;
 366     else if (IN_CLASSB(b.s_addr))
 367         b.s_addr &= IN_CLASSB_NET;
 368     else if (IN_CLASSA(b.s_addr))
 369         b.s_addr &= IN_CLASSA_NET;
 370
 371     out = b;
 372
 373 #endif
 374
 375     debugs(14, 5, "networkFromInaddr : Masked IPv4 Address to " << out << "/24.");
 376
 377     /* use /24 for everything under IPv4 */
 378     out.ApplyMask(24, AF_INET);
 379     debugs(14, 5, "networkFromInaddr : Masked IPv4 Address to " << in << "/24.");
 380
 381     return out;
 382 }
 383
 384 static int
 385 sortByRtt(const void *A, const void *B)
 386 {
 387     const netdbEntry *const *n1 = (const netdbEntry *const *)A;
 388     const netdbEntry *const *n2 = (const netdbEntry *const *)B;
 389
 390     if ((*n1)->rtt > (*n2)->rtt)
 391         return 1;
 392     else if ((*n1)->rtt < (*n2)->rtt)
 393         return -1;
 394     else
 395         return 0;
 396 }
 397
 398 static net_db_peer *
 399 netdbPeerByName(const netdbEntry * n, const char *peername)
 400 {
 401     int i;
 402     net_db_peer *p = n->peers;
 403
 404     for (i = 0; i < n->n_peers; i++, p++) {
 405         if (!strcmp(p->peername, peername))
 406             return p;
 407     }
 408
 409     return NULL;
 410 }
 411
 412 static net_db_peer *
 413 netdbPeerAdd(netdbEntry * n, peer * e)
 414 {
 415     net_db_peer *p;
 416     net_db_peer *o;
 417     int osize;
 418     int i;
 419
 420     if (n->n_peers == n->n_peers_alloc) {
 421         o = n->peers;
 422         osize = n->n_peers_alloc;
 423
 424         if (n->n_peers_alloc == 0)
 425             n->n_peers_alloc = 2;
 426         else
 427             n->n_peers_alloc <<= 1;
 428
 429         debugs(38, 3, "netdbPeerAdd: Growing peer list for '" << n->network << "' to " << n->n_peers_alloc);
 430
 431         n->peers = (net_db_peer *)xcalloc(n->n_peers_alloc, sizeof(net_db_peer));
 432
 433         for (i = 0; i < osize; i++)
 434             *(n->peers + i) = *(o + i);
 435
 436         if (osize) {
 437             safe_free(o);
 438         }
 439     }
 440
 441     p = n->peers + n->n_peers;
 442     p->peername = netdbPeerName(e->host);
 443     n->n_peers++;
 444     return p;
 445 }
 446
 447 static int
 448 sortPeerByRtt(const void *A, const void *B)
 449 {
 450     const net_db_peer *p1 = (net_db_peer *)A;
 451     const net_db_peer *p2 = (net_db_peer *)B;
 452
 453     if (p1->rtt > p2->rtt)
 454         return 1;
 455     else if (p1->rtt < p2->rtt)
 456         return -1;
 457     else
 458         return 0;
 459 }
 460
 461 static void
 462 netdbSaveState(void *foo)
 463 {
 464     if (strcmp(Config.netdbFilename, "none") == 0)
 465         return;
 466
 467     Logfile *lf;
 468     netdbEntry *n;
 469     net_db_name *x;
 470
 471     struct timeval start = current_time;
 472     int count = 0;
 473     /*
 474      * This was nicer when we were using stdio, but thanks to
 475      * Solaris bugs, its a bad idea.  fopen can fail if more than
 476      * 256 FDs are open.
 477      */
 478     /*
 479      * unlink() is here because there is currently no way to make
 480      * logfileOpen() use O_TRUNC.
 481      */
 482     unlink(Config.netdbFilename);
 483     lf = logfileOpen(Config.netdbFilename, 4096, 0);
 484
 485     if (NULL == lf) {
 486         debugs(50, 1, "netdbSaveState: " << Config.netdbFilename << ": " << xstrerror());
 487         return;
 488     }
 489
 490     hash_first(addr_table);
 491
 492     while ((n = (netdbEntry *) hash_next(addr_table))) {
 493         if (n->pings_recv == 0)
 494             continue;
 495
 496         logfilePrintf(lf, "%s %d %d %10.5f %10.5f %d %d",
 497                       n->network,
 498                       n->pings_sent,
 499                       n->pings_recv,
 500                       n->hops,
 501                       n->rtt,
 502                       (int) n->next_ping_time,
 503                       (int) n->last_use_time);
 504
 505         for (x = n->hosts; x; x = x->next)
 506             logfilePrintf(lf, " %s", hashKeyStr(&x->hash));
 507
 508         logfilePrintf(lf, "\n");
 509
 510         count++;
 511
 512 #undef RBUF_SZ
 513
 514     }
 515
 516     logfileClose(lf);
 517     getCurrentTime();
 518     debugs(38, 1, "NETDB state saved; " <<
 519            count << " entries, " <<
 520            tvSubMsec(start, current_time) << " msec" );
 521     eventAddIsh("netdbSaveState", netdbSaveState, NULL, 3600.0, 1);
 522 }
 523
 524 static void
 525 netdbReloadState(void)
 526 {
 527     if (strcmp(Config.netdbFilename, "none") == 0)
 528         return;
 529
 530     char *s;
 531     int fd;
 532     int l;
 533
 534     struct stat sb;
 535     netdbEntry *n;
 536     netdbEntry N;
 537
 538     Ip::Address addr;
 539     int count = 0;
 540
 541     struct timeval start = current_time;
 542     /*
 543      * This was nicer when we were using stdio, but thanks to
 544      * Solaris bugs, its a bad idea.  fopen can fail if more than
 545      * 256 FDs are open.
 546      */
 547     fd = file_open(Config.netdbFilename, O_RDONLY | O_BINARY);
 548
 549     if (fd < 0)
 550         return;
 551
 552     if (fstat(fd, &sb) < 0) {
 553         file_close(fd);
 554         return;
 555     }
 556
 557     char *t;
 558     char *buf = (char *)xcalloc(1, sb.st_size + 1);
 559     t = buf;
 560     l = FD_READ_METHOD(fd, buf, sb.st_size);
 561     file_close(fd);
 562
 563     if (l <= 0) {
 564         safe_free (buf);
 565         return;
 566     };
 567
 568     while ((s = strchr(t, '\n'))) {
 569         char *q;
 570         assert(s - buf < l);
 571         *s = '\0';
 572         memset(&N, '\0', sizeof(netdbEntry));
 573         q = strtok(t, w_space);
 574         t = s + 1;
 575
 576         if (NULL == q)
 577             continue;
 578
 579         if (! (addr = q) )
 580             continue;
 581
 582         if (netdbLookupAddr(addr) != NULL)      /* no dups! */
 583             continue;
 584
 585         if ((q = strtok(NULL, w_space)) == NULL)
 586             continue;
 587
 588         N.pings_sent = atoi(q);
 589
 590         if ((q = strtok(NULL, w_space)) == NULL)
 591             continue;
 592
 593         N.pings_recv = atoi(q);
 594
 595         if (N.pings_recv == 0)
 596             continue;
 597
 598         /* give this measurement low weight */
 599         N.pings_sent = 1;
 600
 601         N.pings_recv = 1;
 602
 603         if ((q = strtok(NULL, w_space)) == NULL)
 604             continue;
 605
 606         N.hops = atof(q);
 607
 608         if ((q = strtok(NULL, w_space)) == NULL)
 609             continue;
 610
 611         N.rtt = atof(q);
 612
 613         if ((q = strtok(NULL, w_space)) == NULL)
 614             continue;
 615
 616         N.next_ping_time = (time_t) atoi(q);
 617
 618         if ((q = strtok(NULL, w_space)) == NULL)
 619             continue;
 620
 621         N.last_use_time = (time_t) atoi(q);
 622
 623         n = (netdbEntry *)memAllocate(MEM_NETDBENTRY);
 624
 625         xmemcpy(n, &N, sizeof(netdbEntry));
 626
 627         netdbHashInsert(n, addr);
 628
 629         while ((q = strtok(NULL, w_space)) != NULL) {
 630             if (netdbLookupHost(q) != NULL)     /* no dups! */
 631                 continue;
 632
 633             netdbHostInsert(n, q);
 634         }
 635
 636         count++;
 637     }
 638
 639     xfree(buf);
 640     getCurrentTime();
 641     debugs(38, 1, "NETDB state reloaded; " <<
 642            count << " entries, " <<
 643            tvSubMsec(start, current_time) << " msec" );
 644 }
 645
 646 static const char *
 647 netdbPeerName(const char *name)
 648 {
 649     const wordlist *w;
 650
 651     for (w = peer_names; w; w = w->next) {
 652         if (!strcmp(w->key, name))
 653             return w->key;
 654     }
 655
 656     return wordlistAdd(&peer_names, name);
 657 }
 658
 659 static void
 660 netdbFreeNetdbEntry(void *data)
 661 {
 662     netdbEntry *n = (netdbEntry *)data;
 663     safe_free(n->peers);
 664     memFree(n, MEM_NETDBENTRY);
 665 }
 666
 667 static void
 668 netdbFreeNameEntry(void *data)
 669 {
 670     net_db_name *x = (net_db_name *)data;
 671     xfree(x->hash.key);
 672     memFree(x, MEM_NET_DB_NAME);
 673 }
 674
 675
 676 static void
 677 netdbExchangeHandleReply(void *data, StoreIOBuffer receivedData)
 678 {
 679     Ip::Address addr;
 680
 681     netdbExchangeState *ex = (netdbExchangeState *)data;
 682     int rec_sz = 0;
 683     int o;
 684
 685     struct in_addr line_addr;
 686     double rtt;
 687     double hops;
 688     char *p;
 689     int j;
 690     HttpReply const *rep;
 691     size_t hdr_sz;
 692     int nused = 0;
 693     int size;
 694     int oldbufofs = ex->buf_ofs;
 695
 696     rec_sz = 0;
 697     rec_sz += 1 + sizeof(struct in_addr);
 698     rec_sz += 1 + sizeof(int);
 699     rec_sz += 1 + sizeof(int);
 700     debugs(38, 3, "netdbExchangeHandleReply: " << receivedData.length << " read bytes");
 701
 702     if (!cbdataReferenceValid(ex->p)) {
 703         debugs(38, 3, "netdbExchangeHandleReply: Peer became invalid");
 704         netdbExchangeDone(ex);
 705         return;
 706     }
 707
 708     debugs(38, 3, "netdbExchangeHandleReply: for '" << ex->p->host << ":" << ex->p->http_port << "'");
 709
 710     if (receivedData.length == 0 &&
 711             !receivedData.flags.error) {
 712         debugs(38, 3, "netdbExchangeHandleReply: Done");
 713         netdbExchangeDone(ex);
 714         return;
 715     }
 716
 717     p = ex->buf;
 718
 719     /* Get the size of the buffer now */
 720     size = ex->buf_ofs + receivedData.length;
 721     debugs(38, 3, "netdbExchangeHandleReply: " << size << " bytes buf");
 722
 723     /* Check if we're still doing headers */
 724
 725     if (ex->connstate == STATE_HEADER) {
 726
 727         ex->buf_ofs += receivedData.length;
 728
 729         /* skip reply headers */
 730
 731         if ((hdr_sz = headersEnd(p, ex->buf_ofs))) {
 732             debugs(38, 5, "netdbExchangeHandleReply: hdr_sz = " << hdr_sz);
 733             rep = ex->e->getReply();
 734             assert (0 != rep->sline.status);
 735             debugs(38, 3, "netdbExchangeHandleReply: reply status " << rep->sline.status);
 736
 737             if (HTTP_OK != rep->sline.status) {
 738                 netdbExchangeDone(ex);
 739                 return;
 740             }
 741
 742             assert((size_t)ex->buf_ofs >= hdr_sz);
 743
 744             /*
 745              * Now, point p to the part of the buffer where the data
 746              * starts, and update the size accordingly
 747              */
 748             assert(ex->used == 0);
 749             ex->used = hdr_sz;
 750             size = ex->buf_ofs - hdr_sz;
 751             p += hdr_sz;
 752
 753             /* Finally, set the conn state mode to STATE_BODY */
 754             ex->connstate = STATE_BODY;
 755         } else {
 756             StoreIOBuffer tempBuffer;
 757             tempBuffer.offset = ex->buf_ofs;
 758             tempBuffer.length = ex->buf_sz - ex->buf_ofs;
 759             tempBuffer.data = ex->buf + ex->buf_ofs;
 760             /* Have more headers .. */
 761             storeClientCopy(ex->sc, ex->e, tempBuffer,
 762                             netdbExchangeHandleReply, ex);
 763             return;
 764         }
 765     }
 766
 767     assert(ex->connstate == STATE_BODY);
 768
 769     /* If we get here, we have some body to parse .. */
 770     debugs(38, 5, "netdbExchangeHandleReply: start parsing loop, size = " << size);
 771
 772     while (size >= rec_sz) {
 773         debugs(38, 5, "netdbExchangeHandleReply: in parsing loop, size = " << size);
 774         addr.SetAnyAddr();
 775         hops = rtt = 0.0;
 776
 777         for (o = 0; o < rec_sz;) {
 778             switch ((int) *(p + o)) {
 779
 780             case NETDB_EX_NETWORK:
 781                 o++;
 782                 /* FIXME INET6 : NetDB can still ony send IPv4 */
 783                 xmemcpy(&line_addr, p + o, sizeof(struct in_addr));
 784                 addr = line_addr;
 785                 o += sizeof(struct in_addr);
 786                 break;
 787
 788             case NETDB_EX_RTT:
 789                 o++;
 790                 xmemcpy(&j, p + o, sizeof(int));
 791                 o += sizeof(int);
 792                 rtt = (double) ntohl(j) / 1000.0;
 793                 break;
 794
 795             case NETDB_EX_HOPS:
 796                 o++;
 797                 xmemcpy(&j, p + o, sizeof(int));
 798                 o += sizeof(int);
 799                 hops = (double) ntohl(j) / 1000.0;
 800                 break;
 801
 802             default:
 803                 debugs(38, 1, "netdbExchangeHandleReply: corrupt data, aborting");
 804                 netdbExchangeDone(ex);
 805                 return;
 806             }
 807         }
 808
 809         if (!addr.IsAnyAddr() && rtt > 0)
 810             netdbExchangeUpdatePeer(addr, ex->p, rtt, hops);
 811
 812         assert(o == rec_sz);
 813
 814         ex->used += rec_sz;
 815
 816         size -= rec_sz;
 817
 818         p += rec_sz;
 819
 820         nused++;
 821     }
 822
 823     /*
 824      * Copy anything that is left over to the beginning of the buffer,
 825      * and adjust buf_ofs accordingly
 826      */
 827
 828     /*
 829      * Evilly, size refers to the buf size left now,
 830      * ex->buf_ofs is the original buffer size, so just copy that
 831      * much data over
 832      */
 833     memmove(ex->buf, ex->buf + (ex->buf_ofs - size), size);
 834
 835     ex->buf_ofs = size;
 836
 837     /*
 838      * And don't re-copy the remaining data ..
 839      */
 840     ex->used += size;
 841
 842     /*
 843      * Now the tricky bit - size _included_ the leftover bit from the _last_
 844      * storeClientCopy. We don't want to include that, or our offset will be wrong.
 845      * So, don't count the size of the leftover buffer we began with.
 846      * This can _disappear_ when we're not tracking offsets ..
 847      */
 848     ex->used -= oldbufofs;
 849
 850     debugs(38, 3, "netdbExchangeHandleReply: size left over in this buffer: " << size << " bytes");
 851
 852     debugs(38, 3, "netdbExchangeHandleReply: used " << nused <<
 853            " entries, (x " << rec_sz << " bytes) == " << nused * rec_sz <<
 854            " bytes total");
 855
 856     debugs(38, 3, "netdbExchangeHandleReply: used " << ex->used);
 857
 858     if (EBIT_TEST(ex->e->flags, ENTRY_ABORTED)) {
 859         debugs(38, 3, "netdbExchangeHandleReply: ENTRY_ABORTED");
 860         netdbExchangeDone(ex);
 861     } else if (ex->e->store_status == STORE_PENDING) {
 862         StoreIOBuffer tempBuffer;
 863         tempBuffer.offset = ex->used;
 864         tempBuffer.length = ex->buf_sz - ex->buf_ofs;
 865         tempBuffer.data = ex->buf + ex->buf_ofs;
 866         debugs(38, 3, "netdbExchangeHandleReply: EOF not received");
 867         storeClientCopy(ex->sc, ex->e, tempBuffer,
 868                         netdbExchangeHandleReply, ex);
 869     }
 870 }
 871
 872 static void
 873 netdbExchangeDone(void *data)
 874 {
 875     netdbExchangeState *ex = (netdbExchangeState *)data;
 876     debugs(38, 3, "netdbExchangeDone: " << ex->e->url()  );
 877     HTTPMSGUNLOCK(ex->r);
 878     storeUnregister(ex->sc, ex->e, ex);
 879     ex->e->unlock();
 880     cbdataReferenceDone(ex->p);
 881     cbdataFree(ex);
 882 }
 883
 884 static void
 885 netdbRegisterWithCacheManager(void)
 886 {
 887     CacheManager::GetInstance()->
 888     registerAction("netdb", "Network Measurement Database", netdbDump, 0, 1);
 889 }
 890
 891 #endif /* USE_ICMP */
 892
 893
 894 /* PUBLIC FUNCTIONS */
 895
 896 void
 897 netdbInit(void)
 898 {
 899 #if USE_ICMP
 900     int n;
 901
 902     netdbRegisterWithCacheManager();
 903
 904     if (addr_table)
 905         return;
 906
 907     n = hashPrime(Config.Netdb.high / 4);
 908
 909     addr_table = hash_create((HASHCMP *) strcmp, n, hash_string);
 910
 911     n = hashPrime(3 * Config.Netdb.high / 4);
 912
 913     host_table = hash_create((HASHCMP *) strcmp, n, hash_string);
 914
 915     eventAddIsh("netdbSaveState", netdbSaveState, NULL, 3600.0, 1);
 916
 917     netdbReloadState();
 918
 919 #endif
 920 }
 921
 922 void
 923 netdbPingSite(const char *hostname)
 924 {
 925 #if USE_ICMP
 926     netdbEntry *n;
 927
 928     if ((n = netdbLookupHost(hostname)) != NULL)
 929         if (n->next_ping_time > squid_curtime)
 930             return;
 931
 932     ipcache_nbgethostbyname(hostname, netdbSendPing,
 933                             new generic_cbdata(xstrdup(hostname)));
 934
 935 #endif
 936 }
 937
 938 void
 939 netdbHandlePingReply(const Ip::Address &from, int hops, int rtt)
 940 {
 941 #if USE_ICMP
 942     netdbEntry *n;
 943     int N;
 944     debugs(38, 3, "netdbHandlePingReply: from " << from);
 945
 946     if ((n = netdbLookupAddr(from)) == NULL)
 947         return;
 948
 949     N = ++n->pings_recv;
 950
 951     if (N > 5)
 952         N = 5;
 953
 954     if (rtt < 1)
 955         rtt = 1;
 956
 957     n->hops = ((n->hops * (N - 1)) + hops) / N;
 958
 959     n->rtt = ((n->rtt * (N - 1)) + rtt) / N;
 960
 961     debugs(38, 3, "netdbHandlePingReply: " << n->network  << "; rtt="<<
 962            std::setw(5)<< std::setprecision(2) << n->rtt << "  hops="<<
 963            std::setw(4) << n->hops);
 964
 965 #endif
 966 }
 967
 968 void
 969 netdbFreeMemory(void)
 970 {
 971 #if USE_ICMP
 972     hashFreeItems(addr_table, netdbFreeNetdbEntry);
 973     hashFreeMemory(addr_table);
 974     addr_table = NULL;
 975     hashFreeItems(host_table, netdbFreeNameEntry);
 976     hashFreeMemory(host_table);
 977     host_table = NULL;
 978     wordlistDestroy(&peer_names);
 979     peer_names = NULL;
 980 #endif
 981 }
 982
 983
 984 #if 0 // AYJ: Looks to be unused code.
 985 int
 986 netdbHops(Ip::Address &addr)
 987 {
 988 #if USE_ICMP
 989     netdbEntry *n = netdbLookupAddr(addr);
 990
 991     if (n && n->pings_recv) {
 992         n->last_use_time = squid_curtime;
 993         return (int) (n->hops + 0.5);
 994     }
 995
 996 #endif
 997     return 256;
 998 }
 999 #endif
1000
1001 void
1002 netdbDump(StoreEntry * sentry)
1003 {
1004 #if USE_ICMP
1005     netdbEntry *n;
1006     netdbEntry **list;
1007     net_db_name *x;
1008     int k;
1009     int i;
1010     int j;
1011     net_db_peer *p;
1012     storeAppendPrintf(sentry, "Network DB Statistics:\n");
1013     storeAppendPrintf(sentry, "%-46.46s %9s %7s %5s %s\n",  /* Max between 16 (IPv4) or 46 (IPv6)   */
1014                       "Network",
1015                       "recv/sent",
1016                       "RTT",
1017                       "Hops",
1018                       "Hostnames");
1019     list = (netdbEntry **)xcalloc(memInUse(MEM_NETDBENTRY), sizeof(netdbEntry *));
1020     i = 0;
1021     hash_first(addr_table);
1022
1023     while ((n = (netdbEntry *) hash_next(addr_table)))
1024         *(list + i++) = n;
1025
1026     if (i != memInUse(MEM_NETDBENTRY))
1027         debugs(38, 0, "WARNING: netdb_addrs count off, found " << i <<
1028                ", expected " << memInUse(MEM_NETDBENTRY));
1029
1030     qsort((char *) list,
1031           i,
1032           sizeof(netdbEntry *),
1033           sortByRtt);
1034
1035     for (k = 0; k < i; k++) {
1036         n = *(list + k);
1037         storeAppendPrintf(sentry, "%-46.46s %4d/%4d %7.1f %5.1f", /* Max between 16 (IPv4) or 46 (IPv6)   */
1038                           n->network,
1039                           n->pings_recv,
1040                           n->pings_sent,
1041                           n->rtt,
1042                           n->hops);
1043
1044         for (x = n->hosts; x; x = x->next)
1045             storeAppendPrintf(sentry, " %s", hashKeyStr(&x->hash));
1046
1047         storeAppendPrintf(sentry, "\n");
1048
1049         p = n->peers;
1050
1051         for (j = 0; j < n->n_peers; j++, p++) {
1052             storeAppendPrintf(sentry, "    %-22.22s %7.1f %5.1f\n",
1053                               p->peername,
1054                               p->rtt,
1055                               p->hops);
1056         }
1057     }
1058
1059     xfree(list);
1060 #else
1061
1062     storeAppendPrintf(sentry,"NETDB support not compiled into this Squid cache.\n");
1063 #endif
1064 }
1065
1066 int
1067 netdbHostHops(const char *host)
1068 {
1069 #if USE_ICMP
1070     netdbEntry *n = netdbLookupHost(host);
1071
1072     if (n) {
1073         n->last_use_time = squid_curtime;
1074         return (int) (n->hops + 0.5);
1075     }
1076
1077 #endif
1078     return 0;
1079 }
1080
1081 int
1082 netdbHostRtt(const char *host)
1083 {
1084 #if USE_ICMP
1085     netdbEntry *n = netdbLookupHost(host);
1086
1087     if (n) {
1088         n->last_use_time = squid_curtime;
1089         return (int) (n->rtt + 0.5);
1090     }
1091
1092 #endif
1093     return 0;
1094 }
1095
1096 void
1097 netdbHostData(const char *host, int *samp, int *rtt, int *hops)
1098 {
1099 #if USE_ICMP
1100     netdbEntry *n = netdbLookupHost(host);
1101
1102     if (n == NULL)
1103         return;
1104
1105     *samp = n->pings_recv;
1106
1107     *rtt = (int) (n->rtt + 0.5);
1108
1109     *hops = (int) (n->hops + 0.5);
1110
1111     n->last_use_time = squid_curtime;
1112
1113 #endif
1114 }
1115
1116 void
1117 netdbUpdatePeer(HttpRequest * r, peer * e, int irtt, int ihops)
1118 {
1119 #if USE_ICMP
1120     netdbEntry *n;
1121     double rtt = (double) irtt;
1122     double hops = (double) ihops;
1123     net_db_peer *p;
1124     debugs(38, 3, "netdbUpdatePeer: '" << r->GetHost() << "', " << ihops << " hops, " << irtt << " rtt");
1125     n = netdbLookupHost(r->GetHost());
1126
1127     if (n == NULL) {
1128         debugs(38, 3, "netdbUpdatePeer: host '" << r->GetHost() << "' not found");
1129         return;
1130     }
1131
1132     if ((p = netdbPeerByName(n, e->host)) == NULL)
1133         p = netdbPeerAdd(n, e);
1134
1135     p->rtt = rtt;
1136
1137     p->hops = hops;
1138
1139     p->expires = squid_curtime + 3600;
1140
1141     if (n->n_peers < 2)
1142         return;
1143
1144     qsort((char *) n->peers,
1145           n->n_peers,
1146           sizeof(net_db_peer),
1147           sortPeerByRtt);
1148
1149 #endif
1150 }
1151
1152 void
1153 netdbExchangeUpdatePeer(Ip::Address &addr, peer * e, double rtt, double hops)
1154 {
1155 #if USE_ICMP
1156     netdbEntry *n;
1157     net_db_peer *p;
1158     debugs(38, 5, "netdbExchangeUpdatePeer: '" << addr << "', "<<
1159            std::setfill('0')<< std::setprecision(2) << hops << " hops, " <<
1160            rtt << " rtt");
1161
1162     if ( !addr.IsIPv4() ) {
1163         debugs(38, 5, "netdbExchangeUpdatePeer: Aborting peer update for '" << addr << "', NetDB cannot handle IPv6.");
1164         return;
1165     }
1166
1167     n = netdbLookupAddr(addr);
1168
1169     if (n == NULL)
1170         n = netdbAdd(addr);
1171
1172     assert(NULL != n);
1173
1174     if ((p = netdbPeerByName(n, e->host)) == NULL)
1175         p = netdbPeerAdd(n, e);
1176
1177     p->rtt = rtt;
1178
1179     p->hops = hops;
1180
1181     p->expires = squid_curtime + 3600;  /* XXX ? */
1182
1183     if (n->n_peers < 2)
1184         return;
1185
1186     qsort((char *) n->peers,
1187           n->n_peers,
1188           sizeof(net_db_peer),
1189           sortPeerByRtt);
1190
1191 #endif
1192 }
1193
1194 void
1195 netdbDeleteAddrNetwork(Ip::Address &addr)
1196 {
1197 #if USE_ICMP
1198     netdbEntry *n = netdbLookupAddr(addr);
1199
1200     if (n == NULL)
1201         return;
1202
1203     debugs(38, 3, "netdbDeleteAddrNetwork: " << n->network);
1204
1205     netdbRelease(n);
1206 #endif
1207 }
1208
1209
1210 void
1211 netdbBinaryExchange(StoreEntry * s)
1212 {
1213     HttpReply *reply = new HttpReply;
1214 #if USE_ICMP
1215
1216     Ip::Address addr;
1217
1218     netdbEntry *n;
1219     int i;
1220     int j;
1221     int rec_sz;
1222     char *buf;
1223
1224     struct in_addr line_addr;
1225     s->buffer();
1226     reply->setHeaders(HTTP_OK, "OK", NULL, -1, squid_curtime, -2);
1227     s->replaceHttpReply(reply);
1228     rec_sz = 0;
1229     rec_sz += 1 + sizeof(struct in_addr);
1230     rec_sz += 1 + sizeof(int);
1231     rec_sz += 1 + sizeof(int);
1232     buf = (char *)memAllocate(MEM_4K_BUF);
1233     i = 0;
1234     hash_first(addr_table);
1235
1236     while ((n = (netdbEntry *) hash_next(addr_table))) {
1237         if (0.0 == n->rtt)
1238             continue;
1239
1240         if (n->rtt > 60000)     /* RTT > 1 MIN probably bogus */
1241             continue;
1242
1243         if (! (addr = n->network) )
1244             continue;
1245
1246         /* FIXME INET6 : NetDB cannot yet handle IPv6 addresses. Ensure only IPv4 get sent. */
1247         if ( !addr.IsIPv4() )
1248             continue;
1249
1250         buf[i++] = (char) NETDB_EX_NETWORK;
1251
1252         addr.GetInAddr(line_addr);
1253         xmemcpy(&buf[i], &line_addr, sizeof(struct in_addr));
1254
1255         i += sizeof(struct in_addr);
1256
1257         buf[i++] = (char) NETDB_EX_RTT;
1258
1259         j = htonl((int) (n->rtt * 1000));
1260
1261         xmemcpy(&buf[i], &j, sizeof(int));
1262
1263         i += sizeof(int);
1264
1265         buf[i++] = (char) NETDB_EX_HOPS;
1266
1267         j = htonl((int) (n->hops * 1000));
1268
1269         xmemcpy(&buf[i], &j, sizeof(int));
1270
1271         i += sizeof(int);
1272
1273         if (i + rec_sz > 4096) {
1274             s->append(buf, i);
1275             i = 0;
1276         }
1277     }
1278
1279     if (i > 0) {
1280         s->append(buf, i);
1281         i = 0;
1282     }
1283
1284     assert(0 == i);
1285     s->flush();
1286     memFree(buf, MEM_4K_BUF);
1287 #else
1288
1289     reply->setHeaders(HTTP_BAD_REQUEST, "Bad Request", NULL, -1, squid_curtime, -2);
1290     s->replaceHttpReply(reply);
1291     storeAppendPrintf(s, "NETDB support not compiled into this Squid cache.\n");
1292 #endif
1293
1294     s->complete();
1295 }
1296
1297 #if USE_ICMP
1298 CBDATA_TYPE(netdbExchangeState);
1299 #endif
1300
1301 void
1302 netdbExchangeStart(void *data)
1303 {
1304 #if USE_ICMP
1305     peer *p = (peer *)data;
1306     char *uri;
1307     netdbExchangeState *ex;
1308     StoreIOBuffer tempBuffer;
1309     CBDATA_INIT_TYPE(netdbExchangeState);
1310     ex = cbdataAlloc(netdbExchangeState);
1311     ex->p = cbdataReference(p);
1312     uri = internalRemoteUri(p->host, p->http_port, "/squid-internal-dynamic/", "netdb");
1313     debugs(38, 3, "netdbExchangeStart: Requesting '" << uri << "'");
1314     assert(NULL != uri);
1315     ex->r = HttpRequest::CreateFromUrl(uri);
1316
1317     if (NULL == ex->r) {
1318         debugs(38, 1, "netdbExchangeStart: Bad URI " << uri);
1319         return;
1320     }
1321
1322     HTTPMSGLOCK(ex->r);
1323     assert(NULL != ex->r);
1324     ex->r->http_ver = HttpVersion(1,1);
1325     ex->connstate = STATE_HEADER;
1326     ex->e = storeCreateEntry(uri, uri, request_flags(), METHOD_GET);
1327     ex->buf_sz = NETDB_REQBUF_SZ;
1328     assert(NULL != ex->e);
1329     ex->sc = storeClientListAdd(ex->e, ex);
1330     tempBuffer.offset = 0;
1331     tempBuffer.length = ex->buf_sz;
1332     tempBuffer.data = ex->buf;
1333     storeClientCopy(ex->sc, ex->e, tempBuffer,
1334                     netdbExchangeHandleReply, ex);
1335     ex->r->flags.loopdetect = 1;        /* cheat! -- force direct */
1336
1337     if (p->login)
1338         xstrncpy(ex->r->login, p->login, MAX_LOGIN_SZ);
1339
1340     urlCanonical(ex->r);
1341
1342     FwdState::fwdStart(-1, ex->e, ex->r);
1343
1344 #endif
1345 }
1346
1347 peer *
1348 netdbClosestParent(HttpRequest * request)
1349 {
1350 #if USE_ICMP
1351     peer *p = NULL;
1352     netdbEntry *n;
1353     const ipcache_addrs *ia;
1354     net_db_peer *h;
1355     int i;
1356     n = netdbLookupHost(request->GetHost());
1357
1358     if (NULL == n) {
1359         /* try IP addr */
1360         ia = ipcache_gethostbyname(request->GetHost(), 0);
1361
1362         if (NULL != ia)
1363             n = netdbLookupAddr(ia->in_addrs[ia->cur]);
1364     }
1365
1366     if (NULL == n)
1367         return NULL;
1368
1369     if (0 == n->n_peers)
1370         return NULL;
1371
1372     n->last_use_time = squid_curtime;
1373
1374     /*
1375      * Find the parent with the least RTT to the origin server.
1376      * Make sure we don't return a parent who is farther away than
1377      * we are.  Note, the n->peers list is pre-sorted by RTT.
1378      */
1379     for (i = 0; i < n->n_peers; i++) {
1380         h = &n->peers[i];
1381
1382         if (n->rtt > 0)
1383             if (n->rtt < h->rtt)
1384                 break;
1385
1386         p = peerFindByName(h->peername);
1387
1388         if (NULL == p)          /* not found */
1389             continue;
1390
1391         if (neighborType(p, request) != PEER_PARENT)
1392             continue;
1393
1394         if (!peerHTTPOkay(p, request))  /* not allowed */
1395             continue;
1396
1397         return p;
1398     }
1399
1400 #endif
1401     return NULL;
1402 }