src/icmp/net_db.cc

   1 /*
   2  * DEBUG: section 38    Network Measurement Database
   3  * AUTHOR: Duane Wessels
   4  *
   5  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   6  * ----------------------------------------------------------
   7  *
   8  *  Squid is the result of efforts by numerous individuals from
   9  *  the Internet community; see the CONTRIBUTORS file for full
  10  *  details.   Many organizations have provided support for Squid's
  11  *  development; see the SPONSORS file for full details.  Squid is
  12  *  Copyrighted (C) 2001 by the Regents of the University of
  13  *  California; see the COPYRIGHT file for full details.  Squid
  14  *  incorporates software developed and/or copyrighted by other
  15  *  sources; see the CREDITS file for full details.
  16  *
  17  *  This program is free software; you can redistribute it and/or modify
  18  *  it under the terms of the GNU General Public License as published by
  19  *  the Free Software Foundation; either version 2 of the License, or
  20  *  (at your option) any later version.
  21  *
  22  *  This program is distributed in the hope that it will be useful,
  23  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  24  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  25  *  GNU General Public License for more details.
  26  *
  27  *  You should have received a copy of the GNU General Public License
  28  *  along with this program; if not, write to the Free Software
  29  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  30  *
  31  */
  32
  33 /*
  34  * XXX XXX XXX
  35  *
  36  * This code may be slightly broken now. If you're getting consistent
  37  * (sometimes working) corrupt data exchanges, please contact adrian
  38  * (adrian@squid-cache.org) to sort them out.
  39  */
  40
  41 #include "squid.h"
  42 #include "icmp/net_db.h"
  43 #include "log/File.h"
  44 #include "cbdata.h"
  45 #include "event.h"
  46 #include "mgr/Registration.h"
  47 #include "Store.h"
  48 #include "SwapDir.h"
  49 #include "HttpRequest.h"
  50 #include "HttpReply.h"
  51 #include "MemObject.h"
  52 #include "fde.h"
  53 #include "forward.h"
  54 #include "protos.h"
  55 #include "SquidTime.h"
  56 #include "wordlist.h"
  57 #include "ip/Address.h"
  58
  59 #if HAVE_SYS_STAT_H
  60 #include <sys/stat.h>
  61 #endif
  62
  63 #if USE_ICMP
  64 #include "icmp/IcmpSquid.h"
  65 #include "ipcache.h"
  66 #include "StoreClient.h"
  67
  68 #define NETDB_REQBUF_SZ 4096
  69
  70 typedef enum {
  71     STATE_NONE,
  72     STATE_HEADER,
  73     STATE_BODY
  74 } netdb_conn_state_t;
  75
  76 typedef struct {
  77     peer *p;
  78     StoreEntry *e;
  79     store_client *sc;
  80     HttpRequest *r;
  81     int64_t used;
  82     size_t buf_sz;
  83     char buf[NETDB_REQBUF_SZ];
  84     int buf_ofs;
  85     netdb_conn_state_t connstate;
  86 } netdbExchangeState;
  87
  88 static hash_table *addr_table = NULL;
  89 static hash_table *host_table = NULL;
  90
  91 Ip::Address networkFromInaddr(const Ip::Address &a);
  92 static void netdbRelease(netdbEntry * n);
  93
  94 static void netdbHashInsert(netdbEntry * n, Ip::Address &addr);
  95 static void netdbHashDelete(const char *key);
  96 static void netdbHostInsert(netdbEntry * n, const char *hostname);
  97 static void netdbHostDelete(const net_db_name * x);
  98 static void netdbPurgeLRU(void);
  99 static netdbEntry *netdbLookupHost(const char *key);
 100 static net_db_peer *netdbPeerByName(const netdbEntry * n, const char *);
 101 static net_db_peer *netdbPeerAdd(netdbEntry * n, peer * e);
 102 static const char *netdbPeerName(const char *name);
 103 static IPH netdbSendPing;
 104 static QS sortPeerByRtt;
 105 static QS sortByRtt;
 106 static QS netdbLRU;
 107 static FREE netdbFreeNameEntry;
 108 static FREE netdbFreeNetdbEntry;
 109 static STCB netdbExchangeHandleReply;
 110 static void netdbExchangeDone(void *);
 111
 112 /* We have to keep a local list of peer names.  The Peers structure
 113  * gets freed during a reconfigure.  We want this database to
 114  * remain persisitent, so _net_db_peer->peername points into this
 115  * linked list */
 116 static wordlist *peer_names = NULL;
 117
 118 static void
 119 netdbHashInsert(netdbEntry * n, Ip::Address &addr)
 120 {
 121     networkFromInaddr(addr).NtoA(n->network, MAX_IPSTRLEN);
 122     n->hash.key = n->network;
 123     assert(hash_lookup(addr_table, n->network) == NULL);
 124     hash_join(addr_table, &n->hash);
 125 }
 126
 127 static void
 128 netdbHashDelete(const char *key)
 129 {
 130     hash_link *hptr = (hash_link *)hash_lookup(addr_table, key);
 131
 132     if (hptr == NULL) {
 133         debug_trap("netdbHashDelete: key not found");
 134         return;
 135     }
 136
 137     hash_remove_link(addr_table, hptr);
 138 }
 139
 140 static void
 141 netdbHostInsert(netdbEntry * n, const char *hostname)
 142 {
 143     net_db_name *x = (net_db_name *)memAllocate(MEM_NET_DB_NAME);
 144     x->hash.key = xstrdup(hostname);
 145     x->next = n->hosts;
 146     n->hosts = x;
 147     x->net_db_entry = n;
 148     assert(hash_lookup(host_table, hostname) == NULL);
 149     hash_join(host_table, &x->hash);
 150     ++ n->link_count;
 151 }
 152
 153 static void
 154 netdbHostDelete(const net_db_name * x)
 155 {
 156     netdbEntry *n;
 157     net_db_name **X;
 158     assert(x != NULL);
 159     assert(x->net_db_entry != NULL);
 160     n = x->net_db_entry;
 161     -- n->link_count;
 162
 163     for (X = &n->hosts; *X; X = &(*X)->next) {
 164         if (*X == x) {
 165             *X = x->next;
 166             break;
 167         }
 168     }
 169
 170     hash_remove_link(host_table, (hash_link *) x);
 171     xfree(x->hash.key);
 172     memFree((void *) x, MEM_NET_DB_NAME);
 173 }
 174
 175 static netdbEntry *
 176 netdbLookupHost(const char *key)
 177 {
 178     net_db_name *x = (net_db_name *) hash_lookup(host_table, key);
 179     return x ? x->net_db_entry : NULL;
 180 }
 181
 182 static void
 183 netdbRelease(netdbEntry * n)
 184 {
 185     net_db_name *x;
 186     net_db_name *next;
 187
 188     for (x = n->hosts; x; x = next) {
 189         next = x->next;
 190         netdbHostDelete(x);
 191     }
 192
 193     n->hosts = NULL;
 194     safe_free(n->peers);
 195     n->peers = NULL;
 196     n->n_peers = 0;
 197     n->n_peers_alloc = 0;
 198
 199     if (n->link_count == 0) {
 200         netdbHashDelete(n->network);
 201         memFree(n, MEM_NETDBENTRY);
 202     }
 203 }
 204
 205 static int
 206 netdbLRU(const void *A, const void *B)
 207 {
 208     const netdbEntry *const *n1 = (const netdbEntry *const *)A;
 209     const netdbEntry *const *n2 = (const netdbEntry *const *)B;
 210
 211     if ((*n1)->last_use_time > (*n2)->last_use_time)
 212         return (1);
 213
 214     if ((*n1)->last_use_time < (*n2)->last_use_time)
 215         return (-1);
 216
 217     return (0);
 218 }
 219
 220 static void
 221 netdbPurgeLRU(void)
 222 {
 223     netdbEntry *n;
 224     netdbEntry **list;
 225     int k = 0;
 226     int list_count = 0;
 227     int removed = 0;
 228     list = (netdbEntry **)xcalloc(memInUse(MEM_NETDBENTRY), sizeof(netdbEntry *));
 229     hash_first(addr_table);
 230
 231     while ((n = (netdbEntry *) hash_next(addr_table))) {
 232         assert(list_count < memInUse(MEM_NETDBENTRY));
 233         *(list + list_count) = n;
 234         ++list_count;
 235     }
 236
 237     qsort((char *) list,
 238           list_count,
 239           sizeof(netdbEntry *),
 240           netdbLRU);
 241
 242     for (k = 0; k < list_count; ++k) {
 243         if (memInUse(MEM_NETDBENTRY) < Config.Netdb.low)
 244             break;
 245
 246         netdbRelease(*(list + k));
 247
 248         ++removed;
 249     }
 250
 251     xfree(list);
 252 }
 253
 254 static netdbEntry *
 255 netdbLookupAddr(const Ip::Address &addr)
 256 {
 257     netdbEntry *n;
 258     char *key = new char[MAX_IPSTRLEN];
 259     networkFromInaddr(addr).NtoA(key,MAX_IPSTRLEN);
 260     n = (netdbEntry *) hash_lookup(addr_table, key);
 261     delete[] key;
 262     return n;
 263 }
 264
 265 static netdbEntry *
 266 netdbAdd(Ip::Address &addr)
 267 {
 268     netdbEntry *n;
 269
 270     if (memInUse(MEM_NETDBENTRY) > Config.Netdb.high)
 271         netdbPurgeLRU();
 272
 273     if ((n = netdbLookupAddr(addr)) == NULL) {
 274         n = (netdbEntry *)memAllocate(MEM_NETDBENTRY);
 275         netdbHashInsert(n, addr);
 276     }
 277
 278     return n;
 279 }
 280
 281 static void
 282 netdbSendPing(const ipcache_addrs *ia, const DnsLookupDetails &, void *data)
 283 {
 284     Ip::Address addr;
 285     char *hostname = NULL;
 286     static_cast<generic_cbdata *>(data)->unwrap(&hostname);
 287     netdbEntry *n;
 288     netdbEntry *na;
 289     net_db_name *x;
 290     net_db_name **X;
 291
 292     if (ia == NULL) {
 293         xfree(hostname);
 294         return;
 295     }
 296
 297     addr = ia->in_addrs[ia->cur];
 298
 299     if ((n = netdbLookupHost(hostname)) == NULL) {
 300         n = netdbAdd(addr);
 301         netdbHostInsert(n, hostname);
 302     } else if ((na = netdbLookupAddr(addr)) != n) {
 303         /*
 304          *hostname moved from 'network n' to 'network na'!
 305          */
 306
 307         if (na == NULL)
 308             na = netdbAdd(addr);
 309
 310         debugs(38, 3, "netdbSendPing: " << hostname << " moved from " << n->network << " to " << na->network);
 311
 312         x = (net_db_name *) hash_lookup(host_table, hostname);
 313
 314         if (x == NULL) {
 315             debugs(38, DBG_IMPORTANT, "netdbSendPing: net_db_name list bug: " << hostname << " not found");
 316             xfree(hostname);
 317             return;
 318         }
 319
 320         /* remove net_db_name from 'network n' linked list */
 321         for (X = &n->hosts; *X; X = &(*X)->next) {
 322             if (*X == x) {
 323                 *X = x->next;
 324                 break;
 325             }
 326         }
 327
 328         -- n->link_count;
 329         /* point to 'network na' from host entry */
 330         x->net_db_entry = na;
 331         /* link net_db_name to 'network na' */
 332         x->next = na->hosts;
 333         na->hosts = x;
 334         ++ na->link_count;
 335         n = na;
 336     }
 337
 338     if (n->next_ping_time <= squid_curtime) {
 339         debugs(38, 3, "netdbSendPing: pinging " << hostname);
 340         icmpEngine.DomainPing(addr, hostname);
 341         ++ n->pings_sent;
 342         n->next_ping_time = squid_curtime + Config.Netdb.period;
 343         n->last_use_time = squid_curtime;
 344     }
 345
 346     xfree(hostname);
 347 }
 348
 349 Ip::Address
 350 networkFromInaddr(const Ip::Address &in)
 351 {
 352     Ip::Address out;
 353
 354     out = in;
 355
 356     /* in IPv6 the 'network' should be the routing section. */
 357     if ( in.IsIPv6() ) {
 358         out.ApplyMask(64, AF_INET6);
 359         debugs(14, 5, "networkFromInaddr : Masked IPv6 Address to " << in << "/64 routing part.");
 360         return out;
 361     }
 362
 363 #if USE_CLASSFUL
 364     struct in_addr b;
 365
 366     in.GetInAddr(b);
 367
 368     if (IN_CLASSC(b.s_addr))
 369         b.s_addr &= IN_CLASSC_NET;
 370     else if (IN_CLASSB(b.s_addr))
 371         b.s_addr &= IN_CLASSB_NET;
 372     else if (IN_CLASSA(b.s_addr))
 373         b.s_addr &= IN_CLASSA_NET;
 374
 375     out = b;
 376
 377 #endif
 378
 379     debugs(14, 5, "networkFromInaddr : Masked IPv4 Address to " << out << "/24.");
 380
 381     /* use /24 for everything under IPv4 */
 382     out.ApplyMask(24, AF_INET);
 383     debugs(14, 5, "networkFromInaddr : Masked IPv4 Address to " << in << "/24.");
 384
 385     return out;
 386 }
 387
 388 static int
 389 sortByRtt(const void *A, const void *B)
 390 {
 391     const netdbEntry *const *n1 = (const netdbEntry *const *)A;
 392     const netdbEntry *const *n2 = (const netdbEntry *const *)B;
 393
 394     if ((*n1)->rtt > (*n2)->rtt)
 395         return 1;
 396     else if ((*n1)->rtt < (*n2)->rtt)
 397         return -1;
 398     else
 399         return 0;
 400 }
 401
 402 static net_db_peer *
 403 netdbPeerByName(const netdbEntry * n, const char *peername)
 404 {
 405     int i;
 406     net_db_peer *p = n->peers;
 407
 408     for (i = 0; i < n->n_peers; ++i, ++p) {
 409         if (!strcmp(p->peername, peername))
 410             return p;
 411     }
 412
 413     return NULL;
 414 }
 415
 416 static net_db_peer *
 417 netdbPeerAdd(netdbEntry * n, peer * e)
 418 {
 419     net_db_peer *p;
 420     net_db_peer *o;
 421     int osize;
 422     int i;
 423
 424     if (n->n_peers == n->n_peers_alloc) {
 425         o = n->peers;
 426         osize = n->n_peers_alloc;
 427
 428         if (n->n_peers_alloc == 0)
 429             n->n_peers_alloc = 2;
 430         else
 431             n->n_peers_alloc <<= 1;
 432
 433         debugs(38, 3, "netdbPeerAdd: Growing peer list for '" << n->network << "' to " << n->n_peers_alloc);
 434
 435         n->peers = (net_db_peer *)xcalloc(n->n_peers_alloc, sizeof(net_db_peer));
 436
 437         for (i = 0; i < osize; ++i)
 438             *(n->peers + i) = *(o + i);
 439
 440         if (osize) {
 441             safe_free(o);
 442         }
 443     }
 444
 445     p = n->peers + n->n_peers;
 446     p->peername = netdbPeerName(e->host);
 447     ++ n->n_peers;
 448     return p;
 449 }
 450
 451 static int
 452 sortPeerByRtt(const void *A, const void *B)
 453 {
 454     const net_db_peer *p1 = (net_db_peer *)A;
 455     const net_db_peer *p2 = (net_db_peer *)B;
 456
 457     if (p1->rtt > p2->rtt)
 458         return 1;
 459     else if (p1->rtt < p2->rtt)
 460         return -1;
 461     else
 462         return 0;
 463 }
 464
 465 static void
 466 netdbSaveState(void *foo)
 467 {
 468     if (strcmp(Config.netdbFilename, "none") == 0)
 469         return;
 470
 471     Logfile *lf;
 472     netdbEntry *n;
 473     net_db_name *x;
 474
 475     struct timeval start = current_time;
 476     int count = 0;
 477     /*
 478      * This was nicer when we were using stdio, but thanks to
 479      * Solaris bugs, its a bad idea.  fopen can fail if more than
 480      * 256 FDs are open.
 481      */
 482     /*
 483      * unlink() is here because there is currently no way to make
 484      * logfileOpen() use O_TRUNC.
 485      */
 486     unlink(Config.netdbFilename);
 487     lf = logfileOpen(Config.netdbFilename, 4096, 0);
 488
 489     if (NULL == lf) {
 490         debugs(50, DBG_IMPORTANT, "netdbSaveState: " << Config.netdbFilename << ": " << xstrerror());
 491         return;
 492     }
 493
 494     hash_first(addr_table);
 495
 496     while ((n = (netdbEntry *) hash_next(addr_table))) {
 497         if (n->pings_recv == 0)
 498             continue;
 499
 500         logfilePrintf(lf, "%s %d %d %10.5f %10.5f %d %d",
 501                       n->network,
 502                       n->pings_sent,
 503                       n->pings_recv,
 504                       n->hops,
 505                       n->rtt,
 506                       (int) n->next_ping_time,
 507                       (int) n->last_use_time);
 508
 509         for (x = n->hosts; x; x = x->next)
 510             logfilePrintf(lf, " %s", hashKeyStr(&x->hash));
 511
 512         logfilePrintf(lf, "\n");
 513
 514         ++count;
 515
 516 #undef RBUF_SZ
 517
 518     }
 519
 520     logfileClose(lf);
 521     getCurrentTime();
 522     debugs(38, DBG_IMPORTANT, "NETDB state saved; " <<
 523            count << " entries, " <<
 524            tvSubMsec(start, current_time) << " msec" );
 525     eventAddIsh("netdbSaveState", netdbSaveState, NULL, 3600.0, 1);
 526 }
 527
 528 static void
 529 netdbReloadState(void)
 530 {
 531     if (strcmp(Config.netdbFilename, "none") == 0)
 532         return;
 533
 534     char *s;
 535     int fd;
 536     int l;
 537
 538     struct stat sb;
 539     netdbEntry *n;
 540     netdbEntry N;
 541
 542     Ip::Address addr;
 543     int count = 0;
 544
 545     struct timeval start = current_time;
 546     /*
 547      * This was nicer when we were using stdio, but thanks to
 548      * Solaris bugs, its a bad idea.  fopen can fail if more than
 549      * 256 FDs are open.
 550      */
 551     fd = file_open(Config.netdbFilename, O_RDONLY | O_BINARY);
 552
 553     if (fd < 0)
 554         return;
 555
 556     if (fstat(fd, &sb) < 0) {
 557         file_close(fd);
 558         return;
 559     }
 560
 561     char *t;
 562     char *buf = (char *)xcalloc(1, sb.st_size + 1);
 563     t = buf;
 564     l = FD_READ_METHOD(fd, buf, sb.st_size);
 565     file_close(fd);
 566
 567     if (l <= 0) {
 568         safe_free (buf);
 569         return;
 570     };
 571
 572     while ((s = strchr(t, '\n'))) {
 573         char *q;
 574         assert(s - buf < l);
 575         *s = '\0';
 576         memset(&N, '\0', sizeof(netdbEntry));
 577         q = strtok(t, w_space);
 578         t = s + 1;
 579
 580         if (NULL == q)
 581             continue;
 582
 583         if (! (addr = q) )
 584             continue;
 585
 586         if (netdbLookupAddr(addr) != NULL)      /* no dups! */
 587             continue;
 588
 589         if ((q = strtok(NULL, w_space)) == NULL)
 590             continue;
 591
 592         N.pings_sent = atoi(q);
 593
 594         if ((q = strtok(NULL, w_space)) == NULL)
 595             continue;
 596
 597         N.pings_recv = atoi(q);
 598
 599         if (N.pings_recv == 0)
 600             continue;
 601
 602         /* give this measurement low weight */
 603         N.pings_sent = 1;
 604
 605         N.pings_recv = 1;
 606
 607         if ((q = strtok(NULL, w_space)) == NULL)
 608             continue;
 609
 610         N.hops = atof(q);
 611
 612         if ((q = strtok(NULL, w_space)) == NULL)
 613             continue;
 614
 615         N.rtt = atof(q);
 616
 617         if ((q = strtok(NULL, w_space)) == NULL)
 618             continue;
 619
 620         N.next_ping_time = (time_t) atoi(q);
 621
 622         if ((q = strtok(NULL, w_space)) == NULL)
 623             continue;
 624
 625         N.last_use_time = (time_t) atoi(q);
 626
 627         n = (netdbEntry *)memAllocate(MEM_NETDBENTRY);
 628
 629         memcpy(n, &N, sizeof(netdbEntry));
 630
 631         netdbHashInsert(n, addr);
 632
 633         while ((q = strtok(NULL, w_space)) != NULL) {
 634             if (netdbLookupHost(q) != NULL)     /* no dups! */
 635                 continue;
 636
 637             netdbHostInsert(n, q);
 638         }
 639
 640         ++count;
 641     }
 642
 643     xfree(buf);
 644     getCurrentTime();
 645     debugs(38, DBG_IMPORTANT, "NETDB state reloaded; " <<
 646            count << " entries, " <<
 647            tvSubMsec(start, current_time) << " msec" );
 648 }
 649
 650 static const char *
 651 netdbPeerName(const char *name)
 652 {
 653     const wordlist *w;
 654
 655     for (w = peer_names; w; w = w->next) {
 656         if (!strcmp(w->key, name))
 657             return w->key;
 658     }
 659
 660     return wordlistAdd(&peer_names, name);
 661 }
 662
 663 static void
 664 netdbFreeNetdbEntry(void *data)
 665 {
 666     netdbEntry *n = (netdbEntry *)data;
 667     safe_free(n->peers);
 668     memFree(n, MEM_NETDBENTRY);
 669 }
 670
 671 static void
 672 netdbFreeNameEntry(void *data)
 673 {
 674     net_db_name *x = (net_db_name *)data;
 675     xfree(x->hash.key);
 676     memFree(x, MEM_NET_DB_NAME);
 677 }
 678
 679 static void
 680 netdbExchangeHandleReply(void *data, StoreIOBuffer receivedData)
 681 {
 682     Ip::Address addr;
 683
 684     netdbExchangeState *ex = (netdbExchangeState *)data;
 685     int rec_sz = 0;
 686     int o;
 687
 688     struct in_addr line_addr;
 689     double rtt;
 690     double hops;
 691     char *p;
 692     int j;
 693     HttpReply const *rep;
 694     size_t hdr_sz;
 695     int nused = 0;
 696     int size;
 697     int oldbufofs = ex->buf_ofs;
 698
 699     rec_sz = 0;
 700     rec_sz += 1 + sizeof(struct in_addr);
 701     rec_sz += 1 + sizeof(int);
 702     rec_sz += 1 + sizeof(int);
 703     debugs(38, 3, "netdbExchangeHandleReply: " << receivedData.length << " read bytes");
 704
 705     if (!cbdataReferenceValid(ex->p)) {
 706         debugs(38, 3, "netdbExchangeHandleReply: Peer became invalid");
 707         netdbExchangeDone(ex);
 708         return;
 709     }
 710
 711     debugs(38, 3, "netdbExchangeHandleReply: for '" << ex->p->host << ":" << ex->p->http_port << "'");
 712
 713     if (receivedData.length == 0 &&
 714             !receivedData.flags.error) {
 715         debugs(38, 3, "netdbExchangeHandleReply: Done");
 716         netdbExchangeDone(ex);
 717         return;
 718     }
 719
 720     p = ex->buf;
 721
 722     /* Get the size of the buffer now */
 723     size = ex->buf_ofs + receivedData.length;
 724     debugs(38, 3, "netdbExchangeHandleReply: " << size << " bytes buf");
 725
 726     /* Check if we're still doing headers */
 727
 728     if (ex->connstate == STATE_HEADER) {
 729
 730         ex->buf_ofs += receivedData.length;
 731
 732         /* skip reply headers */
 733
 734         if ((hdr_sz = headersEnd(p, ex->buf_ofs))) {
 735             debugs(38, 5, "netdbExchangeHandleReply: hdr_sz = " << hdr_sz);
 736             rep = ex->e->getReply();
 737             assert (0 != rep->sline.status);
 738             debugs(38, 3, "netdbExchangeHandleReply: reply status " << rep->sline.status);
 739
 740             if (HTTP_OK != rep->sline.status) {
 741                 netdbExchangeDone(ex);
 742                 return;
 743             }
 744
 745             assert((size_t)ex->buf_ofs >= hdr_sz);
 746
 747             /*
 748              * Now, point p to the part of the buffer where the data
 749              * starts, and update the size accordingly
 750              */
 751             assert(ex->used == 0);
 752             ex->used = hdr_sz;
 753             size = ex->buf_ofs - hdr_sz;
 754             p += hdr_sz;
 755
 756             /* Finally, set the conn state mode to STATE_BODY */
 757             ex->connstate = STATE_BODY;
 758         } else {
 759             StoreIOBuffer tempBuffer;
 760             tempBuffer.offset = ex->buf_ofs;
 761             tempBuffer.length = ex->buf_sz - ex->buf_ofs;
 762             tempBuffer.data = ex->buf + ex->buf_ofs;
 763             /* Have more headers .. */
 764             storeClientCopy(ex->sc, ex->e, tempBuffer,
 765                             netdbExchangeHandleReply, ex);
 766             return;
 767         }
 768     }
 769
 770     assert(ex->connstate == STATE_BODY);
 771
 772     /* If we get here, we have some body to parse .. */
 773     debugs(38, 5, "netdbExchangeHandleReply: start parsing loop, size = " << size);
 774
 775     while (size >= rec_sz) {
 776         debugs(38, 5, "netdbExchangeHandleReply: in parsing loop, size = " << size);
 777         addr.SetAnyAddr();
 778         hops = rtt = 0.0;
 779
 780         for (o = 0; o < rec_sz;) {
 781             switch ((int) *(p + o)) {
 782
 783             case NETDB_EX_NETWORK:
 784                 ++o;
 785                 /* FIXME INET6 : NetDB can still ony send IPv4 */
 786                 memcpy(&line_addr, p + o, sizeof(struct in_addr));
 787                 addr = line_addr;
 788                 o += sizeof(struct in_addr);
 789                 break;
 790
 791             case NETDB_EX_RTT:
 792                 ++o;
 793                 memcpy(&j, p + o, sizeof(int));
 794                 o += sizeof(int);
 795                 rtt = (double) ntohl(j) / 1000.0;
 796                 break;
 797
 798             case NETDB_EX_HOPS:
 799                 ++o;
 800                 memcpy(&j, p + o, sizeof(int));
 801                 o += sizeof(int);
 802                 hops = (double) ntohl(j) / 1000.0;
 803                 break;
 804
 805             default:
 806                 debugs(38, DBG_IMPORTANT, "netdbExchangeHandleReply: corrupt data, aborting");
 807                 netdbExchangeDone(ex);
 808                 return;
 809             }
 810         }
 811
 812         if (!addr.IsAnyAddr() && rtt > 0)
 813             netdbExchangeUpdatePeer(addr, ex->p, rtt, hops);
 814
 815         assert(o == rec_sz);
 816
 817         ex->used += rec_sz;
 818
 819         size -= rec_sz;
 820
 821         p += rec_sz;
 822
 823         ++nused;
 824     }
 825
 826     /*
 827      * Copy anything that is left over to the beginning of the buffer,
 828      * and adjust buf_ofs accordingly
 829      */
 830
 831     /*
 832      * Evilly, size refers to the buf size left now,
 833      * ex->buf_ofs is the original buffer size, so just copy that
 834      * much data over
 835      */
 836     memmove(ex->buf, ex->buf + (ex->buf_ofs - size), size);
 837
 838     ex->buf_ofs = size;
 839
 840     /*
 841      * And don't re-copy the remaining data ..
 842      */
 843     ex->used += size;
 844
 845     /*
 846      * Now the tricky bit - size _included_ the leftover bit from the _last_
 847      * storeClientCopy. We don't want to include that, or our offset will be wrong.
 848      * So, don't count the size of the leftover buffer we began with.
 849      * This can _disappear_ when we're not tracking offsets ..
 850      */
 851     ex->used -= oldbufofs;
 852
 853     debugs(38, 3, "netdbExchangeHandleReply: size left over in this buffer: " << size << " bytes");
 854
 855     debugs(38, 3, "netdbExchangeHandleReply: used " << nused <<
 856            " entries, (x " << rec_sz << " bytes) == " << nused * rec_sz <<
 857            " bytes total");
 858
 859     debugs(38, 3, "netdbExchangeHandleReply: used " << ex->used);
 860
 861     if (EBIT_TEST(ex->e->flags, ENTRY_ABORTED)) {
 862         debugs(38, 3, "netdbExchangeHandleReply: ENTRY_ABORTED");
 863         netdbExchangeDone(ex);
 864     } else if (ex->e->store_status == STORE_PENDING) {
 865         StoreIOBuffer tempBuffer;
 866         tempBuffer.offset = ex->used;
 867         tempBuffer.length = ex->buf_sz - ex->buf_ofs;
 868         tempBuffer.data = ex->buf + ex->buf_ofs;
 869         debugs(38, 3, "netdbExchangeHandleReply: EOF not received");
 870         storeClientCopy(ex->sc, ex->e, tempBuffer,
 871                         netdbExchangeHandleReply, ex);
 872     }
 873 }
 874
 875 static void
 876 netdbExchangeDone(void *data)
 877 {
 878     netdbExchangeState *ex = (netdbExchangeState *)data;
 879     debugs(38, 3, "netdbExchangeDone: " << ex->e->url()  );
 880     HTTPMSGUNLOCK(ex->r);
 881     storeUnregister(ex->sc, ex->e, ex);
 882     ex->e->unlock();
 883     cbdataReferenceDone(ex->p);
 884     cbdataFree(ex);
 885 }
 886
 887 static void
 888 netdbRegisterWithCacheManager(void)
 889 {
 890     Mgr::RegisterAction("netdb", "Network Measurement Database", netdbDump, 0, 1);
 891 }
 892
 893 #endif /* USE_ICMP */
 894
 895 /* PUBLIC FUNCTIONS */
 896
 897 void
 898 netdbInit(void)
 899 {
 900 #if USE_ICMP
 901     int n;
 902
 903     netdbRegisterWithCacheManager();
 904
 905     if (addr_table)
 906         return;
 907
 908     n = hashPrime(Config.Netdb.high / 4);
 909
 910     addr_table = hash_create((HASHCMP *) strcmp, n, hash_string);
 911
 912     n = hashPrime(3 * Config.Netdb.high / 4);
 913
 914     host_table = hash_create((HASHCMP *) strcmp, n, hash_string);
 915
 916     eventAddIsh("netdbSaveState", netdbSaveState, NULL, 3600.0, 1);
 917
 918     netdbReloadState();
 919
 920 #endif
 921 }
 922
 923 void
 924 netdbPingSite(const char *hostname)
 925 {
 926 #if USE_ICMP
 927     netdbEntry *n;
 928
 929     if ((n = netdbLookupHost(hostname)) != NULL)
 930         if (n->next_ping_time > squid_curtime)
 931             return;
 932
 933     ipcache_nbgethostbyname(hostname, netdbSendPing,
 934                             new generic_cbdata(xstrdup(hostname)));
 935
 936 #endif
 937 }
 938
 939 void
 940 netdbHandlePingReply(const Ip::Address &from, int hops, int rtt)
 941 {
 942 #if USE_ICMP
 943     netdbEntry *n;
 944     int N;
 945     debugs(38, 3, "netdbHandlePingReply: from " << from);
 946
 947     if ((n = netdbLookupAddr(from)) == NULL)
 948         return;
 949
 950     N = ++n->pings_recv;
 951
 952     if (N > 5)
 953         N = 5;
 954
 955     if (rtt < 1)
 956         rtt = 1;
 957
 958     n->hops = ((n->hops * (N - 1)) + hops) / N;
 959
 960     n->rtt = ((n->rtt * (N - 1)) + rtt) / N;
 961
 962     debugs(38, 3, "netdbHandlePingReply: " << n->network  << "; rtt="<<
 963            std::setw(5)<< std::setprecision(2) << n->rtt << "  hops="<<
 964            std::setw(4) << n->hops);
 965
 966 #endif
 967 }
 968
 969 void
 970 netdbFreeMemory(void)
 971 {
 972 #if USE_ICMP
 973     hashFreeItems(addr_table, netdbFreeNetdbEntry);
 974     hashFreeMemory(addr_table);
 975     addr_table = NULL;
 976     hashFreeItems(host_table, netdbFreeNameEntry);
 977     hashFreeMemory(host_table);
 978     host_table = NULL;
 979     wordlistDestroy(&peer_names);
 980     peer_names = NULL;
 981 #endif
 982 }
 983
 984 #if 0 // AYJ: Looks to be unused code.
 985 int
 986 netdbHops(Ip::Address &addr)
 987 {
 988 #if USE_ICMP
 989     netdbEntry *n = netdbLookupAddr(addr);
 990
 991     if (n && n->pings_recv) {
 992         n->last_use_time = squid_curtime;
 993         return (int) (n->hops + 0.5);
 994     }
 995
 996 #endif
 997     return 256;
 998 }
 999 #endif
1000
1001 void
1002 netdbDump(StoreEntry * sentry)
1003 {
1004 #if USE_ICMP
1005     netdbEntry *n;
1006     netdbEntry **list;
1007     net_db_name *x;
1008     int k;
1009     int i;
1010     int j;
1011     net_db_peer *p;
1012     storeAppendPrintf(sentry, "Network DB Statistics:\n");
1013     storeAppendPrintf(sentry, "%-46.46s %9s %7s %5s %s\n",  /* Max between 16 (IPv4) or 46 (IPv6)   */
1014                       "Network",
1015                       "recv/sent",
1016                       "RTT",
1017                       "Hops",
1018                       "Hostnames");
1019     list = (netdbEntry **)xcalloc(memInUse(MEM_NETDBENTRY), sizeof(netdbEntry *));
1020     i = 0;
1021     hash_first(addr_table);
1022
1023     while ((n = (netdbEntry *) hash_next(addr_table))) {
1024         *(list + i) = n;
1025         ++i;
1026     }
1027
1028     if (i != memInUse(MEM_NETDBENTRY))
1029         debugs(38, DBG_CRITICAL, "WARNING: netdb_addrs count off, found " << i <<
1030                ", expected " << memInUse(MEM_NETDBENTRY));
1031
1032     qsort((char *) list,
1033           i,
1034           sizeof(netdbEntry *),
1035           sortByRtt);
1036
1037     for (k = 0; k < i; ++k) {
1038         n = *(list + k);
1039         storeAppendPrintf(sentry, "%-46.46s %4d/%4d %7.1f %5.1f", /* Max between 16 (IPv4) or 46 (IPv6)   */
1040                           n->network,
1041                           n->pings_recv,
1042                           n->pings_sent,
1043                           n->rtt,
1044                           n->hops);
1045
1046         for (x = n->hosts; x; x = x->next)
1047             storeAppendPrintf(sentry, " %s", hashKeyStr(&x->hash));
1048
1049         storeAppendPrintf(sentry, "\n");
1050
1051         p = n->peers;
1052
1053         for (j = 0; j < n->n_peers; ++j, ++p) {
1054             storeAppendPrintf(sentry, "    %-22.22s %7.1f %5.1f\n",
1055                               p->peername,
1056                               p->rtt,
1057                               p->hops);
1058         }
1059     }
1060
1061     xfree(list);
1062 #else
1063
1064     storeAppendPrintf(sentry,"NETDB support not compiled into this Squid cache.\n");
1065 #endif
1066 }
1067
1068 int
1069 netdbHostHops(const char *host)
1070 {
1071 #if USE_ICMP
1072     netdbEntry *n = netdbLookupHost(host);
1073
1074     if (n) {
1075         n->last_use_time = squid_curtime;
1076         return (int) (n->hops + 0.5);
1077     }
1078
1079 #endif
1080     return 0;
1081 }
1082
1083 int
1084 netdbHostRtt(const char *host)
1085 {
1086 #if USE_ICMP
1087     netdbEntry *n = netdbLookupHost(host);
1088
1089     if (n) {
1090         n->last_use_time = squid_curtime;
1091         return (int) (n->rtt + 0.5);
1092     }
1093
1094 #endif
1095     return 0;
1096 }
1097
1098 void
1099 netdbHostData(const char *host, int *samp, int *rtt, int *hops)
1100 {
1101 #if USE_ICMP
1102     netdbEntry *n = netdbLookupHost(host);
1103
1104     if (n == NULL)
1105         return;
1106
1107     *samp = n->pings_recv;
1108
1109     *rtt = (int) (n->rtt + 0.5);
1110
1111     *hops = (int) (n->hops + 0.5);
1112
1113     n->last_use_time = squid_curtime;
1114
1115 #endif
1116 }
1117
1118 void
1119 netdbUpdatePeer(HttpRequest * r, peer * e, int irtt, int ihops)
1120 {
1121 #if USE_ICMP
1122     netdbEntry *n;
1123     double rtt = (double) irtt;
1124     double hops = (double) ihops;
1125     net_db_peer *p;
1126     debugs(38, 3, "netdbUpdatePeer: '" << r->GetHost() << "', " << ihops << " hops, " << irtt << " rtt");
1127     n = netdbLookupHost(r->GetHost());
1128
1129     if (n == NULL) {
1130         debugs(38, 3, "netdbUpdatePeer: host '" << r->GetHost() << "' not found");
1131         return;
1132     }
1133
1134     if ((p = netdbPeerByName(n, e->host)) == NULL)
1135         p = netdbPeerAdd(n, e);
1136
1137     p->rtt = rtt;
1138
1139     p->hops = hops;
1140
1141     p->expires = squid_curtime + 3600;
1142
1143     if (n->n_peers < 2)
1144         return;
1145
1146     qsort((char *) n->peers,
1147           n->n_peers,
1148           sizeof(net_db_peer),
1149           sortPeerByRtt);
1150
1151 #endif
1152 }
1153
1154 void
1155 netdbExchangeUpdatePeer(Ip::Address &addr, peer * e, double rtt, double hops)
1156 {
1157 #if USE_ICMP
1158     netdbEntry *n;
1159     net_db_peer *p;
1160     debugs(38, 5, "netdbExchangeUpdatePeer: '" << addr << "', "<<
1161            std::setfill('0')<< std::setprecision(2) << hops << " hops, " <<
1162            rtt << " rtt");
1163
1164     if ( !addr.IsIPv4() ) {
1165         debugs(38, 5, "netdbExchangeUpdatePeer: Aborting peer update for '" << addr << "', NetDB cannot handle IPv6.");
1166         return;
1167     }
1168
1169     n = netdbLookupAddr(addr);
1170
1171     if (n == NULL)
1172         n = netdbAdd(addr);
1173
1174     assert(NULL != n);
1175
1176     if ((p = netdbPeerByName(n, e->host)) == NULL)
1177         p = netdbPeerAdd(n, e);
1178
1179     p->rtt = rtt;
1180
1181     p->hops = hops;
1182
1183     p->expires = squid_curtime + 3600;  /* XXX ? */
1184
1185     if (n->n_peers < 2)
1186         return;
1187
1188     qsort((char *) n->peers,
1189           n->n_peers,
1190           sizeof(net_db_peer),
1191           sortPeerByRtt);
1192
1193 #endif
1194 }
1195
1196 void
1197 netdbDeleteAddrNetwork(Ip::Address &addr)
1198 {
1199 #if USE_ICMP
1200     netdbEntry *n = netdbLookupAddr(addr);
1201
1202     if (n == NULL)
1203         return;
1204
1205     debugs(38, 3, "netdbDeleteAddrNetwork: " << n->network);
1206
1207     netdbRelease(n);
1208 #endif
1209 }
1210
1211 void
1212 netdbBinaryExchange(StoreEntry * s)
1213 {
1214     HttpReply *reply = new HttpReply;
1215 #if USE_ICMP
1216
1217     Ip::Address addr;
1218
1219     netdbEntry *n;
1220     int i;
1221     int j;
1222     int rec_sz;
1223     char *buf;
1224
1225     struct in_addr line_addr;
1226     s->buffer();
1227     reply->setHeaders(HTTP_OK, "OK", NULL, -1, squid_curtime, -2);
1228     s->replaceHttpReply(reply);
1229     rec_sz = 0;
1230     rec_sz += 1 + sizeof(struct in_addr);
1231     rec_sz += 1 + sizeof(int);
1232     rec_sz += 1 + sizeof(int);
1233     buf = (char *)memAllocate(MEM_4K_BUF);
1234     i = 0;
1235     hash_first(addr_table);
1236
1237     while ((n = (netdbEntry *) hash_next(addr_table))) {
1238         if (0.0 == n->rtt)
1239             continue;
1240
1241         if (n->rtt > 60000)     /* RTT > 1 MIN probably bogus */
1242             continue;
1243
1244         if (! (addr = n->network) )
1245             continue;
1246
1247         /* FIXME INET6 : NetDB cannot yet handle IPv6 addresses. Ensure only IPv4 get sent. */
1248         if ( !addr.IsIPv4() )
1249             continue;
1250
1251         buf[i] = (char) NETDB_EX_NETWORK;
1252         ++i;
1253
1254         addr.GetInAddr(line_addr);
1255         memcpy(&buf[i], &line_addr, sizeof(struct in_addr));
1256
1257         i += sizeof(struct in_addr);
1258
1259         buf[i] = (char) NETDB_EX_RTT;
1260         ++i;
1261
1262         j = htonl((int) (n->rtt * 1000));
1263
1264         memcpy(&buf[i], &j, sizeof(int));
1265
1266         i += sizeof(int);
1267
1268         buf[i] = (char) NETDB_EX_HOPS;
1269         ++i;
1270
1271         j = htonl((int) (n->hops * 1000));
1272
1273         memcpy(&buf[i], &j, sizeof(int));
1274
1275         i += sizeof(int);
1276
1277         if (i + rec_sz > 4096) {
1278             s->append(buf, i);
1279             i = 0;
1280         }
1281     }
1282
1283     if (i > 0) {
1284         s->append(buf, i);
1285         i = 0;
1286     }
1287
1288     assert(0 == i);
1289     s->flush();
1290     memFree(buf, MEM_4K_BUF);
1291 #else
1292
1293     reply->setHeaders(HTTP_BAD_REQUEST, "Bad Request", NULL, -1, squid_curtime, -2);
1294     s->replaceHttpReply(reply);
1295     storeAppendPrintf(s, "NETDB support not compiled into this Squid cache.\n");
1296 #endif
1297
1298     s->complete();
1299 }
1300
1301 #if USE_ICMP
1302 CBDATA_TYPE(netdbExchangeState);
1303 #endif
1304
1305 void
1306 netdbExchangeStart(void *data)
1307 {
1308 #if USE_ICMP
1309     peer *p = (peer *)data;
1310     char *uri;
1311     netdbExchangeState *ex;
1312     StoreIOBuffer tempBuffer;
1313     CBDATA_INIT_TYPE(netdbExchangeState);
1314     ex = cbdataAlloc(netdbExchangeState);
1315     ex->p = cbdataReference(p);
1316     uri = internalRemoteUri(p->host, p->http_port, "/squid-internal-dynamic/", "netdb");
1317     debugs(38, 3, "netdbExchangeStart: Requesting '" << uri << "'");
1318     assert(NULL != uri);
1319     ex->r = HttpRequest::CreateFromUrl(uri);
1320
1321     if (NULL == ex->r) {
1322         debugs(38, DBG_IMPORTANT, "netdbExchangeStart: Bad URI " << uri);
1323         return;
1324     }
1325
1326     HTTPMSGLOCK(ex->r);
1327     assert(NULL != ex->r);
1328     ex->r->http_ver = HttpVersion(1,1);
1329     ex->connstate = STATE_HEADER;
1330     ex->e = storeCreateEntry(uri, uri, request_flags(), METHOD_GET);
1331     ex->buf_sz = NETDB_REQBUF_SZ;
1332     assert(NULL != ex->e);
1333     ex->sc = storeClientListAdd(ex->e, ex);
1334     tempBuffer.offset = 0;
1335     tempBuffer.length = ex->buf_sz;
1336     tempBuffer.data = ex->buf;
1337     storeClientCopy(ex->sc, ex->e, tempBuffer,
1338                     netdbExchangeHandleReply, ex);
1339     ex->r->flags.loopdetect = 1;        /* cheat! -- force direct */
1340
1341     if (p->login)
1342         xstrncpy(ex->r->login, p->login, MAX_LOGIN_SZ);
1343
1344     urlCanonical(ex->r);
1345
1346     FwdState::fwdStart(Comm::ConnectionPointer(), ex->e, ex->r);
1347
1348 #endif
1349 }
1350
1351 peer *
1352 netdbClosestParent(HttpRequest * request)
1353 {
1354 #if USE_ICMP
1355     peer *p = NULL;
1356     netdbEntry *n;
1357     const ipcache_addrs *ia;
1358     net_db_peer *h;
1359     int i;
1360     n = netdbLookupHost(request->GetHost());
1361
1362     if (NULL == n) {
1363         /* try IP addr */
1364         ia = ipcache_gethostbyname(request->GetHost(), 0);
1365
1366         if (NULL != ia)
1367             n = netdbLookupAddr(ia->in_addrs[ia->cur]);
1368     }
1369
1370     if (NULL == n)
1371         return NULL;
1372
1373     if (0 == n->n_peers)
1374         return NULL;
1375
1376     n->last_use_time = squid_curtime;
1377
1378     /*
1379      * Find the parent with the least RTT to the origin server.
1380      * Make sure we don't return a parent who is farther away than
1381      * we are.  Note, the n->peers list is pre-sorted by RTT.
1382      */
1383     for (i = 0; i < n->n_peers; ++i) {
1384         h = &n->peers[i];
1385
1386         if (n->rtt > 0)
1387             if (n->rtt < h->rtt)
1388                 break;
1389
1390         p = peerFindByName(h->peername);
1391
1392         if (NULL == p)          /* not found */
1393             continue;
1394
1395         if (neighborType(p, request) != PEER_PARENT)
1396             continue;
1397
1398         if (!peerHTTPOkay(p, request))  /* not allowed */
1399             continue;
1400
1401         return p;
1402     }
1403
1404 #endif
1405     return NULL;
1406 }