src/icmp/net_db.cc

   1 /*
   2  * DEBUG: section 38    Network Measurement Database
   3  * AUTHOR: Duane Wessels
   4  *
   5  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   6  * ----------------------------------------------------------
   7  *
   8  *  Squid is the result of efforts by numerous individuals from
   9  *  the Internet community; see the CONTRIBUTORS file for full
  10  *  details.   Many organizations have provided support for Squid's
  11  *  development; see the SPONSORS file for full details.  Squid is
  12  *  Copyrighted (C) 2001 by the Regents of the University of
  13  *  California; see the COPYRIGHT file for full details.  Squid
  14  *  incorporates software developed and/or copyrighted by other
  15  *  sources; see the CREDITS file for full details.
  16  *
  17  *  This program is free software; you can redistribute it and/or modify
  18  *  it under the terms of the GNU General Public License as published by
  19  *  the Free Software Foundation; either version 2 of the License, or
  20  *  (at your option) any later version.
  21  *
  22  *  This program is distributed in the hope that it will be useful,
  23  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  24  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  25  *  GNU General Public License for more details.
  26  *
  27  *  You should have received a copy of the GNU General Public License
  28  *  along with this program; if not, write to the Free Software
  29  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  30  *
  31  */
  32
  33 /*
  34  * XXX XXX XXX
  35  *
  36  * This code may be slightly broken now. If you're getting consistent
  37  * (sometimes working) corrupt data exchanges, please contact adrian
  38  * (adrian@squid-cache.org) to sort them out.
  39  */
  40
  41 #include "squid.h"
  42 #include "CachePeer.h"
  43 #include "cbdata.h"
  44 #include "disk.h"
  45 #include "event.h"
  46 #include "fde.h"
  47 #include "forward.h"
  48 #include "HttpReply.h"
  49 #include "HttpRequest.h"
  50 #include "icmp/net_db.h"
  51 #include "internal.h"
  52 #include "ip/Address.h"
  53 #include "log/File.h"
  54 #include "Mem.h"
  55 #include "MemObject.h"
  56 #include "mgr/Registration.h"
  57 #include "mime_header.h"
  58 #include "neighbors.h"
  59 #include "SquidTime.h"
  60 #include "Store.h"
  61 #include "StoreClient.h"
  62 #include "SwapDir.h"
  63 #include "tools.h"
  64 #include "URL.h"
  65 #include "wordlist.h"
  66
  67 #if HAVE_SYS_STAT_H
  68 #include <sys/stat.h>
  69 #endif
  70
  71 #if USE_ICMP
  72 #include "icmp/IcmpSquid.h"
  73 #include "ipcache.h"
  74 #include "StoreClient.h"
  75
  76 #define NETDB_REQBUF_SZ 4096
  77
  78 typedef enum {
  79     STATE_NONE,
  80     STATE_HEADER,
  81     STATE_BODY
  82 } netdb_conn_state_t;
  83
  84 typedef struct {
  85     CachePeer *p;
  86     StoreEntry *e;
  87     store_client *sc;
  88     HttpRequest *r;
  89     int64_t used;
  90     size_t buf_sz;
  91     char buf[NETDB_REQBUF_SZ];
  92     int buf_ofs;
  93     netdb_conn_state_t connstate;
  94 } netdbExchangeState;
  95
  96 static hash_table *addr_table = NULL;
  97 static hash_table *host_table = NULL;
  98
  99 Ip::Address networkFromInaddr(const Ip::Address &a);
 100 static void netdbRelease(netdbEntry * n);
 101
 102 static void netdbHashInsert(netdbEntry * n, Ip::Address &addr);
 103 static void netdbHashDelete(const char *key);
 104 static void netdbHostInsert(netdbEntry * n, const char *hostname);
 105 static void netdbHostDelete(const net_db_name * x);
 106 static void netdbPurgeLRU(void);
 107 static netdbEntry *netdbLookupHost(const char *key);
 108 static net_db_peer *netdbPeerByName(const netdbEntry * n, const char *);
 109 static net_db_peer *netdbPeerAdd(netdbEntry * n, CachePeer * e);
 110 static const char *netdbPeerName(const char *name);
 111 static IPH netdbSendPing;
 112 static QS sortPeerByRtt;
 113 static QS sortByRtt;
 114 static QS netdbLRU;
 115 static FREE netdbFreeNameEntry;
 116 static FREE netdbFreeNetdbEntry;
 117 static STCB netdbExchangeHandleReply;
 118 static void netdbExchangeDone(void *);
 119
 120 /* We have to keep a local list of CachePeer names.  The Peers structure
 121  * gets freed during a reconfigure.  We want this database to
 122  * remain persisitent, so _net_db_peer->peername points into this
 123  * linked list */
 124 static wordlist *peer_names = NULL;
 125
 126 static void
 127 netdbHashInsert(netdbEntry * n, Ip::Address &addr)
 128 {
 129     networkFromInaddr(addr).NtoA(n->network, MAX_IPSTRLEN);
 130     n->hash.key = n->network;
 131     assert(hash_lookup(addr_table, n->network) == NULL);
 132     hash_join(addr_table, &n->hash);
 133 }
 134
 135 static void
 136 netdbHashDelete(const char *key)
 137 {
 138     hash_link *hptr = (hash_link *)hash_lookup(addr_table, key);
 139
 140     if (hptr == NULL) {
 141         debug_trap("netdbHashDelete: key not found");
 142         return;
 143     }
 144
 145     hash_remove_link(addr_table, hptr);
 146 }
 147
 148 static void
 149 netdbHostInsert(netdbEntry * n, const char *hostname)
 150 {
 151     net_db_name *x = (net_db_name *)memAllocate(MEM_NET_DB_NAME);
 152     x->hash.key = xstrdup(hostname);
 153     x->next = n->hosts;
 154     n->hosts = x;
 155     x->net_db_entry = n;
 156     assert(hash_lookup(host_table, hostname) == NULL);
 157     hash_join(host_table, &x->hash);
 158     ++ n->link_count;
 159 }
 160
 161 static void
 162 netdbHostDelete(const net_db_name * x)
 163 {
 164     netdbEntry *n;
 165     net_db_name **X;
 166     assert(x != NULL);
 167     assert(x->net_db_entry != NULL);
 168     n = x->net_db_entry;
 169     -- n->link_count;
 170
 171     for (X = &n->hosts; *X; X = &(*X)->next) {
 172         if (*X == x) {
 173             *X = x->next;
 174             break;
 175         }
 176     }
 177
 178     hash_remove_link(host_table, (hash_link *) x);
 179     xfree(x->hash.key);
 180     memFree((void *) x, MEM_NET_DB_NAME);
 181 }
 182
 183 static netdbEntry *
 184 netdbLookupHost(const char *key)
 185 {
 186     net_db_name *x = (net_db_name *) hash_lookup(host_table, key);
 187     return x ? x->net_db_entry : NULL;
 188 }
 189
 190 static void
 191 netdbRelease(netdbEntry * n)
 192 {
 193     net_db_name *x;
 194     net_db_name *next;
 195
 196     for (x = n->hosts; x; x = next) {
 197         next = x->next;
 198         netdbHostDelete(x);
 199     }
 200
 201     n->hosts = NULL;
 202     safe_free(n->peers);
 203     n->peers = NULL;
 204     n->n_peers = 0;
 205     n->n_peers_alloc = 0;
 206
 207     if (n->link_count == 0) {
 208         netdbHashDelete(n->network);
 209         memFree(n, MEM_NETDBENTRY);
 210     }
 211 }
 212
 213 static int
 214 netdbLRU(const void *A, const void *B)
 215 {
 216     const netdbEntry *const *n1 = (const netdbEntry *const *)A;
 217     const netdbEntry *const *n2 = (const netdbEntry *const *)B;
 218
 219     if ((*n1)->last_use_time > (*n2)->last_use_time)
 220         return (1);
 221
 222     if ((*n1)->last_use_time < (*n2)->last_use_time)
 223         return (-1);
 224
 225     return (0);
 226 }
 227
 228 static void
 229 netdbPurgeLRU(void)
 230 {
 231     netdbEntry *n;
 232     netdbEntry **list;
 233     int k = 0;
 234     int list_count = 0;
 235     int removed = 0;
 236     list = (netdbEntry **)xcalloc(memInUse(MEM_NETDBENTRY), sizeof(netdbEntry *));
 237     hash_first(addr_table);
 238
 239     while ((n = (netdbEntry *) hash_next(addr_table))) {
 240         assert(list_count < memInUse(MEM_NETDBENTRY));
 241         *(list + list_count) = n;
 242         ++list_count;
 243     }
 244
 245     qsort((char *) list,
 246           list_count,
 247           sizeof(netdbEntry *),
 248           netdbLRU);
 249
 250     for (k = 0; k < list_count; ++k) {
 251         if (memInUse(MEM_NETDBENTRY) < Config.Netdb.low)
 252             break;
 253
 254         netdbRelease(*(list + k));
 255
 256         ++removed;
 257     }
 258
 259     xfree(list);
 260 }
 261
 262 static netdbEntry *
 263 netdbLookupAddr(const Ip::Address &addr)
 264 {
 265     netdbEntry *n;
 266     char *key = new char[MAX_IPSTRLEN];
 267     networkFromInaddr(addr).NtoA(key,MAX_IPSTRLEN);
 268     n = (netdbEntry *) hash_lookup(addr_table, key);
 269     delete[] key;
 270     return n;
 271 }
 272
 273 static netdbEntry *
 274 netdbAdd(Ip::Address &addr)
 275 {
 276     netdbEntry *n;
 277
 278     if (memInUse(MEM_NETDBENTRY) > Config.Netdb.high)
 279         netdbPurgeLRU();
 280
 281     if ((n = netdbLookupAddr(addr)) == NULL) {
 282         n = (netdbEntry *)memAllocate(MEM_NETDBENTRY);
 283         netdbHashInsert(n, addr);
 284     }
 285
 286     return n;
 287 }
 288
 289 static void
 290 netdbSendPing(const ipcache_addrs *ia, const DnsLookupDetails &, void *data)
 291 {
 292     Ip::Address addr;
 293     char *hostname = NULL;
 294     static_cast<generic_cbdata *>(data)->unwrap(&hostname);
 295     netdbEntry *n;
 296     netdbEntry *na;
 297     net_db_name *x;
 298     net_db_name **X;
 299
 300     if (ia == NULL) {
 301         xfree(hostname);
 302         return;
 303     }
 304
 305     addr = ia->in_addrs[ia->cur];
 306
 307     if ((n = netdbLookupHost(hostname)) == NULL) {
 308         n = netdbAdd(addr);
 309         netdbHostInsert(n, hostname);
 310     } else if ((na = netdbLookupAddr(addr)) != n) {
 311         /*
 312          *hostname moved from 'network n' to 'network na'!
 313          */
 314
 315         if (na == NULL)
 316             na = netdbAdd(addr);
 317
 318         debugs(38, 3, "netdbSendPing: " << hostname << " moved from " << n->network << " to " << na->network);
 319
 320         x = (net_db_name *) hash_lookup(host_table, hostname);
 321
 322         if (x == NULL) {
 323             debugs(38, DBG_IMPORTANT, "netdbSendPing: net_db_name list bug: " << hostname << " not found");
 324             xfree(hostname);
 325             return;
 326         }
 327
 328         /* remove net_db_name from 'network n' linked list */
 329         for (X = &n->hosts; *X; X = &(*X)->next) {
 330             if (*X == x) {
 331                 *X = x->next;
 332                 break;
 333             }
 334         }
 335
 336         -- n->link_count;
 337         /* point to 'network na' from host entry */
 338         x->net_db_entry = na;
 339         /* link net_db_name to 'network na' */
 340         x->next = na->hosts;
 341         na->hosts = x;
 342         ++ na->link_count;
 343         n = na;
 344     }
 345
 346     if (n->next_ping_time <= squid_curtime) {
 347         debugs(38, 3, "netdbSendPing: pinging " << hostname);
 348         icmpEngine.DomainPing(addr, hostname);
 349         ++ n->pings_sent;
 350         n->next_ping_time = squid_curtime + Config.Netdb.period;
 351         n->last_use_time = squid_curtime;
 352     }
 353
 354     xfree(hostname);
 355 }
 356
 357 Ip::Address
 358 networkFromInaddr(const Ip::Address &in)
 359 {
 360     Ip::Address out;
 361
 362     out = in;
 363
 364     /* in IPv6 the 'network' should be the routing section. */
 365     if ( in.IsIPv6() ) {
 366         out.ApplyMask(64, AF_INET6);
 367         debugs(14, 5, "networkFromInaddr : Masked IPv6 Address to " << in << "/64 routing part.");
 368         return out;
 369     }
 370
 371 #if USE_CLASSFUL
 372     struct in_addr b;
 373
 374     in.GetInAddr(b);
 375
 376     if (IN_CLASSC(b.s_addr))
 377         b.s_addr &= IN_CLASSC_NET;
 378     else if (IN_CLASSB(b.s_addr))
 379         b.s_addr &= IN_CLASSB_NET;
 380     else if (IN_CLASSA(b.s_addr))
 381         b.s_addr &= IN_CLASSA_NET;
 382
 383     out = b;
 384
 385 #endif
 386
 387     debugs(14, 5, "networkFromInaddr : Masked IPv4 Address to " << out << "/24.");
 388
 389     /* use /24 for everything under IPv4 */
 390     out.ApplyMask(24, AF_INET);
 391     debugs(14, 5, "networkFromInaddr : Masked IPv4 Address to " << in << "/24.");
 392
 393     return out;
 394 }
 395
 396 static int
 397 sortByRtt(const void *A, const void *B)
 398 {
 399     const netdbEntry *const *n1 = (const netdbEntry *const *)A;
 400     const netdbEntry *const *n2 = (const netdbEntry *const *)B;
 401
 402     if ((*n1)->rtt > (*n2)->rtt)
 403         return 1;
 404     else if ((*n1)->rtt < (*n2)->rtt)
 405         return -1;
 406     else
 407         return 0;
 408 }
 409
 410 static net_db_peer *
 411 netdbPeerByName(const netdbEntry * n, const char *peername)
 412 {
 413     int i;
 414     net_db_peer *p = n->peers;
 415
 416     for (i = 0; i < n->n_peers; ++i, ++p) {
 417         if (!strcmp(p->peername, peername))
 418             return p;
 419     }
 420
 421     return NULL;
 422 }
 423
 424 static net_db_peer *
 425 netdbPeerAdd(netdbEntry * n, CachePeer * e)
 426 {
 427     net_db_peer *p;
 428     net_db_peer *o;
 429     int osize;
 430     int i;
 431
 432     if (n->n_peers == n->n_peers_alloc) {
 433         o = n->peers;
 434         osize = n->n_peers_alloc;
 435
 436         if (n->n_peers_alloc == 0)
 437             n->n_peers_alloc = 2;
 438         else
 439             n->n_peers_alloc <<= 1;
 440
 441         debugs(38, 3, "netdbPeerAdd: Growing peer list for '" << n->network << "' to " << n->n_peers_alloc);
 442
 443         n->peers = (net_db_peer *)xcalloc(n->n_peers_alloc, sizeof(net_db_peer));
 444
 445         for (i = 0; i < osize; ++i)
 446             *(n->peers + i) = *(o + i);
 447
 448         if (osize) {
 449             safe_free(o);
 450         }
 451     }
 452
 453     p = n->peers + n->n_peers;
 454     p->peername = netdbPeerName(e->host);
 455     ++ n->n_peers;
 456     return p;
 457 }
 458
 459 static int
 460 sortPeerByRtt(const void *A, const void *B)
 461 {
 462     const net_db_peer *p1 = (net_db_peer *)A;
 463     const net_db_peer *p2 = (net_db_peer *)B;
 464
 465     if (p1->rtt > p2->rtt)
 466         return 1;
 467     else if (p1->rtt < p2->rtt)
 468         return -1;
 469     else
 470         return 0;
 471 }
 472
 473 static void
 474 netdbSaveState(void *foo)
 475 {
 476     if (strcmp(Config.netdbFilename, "none") == 0)
 477         return;
 478
 479     Logfile *lf;
 480     netdbEntry *n;
 481     net_db_name *x;
 482
 483     struct timeval start = current_time;
 484     int count = 0;
 485     /*
 486      * This was nicer when we were using stdio, but thanks to
 487      * Solaris bugs, its a bad idea.  fopen can fail if more than
 488      * 256 FDs are open.
 489      */
 490     /*
 491      * unlink() is here because there is currently no way to make
 492      * logfileOpen() use O_TRUNC.
 493      */
 494     unlink(Config.netdbFilename);
 495     lf = logfileOpen(Config.netdbFilename, 4096, 0);
 496
 497     if (NULL == lf) {
 498         debugs(50, DBG_IMPORTANT, "netdbSaveState: " << Config.netdbFilename << ": " << xstrerror());
 499         return;
 500     }
 501
 502     hash_first(addr_table);
 503
 504     while ((n = (netdbEntry *) hash_next(addr_table))) {
 505         if (n->pings_recv == 0)
 506             continue;
 507
 508         logfilePrintf(lf, "%s %d %d %10.5f %10.5f %d %d",
 509                       n->network,
 510                       n->pings_sent,
 511                       n->pings_recv,
 512                       n->hops,
 513                       n->rtt,
 514                       (int) n->next_ping_time,
 515                       (int) n->last_use_time);
 516
 517         for (x = n->hosts; x; x = x->next)
 518             logfilePrintf(lf, " %s", hashKeyStr(&x->hash));
 519
 520         logfilePrintf(lf, "\n");
 521
 522         ++count;
 523
 524 #undef RBUF_SZ
 525
 526     }
 527
 528     logfileClose(lf);
 529     getCurrentTime();
 530     debugs(38, DBG_IMPORTANT, "NETDB state saved; " <<
 531            count << " entries, " <<
 532            tvSubMsec(start, current_time) << " msec" );
 533     eventAddIsh("netdbSaveState", netdbSaveState, NULL, 3600.0, 1);
 534 }
 535
 536 static void
 537 netdbReloadState(void)
 538 {
 539     if (strcmp(Config.netdbFilename, "none") == 0)
 540         return;
 541
 542     char *s;
 543     int fd;
 544     int l;
 545
 546     struct stat sb;
 547     netdbEntry *n;
 548     netdbEntry N;
 549
 550     Ip::Address addr;
 551     int count = 0;
 552
 553     struct timeval start = current_time;
 554     /*
 555      * This was nicer when we were using stdio, but thanks to
 556      * Solaris bugs, its a bad idea.  fopen can fail if more than
 557      * 256 FDs are open.
 558      */
 559     fd = file_open(Config.netdbFilename, O_RDONLY | O_BINARY);
 560
 561     if (fd < 0)
 562         return;
 563
 564     if (fstat(fd, &sb) < 0) {
 565         file_close(fd);
 566         return;
 567     }
 568
 569     char *t;
 570     char *buf = (char *)xcalloc(1, sb.st_size + 1);
 571     t = buf;
 572     l = FD_READ_METHOD(fd, buf, sb.st_size);
 573     file_close(fd);
 574
 575     if (l <= 0) {
 576         safe_free (buf);
 577         return;
 578     };
 579
 580     while ((s = strchr(t, '\n'))) {
 581         char *q;
 582         assert(s - buf < l);
 583         *s = '\0';
 584         memset(&N, '\0', sizeof(netdbEntry));
 585         q = strtok(t, w_space);
 586         t = s + 1;
 587
 588         if (NULL == q)
 589             continue;
 590
 591         if (! (addr = q) )
 592             continue;
 593
 594         if (netdbLookupAddr(addr) != NULL)      /* no dups! */
 595             continue;
 596
 597         if ((q = strtok(NULL, w_space)) == NULL)
 598             continue;
 599
 600         N.pings_sent = atoi(q);
 601
 602         if ((q = strtok(NULL, w_space)) == NULL)
 603             continue;
 604
 605         N.pings_recv = atoi(q);
 606
 607         if (N.pings_recv == 0)
 608             continue;
 609
 610         /* give this measurement low weight */
 611         N.pings_sent = 1;
 612
 613         N.pings_recv = 1;
 614
 615         if ((q = strtok(NULL, w_space)) == NULL)
 616             continue;
 617
 618         N.hops = atof(q);
 619
 620         if ((q = strtok(NULL, w_space)) == NULL)
 621             continue;
 622
 623         N.rtt = atof(q);
 624
 625         if ((q = strtok(NULL, w_space)) == NULL)
 626             continue;
 627
 628         N.next_ping_time = (time_t) atoi(q);
 629
 630         if ((q = strtok(NULL, w_space)) == NULL)
 631             continue;
 632
 633         N.last_use_time = (time_t) atoi(q);
 634
 635         n = (netdbEntry *)memAllocate(MEM_NETDBENTRY);
 636
 637         memcpy(n, &N, sizeof(netdbEntry));
 638
 639         netdbHashInsert(n, addr);
 640
 641         while ((q = strtok(NULL, w_space)) != NULL) {
 642             if (netdbLookupHost(q) != NULL)     /* no dups! */
 643                 continue;
 644
 645             netdbHostInsert(n, q);
 646         }
 647
 648         ++count;
 649     }
 650
 651     xfree(buf);
 652     getCurrentTime();
 653     debugs(38, DBG_IMPORTANT, "NETDB state reloaded; " <<
 654            count << " entries, " <<
 655            tvSubMsec(start, current_time) << " msec" );
 656 }
 657
 658 static const char *
 659 netdbPeerName(const char *name)
 660 {
 661     const wordlist *w;
 662
 663     for (w = peer_names; w; w = w->next) {
 664         if (!strcmp(w->key, name))
 665             return w->key;
 666     }
 667
 668     return wordlistAdd(&peer_names, name);
 669 }
 670
 671 static void
 672 netdbFreeNetdbEntry(void *data)
 673 {
 674     netdbEntry *n = (netdbEntry *)data;
 675     safe_free(n->peers);
 676     memFree(n, MEM_NETDBENTRY);
 677 }
 678
 679 static void
 680 netdbFreeNameEntry(void *data)
 681 {
 682     net_db_name *x = (net_db_name *)data;
 683     xfree(x->hash.key);
 684     memFree(x, MEM_NET_DB_NAME);
 685 }
 686
 687 static void
 688 netdbExchangeHandleReply(void *data, StoreIOBuffer receivedData)
 689 {
 690     Ip::Address addr;
 691
 692     netdbExchangeState *ex = (netdbExchangeState *)data;
 693     int rec_sz = 0;
 694     int o;
 695
 696     struct in_addr line_addr;
 697     double rtt;
 698     double hops;
 699     char *p;
 700     int j;
 701     HttpReply const *rep;
 702     size_t hdr_sz;
 703     int nused = 0;
 704     int size;
 705     int oldbufofs = ex->buf_ofs;
 706
 707     rec_sz = 0;
 708     rec_sz += 1 + sizeof(struct in_addr);
 709     rec_sz += 1 + sizeof(int);
 710     rec_sz += 1 + sizeof(int);
 711     debugs(38, 3, "netdbExchangeHandleReply: " << receivedData.length << " read bytes");
 712
 713     if (!cbdataReferenceValid(ex->p)) {
 714         debugs(38, 3, "netdbExchangeHandleReply: Peer became invalid");
 715         netdbExchangeDone(ex);
 716         return;
 717     }
 718
 719     debugs(38, 3, "netdbExchangeHandleReply: for '" << ex->p->host << ":" << ex->p->http_port << "'");
 720
 721     if (receivedData.length == 0 &&
 722             !receivedData.flags.error) {
 723         debugs(38, 3, "netdbExchangeHandleReply: Done");
 724         netdbExchangeDone(ex);
 725         return;
 726     }
 727
 728     p = ex->buf;
 729
 730     /* Get the size of the buffer now */
 731     size = ex->buf_ofs + receivedData.length;
 732     debugs(38, 3, "netdbExchangeHandleReply: " << size << " bytes buf");
 733
 734     /* Check if we're still doing headers */
 735
 736     if (ex->connstate == STATE_HEADER) {
 737
 738         ex->buf_ofs += receivedData.length;
 739
 740         /* skip reply headers */
 741
 742         if ((hdr_sz = headersEnd(p, ex->buf_ofs))) {
 743             debugs(38, 5, "netdbExchangeHandleReply: hdr_sz = " << hdr_sz);
 744             rep = ex->e->getReply();
 745             assert(rep->sline.status() != Http::scNone);
 746             debugs(38, 3, "netdbExchangeHandleReply: reply status " << rep->sline.status());
 747
 748             if (rep->sline.status() != Http::scOkay) {
 749                 netdbExchangeDone(ex);
 750                 return;
 751             }
 752
 753             assert((size_t)ex->buf_ofs >= hdr_sz);
 754
 755             /*
 756              * Now, point p to the part of the buffer where the data
 757              * starts, and update the size accordingly
 758              */
 759             assert(ex->used == 0);
 760             ex->used = hdr_sz;
 761             size = ex->buf_ofs - hdr_sz;
 762             p += hdr_sz;
 763
 764             /* Finally, set the conn state mode to STATE_BODY */
 765             ex->connstate = STATE_BODY;
 766         } else {
 767             StoreIOBuffer tempBuffer;
 768             tempBuffer.offset = ex->buf_ofs;
 769             tempBuffer.length = ex->buf_sz - ex->buf_ofs;
 770             tempBuffer.data = ex->buf + ex->buf_ofs;
 771             /* Have more headers .. */
 772             storeClientCopy(ex->sc, ex->e, tempBuffer,
 773                             netdbExchangeHandleReply, ex);
 774             return;
 775         }
 776     }
 777
 778     assert(ex->connstate == STATE_BODY);
 779
 780     /* If we get here, we have some body to parse .. */
 781     debugs(38, 5, "netdbExchangeHandleReply: start parsing loop, size = " << size);
 782
 783     while (size >= rec_sz) {
 784         debugs(38, 5, "netdbExchangeHandleReply: in parsing loop, size = " << size);
 785         addr.SetAnyAddr();
 786         hops = rtt = 0.0;
 787
 788         for (o = 0; o < rec_sz;) {
 789             switch ((int) *(p + o)) {
 790
 791             case NETDB_EX_NETWORK:
 792                 ++o;
 793                 /* FIXME INET6 : NetDB can still ony send IPv4 */
 794                 memcpy(&line_addr, p + o, sizeof(struct in_addr));
 795                 addr = line_addr;
 796                 o += sizeof(struct in_addr);
 797                 break;
 798
 799             case NETDB_EX_RTT:
 800                 ++o;
 801                 memcpy(&j, p + o, sizeof(int));
 802                 o += sizeof(int);
 803                 rtt = (double) ntohl(j) / 1000.0;
 804                 break;
 805
 806             case NETDB_EX_HOPS:
 807                 ++o;
 808                 memcpy(&j, p + o, sizeof(int));
 809                 o += sizeof(int);
 810                 hops = (double) ntohl(j) / 1000.0;
 811                 break;
 812
 813             default:
 814                 debugs(38, DBG_IMPORTANT, "netdbExchangeHandleReply: corrupt data, aborting");
 815                 netdbExchangeDone(ex);
 816                 return;
 817             }
 818         }
 819
 820         if (!addr.IsAnyAddr() && rtt > 0)
 821             netdbExchangeUpdatePeer(addr, ex->p, rtt, hops);
 822
 823         assert(o == rec_sz);
 824
 825         ex->used += rec_sz;
 826
 827         size -= rec_sz;
 828
 829         p += rec_sz;
 830
 831         ++nused;
 832     }
 833
 834     /*
 835      * Copy anything that is left over to the beginning of the buffer,
 836      * and adjust buf_ofs accordingly
 837      */
 838
 839     /*
 840      * Evilly, size refers to the buf size left now,
 841      * ex->buf_ofs is the original buffer size, so just copy that
 842      * much data over
 843      */
 844     memmove(ex->buf, ex->buf + (ex->buf_ofs - size), size);
 845
 846     ex->buf_ofs = size;
 847
 848     /*
 849      * And don't re-copy the remaining data ..
 850      */
 851     ex->used += size;
 852
 853     /*
 854      * Now the tricky bit - size _included_ the leftover bit from the _last_
 855      * storeClientCopy. We don't want to include that, or our offset will be wrong.
 856      * So, don't count the size of the leftover buffer we began with.
 857      * This can _disappear_ when we're not tracking offsets ..
 858      */
 859     ex->used -= oldbufofs;
 860
 861     debugs(38, 3, "netdbExchangeHandleReply: size left over in this buffer: " << size << " bytes");
 862
 863     debugs(38, 3, "netdbExchangeHandleReply: used " << nused <<
 864            " entries, (x " << rec_sz << " bytes) == " << nused * rec_sz <<
 865            " bytes total");
 866
 867     debugs(38, 3, "netdbExchangeHandleReply: used " << ex->used);
 868
 869     if (EBIT_TEST(ex->e->flags, ENTRY_ABORTED)) {
 870         debugs(38, 3, "netdbExchangeHandleReply: ENTRY_ABORTED");
 871         netdbExchangeDone(ex);
 872     } else if (ex->e->store_status == STORE_PENDING) {
 873         StoreIOBuffer tempBuffer;
 874         tempBuffer.offset = ex->used;
 875         tempBuffer.length = ex->buf_sz - ex->buf_ofs;
 876         tempBuffer.data = ex->buf + ex->buf_ofs;
 877         debugs(38, 3, "netdbExchangeHandleReply: EOF not received");
 878         storeClientCopy(ex->sc, ex->e, tempBuffer,
 879                         netdbExchangeHandleReply, ex);
 880     }
 881 }
 882
 883 static void
 884 netdbExchangeDone(void *data)
 885 {
 886     netdbExchangeState *ex = (netdbExchangeState *)data;
 887     debugs(38, 3, "netdbExchangeDone: " << ex->e->url()  );
 888     HTTPMSGUNLOCK(ex->r);
 889     storeUnregister(ex->sc, ex->e, ex);
 890     ex->e->unlock();
 891     cbdataReferenceDone(ex->p);
 892     cbdataFree(ex);
 893 }
 894
 895 static void
 896 netdbRegisterWithCacheManager(void)
 897 {
 898     Mgr::RegisterAction("netdb", "Network Measurement Database", netdbDump, 0, 1);
 899 }
 900
 901 #endif /* USE_ICMP */
 902
 903 /* PUBLIC FUNCTIONS */
 904
 905 void
 906 netdbInit(void)
 907 {
 908 #if USE_ICMP
 909     int n;
 910
 911     netdbRegisterWithCacheManager();
 912
 913     if (addr_table)
 914         return;
 915
 916     n = hashPrime(Config.Netdb.high / 4);
 917
 918     addr_table = hash_create((HASHCMP *) strcmp, n, hash_string);
 919
 920     n = hashPrime(3 * Config.Netdb.high / 4);
 921
 922     host_table = hash_create((HASHCMP *) strcmp, n, hash_string);
 923
 924     eventAddIsh("netdbSaveState", netdbSaveState, NULL, 3600.0, 1);
 925
 926     netdbReloadState();
 927
 928 #endif
 929 }
 930
 931 void
 932 netdbPingSite(const char *hostname)
 933 {
 934 #if USE_ICMP
 935     netdbEntry *n;
 936
 937     if ((n = netdbLookupHost(hostname)) != NULL)
 938         if (n->next_ping_time > squid_curtime)
 939             return;
 940
 941     ipcache_nbgethostbyname(hostname, netdbSendPing,
 942                             new generic_cbdata(xstrdup(hostname)));
 943
 944 #endif
 945 }
 946
 947 void
 948 netdbHandlePingReply(const Ip::Address &from, int hops, int rtt)
 949 {
 950 #if USE_ICMP
 951     netdbEntry *n;
 952     int N;
 953     debugs(38, 3, "netdbHandlePingReply: from " << from);
 954
 955     if ((n = netdbLookupAddr(from)) == NULL)
 956         return;
 957
 958     N = ++n->pings_recv;
 959
 960     if (N > 5)
 961         N = 5;
 962
 963     if (rtt < 1)
 964         rtt = 1;
 965
 966     n->hops = ((n->hops * (N - 1)) + hops) / N;
 967
 968     n->rtt = ((n->rtt * (N - 1)) + rtt) / N;
 969
 970     debugs(38, 3, "netdbHandlePingReply: " << n->network  << "; rtt="<<
 971            std::setw(5)<< std::setprecision(2) << n->rtt << "  hops="<<
 972            std::setw(4) << n->hops);
 973
 974 #endif
 975 }
 976
 977 void
 978 netdbFreeMemory(void)
 979 {
 980 #if USE_ICMP
 981     hashFreeItems(addr_table, netdbFreeNetdbEntry);
 982     hashFreeMemory(addr_table);
 983     addr_table = NULL;
 984     hashFreeItems(host_table, netdbFreeNameEntry);
 985     hashFreeMemory(host_table);
 986     host_table = NULL;
 987     wordlistDestroy(&peer_names);
 988     peer_names = NULL;
 989 #endif
 990 }
 991
 992 #if 0 // AYJ: Looks to be unused code.
 993 int
 994 netdbHops(Ip::Address &addr)
 995 {
 996 #if USE_ICMP
 997     netdbEntry *n = netdbLookupAddr(addr);
 998
 999     if (n && n->pings_recv) {
1000         n->last_use_time = squid_curtime;
1001         return (int) (n->hops + 0.5);
1002     }
1003
1004 #endif
1005     return 256;
1006 }
1007 #endif
1008
1009 void
1010 netdbDump(StoreEntry * sentry)
1011 {
1012 #if USE_ICMP
1013     netdbEntry *n;
1014     netdbEntry **list;
1015     net_db_name *x;
1016     int k;
1017     int i;
1018     int j;
1019     net_db_peer *p;
1020     storeAppendPrintf(sentry, "Network DB Statistics:\n");
1021     storeAppendPrintf(sentry, "%-46.46s %9s %7s %5s %s\n",  /* Max between 16 (IPv4) or 46 (IPv6)   */
1022                       "Network",
1023                       "recv/sent",
1024                       "RTT",
1025                       "Hops",
1026                       "Hostnames");
1027     list = (netdbEntry **)xcalloc(memInUse(MEM_NETDBENTRY), sizeof(netdbEntry *));
1028     i = 0;
1029     hash_first(addr_table);
1030
1031     while ((n = (netdbEntry *) hash_next(addr_table))) {
1032         *(list + i) = n;
1033         ++i;
1034     }
1035
1036     if (i != memInUse(MEM_NETDBENTRY))
1037         debugs(38, DBG_CRITICAL, "WARNING: netdb_addrs count off, found " << i <<
1038                ", expected " << memInUse(MEM_NETDBENTRY));
1039
1040     qsort((char *) list,
1041           i,
1042           sizeof(netdbEntry *),
1043           sortByRtt);
1044
1045     for (k = 0; k < i; ++k) {
1046         n = *(list + k);
1047         storeAppendPrintf(sentry, "%-46.46s %4d/%4d %7.1f %5.1f", /* Max between 16 (IPv4) or 46 (IPv6)   */
1048                           n->network,
1049                           n->pings_recv,
1050                           n->pings_sent,
1051                           n->rtt,
1052                           n->hops);
1053
1054         for (x = n->hosts; x; x = x->next)
1055             storeAppendPrintf(sentry, " %s", hashKeyStr(&x->hash));
1056
1057         storeAppendPrintf(sentry, "\n");
1058
1059         p = n->peers;
1060
1061         for (j = 0; j < n->n_peers; ++j, ++p) {
1062             storeAppendPrintf(sentry, "    %-22.22s %7.1f %5.1f\n",
1063                               p->peername,
1064                               p->rtt,
1065                               p->hops);
1066         }
1067     }
1068
1069     xfree(list);
1070 #else
1071
1072     storeAppendPrintf(sentry,"NETDB support not compiled into this Squid cache.\n");
1073 #endif
1074 }
1075
1076 int
1077 netdbHostHops(const char *host)
1078 {
1079 #if USE_ICMP
1080     netdbEntry *n = netdbLookupHost(host);
1081
1082     if (n) {
1083         n->last_use_time = squid_curtime;
1084         return (int) (n->hops + 0.5);
1085     }
1086
1087 #endif
1088     return 0;
1089 }
1090
1091 int
1092 netdbHostRtt(const char *host)
1093 {
1094 #if USE_ICMP
1095     netdbEntry *n = netdbLookupHost(host);
1096
1097     if (n) {
1098         n->last_use_time = squid_curtime;
1099         return (int) (n->rtt + 0.5);
1100     }
1101
1102 #endif
1103     return 0;
1104 }
1105
1106 void
1107 netdbHostData(const char *host, int *samp, int *rtt, int *hops)
1108 {
1109 #if USE_ICMP
1110     netdbEntry *n = netdbLookupHost(host);
1111
1112     if (n == NULL)
1113         return;
1114
1115     *samp = n->pings_recv;
1116
1117     *rtt = (int) (n->rtt + 0.5);
1118
1119     *hops = (int) (n->hops + 0.5);
1120
1121     n->last_use_time = squid_curtime;
1122
1123 #endif
1124 }
1125
1126 void
1127 netdbUpdatePeer(HttpRequest * r, CachePeer * e, int irtt, int ihops)
1128 {
1129 #if USE_ICMP
1130     netdbEntry *n;
1131     double rtt = (double) irtt;
1132     double hops = (double) ihops;
1133     net_db_peer *p;
1134     debugs(38, 3, "netdbUpdatePeer: '" << r->GetHost() << "', " << ihops << " hops, " << irtt << " rtt");
1135     n = netdbLookupHost(r->GetHost());
1136
1137     if (n == NULL) {
1138         debugs(38, 3, "netdbUpdatePeer: host '" << r->GetHost() << "' not found");
1139         return;
1140     }
1141
1142     if ((p = netdbPeerByName(n, e->host)) == NULL)
1143         p = netdbPeerAdd(n, e);
1144
1145     p->rtt = rtt;
1146
1147     p->hops = hops;
1148
1149     p->expires = squid_curtime + 3600;
1150
1151     if (n->n_peers < 2)
1152         return;
1153
1154     qsort((char *) n->peers,
1155           n->n_peers,
1156           sizeof(net_db_peer),
1157           sortPeerByRtt);
1158
1159 #endif
1160 }
1161
1162 void
1163 netdbExchangeUpdatePeer(Ip::Address &addr, CachePeer * e, double rtt, double hops)
1164 {
1165 #if USE_ICMP
1166     netdbEntry *n;
1167     net_db_peer *p;
1168     debugs(38, 5, "netdbExchangeUpdatePeer: '" << addr << "', "<<
1169            std::setfill('0')<< std::setprecision(2) << hops << " hops, " <<
1170            rtt << " rtt");
1171
1172     if ( !addr.IsIPv4() ) {
1173         debugs(38, 5, "netdbExchangeUpdatePeer: Aborting peer update for '" << addr << "', NetDB cannot handle IPv6.");
1174         return;
1175     }
1176
1177     n = netdbLookupAddr(addr);
1178
1179     if (n == NULL)
1180         n = netdbAdd(addr);
1181
1182     assert(NULL != n);
1183
1184     if ((p = netdbPeerByName(n, e->host)) == NULL)
1185         p = netdbPeerAdd(n, e);
1186
1187     p->rtt = rtt;
1188
1189     p->hops = hops;
1190
1191     p->expires = squid_curtime + 3600;  /* XXX ? */
1192
1193     if (n->n_peers < 2)
1194         return;
1195
1196     qsort((char *) n->peers,
1197           n->n_peers,
1198           sizeof(net_db_peer),
1199           sortPeerByRtt);
1200
1201 #endif
1202 }
1203
1204 void
1205 netdbDeleteAddrNetwork(Ip::Address &addr)
1206 {
1207 #if USE_ICMP
1208     netdbEntry *n = netdbLookupAddr(addr);
1209
1210     if (n == NULL)
1211         return;
1212
1213     debugs(38, 3, "netdbDeleteAddrNetwork: " << n->network);
1214
1215     netdbRelease(n);
1216 #endif
1217 }
1218
1219 void
1220 netdbBinaryExchange(StoreEntry * s)
1221 {
1222     HttpReply *reply = new HttpReply;
1223 #if USE_ICMP
1224
1225     Ip::Address addr;
1226
1227     netdbEntry *n;
1228     int i;
1229     int j;
1230     int rec_sz;
1231     char *buf;
1232
1233     struct in_addr line_addr;
1234     s->buffer();
1235     reply->setHeaders(Http::scOkay, "OK", NULL, -1, squid_curtime, -2);
1236     s->replaceHttpReply(reply);
1237     rec_sz = 0;
1238     rec_sz += 1 + sizeof(struct in_addr);
1239     rec_sz += 1 + sizeof(int);
1240     rec_sz += 1 + sizeof(int);
1241     buf = (char *)memAllocate(MEM_4K_BUF);
1242     i = 0;
1243     hash_first(addr_table);
1244
1245     while ((n = (netdbEntry *) hash_next(addr_table))) {
1246         if (0.0 == n->rtt)
1247             continue;
1248
1249         if (n->rtt > 60000)     /* RTT > 1 MIN probably bogus */
1250             continue;
1251
1252         if (! (addr = n->network) )
1253             continue;
1254
1255         /* FIXME INET6 : NetDB cannot yet handle IPv6 addresses. Ensure only IPv4 get sent. */
1256         if ( !addr.IsIPv4() )
1257             continue;
1258
1259         buf[i] = (char) NETDB_EX_NETWORK;
1260         ++i;
1261
1262         addr.GetInAddr(line_addr);
1263         memcpy(&buf[i], &line_addr, sizeof(struct in_addr));
1264
1265         i += sizeof(struct in_addr);
1266
1267         buf[i] = (char) NETDB_EX_RTT;
1268         ++i;
1269
1270         j = htonl((int) (n->rtt * 1000));
1271
1272         memcpy(&buf[i], &j, sizeof(int));
1273
1274         i += sizeof(int);
1275
1276         buf[i] = (char) NETDB_EX_HOPS;
1277         ++i;
1278
1279         j = htonl((int) (n->hops * 1000));
1280
1281         memcpy(&buf[i], &j, sizeof(int));
1282
1283         i += sizeof(int);
1284
1285         if (i + rec_sz > 4096) {
1286             s->append(buf, i);
1287             i = 0;
1288         }
1289     }
1290
1291     if (i > 0) {
1292         s->append(buf, i);
1293         i = 0;
1294     }
1295
1296     assert(0 == i);
1297     s->flush();
1298     memFree(buf, MEM_4K_BUF);
1299 #else
1300
1301     reply->setHeaders(Http::scBadRequest, "Bad Request", NULL, -1, squid_curtime, -2);
1302     s->replaceHttpReply(reply);
1303     storeAppendPrintf(s, "NETDB support not compiled into this Squid cache.\n");
1304 #endif
1305
1306     s->complete();
1307 }
1308
1309 #if USE_ICMP
1310 CBDATA_TYPE(netdbExchangeState);
1311 #endif
1312
1313 void
1314 netdbExchangeStart(void *data)
1315 {
1316 #if USE_ICMP
1317     CachePeer *p = (CachePeer *)data;
1318     char *uri;
1319     netdbExchangeState *ex;
1320     StoreIOBuffer tempBuffer;
1321     CBDATA_INIT_TYPE(netdbExchangeState);
1322     ex = cbdataAlloc(netdbExchangeState);
1323     ex->p = cbdataReference(p);
1324     uri = internalRemoteUri(p->host, p->http_port, "/squid-internal-dynamic/", "netdb");
1325     debugs(38, 3, "netdbExchangeStart: Requesting '" << uri << "'");
1326     assert(NULL != uri);
1327     ex->r = HttpRequest::CreateFromUrl(uri);
1328
1329     if (NULL == ex->r) {
1330         debugs(38, DBG_IMPORTANT, "netdbExchangeStart: Bad URI " << uri);
1331         return;
1332     }
1333
1334     HTTPMSGLOCK(ex->r);
1335     assert(NULL != ex->r);
1336     ex->r->http_ver = Http::ProtocolVersion(1,1);
1337     ex->connstate = STATE_HEADER;
1338     ex->e = storeCreateEntry(uri, uri, RequestFlags(), Http::METHOD_GET);
1339     ex->buf_sz = NETDB_REQBUF_SZ;
1340     assert(NULL != ex->e);
1341     ex->sc = storeClientListAdd(ex->e, ex);
1342     tempBuffer.offset = 0;
1343     tempBuffer.length = ex->buf_sz;
1344     tempBuffer.data = ex->buf;
1345     storeClientCopy(ex->sc, ex->e, tempBuffer,
1346                     netdbExchangeHandleReply, ex);
1347     ex->r->flags.loopDetected = true;   /* cheat! -- force direct */
1348
1349     if (p->login)
1350         xstrncpy(ex->r->login, p->login, MAX_LOGIN_SZ);
1351
1352     urlCanonical(ex->r);
1353
1354     FwdState::fwdStart(Comm::ConnectionPointer(), ex->e, ex->r);
1355
1356 #endif
1357 }
1358
1359 CachePeer *
1360 netdbClosestParent(HttpRequest * request)
1361 {
1362 #if USE_ICMP
1363     CachePeer *p = NULL;
1364     netdbEntry *n;
1365     const ipcache_addrs *ia;
1366     net_db_peer *h;
1367     int i;
1368     n = netdbLookupHost(request->GetHost());
1369
1370     if (NULL == n) {
1371         /* try IP addr */
1372         ia = ipcache_gethostbyname(request->GetHost(), 0);
1373
1374         if (NULL != ia)
1375             n = netdbLookupAddr(ia->in_addrs[ia->cur]);
1376     }
1377
1378     if (NULL == n)
1379         return NULL;
1380
1381     if (0 == n->n_peers)
1382         return NULL;
1383
1384     n->last_use_time = squid_curtime;
1385
1386     /*
1387      * Find the parent with the least RTT to the origin server.
1388      * Make sure we don't return a parent who is farther away than
1389      * we are.  Note, the n->peers list is pre-sorted by RTT.
1390      */
1391     for (i = 0; i < n->n_peers; ++i) {
1392         h = &n->peers[i];
1393
1394         if (n->rtt > 0)
1395             if (n->rtt < h->rtt)
1396                 break;
1397
1398         p = peerFindByName(h->peername);
1399
1400         if (NULL == p)          /* not found */
1401             continue;
1402
1403         if (neighborType(p, request) != PEER_PARENT)
1404             continue;
1405
1406         if (!peerHTTPOkay(p, request))  /* not allowed */
1407             continue;
1408
1409         return p;
1410     }
1411
1412 #endif
1413     return NULL;
1414 }