src/icmp/net_db.cc

   1 /*
   2  * DEBUG: section 38    Network Measurement Database
   3  * AUTHOR: Duane Wessels
   4  *
   5  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   6  * ----------------------------------------------------------
   7  *
   8  *  Squid is the result of efforts by numerous individuals from
   9  *  the Internet community; see the CONTRIBUTORS file for full
  10  *  details.   Many organizations have provided support for Squid's
  11  *  development; see the SPONSORS file for full details.  Squid is
  12  *  Copyrighted (C) 2001 by the Regents of the University of
  13  *  California; see the COPYRIGHT file for full details.  Squid
  14  *  incorporates software developed and/or copyrighted by other
  15  *  sources; see the CREDITS file for full details.
  16  *
  17  *  This program is free software; you can redistribute it and/or modify
  18  *  it under the terms of the GNU General Public License as published by
  19  *  the Free Software Foundation; either version 2 of the License, or
  20  *  (at your option) any later version.
  21  *
  22  *  This program is distributed in the hope that it will be useful,
  23  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  24  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  25  *  GNU General Public License for more details.
  26  *
  27  *  You should have received a copy of the GNU General Public License
  28  *  along with this program; if not, write to the Free Software
  29  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  30  *
  31  */
  32
  33 /*
  34  * XXX XXX XXX
  35  *
  36  * This code may be slightly broken now. If you're getting consistent
  37  * (sometimes working) corrupt data exchanges, please contact adrian
  38  * (adrian@squid-cache.org) to sort them out.
  39  */
  40
  41 #include "squid.h"
  42 #include "CachePeer.h"
  43 #include "cbdata.h"
  44 #include "disk.h"
  45 #include "event.h"
  46 #include "fde.h"
  47 #include "FwdState.h"
  48 #include "HttpReply.h"
  49 #include "HttpRequest.h"
  50 #include "icmp/net_db.h"
  51 #include "internal.h"
  52 #include "ip/Address.h"
  53 #include "log/File.h"
  54 #include "Mem.h"
  55 #include "MemObject.h"
  56 #include "mgr/Registration.h"
  57 #include "mime_header.h"
  58 #include "neighbors.h"
  59 #include "SquidTime.h"
  60 #include "Store.h"
  61 #include "StoreClient.h"
  62 #include "SwapDir.h"
  63 #include "tools.h"
  64 #include "URL.h"
  65 #include "wordlist.h"
  66
  67 #if HAVE_SYS_STAT_H
  68 #include <sys/stat.h>
  69 #endif
  70
  71 #if USE_ICMP
  72 #include "icmp/IcmpSquid.h"
  73 #include "ipcache.h"
  74 #include "StoreClient.h"
  75
  76 #define NETDB_REQBUF_SZ 4096
  77
  78 typedef enum {
  79     STATE_NONE,
  80     STATE_HEADER,
  81     STATE_BODY
  82 } netdb_conn_state_t;
  83
  84 typedef struct {
  85     CachePeer *p;
  86     StoreEntry *e;
  87     store_client *sc;
  88     HttpRequest *r;
  89     int64_t used;
  90     size_t buf_sz;
  91     char buf[NETDB_REQBUF_SZ];
  92     int buf_ofs;
  93     netdb_conn_state_t connstate;
  94 } netdbExchangeState;
  95
  96 static hash_table *addr_table = NULL;
  97 static hash_table *host_table = NULL;
  98
  99 Ip::Address networkFromInaddr(const Ip::Address &a);
 100 static void netdbRelease(netdbEntry * n);
 101
 102 static void netdbHashInsert(netdbEntry * n, Ip::Address &addr);
 103 static void netdbHashDelete(const char *key);
 104 static void netdbHostInsert(netdbEntry * n, const char *hostname);
 105 static void netdbHostDelete(const net_db_name * x);
 106 static void netdbPurgeLRU(void);
 107 static netdbEntry *netdbLookupHost(const char *key);
 108 static net_db_peer *netdbPeerByName(const netdbEntry * n, const char *);
 109 static net_db_peer *netdbPeerAdd(netdbEntry * n, CachePeer * e);
 110 static const char *netdbPeerName(const char *name);
 111 static IPH netdbSendPing;
 112 static QS sortPeerByRtt;
 113 static QS sortByRtt;
 114 static QS netdbLRU;
 115 static FREE netdbFreeNameEntry;
 116 static FREE netdbFreeNetdbEntry;
 117 static STCB netdbExchangeHandleReply;
 118 static void netdbExchangeDone(void *);
 119
 120 /* We have to keep a local list of CachePeer names.  The Peers structure
 121  * gets freed during a reconfigure.  We want this database to
 122  * remain persisitent, so _net_db_peer->peername points into this
 123  * linked list */
 124 static wordlist *peer_names = NULL;
 125
 126 static void
 127 netdbHashInsert(netdbEntry * n, Ip::Address &addr)
 128 {
 129     networkFromInaddr(addr).toStr(n->network, MAX_IPSTRLEN);
 130     n->hash.key = n->network;
 131     assert(hash_lookup(addr_table, n->network) == NULL);
 132     hash_join(addr_table, &n->hash);
 133 }
 134
 135 static void
 136 netdbHashDelete(const char *key)
 137 {
 138     hash_link *hptr = (hash_link *)hash_lookup(addr_table, key);
 139
 140     if (hptr == NULL) {
 141         debug_trap("netdbHashDelete: key not found");
 142         return;
 143     }
 144
 145     hash_remove_link(addr_table, hptr);
 146 }
 147
 148 static void
 149 netdbHostInsert(netdbEntry * n, const char *hostname)
 150 {
 151     net_db_name *x = (net_db_name *)memAllocate(MEM_NET_DB_NAME);
 152     x->hash.key = xstrdup(hostname);
 153     x->next = n->hosts;
 154     n->hosts = x;
 155     x->net_db_entry = n;
 156     assert(hash_lookup(host_table, hostname) == NULL);
 157     hash_join(host_table, &x->hash);
 158     ++ n->link_count;
 159 }
 160
 161 static void
 162 netdbHostDelete(const net_db_name * x)
 163 {
 164     netdbEntry *n;
 165     net_db_name **X;
 166     assert(x != NULL);
 167     assert(x->net_db_entry != NULL);
 168     n = x->net_db_entry;
 169     -- n->link_count;
 170
 171     for (X = &n->hosts; *X; X = &(*X)->next) {
 172         if (*X == x) {
 173             *X = x->next;
 174             break;
 175         }
 176     }
 177
 178     hash_remove_link(host_table, (hash_link *) x);
 179     xfree(x->hash.key);
 180     memFree((void *) x, MEM_NET_DB_NAME);
 181 }
 182
 183 static netdbEntry *
 184 netdbLookupHost(const char *key)
 185 {
 186     net_db_name *x = (net_db_name *) hash_lookup(host_table, key);
 187     return x ? x->net_db_entry : NULL;
 188 }
 189
 190 static void
 191 netdbRelease(netdbEntry * n)
 192 {
 193     net_db_name *x;
 194     net_db_name *next;
 195
 196     for (x = n->hosts; x; x = next) {
 197         next = x->next;
 198         netdbHostDelete(x);
 199     }
 200
 201     n->hosts = NULL;
 202     safe_free(n->peers);
 203     n->peers = NULL;
 204     n->n_peers = 0;
 205     n->n_peers_alloc = 0;
 206
 207     if (n->link_count == 0) {
 208         netdbHashDelete(n->network);
 209         memFree(n, MEM_NETDBENTRY);
 210     }
 211 }
 212
 213 static int
 214 netdbLRU(const void *A, const void *B)
 215 {
 216     const netdbEntry *const *n1 = (const netdbEntry *const *)A;
 217     const netdbEntry *const *n2 = (const netdbEntry *const *)B;
 218
 219     if ((*n1)->last_use_time > (*n2)->last_use_time)
 220         return (1);
 221
 222     if ((*n1)->last_use_time < (*n2)->last_use_time)
 223         return (-1);
 224
 225     return (0);
 226 }
 227
 228 static void
 229 netdbPurgeLRU(void)
 230 {
 231     netdbEntry *n;
 232     netdbEntry **list;
 233     int k = 0;
 234     int list_count = 0;
 235     int removed = 0;
 236     list = (netdbEntry **)xcalloc(memInUse(MEM_NETDBENTRY), sizeof(netdbEntry *));
 237     hash_first(addr_table);
 238
 239     while ((n = (netdbEntry *) hash_next(addr_table))) {
 240         assert(list_count < memInUse(MEM_NETDBENTRY));
 241         *(list + list_count) = n;
 242         ++list_count;
 243     }
 244
 245     qsort((char *) list,
 246           list_count,
 247           sizeof(netdbEntry *),
 248           netdbLRU);
 249
 250     for (k = 0; k < list_count; ++k) {
 251         if (memInUse(MEM_NETDBENTRY) < Config.Netdb.low)
 252             break;
 253
 254         netdbRelease(*(list + k));
 255
 256         ++removed;
 257     }
 258
 259     xfree(list);
 260 }
 261
 262 static netdbEntry *
 263 netdbLookupAddr(const Ip::Address &addr)
 264 {
 265     netdbEntry *n;
 266     char *key = new char[MAX_IPSTRLEN];
 267     networkFromInaddr(addr).toStr(key,MAX_IPSTRLEN);
 268     n = (netdbEntry *) hash_lookup(addr_table, key);
 269     delete[] key;
 270     return n;
 271 }
 272
 273 static netdbEntry *
 274 netdbAdd(Ip::Address &addr)
 275 {
 276     netdbEntry *n;
 277
 278     if (memInUse(MEM_NETDBENTRY) > Config.Netdb.high)
 279         netdbPurgeLRU();
 280
 281     if ((n = netdbLookupAddr(addr)) == NULL) {
 282         n = (netdbEntry *)memAllocate(MEM_NETDBENTRY);
 283         netdbHashInsert(n, addr);
 284     }
 285
 286     return n;
 287 }
 288
 289 static void
 290 netdbSendPing(const ipcache_addrs *ia, const DnsLookupDetails &, void *data)
 291 {
 292     Ip::Address addr;
 293     char *hostname = NULL;
 294     static_cast<generic_cbdata *>(data)->unwrap(&hostname);
 295     netdbEntry *n;
 296     netdbEntry *na;
 297     net_db_name *x;
 298     net_db_name **X;
 299
 300     if (ia == NULL) {
 301         xfree(hostname);
 302         return;
 303     }
 304
 305     addr = ia->in_addrs[ia->cur];
 306
 307     if ((n = netdbLookupHost(hostname)) == NULL) {
 308         n = netdbAdd(addr);
 309         netdbHostInsert(n, hostname);
 310     } else if ((na = netdbLookupAddr(addr)) != n) {
 311         /*
 312          *hostname moved from 'network n' to 'network na'!
 313          */
 314
 315         if (na == NULL)
 316             na = netdbAdd(addr);
 317
 318         debugs(38, 3, "netdbSendPing: " << hostname << " moved from " << n->network << " to " << na->network);
 319
 320         x = (net_db_name *) hash_lookup(host_table, hostname);
 321
 322         if (x == NULL) {
 323             debugs(38, DBG_IMPORTANT, "netdbSendPing: net_db_name list bug: " << hostname << " not found");
 324             xfree(hostname);
 325             return;
 326         }
 327
 328         /* remove net_db_name from 'network n' linked list */
 329         for (X = &n->hosts; *X; X = &(*X)->next) {
 330             if (*X == x) {
 331                 *X = x->next;
 332                 break;
 333             }
 334         }
 335
 336         -- n->link_count;
 337         /* point to 'network na' from host entry */
 338         x->net_db_entry = na;
 339         /* link net_db_name to 'network na' */
 340         x->next = na->hosts;
 341         na->hosts = x;
 342         ++ na->link_count;
 343         n = na;
 344     }
 345
 346     if (n->next_ping_time <= squid_curtime) {
 347         debugs(38, 3, "netdbSendPing: pinging " << hostname);
 348         icmpEngine.DomainPing(addr, hostname);
 349         ++ n->pings_sent;
 350         n->next_ping_time = squid_curtime + Config.Netdb.period;
 351         n->last_use_time = squid_curtime;
 352     }
 353
 354     xfree(hostname);
 355 }
 356
 357 Ip::Address
 358 networkFromInaddr(const Ip::Address &in)
 359 {
 360     Ip::Address out;
 361
 362     out = in;
 363
 364     /* in IPv6 the 'network' should be the routing section. */
 365     if ( in.isIPv6() ) {
 366         out.applyMask(64, AF_INET6);
 367         debugs(14, 5, "networkFromInaddr : Masked IPv6 Address to " << in << "/64 routing part.");
 368         return out;
 369     }
 370
 371 #if USE_CLASSFUL
 372     struct in_addr b;
 373
 374     in.getInAddr(b);
 375
 376     if (IN_CLASSC(b.s_addr))
 377         b.s_addr &= IN_CLASSC_NET;
 378     else if (IN_CLASSB(b.s_addr))
 379         b.s_addr &= IN_CLASSB_NET;
 380     else if (IN_CLASSA(b.s_addr))
 381         b.s_addr &= IN_CLASSA_NET;
 382
 383     out = b;
 384
 385 #endif
 386
 387     debugs(14, 5, "networkFromInaddr : Masked IPv4 Address to " << out << "/24.");
 388
 389     /* use /24 for everything under IPv4 */
 390     out.applyMask(24, AF_INET);
 391     debugs(14, 5, "networkFromInaddr : Masked IPv4 Address to " << in << "/24.");
 392
 393     return out;
 394 }
 395
 396 static int
 397 sortByRtt(const void *A, const void *B)
 398 {
 399     const netdbEntry *const *n1 = (const netdbEntry *const *)A;
 400     const netdbEntry *const *n2 = (const netdbEntry *const *)B;
 401
 402     if ((*n1)->rtt > (*n2)->rtt)
 403         return 1;
 404     else if ((*n1)->rtt < (*n2)->rtt)
 405         return -1;
 406     else
 407         return 0;
 408 }
 409
 410 static net_db_peer *
 411 netdbPeerByName(const netdbEntry * n, const char *peername)
 412 {
 413     int i;
 414     net_db_peer *p = n->peers;
 415
 416     for (i = 0; i < n->n_peers; ++i, ++p) {
 417         if (!strcmp(p->peername, peername))
 418             return p;
 419     }
 420
 421     return NULL;
 422 }
 423
 424 static net_db_peer *
 425 netdbPeerAdd(netdbEntry * n, CachePeer * e)
 426 {
 427     net_db_peer *p;
 428     net_db_peer *o;
 429     int osize;
 430     int i;
 431
 432     if (n->n_peers == n->n_peers_alloc) {
 433         o = n->peers;
 434         osize = n->n_peers_alloc;
 435
 436         if (n->n_peers_alloc == 0)
 437             n->n_peers_alloc = 2;
 438         else
 439             n->n_peers_alloc <<= 1;
 440
 441         debugs(38, 3, "netdbPeerAdd: Growing peer list for '" << n->network << "' to " << n->n_peers_alloc);
 442
 443         n->peers = (net_db_peer *)xcalloc(n->n_peers_alloc, sizeof(net_db_peer));
 444
 445         for (i = 0; i < osize; ++i)
 446             *(n->peers + i) = *(o + i);
 447
 448         if (osize) {
 449             safe_free(o);
 450         }
 451     }
 452
 453     p = n->peers + n->n_peers;
 454     p->peername = netdbPeerName(e->host);
 455     ++ n->n_peers;
 456     return p;
 457 }
 458
 459 static int
 460 sortPeerByRtt(const void *A, const void *B)
 461 {
 462     const net_db_peer *p1 = (net_db_peer *)A;
 463     const net_db_peer *p2 = (net_db_peer *)B;
 464
 465     if (p1->rtt > p2->rtt)
 466         return 1;
 467     else if (p1->rtt < p2->rtt)
 468         return -1;
 469     else
 470         return 0;
 471 }
 472
 473 static void
 474 netdbSaveState(void *foo)
 475 {
 476     if (strcmp(Config.netdbFilename, "none") == 0)
 477         return;
 478
 479     Logfile *lf;
 480     netdbEntry *n;
 481     net_db_name *x;
 482
 483     struct timeval start = current_time;
 484     int count = 0;
 485     /*
 486      * This was nicer when we were using stdio, but thanks to
 487      * Solaris bugs, its a bad idea.  fopen can fail if more than
 488      * 256 FDs are open.
 489      */
 490     /*
 491      * unlink() is here because there is currently no way to make
 492      * logfileOpen() use O_TRUNC.
 493      */
 494     unlink(Config.netdbFilename);
 495     lf = logfileOpen(Config.netdbFilename, 4096, 0);
 496
 497     if (NULL == lf) {
 498         debugs(50, DBG_IMPORTANT, "netdbSaveState: " << Config.netdbFilename << ": " << xstrerror());
 499         return;
 500     }
 501
 502     hash_first(addr_table);
 503
 504     while ((n = (netdbEntry *) hash_next(addr_table))) {
 505         if (n->pings_recv == 0)
 506             continue;
 507
 508         logfilePrintf(lf, "%s %d %d %10.5f %10.5f %d %d",
 509                       n->network,
 510                       n->pings_sent,
 511                       n->pings_recv,
 512                       n->hops,
 513                       n->rtt,
 514                       (int) n->next_ping_time,
 515                       (int) n->last_use_time);
 516
 517         for (x = n->hosts; x; x = x->next)
 518             logfilePrintf(lf, " %s", hashKeyStr(&x->hash));
 519
 520         logfilePrintf(lf, "\n");
 521
 522         ++count;
 523
 524 #undef RBUF_SZ
 525
 526     }
 527
 528     logfileClose(lf);
 529     getCurrentTime();
 530     debugs(38, DBG_IMPORTANT, "NETDB state saved; " <<
 531            count << " entries, " <<
 532            tvSubMsec(start, current_time) << " msec" );
 533     eventAddIsh("netdbSaveState", netdbSaveState, NULL, 3600.0, 1);
 534 }
 535
 536 static void
 537 netdbReloadState(void)
 538 {
 539     if (strcmp(Config.netdbFilename, "none") == 0)
 540         return;
 541
 542     char *s;
 543     int fd;
 544     int l;
 545
 546     struct stat sb;
 547     netdbEntry *n;
 548     netdbEntry N;
 549
 550     Ip::Address addr;
 551     int count = 0;
 552
 553     struct timeval start = current_time;
 554     /*
 555      * This was nicer when we were using stdio, but thanks to
 556      * Solaris bugs, its a bad idea.  fopen can fail if more than
 557      * 256 FDs are open.
 558      */
 559     fd = file_open(Config.netdbFilename, O_RDONLY | O_BINARY);
 560
 561     if (fd < 0)
 562         return;
 563
 564     if (fstat(fd, &sb) < 0) {
 565         file_close(fd);
 566         return;
 567     }
 568
 569     char *t;
 570     char *buf = (char *)xcalloc(1, sb.st_size + 1);
 571     t = buf;
 572     l = FD_READ_METHOD(fd, buf, sb.st_size);
 573     file_close(fd);
 574
 575     if (l <= 0) {
 576         safe_free (buf);
 577         return;
 578     };
 579
 580     while ((s = strchr(t, '\n'))) {
 581         char *q;
 582         assert(s - buf < l);
 583         *s = '\0';
 584         memset(&N, '\0', sizeof(netdbEntry));
 585         q = strtok(t, w_space);
 586         t = s + 1;
 587
 588         if (NULL == q)
 589             continue;
 590
 591         if (! (addr = q) )
 592             continue;
 593
 594         if (netdbLookupAddr(addr) != NULL)      /* no dups! */
 595             continue;
 596
 597         if ((q = strtok(NULL, w_space)) == NULL)
 598             continue;
 599
 600         N.pings_sent = atoi(q);
 601
 602         if ((q = strtok(NULL, w_space)) == NULL)
 603             continue;
 604
 605         N.pings_recv = atoi(q);
 606
 607         if (N.pings_recv == 0)
 608             continue;
 609
 610         /* give this measurement low weight */
 611         N.pings_sent = 1;
 612
 613         N.pings_recv = 1;
 614
 615         if ((q = strtok(NULL, w_space)) == NULL)
 616             continue;
 617
 618         N.hops = atof(q);
 619
 620         if ((q = strtok(NULL, w_space)) == NULL)
 621             continue;
 622
 623         N.rtt = atof(q);
 624
 625         if ((q = strtok(NULL, w_space)) == NULL)
 626             continue;
 627
 628         N.next_ping_time = (time_t) atoi(q);
 629
 630         if ((q = strtok(NULL, w_space)) == NULL)
 631             continue;
 632
 633         N.last_use_time = (time_t) atoi(q);
 634
 635         n = (netdbEntry *)memAllocate(MEM_NETDBENTRY);
 636
 637         memcpy(n, &N, sizeof(netdbEntry));
 638
 639         netdbHashInsert(n, addr);
 640
 641         while ((q = strtok(NULL, w_space)) != NULL) {
 642             if (netdbLookupHost(q) != NULL)     /* no dups! */
 643                 continue;
 644
 645             netdbHostInsert(n, q);
 646         }
 647
 648         ++count;
 649     }
 650
 651     xfree(buf);
 652     getCurrentTime();
 653     debugs(38, DBG_IMPORTANT, "NETDB state reloaded; " <<
 654            count << " entries, " <<
 655            tvSubMsec(start, current_time) << " msec" );
 656 }
 657
 658 static const char *
 659 netdbPeerName(const char *name)
 660 {
 661     const wordlist *w;
 662
 663     for (w = peer_names; w; w = w->next) {
 664         if (!strcmp(w->key, name))
 665             return w->key;
 666     }
 667
 668     return wordlistAdd(&peer_names, name);
 669 }
 670
 671 static void
 672 netdbFreeNetdbEntry(void *data)
 673 {
 674     netdbEntry *n = (netdbEntry *)data;
 675     safe_free(n->peers);
 676     memFree(n, MEM_NETDBENTRY);
 677 }
 678
 679 static void
 680 netdbFreeNameEntry(void *data)
 681 {
 682     net_db_name *x = (net_db_name *)data;
 683     xfree(x->hash.key);
 684     memFree(x, MEM_NET_DB_NAME);
 685 }
 686
 687 static void
 688 netdbExchangeHandleReply(void *data, StoreIOBuffer receivedData)
 689 {
 690     Ip::Address addr;
 691
 692     netdbExchangeState *ex = (netdbExchangeState *)data;
 693     int rec_sz = 0;
 694     int o;
 695
 696     struct in_addr line_addr;
 697     double rtt;
 698     double hops;
 699     char *p;
 700     int j;
 701     HttpReply const *rep;
 702     size_t hdr_sz;
 703     int nused = 0;
 704     int size;
 705     int oldbufofs = ex->buf_ofs;
 706
 707     rec_sz = 0;
 708     rec_sz += 1 + sizeof(struct in_addr);
 709     rec_sz += 1 + sizeof(int);
 710     rec_sz += 1 + sizeof(int);
 711     debugs(38, 3, "netdbExchangeHandleReply: " << receivedData.length << " read bytes");
 712
 713     if (!cbdataReferenceValid(ex->p)) {
 714         debugs(38, 3, "netdbExchangeHandleReply: Peer became invalid");
 715         netdbExchangeDone(ex);
 716         return;
 717     }
 718
 719     debugs(38, 3, "netdbExchangeHandleReply: for '" << ex->p->host << ":" << ex->p->http_port << "'");
 720
 721     if (receivedData.length == 0 &&
 722             !receivedData.flags.error) {
 723         debugs(38, 3, "netdbExchangeHandleReply: Done");
 724         netdbExchangeDone(ex);
 725         return;
 726     }
 727
 728     p = ex->buf;
 729
 730     /* Get the size of the buffer now */
 731     size = ex->buf_ofs + receivedData.length;
 732     debugs(38, 3, "netdbExchangeHandleReply: " << size << " bytes buf");
 733
 734     /* Check if we're still doing headers */
 735
 736     if (ex->connstate == STATE_HEADER) {
 737
 738         ex->buf_ofs += receivedData.length;
 739
 740         /* skip reply headers */
 741
 742         if ((hdr_sz = headersEnd(p, ex->buf_ofs))) {
 743             debugs(38, 5, "netdbExchangeHandleReply: hdr_sz = " << hdr_sz);
 744             rep = ex->e->getReply();
 745             assert(rep->sline.status() != Http::scNone);
 746             debugs(38, 3, "netdbExchangeHandleReply: reply status " << rep->sline.status());
 747
 748             if (rep->sline.status() != Http::scOkay) {
 749                 netdbExchangeDone(ex);
 750                 return;
 751             }
 752
 753             assert((size_t)ex->buf_ofs >= hdr_sz);
 754
 755             /*
 756              * Now, point p to the part of the buffer where the data
 757              * starts, and update the size accordingly
 758              */
 759             assert(ex->used == 0);
 760             ex->used = hdr_sz;
 761             size = ex->buf_ofs - hdr_sz;
 762             p += hdr_sz;
 763
 764             /* Finally, set the conn state mode to STATE_BODY */
 765             ex->connstate = STATE_BODY;
 766         } else {
 767             StoreIOBuffer tempBuffer;
 768             tempBuffer.offset = ex->buf_ofs;
 769             tempBuffer.length = ex->buf_sz - ex->buf_ofs;
 770             tempBuffer.data = ex->buf + ex->buf_ofs;
 771             /* Have more headers .. */
 772             storeClientCopy(ex->sc, ex->e, tempBuffer,
 773                             netdbExchangeHandleReply, ex);
 774             return;
 775         }
 776     }
 777
 778     assert(ex->connstate == STATE_BODY);
 779
 780     /* If we get here, we have some body to parse .. */
 781     debugs(38, 5, "netdbExchangeHandleReply: start parsing loop, size = " << size);
 782
 783     while (size >= rec_sz) {
 784         debugs(38, 5, "netdbExchangeHandleReply: in parsing loop, size = " << size);
 785         addr.setAnyAddr();
 786         hops = rtt = 0.0;
 787
 788         for (o = 0; o < rec_sz;) {
 789             switch ((int) *(p + o)) {
 790
 791             case NETDB_EX_NETWORK:
 792                 ++o;
 793                 /* FIXME INET6 : NetDB can still ony send IPv4 */
 794                 memcpy(&line_addr, p + o, sizeof(struct in_addr));
 795                 addr = line_addr;
 796                 o += sizeof(struct in_addr);
 797                 break;
 798
 799             case NETDB_EX_RTT:
 800                 ++o;
 801                 memcpy(&j, p + o, sizeof(int));
 802                 o += sizeof(int);
 803                 rtt = (double) ntohl(j) / 1000.0;
 804                 break;
 805
 806             case NETDB_EX_HOPS:
 807                 ++o;
 808                 memcpy(&j, p + o, sizeof(int));
 809                 o += sizeof(int);
 810                 hops = (double) ntohl(j) / 1000.0;
 811                 break;
 812
 813             default:
 814                 debugs(38, DBG_IMPORTANT, "netdbExchangeHandleReply: corrupt data, aborting");
 815                 netdbExchangeDone(ex);
 816                 return;
 817             }
 818         }
 819
 820         if (!addr.isAnyAddr() && rtt > 0)
 821             netdbExchangeUpdatePeer(addr, ex->p, rtt, hops);
 822
 823         assert(o == rec_sz);
 824
 825         ex->used += rec_sz;
 826
 827         size -= rec_sz;
 828
 829         p += rec_sz;
 830
 831         ++nused;
 832     }
 833
 834     /*
 835      * Copy anything that is left over to the beginning of the buffer,
 836      * and adjust buf_ofs accordingly
 837      */
 838
 839     /*
 840      * Evilly, size refers to the buf size left now,
 841      * ex->buf_ofs is the original buffer size, so just copy that
 842      * much data over
 843      */
 844     memmove(ex->buf, ex->buf + (ex->buf_ofs - size), size);
 845
 846     ex->buf_ofs = size;
 847
 848     /*
 849      * And don't re-copy the remaining data ..
 850      */
 851     ex->used += size;
 852
 853     /*
 854      * Now the tricky bit - size _included_ the leftover bit from the _last_
 855      * storeClientCopy. We don't want to include that, or our offset will be wrong.
 856      * So, don't count the size of the leftover buffer we began with.
 857      * This can _disappear_ when we're not tracking offsets ..
 858      */
 859     ex->used -= oldbufofs;
 860
 861     debugs(38, 3, "netdbExchangeHandleReply: size left over in this buffer: " << size << " bytes");
 862
 863     debugs(38, 3, "netdbExchangeHandleReply: used " << nused <<
 864            " entries, (x " << rec_sz << " bytes) == " << nused * rec_sz <<
 865            " bytes total");
 866
 867     debugs(38, 3, "netdbExchangeHandleReply: used " << ex->used);
 868
 869     if (EBIT_TEST(ex->e->flags, ENTRY_ABORTED)) {
 870         debugs(38, 3, "netdbExchangeHandleReply: ENTRY_ABORTED");
 871         netdbExchangeDone(ex);
 872     } else if (ex->e->store_status == STORE_PENDING) {
 873         StoreIOBuffer tempBuffer;
 874         tempBuffer.offset = ex->used;
 875         tempBuffer.length = ex->buf_sz - ex->buf_ofs;
 876         tempBuffer.data = ex->buf + ex->buf_ofs;
 877         debugs(38, 3, "netdbExchangeHandleReply: EOF not received");
 878         storeClientCopy(ex->sc, ex->e, tempBuffer,
 879                         netdbExchangeHandleReply, ex);
 880     }
 881 }
 882
 883 static void
 884 netdbExchangeDone(void *data)
 885 {
 886     netdbExchangeState *ex = (netdbExchangeState *)data;
 887     debugs(38, 3, "netdbExchangeDone: " << ex->e->url()  );
 888     HTTPMSGUNLOCK(ex->r);
 889     storeUnregister(ex->sc, ex->e, ex);
 890     ex->e->unlock("netdbExchangeDone");
 891     cbdataReferenceDone(ex->p);
 892     cbdataFree(ex);
 893 }
 894
 895 static void
 896 netdbRegisterWithCacheManager(void)
 897 {
 898     Mgr::RegisterAction("netdb", "Network Measurement Database", netdbDump, 0, 1);
 899 }
 900
 901 #endif /* USE_ICMP */
 902
 903 /* PUBLIC FUNCTIONS */
 904
 905 void
 906 netdbInit(void)
 907 {
 908 #if USE_ICMP
 909     int n;
 910
 911     netdbRegisterWithCacheManager();
 912
 913     if (addr_table)
 914         return;
 915
 916     n = hashPrime(Config.Netdb.high / 4);
 917
 918     addr_table = hash_create((HASHCMP *) strcmp, n, hash_string);
 919
 920     n = hashPrime(3 * Config.Netdb.high / 4);
 921
 922     host_table = hash_create((HASHCMP *) strcmp, n, hash_string);
 923
 924     eventAddIsh("netdbSaveState", netdbSaveState, NULL, 3600.0, 1);
 925
 926     netdbReloadState();
 927
 928 #endif
 929 }
 930
 931 void
 932 netdbPingSite(const char *hostname)
 933 {
 934 #if USE_ICMP
 935     netdbEntry *n;
 936
 937     if ((n = netdbLookupHost(hostname)) != NULL)
 938         if (n->next_ping_time > squid_curtime)
 939             return;
 940
 941     ipcache_nbgethostbyname(hostname, netdbSendPing,
 942                             new generic_cbdata(xstrdup(hostname)));
 943
 944 #endif
 945 }
 946
 947 void
 948 netdbHandlePingReply(const Ip::Address &from, int hops, int rtt)
 949 {
 950 #if USE_ICMP
 951     netdbEntry *n;
 952     int N;
 953     debugs(38, 3, "netdbHandlePingReply: from " << from);
 954
 955     if ((n = netdbLookupAddr(from)) == NULL)
 956         return;
 957
 958     N = ++n->pings_recv;
 959
 960     if (N > 5)
 961         N = 5;
 962
 963     if (rtt < 1)
 964         rtt = 1;
 965
 966     n->hops = ((n->hops * (N - 1)) + hops) / N;
 967
 968     n->rtt = ((n->rtt * (N - 1)) + rtt) / N;
 969
 970     debugs(38, 3, "netdbHandlePingReply: " << n->network  << "; rtt="<<
 971            std::setw(5)<< std::setprecision(2) << n->rtt << "  hops="<<
 972            std::setw(4) << n->hops);
 973
 974 #endif
 975 }
 976
 977 void
 978 netdbFreeMemory(void)
 979 {
 980 #if USE_ICMP
 981     hashFreeItems(addr_table, netdbFreeNetdbEntry);
 982     hashFreeMemory(addr_table);
 983     addr_table = NULL;
 984     hashFreeItems(host_table, netdbFreeNameEntry);
 985     hashFreeMemory(host_table);
 986     host_table = NULL;
 987     wordlistDestroy(&peer_names);
 988     peer_names = NULL;
 989 #endif
 990 }
 991
 992 void
 993 netdbDump(StoreEntry * sentry)
 994 {
 995 #if USE_ICMP
 996     netdbEntry *n;
 997     netdbEntry **list;
 998     net_db_name *x;
 999     int k;
1000     int i;
1001     int j;
1002     net_db_peer *p;
1003     storeAppendPrintf(sentry, "Network DB Statistics:\n");
1004     storeAppendPrintf(sentry, "%-46.46s %9s %7s %5s %s\n",  /* Max between 16 (IPv4) or 46 (IPv6)   */
1005                       "Network",
1006                       "recv/sent",
1007                       "RTT",
1008                       "Hops",
1009                       "Hostnames");
1010     list = (netdbEntry **)xcalloc(memInUse(MEM_NETDBENTRY), sizeof(netdbEntry *));
1011     i = 0;
1012     hash_first(addr_table);
1013
1014     while ((n = (netdbEntry *) hash_next(addr_table))) {
1015         *(list + i) = n;
1016         ++i;
1017     }
1018
1019     if (i != memInUse(MEM_NETDBENTRY))
1020         debugs(38, DBG_CRITICAL, "WARNING: netdb_addrs count off, found " << i <<
1021                ", expected " << memInUse(MEM_NETDBENTRY));
1022
1023     qsort((char *) list,
1024           i,
1025           sizeof(netdbEntry *),
1026           sortByRtt);
1027
1028     for (k = 0; k < i; ++k) {
1029         n = *(list + k);
1030         storeAppendPrintf(sentry, "%-46.46s %4d/%4d %7.1f %5.1f", /* Max between 16 (IPv4) or 46 (IPv6)   */
1031                           n->network,
1032                           n->pings_recv,
1033                           n->pings_sent,
1034                           n->rtt,
1035                           n->hops);
1036
1037         for (x = n->hosts; x; x = x->next)
1038             storeAppendPrintf(sentry, " %s", hashKeyStr(&x->hash));
1039
1040         storeAppendPrintf(sentry, "\n");
1041
1042         p = n->peers;
1043
1044         for (j = 0; j < n->n_peers; ++j, ++p) {
1045             storeAppendPrintf(sentry, "    %-22.22s %7.1f %5.1f\n",
1046                               p->peername,
1047                               p->rtt,
1048                               p->hops);
1049         }
1050     }
1051
1052     xfree(list);
1053 #else
1054
1055     storeAppendPrintf(sentry,"NETDB support not compiled into this Squid cache.\n");
1056 #endif
1057 }
1058
1059 int
1060 netdbHostHops(const char *host)
1061 {
1062 #if USE_ICMP
1063     netdbEntry *n = netdbLookupHost(host);
1064
1065     if (n) {
1066         n->last_use_time = squid_curtime;
1067         return (int) (n->hops + 0.5);
1068     }
1069
1070 #endif
1071     return 0;
1072 }
1073
1074 int
1075 netdbHostRtt(const char *host)
1076 {
1077 #if USE_ICMP
1078     netdbEntry *n = netdbLookupHost(host);
1079
1080     if (n) {
1081         n->last_use_time = squid_curtime;
1082         return (int) (n->rtt + 0.5);
1083     }
1084
1085 #endif
1086     return 0;
1087 }
1088
1089 void
1090 netdbHostData(const char *host, int *samp, int *rtt, int *hops)
1091 {
1092 #if USE_ICMP
1093     netdbEntry *n = netdbLookupHost(host);
1094
1095     if (n == NULL)
1096         return;
1097
1098     *samp = n->pings_recv;
1099
1100     *rtt = (int) (n->rtt + 0.5);
1101
1102     *hops = (int) (n->hops + 0.5);
1103
1104     n->last_use_time = squid_curtime;
1105
1106 #endif
1107 }
1108
1109 void
1110 netdbUpdatePeer(HttpRequest * r, CachePeer * e, int irtt, int ihops)
1111 {
1112 #if USE_ICMP
1113     netdbEntry *n;
1114     double rtt = (double) irtt;
1115     double hops = (double) ihops;
1116     net_db_peer *p;
1117     debugs(38, 3, "netdbUpdatePeer: '" << r->GetHost() << "', " << ihops << " hops, " << irtt << " rtt");
1118     n = netdbLookupHost(r->GetHost());
1119
1120     if (n == NULL) {
1121         debugs(38, 3, "netdbUpdatePeer: host '" << r->GetHost() << "' not found");
1122         return;
1123     }
1124
1125     if ((p = netdbPeerByName(n, e->host)) == NULL)
1126         p = netdbPeerAdd(n, e);
1127
1128     p->rtt = rtt;
1129
1130     p->hops = hops;
1131
1132     p->expires = squid_curtime + 3600;
1133
1134     if (n->n_peers < 2)
1135         return;
1136
1137     qsort((char *) n->peers,
1138           n->n_peers,
1139           sizeof(net_db_peer),
1140           sortPeerByRtt);
1141
1142 #endif
1143 }
1144
1145 void
1146 netdbExchangeUpdatePeer(Ip::Address &addr, CachePeer * e, double rtt, double hops)
1147 {
1148 #if USE_ICMP
1149     netdbEntry *n;
1150     net_db_peer *p;
1151     debugs(38, 5, "netdbExchangeUpdatePeer: '" << addr << "', "<<
1152            std::setfill('0')<< std::setprecision(2) << hops << " hops, " <<
1153            rtt << " rtt");
1154
1155     if ( !addr.isIPv4() ) {
1156         debugs(38, 5, "netdbExchangeUpdatePeer: Aborting peer update for '" << addr << "', NetDB cannot handle IPv6.");
1157         return;
1158     }
1159
1160     n = netdbLookupAddr(addr);
1161
1162     if (n == NULL)
1163         n = netdbAdd(addr);
1164
1165     assert(NULL != n);
1166
1167     if ((p = netdbPeerByName(n, e->host)) == NULL)
1168         p = netdbPeerAdd(n, e);
1169
1170     p->rtt = rtt;
1171
1172     p->hops = hops;
1173
1174     p->expires = squid_curtime + 3600;  /* XXX ? */
1175
1176     if (n->n_peers < 2)
1177         return;
1178
1179     qsort((char *) n->peers,
1180           n->n_peers,
1181           sizeof(net_db_peer),
1182           sortPeerByRtt);
1183
1184 #endif
1185 }
1186
1187 void
1188 netdbDeleteAddrNetwork(Ip::Address &addr)
1189 {
1190 #if USE_ICMP
1191     netdbEntry *n = netdbLookupAddr(addr);
1192
1193     if (n == NULL)
1194         return;
1195
1196     debugs(38, 3, "netdbDeleteAddrNetwork: " << n->network);
1197
1198     netdbRelease(n);
1199 #endif
1200 }
1201
1202 void
1203 netdbBinaryExchange(StoreEntry * s)
1204 {
1205     HttpReply *reply = new HttpReply;
1206 #if USE_ICMP
1207
1208     Ip::Address addr;
1209
1210     netdbEntry *n;
1211     int i;
1212     int j;
1213     int rec_sz;
1214     char *buf;
1215
1216     struct in_addr line_addr;
1217     s->buffer();
1218     reply->setHeaders(Http::scOkay, "OK", NULL, -1, squid_curtime, -2);
1219     s->replaceHttpReply(reply);
1220     rec_sz = 0;
1221     rec_sz += 1 + sizeof(struct in_addr);
1222     rec_sz += 1 + sizeof(int);
1223     rec_sz += 1 + sizeof(int);
1224     buf = (char *)memAllocate(MEM_4K_BUF);
1225     i = 0;
1226     hash_first(addr_table);
1227
1228     while ((n = (netdbEntry *) hash_next(addr_table))) {
1229         if (0.0 == n->rtt)
1230             continue;
1231
1232         if (n->rtt > 60000)     /* RTT > 1 MIN probably bogus */
1233             continue;
1234
1235         if (! (addr = n->network) )
1236             continue;
1237
1238         /* FIXME INET6 : NetDB cannot yet handle IPv6 addresses. Ensure only IPv4 get sent. */
1239         if ( !addr.isIPv4() )
1240             continue;
1241
1242         buf[i] = (char) NETDB_EX_NETWORK;
1243         ++i;
1244
1245         addr.getInAddr(line_addr);
1246         memcpy(&buf[i], &line_addr, sizeof(struct in_addr));
1247
1248         i += sizeof(struct in_addr);
1249
1250         buf[i] = (char) NETDB_EX_RTT;
1251         ++i;
1252
1253         j = htonl((int) (n->rtt * 1000));
1254
1255         memcpy(&buf[i], &j, sizeof(int));
1256
1257         i += sizeof(int);
1258
1259         buf[i] = (char) NETDB_EX_HOPS;
1260         ++i;
1261
1262         j = htonl((int) (n->hops * 1000));
1263
1264         memcpy(&buf[i], &j, sizeof(int));
1265
1266         i += sizeof(int);
1267
1268         if (i + rec_sz > 4096) {
1269             s->append(buf, i);
1270             i = 0;
1271         }
1272     }
1273
1274     if (i > 0) {
1275         s->append(buf, i);
1276         i = 0;
1277     }
1278
1279     assert(0 == i);
1280     s->flush();
1281     memFree(buf, MEM_4K_BUF);
1282 #else
1283
1284     reply->setHeaders(Http::scBadRequest, "Bad Request", NULL, -1, squid_curtime, -2);
1285     s->replaceHttpReply(reply);
1286     storeAppendPrintf(s, "NETDB support not compiled into this Squid cache.\n");
1287 #endif
1288
1289     s->complete();
1290 }
1291
1292 #if USE_ICMP
1293 CBDATA_TYPE(netdbExchangeState);
1294 #endif
1295
1296 void
1297 netdbExchangeStart(void *data)
1298 {
1299 #if USE_ICMP
1300     CachePeer *p = (CachePeer *)data;
1301     char *uri;
1302     netdbExchangeState *ex;
1303     StoreIOBuffer tempBuffer;
1304     CBDATA_INIT_TYPE(netdbExchangeState);
1305     ex = cbdataAlloc(netdbExchangeState);
1306     ex->p = cbdataReference(p);
1307     uri = internalRemoteUri(p->host, p->http_port, "/squid-internal-dynamic/", "netdb");
1308     debugs(38, 3, "netdbExchangeStart: Requesting '" << uri << "'");
1309     assert(NULL != uri);
1310     ex->r = HttpRequest::CreateFromUrl(uri);
1311
1312     if (NULL == ex->r) {
1313         debugs(38, DBG_IMPORTANT, "netdbExchangeStart: Bad URI " << uri);
1314         return;
1315     }
1316
1317     HTTPMSGLOCK(ex->r);
1318     assert(NULL != ex->r);
1319     ex->r->http_ver = Http::ProtocolVersion(1,1);
1320     ex->connstate = STATE_HEADER;
1321     ex->e = storeCreateEntry(uri, uri, RequestFlags(), Http::METHOD_GET);
1322     ex->buf_sz = NETDB_REQBUF_SZ;
1323     assert(NULL != ex->e);
1324     ex->sc = storeClientListAdd(ex->e, ex);
1325     tempBuffer.offset = 0;
1326     tempBuffer.length = ex->buf_sz;
1327     tempBuffer.data = ex->buf;
1328     storeClientCopy(ex->sc, ex->e, tempBuffer,
1329                     netdbExchangeHandleReply, ex);
1330     ex->r->flags.loopDetected = true;   /* cheat! -- force direct */
1331
1332     if (p->login)
1333         xstrncpy(ex->r->login, p->login, MAX_LOGIN_SZ);
1334
1335     urlCanonical(ex->r);
1336
1337     FwdState::fwdStart(Comm::ConnectionPointer(), ex->e, ex->r);
1338
1339 #endif
1340 }
1341
1342 CachePeer *
1343 netdbClosestParent(HttpRequest * request)
1344 {
1345 #if USE_ICMP
1346     CachePeer *p = NULL;
1347     netdbEntry *n;
1348     const ipcache_addrs *ia;
1349     net_db_peer *h;
1350     int i;
1351     n = netdbLookupHost(request->GetHost());
1352
1353     if (NULL == n) {
1354         /* try IP addr */
1355         ia = ipcache_gethostbyname(request->GetHost(), 0);
1356
1357         if (NULL != ia)
1358             n = netdbLookupAddr(ia->in_addrs[ia->cur]);
1359     }
1360
1361     if (NULL == n)
1362         return NULL;
1363
1364     if (0 == n->n_peers)
1365         return NULL;
1366
1367     n->last_use_time = squid_curtime;
1368
1369     /*
1370      * Find the parent with the least RTT to the origin server.
1371      * Make sure we don't return a parent who is farther away than
1372      * we are.  Note, the n->peers list is pre-sorted by RTT.
1373      */
1374     for (i = 0; i < n->n_peers; ++i) {
1375         h = &n->peers[i];
1376
1377         if (n->rtt > 0)
1378             if (n->rtt < h->rtt)
1379                 break;
1380
1381         p = peerFindByName(h->peername);
1382
1383         if (NULL == p)          /* not found */
1384             continue;
1385
1386         if (neighborType(p, request) != PEER_PARENT)
1387             continue;
1388
1389         if (!peerHTTPOkay(p, request))  /* not allowed */
1390             continue;
1391
1392         return p;
1393     }
1394
1395 #endif
1396     return NULL;
1397 }