src/icmp/net_db.cc

   1 /*
   2  * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 38    Network Measurement Database */
  10
  11 /*
  12  * XXX XXX XXX
  13  *
  14  * This code may be slightly broken now. If you're getting consistent
  15  * (sometimes working) corrupt data exchanges, please contact adrian
  16  * (adrian@squid-cache.org) to sort them out.
  17  */
  18
  19 #include "squid.h"
  20 #include "CachePeer.h"
  21 #include "cbdata.h"
  22 #include "disk.h"
  23 #include "event.h"
  24 #include "fde.h"
  25 #include "FwdState.h"
  26 #include "HttpReply.h"
  27 #include "HttpRequest.h"
  28 #include "icmp/net_db.h"
  29 #include "internal.h"
  30 #include "ip/Address.h"
  31 #include "log/File.h"
  32 #include "MemObject.h"
  33 #include "mgr/Registration.h"
  34 #include "mime_header.h"
  35 #include "neighbors.h"
  36 #include "SquidTime.h"
  37 #include "Store.h"
  38 #include "StoreClient.h"
  39 #include "SwapDir.h"
  40 #include "tools.h"
  41 #include "URL.h"
  42 #include "wordlist.h"
  43
  44 #if HAVE_SYS_STAT_H
  45 #include <sys/stat.h>
  46 #endif
  47
  48 #if USE_ICMP
  49 #include "icmp/IcmpSquid.h"
  50 #include "ipcache.h"
  51 #include "StoreClient.h"
  52
  53 #define NETDB_REQBUF_SZ 4096
  54
  55 typedef enum {
  56     STATE_NONE,
  57     STATE_HEADER,
  58     STATE_BODY
  59 } netdb_conn_state_t;
  60
  61 typedef struct {
  62     CachePeer *p;
  63     StoreEntry *e;
  64     store_client *sc;
  65     HttpRequest *r;
  66     int64_t used;
  67     size_t buf_sz;
  68     char buf[NETDB_REQBUF_SZ];
  69     int buf_ofs;
  70     netdb_conn_state_t connstate;
  71 } netdbExchangeState;
  72
  73 static hash_table *addr_table = NULL;
  74 static hash_table *host_table = NULL;
  75
  76 Ip::Address networkFromInaddr(const Ip::Address &a);
  77 static void netdbRelease(netdbEntry * n);
  78
  79 static void netdbHashInsert(netdbEntry * n, Ip::Address &addr);
  80 static void netdbHashDelete(const char *key);
  81 static void netdbHostInsert(netdbEntry * n, const char *hostname);
  82 static void netdbHostDelete(const net_db_name * x);
  83 static void netdbPurgeLRU(void);
  84 static netdbEntry *netdbLookupHost(const char *key);
  85 static net_db_peer *netdbPeerByName(const netdbEntry * n, const char *);
  86 static net_db_peer *netdbPeerAdd(netdbEntry * n, CachePeer * e);
  87 static const char *netdbPeerName(const char *name);
  88 static IPH netdbSendPing;
  89 static QS sortPeerByRtt;
  90 static QS sortByRtt;
  91 static QS netdbLRU;
  92 static FREE netdbFreeNameEntry;
  93 static FREE netdbFreeNetdbEntry;
  94 static STCB netdbExchangeHandleReply;
  95 static void netdbExchangeDone(void *);
  96
  97 /* We have to keep a local list of CachePeer names.  The Peers structure
  98  * gets freed during a reconfigure.  We want this database to
  99  * remain persisitent, so _net_db_peer->peername points into this
 100  * linked list */
 101 static wordlist *peer_names = NULL;
 102
 103 static void
 104 netdbHashInsert(netdbEntry * n, Ip::Address &addr)
 105 {
 106     networkFromInaddr(addr).toStr(n->network, MAX_IPSTRLEN);
 107     n->hash.key = n->network;
 108     assert(hash_lookup(addr_table, n->network) == NULL);
 109     hash_join(addr_table, &n->hash);
 110 }
 111
 112 static void
 113 netdbHashDelete(const char *key)
 114 {
 115     hash_link *hptr = (hash_link *)hash_lookup(addr_table, key);
 116
 117     if (hptr == NULL) {
 118         debug_trap("netdbHashDelete: key not found");
 119         return;
 120     }
 121
 122     hash_remove_link(addr_table, hptr);
 123 }
 124
 125 static void
 126 netdbHostInsert(netdbEntry * n, const char *hostname)
 127 {
 128     net_db_name *x = (net_db_name *)memAllocate(MEM_NET_DB_NAME);
 129     x->hash.key = xstrdup(hostname);
 130     x->next = n->hosts;
 131     n->hosts = x;
 132     x->net_db_entry = n;
 133     assert(hash_lookup(host_table, hostname) == NULL);
 134     hash_join(host_table, &x->hash);
 135     ++ n->link_count;
 136 }
 137
 138 static void
 139 netdbHostDelete(const net_db_name * x)
 140 {
 141     netdbEntry *n;
 142     net_db_name **X;
 143     assert(x != NULL);
 144     assert(x->net_db_entry != NULL);
 145     n = x->net_db_entry;
 146     -- n->link_count;
 147
 148     for (X = &n->hosts; *X; X = &(*X)->next) {
 149         if (*X == x) {
 150             *X = x->next;
 151             break;
 152         }
 153     }
 154
 155     hash_remove_link(host_table, (hash_link *) x);
 156     xfree(x->hash.key);
 157     memFree((void *) x, MEM_NET_DB_NAME);
 158 }
 159
 160 static netdbEntry *
 161 netdbLookupHost(const char *key)
 162 {
 163     net_db_name *x = (net_db_name *) hash_lookup(host_table, key);
 164     return x ? x->net_db_entry : NULL;
 165 }
 166
 167 static void
 168 netdbRelease(netdbEntry * n)
 169 {
 170     net_db_name *x;
 171     net_db_name *next;
 172
 173     for (x = n->hosts; x; x = next) {
 174         next = x->next;
 175         netdbHostDelete(x);
 176     }
 177
 178     n->hosts = NULL;
 179     safe_free(n->peers);
 180     n->peers = NULL;
 181     n->n_peers = 0;
 182     n->n_peers_alloc = 0;
 183
 184     if (n->link_count == 0) {
 185         netdbHashDelete(n->network);
 186         memFree(n, MEM_NETDBENTRY);
 187     }
 188 }
 189
 190 static int
 191 netdbLRU(const void *A, const void *B)
 192 {
 193     const netdbEntry *const *n1 = (const netdbEntry *const *)A;
 194     const netdbEntry *const *n2 = (const netdbEntry *const *)B;
 195
 196     if ((*n1)->last_use_time > (*n2)->last_use_time)
 197         return (1);
 198
 199     if ((*n1)->last_use_time < (*n2)->last_use_time)
 200         return (-1);
 201
 202     return (0);
 203 }
 204
 205 static void
 206 netdbPurgeLRU(void)
 207 {
 208     netdbEntry *n;
 209     netdbEntry **list;
 210     int k = 0;
 211     int list_count = 0;
 212     int removed = 0;
 213     list = (netdbEntry **)xcalloc(memInUse(MEM_NETDBENTRY), sizeof(netdbEntry *));
 214     hash_first(addr_table);
 215
 216     while ((n = (netdbEntry *) hash_next(addr_table))) {
 217         assert(list_count < memInUse(MEM_NETDBENTRY));
 218         *(list + list_count) = n;
 219         ++list_count;
 220     }
 221
 222     qsort((char *) list,
 223           list_count,
 224           sizeof(netdbEntry *),
 225           netdbLRU);
 226
 227     for (k = 0; k < list_count; ++k) {
 228         if (memInUse(MEM_NETDBENTRY) < Config.Netdb.low)
 229             break;
 230
 231         netdbRelease(*(list + k));
 232
 233         ++removed;
 234     }
 235
 236     xfree(list);
 237 }
 238
 239 static netdbEntry *
 240 netdbLookupAddr(const Ip::Address &addr)
 241 {
 242     netdbEntry *n;
 243     char *key = new char[MAX_IPSTRLEN];
 244     networkFromInaddr(addr).toStr(key,MAX_IPSTRLEN);
 245     n = (netdbEntry *) hash_lookup(addr_table, key);
 246     delete[] key;
 247     return n;
 248 }
 249
 250 static netdbEntry *
 251 netdbAdd(Ip::Address &addr)
 252 {
 253     netdbEntry *n;
 254
 255     if (memInUse(MEM_NETDBENTRY) > Config.Netdb.high)
 256         netdbPurgeLRU();
 257
 258     if ((n = netdbLookupAddr(addr)) == NULL) {
 259         n = (netdbEntry *)memAllocate(MEM_NETDBENTRY);
 260         netdbHashInsert(n, addr);
 261     }
 262
 263     return n;
 264 }
 265
 266 static void
 267 netdbSendPing(const ipcache_addrs *ia, const DnsLookupDetails &, void *data)
 268 {
 269     Ip::Address addr;
 270     char *hostname = NULL;
 271     static_cast<generic_cbdata *>(data)->unwrap(&hostname);
 272     netdbEntry *n;
 273     netdbEntry *na;
 274     net_db_name *x;
 275     net_db_name **X;
 276
 277     if (ia == NULL) {
 278         xfree(hostname);
 279         return;
 280     }
 281
 282     addr = ia->in_addrs[ia->cur];
 283
 284     if ((n = netdbLookupHost(hostname)) == NULL) {
 285         n = netdbAdd(addr);
 286         netdbHostInsert(n, hostname);
 287     } else if ((na = netdbLookupAddr(addr)) != n) {
 288         /*
 289          *hostname moved from 'network n' to 'network na'!
 290          */
 291
 292         if (na == NULL)
 293             na = netdbAdd(addr);
 294
 295         debugs(38, 3, "netdbSendPing: " << hostname << " moved from " << n->network << " to " << na->network);
 296
 297         x = (net_db_name *) hash_lookup(host_table, hostname);
 298
 299         if (x == NULL) {
 300             debugs(38, DBG_IMPORTANT, "netdbSendPing: net_db_name list bug: " << hostname << " not found");
 301             xfree(hostname);
 302             return;
 303         }
 304
 305         /* remove net_db_name from 'network n' linked list */
 306         for (X = &n->hosts; *X; X = &(*X)->next) {
 307             if (*X == x) {
 308                 *X = x->next;
 309                 break;
 310             }
 311         }
 312
 313         -- n->link_count;
 314         /* point to 'network na' from host entry */
 315         x->net_db_entry = na;
 316         /* link net_db_name to 'network na' */
 317         x->next = na->hosts;
 318         na->hosts = x;
 319         ++ na->link_count;
 320         n = na;
 321     }
 322
 323     if (n->next_ping_time <= squid_curtime) {
 324         debugs(38, 3, "netdbSendPing: pinging " << hostname);
 325         icmpEngine.DomainPing(addr, hostname);
 326         ++ n->pings_sent;
 327         n->next_ping_time = squid_curtime + Config.Netdb.period;
 328         n->last_use_time = squid_curtime;
 329     }
 330
 331     xfree(hostname);
 332 }
 333
 334 Ip::Address
 335 networkFromInaddr(const Ip::Address &in)
 336 {
 337     Ip::Address out;
 338
 339     out = in;
 340
 341     /* in IPv6 the 'network' should be the routing section. */
 342     if ( in.isIPv6() ) {
 343         out.applyMask(64, AF_INET6);
 344         debugs(14, 5, "networkFromInaddr : Masked IPv6 Address to " << in << "/64 routing part.");
 345         return out;
 346     }
 347
 348 #if USE_CLASSFUL
 349     struct in_addr b;
 350
 351     in.getInAddr(b);
 352
 353     if (IN_CLASSC(b.s_addr))
 354         b.s_addr &= IN_CLASSC_NET;
 355     else if (IN_CLASSB(b.s_addr))
 356         b.s_addr &= IN_CLASSB_NET;
 357     else if (IN_CLASSA(b.s_addr))
 358         b.s_addr &= IN_CLASSA_NET;
 359
 360     out = b;
 361
 362 #endif
 363
 364     debugs(14, 5, "networkFromInaddr : Masked IPv4 Address to " << out << "/24.");
 365
 366     /* use /24 for everything under IPv4 */
 367     out.applyMask(24, AF_INET);
 368     debugs(14, 5, "networkFromInaddr : Masked IPv4 Address to " << in << "/24.");
 369
 370     return out;
 371 }
 372
 373 static int
 374 sortByRtt(const void *A, const void *B)
 375 {
 376     const netdbEntry *const *n1 = (const netdbEntry *const *)A;
 377     const netdbEntry *const *n2 = (const netdbEntry *const *)B;
 378
 379     if ((*n1)->rtt > (*n2)->rtt)
 380         return 1;
 381     else if ((*n1)->rtt < (*n2)->rtt)
 382         return -1;
 383     else
 384         return 0;
 385 }
 386
 387 static net_db_peer *
 388 netdbPeerByName(const netdbEntry * n, const char *peername)
 389 {
 390     int i;
 391     net_db_peer *p = n->peers;
 392
 393     for (i = 0; i < n->n_peers; ++i, ++p) {
 394         if (!strcmp(p->peername, peername))
 395             return p;
 396     }
 397
 398     return NULL;
 399 }
 400
 401 static net_db_peer *
 402 netdbPeerAdd(netdbEntry * n, CachePeer * e)
 403 {
 404     net_db_peer *p;
 405     net_db_peer *o;
 406     int osize;
 407     int i;
 408
 409     if (n->n_peers == n->n_peers_alloc) {
 410         o = n->peers;
 411         osize = n->n_peers_alloc;
 412
 413         if (n->n_peers_alloc == 0)
 414             n->n_peers_alloc = 2;
 415         else
 416             n->n_peers_alloc <<= 1;
 417
 418         debugs(38, 3, "netdbPeerAdd: Growing peer list for '" << n->network << "' to " << n->n_peers_alloc);
 419
 420         n->peers = (net_db_peer *)xcalloc(n->n_peers_alloc, sizeof(net_db_peer));
 421
 422         for (i = 0; i < osize; ++i)
 423             *(n->peers + i) = *(o + i);
 424
 425         if (osize) {
 426             safe_free(o);
 427         }
 428     }
 429
 430     p = n->peers + n->n_peers;
 431     p->peername = netdbPeerName(e->host);
 432     ++ n->n_peers;
 433     return p;
 434 }
 435
 436 static int
 437 sortPeerByRtt(const void *A, const void *B)
 438 {
 439     const net_db_peer *p1 = (net_db_peer *)A;
 440     const net_db_peer *p2 = (net_db_peer *)B;
 441
 442     if (p1->rtt > p2->rtt)
 443         return 1;
 444     else if (p1->rtt < p2->rtt)
 445         return -1;
 446     else
 447         return 0;
 448 }
 449
 450 static void
 451 netdbSaveState(void *foo)
 452 {
 453     if (strcmp(Config.netdbFilename, "none") == 0)
 454         return;
 455
 456     Logfile *lf;
 457     netdbEntry *n;
 458     net_db_name *x;
 459
 460     struct timeval start = current_time;
 461     int count = 0;
 462     /*
 463      * This was nicer when we were using stdio, but thanks to
 464      * Solaris bugs, its a bad idea.  fopen can fail if more than
 465      * 256 FDs are open.
 466      */
 467     /*
 468      * unlink() is here because there is currently no way to make
 469      * logfileOpen() use O_TRUNC.
 470      */
 471     unlink(Config.netdbFilename);
 472     lf = logfileOpen(Config.netdbFilename, 4096, 0);
 473
 474     if (NULL == lf) {
 475         debugs(50, DBG_IMPORTANT, "netdbSaveState: " << Config.netdbFilename << ": " << xstrerror());
 476         return;
 477     }
 478
 479     hash_first(addr_table);
 480
 481     while ((n = (netdbEntry *) hash_next(addr_table))) {
 482         if (n->pings_recv == 0)
 483             continue;
 484
 485         logfilePrintf(lf, "%s %d %d %10.5f %10.5f %d %d",
 486                       n->network,
 487                       n->pings_sent,
 488                       n->pings_recv,
 489                       n->hops,
 490                       n->rtt,
 491                       (int) n->next_ping_time,
 492                       (int) n->last_use_time);
 493
 494         for (x = n->hosts; x; x = x->next)
 495             logfilePrintf(lf, " %s", hashKeyStr(&x->hash));
 496
 497         logfilePrintf(lf, "\n");
 498
 499         ++count;
 500
 501 #undef RBUF_SZ
 502
 503     }
 504
 505     logfileClose(lf);
 506     getCurrentTime();
 507     debugs(38, DBG_IMPORTANT, "NETDB state saved; " <<
 508            count << " entries, " <<
 509            tvSubMsec(start, current_time) << " msec" );
 510     eventAddIsh("netdbSaveState", netdbSaveState, NULL, 3600.0, 1);
 511 }
 512
 513 static void
 514 netdbReloadState(void)
 515 {
 516     if (strcmp(Config.netdbFilename, "none") == 0)
 517         return;
 518
 519     char *s;
 520     int fd;
 521     int l;
 522
 523     struct stat sb;
 524     netdbEntry *n;
 525     netdbEntry N;
 526
 527     Ip::Address addr;
 528     int count = 0;
 529
 530     struct timeval start = current_time;
 531     /*
 532      * This was nicer when we were using stdio, but thanks to
 533      * Solaris bugs, its a bad idea.  fopen can fail if more than
 534      * 256 FDs are open.
 535      */
 536     fd = file_open(Config.netdbFilename, O_RDONLY | O_BINARY);
 537
 538     if (fd < 0)
 539         return;
 540
 541     if (fstat(fd, &sb) < 0) {
 542         file_close(fd);
 543         return;
 544     }
 545
 546     char *t;
 547     char *buf = (char *)xcalloc(1, sb.st_size + 1);
 548     t = buf;
 549     l = FD_READ_METHOD(fd, buf, sb.st_size);
 550     file_close(fd);
 551
 552     if (l <= 0) {
 553         safe_free (buf);
 554         return;
 555     };
 556
 557     while ((s = strchr(t, '\n'))) {
 558         char *q;
 559         assert(s - buf < l);
 560         *s = '\0';
 561         memset(&N, '\0', sizeof(netdbEntry));
 562         q = strtok(t, w_space);
 563         t = s + 1;
 564
 565         if (NULL == q)
 566             continue;
 567
 568         if (! (addr = q) )
 569             continue;
 570
 571         if (netdbLookupAddr(addr) != NULL)  /* no dups! */
 572             continue;
 573
 574         if ((q = strtok(NULL, w_space)) == NULL)
 575             continue;
 576
 577         N.pings_sent = atoi(q);
 578
 579         if ((q = strtok(NULL, w_space)) == NULL)
 580             continue;
 581
 582         N.pings_recv = atoi(q);
 583
 584         if (N.pings_recv == 0)
 585             continue;
 586
 587         /* give this measurement low weight */
 588         N.pings_sent = 1;
 589
 590         N.pings_recv = 1;
 591
 592         if ((q = strtok(NULL, w_space)) == NULL)
 593             continue;
 594
 595         N.hops = atof(q);
 596
 597         if ((q = strtok(NULL, w_space)) == NULL)
 598             continue;
 599
 600         N.rtt = atof(q);
 601
 602         if ((q = strtok(NULL, w_space)) == NULL)
 603             continue;
 604
 605         N.next_ping_time = (time_t) atoi(q);
 606
 607         if ((q = strtok(NULL, w_space)) == NULL)
 608             continue;
 609
 610         N.last_use_time = (time_t) atoi(q);
 611
 612         n = (netdbEntry *)memAllocate(MEM_NETDBENTRY);
 613
 614         memcpy(n, &N, sizeof(netdbEntry));
 615
 616         netdbHashInsert(n, addr);
 617
 618         while ((q = strtok(NULL, w_space)) != NULL) {
 619             if (netdbLookupHost(q) != NULL) /* no dups! */
 620                 continue;
 621
 622             netdbHostInsert(n, q);
 623         }
 624
 625         ++count;
 626     }
 627
 628     xfree(buf);
 629     getCurrentTime();
 630     debugs(38, DBG_IMPORTANT, "NETDB state reloaded; " <<
 631            count << " entries, " <<
 632            tvSubMsec(start, current_time) << " msec" );
 633 }
 634
 635 static const char *
 636 netdbPeerName(const char *name)
 637 {
 638     const wordlist *w;
 639
 640     for (w = peer_names; w; w = w->next) {
 641         if (!strcmp(w->key, name))
 642             return w->key;
 643     }
 644
 645     return wordlistAdd(&peer_names, name);
 646 }
 647
 648 static void
 649 netdbFreeNetdbEntry(void *data)
 650 {
 651     netdbEntry *n = (netdbEntry *)data;
 652     safe_free(n->peers);
 653     memFree(n, MEM_NETDBENTRY);
 654 }
 655
 656 static void
 657 netdbFreeNameEntry(void *data)
 658 {
 659     net_db_name *x = (net_db_name *)data;
 660     xfree(x->hash.key);
 661     memFree(x, MEM_NET_DB_NAME);
 662 }
 663
 664 static void
 665 netdbExchangeHandleReply(void *data, StoreIOBuffer receivedData)
 666 {
 667     Ip::Address addr;
 668
 669     netdbExchangeState *ex = (netdbExchangeState *)data;
 670     int rec_sz = 0;
 671     int o;
 672
 673     struct in_addr line_addr;
 674     double rtt;
 675     double hops;
 676     char *p;
 677     int j;
 678     HttpReply const *rep;
 679     size_t hdr_sz;
 680     int nused = 0;
 681     int size;
 682     int oldbufofs = ex->buf_ofs;
 683
 684     rec_sz = 0;
 685     rec_sz += 1 + sizeof(struct in_addr);
 686     rec_sz += 1 + sizeof(int);
 687     rec_sz += 1 + sizeof(int);
 688     debugs(38, 3, "netdbExchangeHandleReply: " << receivedData.length << " read bytes");
 689
 690     if (!cbdataReferenceValid(ex->p)) {
 691         debugs(38, 3, "netdbExchangeHandleReply: Peer became invalid");
 692         netdbExchangeDone(ex);
 693         return;
 694     }
 695
 696     debugs(38, 3, "netdbExchangeHandleReply: for '" << ex->p->host << ":" << ex->p->http_port << "'");
 697
 698     if (receivedData.length == 0 &&
 699             !receivedData.flags.error) {
 700         debugs(38, 3, "netdbExchangeHandleReply: Done");
 701         netdbExchangeDone(ex);
 702         return;
 703     }
 704
 705     p = ex->buf;
 706
 707     /* Get the size of the buffer now */
 708     size = ex->buf_ofs + receivedData.length;
 709     debugs(38, 3, "netdbExchangeHandleReply: " << size << " bytes buf");
 710
 711     /* Check if we're still doing headers */
 712
 713     if (ex->connstate == STATE_HEADER) {
 714
 715         ex->buf_ofs += receivedData.length;
 716
 717         /* skip reply headers */
 718
 719         if ((hdr_sz = headersEnd(p, ex->buf_ofs))) {
 720             debugs(38, 5, "netdbExchangeHandleReply: hdr_sz = " << hdr_sz);
 721             rep = ex->e->getReply();
 722             assert(rep->sline.status() != Http::scNone);
 723             debugs(38, 3, "netdbExchangeHandleReply: reply status " << rep->sline.status());
 724
 725             if (rep->sline.status() != Http::scOkay) {
 726                 netdbExchangeDone(ex);
 727                 return;
 728             }
 729
 730             assert((size_t)ex->buf_ofs >= hdr_sz);
 731
 732             /*
 733              * Now, point p to the part of the buffer where the data
 734              * starts, and update the size accordingly
 735              */
 736             assert(ex->used == 0);
 737             ex->used = hdr_sz;
 738             size = ex->buf_ofs - hdr_sz;
 739             p += hdr_sz;
 740
 741             /* Finally, set the conn state mode to STATE_BODY */
 742             ex->connstate = STATE_BODY;
 743         } else {
 744             StoreIOBuffer tempBuffer;
 745             tempBuffer.offset = ex->buf_ofs;
 746             tempBuffer.length = ex->buf_sz - ex->buf_ofs;
 747             tempBuffer.data = ex->buf + ex->buf_ofs;
 748             /* Have more headers .. */
 749             storeClientCopy(ex->sc, ex->e, tempBuffer,
 750                             netdbExchangeHandleReply, ex);
 751             return;
 752         }
 753     }
 754
 755     assert(ex->connstate == STATE_BODY);
 756
 757     /* If we get here, we have some body to parse .. */
 758     debugs(38, 5, "netdbExchangeHandleReply: start parsing loop, size = " << size);
 759
 760     while (size >= rec_sz) {
 761         debugs(38, 5, "netdbExchangeHandleReply: in parsing loop, size = " << size);
 762         addr.setAnyAddr();
 763         hops = rtt = 0.0;
 764
 765         for (o = 0; o < rec_sz;) {
 766             switch ((int) *(p + o)) {
 767
 768             case NETDB_EX_NETWORK:
 769                 ++o;
 770                 /* FIXME INET6 : NetDB can still ony send IPv4 */
 771                 memcpy(&line_addr, p + o, sizeof(struct in_addr));
 772                 addr = line_addr;
 773                 o += sizeof(struct in_addr);
 774                 break;
 775
 776             case NETDB_EX_RTT:
 777                 ++o;
 778                 memcpy(&j, p + o, sizeof(int));
 779                 o += sizeof(int);
 780                 rtt = (double) ntohl(j) / 1000.0;
 781                 break;
 782
 783             case NETDB_EX_HOPS:
 784                 ++o;
 785                 memcpy(&j, p + o, sizeof(int));
 786                 o += sizeof(int);
 787                 hops = (double) ntohl(j) / 1000.0;
 788                 break;
 789
 790             default:
 791                 debugs(38, DBG_IMPORTANT, "netdbExchangeHandleReply: corrupt data, aborting");
 792                 netdbExchangeDone(ex);
 793                 return;
 794             }
 795         }
 796
 797         if (!addr.isAnyAddr() && rtt > 0)
 798             netdbExchangeUpdatePeer(addr, ex->p, rtt, hops);
 799
 800         assert(o == rec_sz);
 801
 802         ex->used += rec_sz;
 803
 804         size -= rec_sz;
 805
 806         p += rec_sz;
 807
 808         ++nused;
 809     }
 810
 811     /*
 812      * Copy anything that is left over to the beginning of the buffer,
 813      * and adjust buf_ofs accordingly
 814      */
 815
 816     /*
 817      * Evilly, size refers to the buf size left now,
 818      * ex->buf_ofs is the original buffer size, so just copy that
 819      * much data over
 820      */
 821     memmove(ex->buf, ex->buf + (ex->buf_ofs - size), size);
 822
 823     ex->buf_ofs = size;
 824
 825     /*
 826      * And don't re-copy the remaining data ..
 827      */
 828     ex->used += size;
 829
 830     /*
 831      * Now the tricky bit - size _included_ the leftover bit from the _last_
 832      * storeClientCopy. We don't want to include that, or our offset will be wrong.
 833      * So, don't count the size of the leftover buffer we began with.
 834      * This can _disappear_ when we're not tracking offsets ..
 835      */
 836     ex->used -= oldbufofs;
 837
 838     debugs(38, 3, "netdbExchangeHandleReply: size left over in this buffer: " << size << " bytes");
 839
 840     debugs(38, 3, "netdbExchangeHandleReply: used " << nused <<
 841            " entries, (x " << rec_sz << " bytes) == " << nused * rec_sz <<
 842            " bytes total");
 843
 844     debugs(38, 3, "netdbExchangeHandleReply: used " << ex->used);
 845
 846     if (EBIT_TEST(ex->e->flags, ENTRY_ABORTED)) {
 847         debugs(38, 3, "netdbExchangeHandleReply: ENTRY_ABORTED");
 848         netdbExchangeDone(ex);
 849     } else if (ex->e->store_status == STORE_PENDING) {
 850         StoreIOBuffer tempBuffer;
 851         tempBuffer.offset = ex->used;
 852         tempBuffer.length = ex->buf_sz - ex->buf_ofs;
 853         tempBuffer.data = ex->buf + ex->buf_ofs;
 854         debugs(38, 3, "netdbExchangeHandleReply: EOF not received");
 855         storeClientCopy(ex->sc, ex->e, tempBuffer,
 856                         netdbExchangeHandleReply, ex);
 857     }
 858 }
 859
 860 static void
 861 netdbExchangeDone(void *data)
 862 {
 863     netdbExchangeState *ex = (netdbExchangeState *)data;
 864     debugs(38, 3, "netdbExchangeDone: " << ex->e->url()  );
 865     HTTPMSGUNLOCK(ex->r);
 866     storeUnregister(ex->sc, ex->e, ex);
 867     ex->e->unlock("netdbExchangeDone");
 868     cbdataReferenceDone(ex->p);
 869     cbdataFree(ex);
 870 }
 871
 872 static void
 873 netdbRegisterWithCacheManager(void)
 874 {
 875     Mgr::RegisterAction("netdb", "Network Measurement Database", netdbDump, 0, 1);
 876 }
 877
 878 #endif /* USE_ICMP */
 879
 880 /* PUBLIC FUNCTIONS */
 881
 882 void
 883 netdbInit(void)
 884 {
 885 #if USE_ICMP
 886     int n;
 887
 888     netdbRegisterWithCacheManager();
 889
 890     if (addr_table)
 891         return;
 892
 893     n = hashPrime(Config.Netdb.high / 4);
 894
 895     addr_table = hash_create((HASHCMP *) strcmp, n, hash_string);
 896
 897     n = hashPrime(3 * Config.Netdb.high / 4);
 898
 899     host_table = hash_create((HASHCMP *) strcmp, n, hash_string);
 900
 901     eventAddIsh("netdbSaveState", netdbSaveState, NULL, 3600.0, 1);
 902
 903     netdbReloadState();
 904
 905 #endif
 906 }
 907
 908 void
 909 netdbPingSite(const char *hostname)
 910 {
 911 #if USE_ICMP
 912     netdbEntry *n;
 913
 914     if ((n = netdbLookupHost(hostname)) != NULL)
 915         if (n->next_ping_time > squid_curtime)
 916             return;
 917
 918     ipcache_nbgethostbyname(hostname, netdbSendPing,
 919                             new generic_cbdata(xstrdup(hostname)));
 920
 921 #endif
 922 }
 923
 924 void
 925 netdbHandlePingReply(const Ip::Address &from, int hops, int rtt)
 926 {
 927 #if USE_ICMP
 928     netdbEntry *n;
 929     int N;
 930     debugs(38, 3, "netdbHandlePingReply: from " << from);
 931
 932     if ((n = netdbLookupAddr(from)) == NULL)
 933         return;
 934
 935     N = ++n->pings_recv;
 936
 937     if (N > 5)
 938         N = 5;
 939
 940     if (rtt < 1)
 941         rtt = 1;
 942
 943     n->hops = ((n->hops * (N - 1)) + hops) / N;
 944
 945     n->rtt = ((n->rtt * (N - 1)) + rtt) / N;
 946
 947     debugs(38, 3, "netdbHandlePingReply: " << n->network  << "; rtt="<<
 948            std::setw(5)<< std::setprecision(2) << n->rtt << "  hops="<<
 949            std::setw(4) << n->hops);
 950
 951 #endif
 952 }
 953
 954 void
 955 netdbFreeMemory(void)
 956 {
 957 #if USE_ICMP
 958     hashFreeItems(addr_table, netdbFreeNetdbEntry);
 959     hashFreeMemory(addr_table);
 960     addr_table = NULL;
 961     hashFreeItems(host_table, netdbFreeNameEntry);
 962     hashFreeMemory(host_table);
 963     host_table = NULL;
 964     wordlistDestroy(&peer_names);
 965     peer_names = NULL;
 966 #endif
 967 }
 968
 969 void
 970 netdbDump(StoreEntry * sentry)
 971 {
 972 #if USE_ICMP
 973     netdbEntry *n;
 974     netdbEntry **list;
 975     net_db_name *x;
 976     int k;
 977     int i;
 978     int j;
 979     net_db_peer *p;
 980     storeAppendPrintf(sentry, "Network DB Statistics:\n");
 981     storeAppendPrintf(sentry, "%-46.46s %9s %7s %5s %s\n",  /* Max between 16 (IPv4) or 46 (IPv6)   */
 982                       "Network",
 983                       "recv/sent",
 984                       "RTT",
 985                       "Hops",
 986                       "Hostnames");
 987     list = (netdbEntry **)xcalloc(memInUse(MEM_NETDBENTRY), sizeof(netdbEntry *));
 988     i = 0;
 989     hash_first(addr_table);
 990
 991     while ((n = (netdbEntry *) hash_next(addr_table))) {
 992         *(list + i) = n;
 993         ++i;
 994     }
 995
 996     if (i != memInUse(MEM_NETDBENTRY))
 997         debugs(38, DBG_CRITICAL, "WARNING: netdb_addrs count off, found " << i <<
 998                ", expected " << memInUse(MEM_NETDBENTRY));
 999
1000     qsort((char *) list,
1001           i,
1002           sizeof(netdbEntry *),
1003           sortByRtt);
1004
1005     for (k = 0; k < i; ++k) {
1006         n = *(list + k);
1007         storeAppendPrintf(sentry, "%-46.46s %4d/%4d %7.1f %5.1f", /* Max between 16 (IPv4) or 46 (IPv6)   */
1008                           n->network,
1009                           n->pings_recv,
1010                           n->pings_sent,
1011                           n->rtt,
1012                           n->hops);
1013
1014         for (x = n->hosts; x; x = x->next)
1015             storeAppendPrintf(sentry, " %s", hashKeyStr(&x->hash));
1016
1017         storeAppendPrintf(sentry, "\n");
1018
1019         p = n->peers;
1020
1021         for (j = 0; j < n->n_peers; ++j, ++p) {
1022             storeAppendPrintf(sentry, "    %-22.22s %7.1f %5.1f\n",
1023                               p->peername,
1024                               p->rtt,
1025                               p->hops);
1026         }
1027     }
1028
1029     xfree(list);
1030 #else
1031
1032     storeAppendPrintf(sentry,"NETDB support not compiled into this Squid cache.\n");
1033 #endif
1034 }
1035
1036 int
1037 netdbHostHops(const char *host)
1038 {
1039 #if USE_ICMP
1040     netdbEntry *n = netdbLookupHost(host);
1041
1042     if (n) {
1043         n->last_use_time = squid_curtime;
1044         return (int) (n->hops + 0.5);
1045     }
1046
1047 #endif
1048     return 0;
1049 }
1050
1051 int
1052 netdbHostRtt(const char *host)
1053 {
1054 #if USE_ICMP
1055     netdbEntry *n = netdbLookupHost(host);
1056
1057     if (n) {
1058         n->last_use_time = squid_curtime;
1059         return (int) (n->rtt + 0.5);
1060     }
1061
1062 #endif
1063     return 0;
1064 }
1065
1066 void
1067 netdbHostData(const char *host, int *samp, int *rtt, int *hops)
1068 {
1069 #if USE_ICMP
1070     netdbEntry *n = netdbLookupHost(host);
1071
1072     if (n == NULL)
1073         return;
1074
1075     *samp = n->pings_recv;
1076
1077     *rtt = (int) (n->rtt + 0.5);
1078
1079     *hops = (int) (n->hops + 0.5);
1080
1081     n->last_use_time = squid_curtime;
1082
1083 #endif
1084 }
1085
1086 void
1087 netdbUpdatePeer(HttpRequest * r, CachePeer * e, int irtt, int ihops)
1088 {
1089 #if USE_ICMP
1090     netdbEntry *n;
1091     double rtt = (double) irtt;
1092     double hops = (double) ihops;
1093     net_db_peer *p;
1094     debugs(38, 3, "netdbUpdatePeer: '" << r->GetHost() << "', " << ihops << " hops, " << irtt << " rtt");
1095     n = netdbLookupHost(r->GetHost());
1096
1097     if (n == NULL) {
1098         debugs(38, 3, "netdbUpdatePeer: host '" << r->GetHost() << "' not found");
1099         return;
1100     }
1101
1102     if ((p = netdbPeerByName(n, e->host)) == NULL)
1103         p = netdbPeerAdd(n, e);
1104
1105     p->rtt = rtt;
1106
1107     p->hops = hops;
1108
1109     p->expires = squid_curtime + 3600;
1110
1111     if (n->n_peers < 2)
1112         return;
1113
1114     qsort((char *) n->peers,
1115           n->n_peers,
1116           sizeof(net_db_peer),
1117           sortPeerByRtt);
1118
1119 #endif
1120 }
1121
1122 void
1123 netdbExchangeUpdatePeer(Ip::Address &addr, CachePeer * e, double rtt, double hops)
1124 {
1125 #if USE_ICMP
1126     netdbEntry *n;
1127     net_db_peer *p;
1128     debugs(38, 5, "netdbExchangeUpdatePeer: '" << addr << "', "<<
1129            std::setfill('0')<< std::setprecision(2) << hops << " hops, " <<
1130            rtt << " rtt");
1131
1132     if ( !addr.isIPv4() ) {
1133         debugs(38, 5, "netdbExchangeUpdatePeer: Aborting peer update for '" << addr << "', NetDB cannot handle IPv6.");
1134         return;
1135     }
1136
1137     n = netdbLookupAddr(addr);
1138
1139     if (n == NULL)
1140         n = netdbAdd(addr);
1141
1142     assert(NULL != n);
1143
1144     if ((p = netdbPeerByName(n, e->host)) == NULL)
1145         p = netdbPeerAdd(n, e);
1146
1147     p->rtt = rtt;
1148
1149     p->hops = hops;
1150
1151     p->expires = squid_curtime + 3600;  /* XXX ? */
1152
1153     if (n->n_peers < 2)
1154         return;
1155
1156     qsort((char *) n->peers,
1157           n->n_peers,
1158           sizeof(net_db_peer),
1159           sortPeerByRtt);
1160
1161 #endif
1162 }
1163
1164 void
1165 netdbDeleteAddrNetwork(Ip::Address &addr)
1166 {
1167 #if USE_ICMP
1168     netdbEntry *n = netdbLookupAddr(addr);
1169
1170     if (n == NULL)
1171         return;
1172
1173     debugs(38, 3, "netdbDeleteAddrNetwork: " << n->network);
1174
1175     netdbRelease(n);
1176 #endif
1177 }
1178
1179 void
1180 netdbBinaryExchange(StoreEntry * s)
1181 {
1182     HttpReply *reply = new HttpReply;
1183 #if USE_ICMP
1184
1185     Ip::Address addr;
1186
1187     netdbEntry *n;
1188     int i;
1189     int j;
1190     int rec_sz;
1191     char *buf;
1192
1193     struct in_addr line_addr;
1194     s->buffer();
1195     reply->setHeaders(Http::scOkay, "OK", NULL, -1, squid_curtime, -2);
1196     s->replaceHttpReply(reply);
1197     rec_sz = 0;
1198     rec_sz += 1 + sizeof(struct in_addr);
1199     rec_sz += 1 + sizeof(int);
1200     rec_sz += 1 + sizeof(int);
1201     buf = (char *)memAllocate(MEM_4K_BUF);
1202     i = 0;
1203     hash_first(addr_table);
1204
1205     while ((n = (netdbEntry *) hash_next(addr_table))) {
1206         if (0.0 == n->rtt)
1207             continue;
1208
1209         if (n->rtt > 60000) /* RTT > 1 MIN probably bogus */
1210             continue;
1211
1212         if (! (addr = n->network) )
1213             continue;
1214
1215         /* FIXME INET6 : NetDB cannot yet handle IPv6 addresses. Ensure only IPv4 get sent. */
1216         if ( !addr.isIPv4() )
1217             continue;
1218
1219         buf[i] = (char) NETDB_EX_NETWORK;
1220         ++i;
1221
1222         addr.getInAddr(line_addr);
1223         memcpy(&buf[i], &line_addr, sizeof(struct in_addr));
1224
1225         i += sizeof(struct in_addr);
1226
1227         buf[i] = (char) NETDB_EX_RTT;
1228         ++i;
1229
1230         j = htonl((int) (n->rtt * 1000));
1231
1232         memcpy(&buf[i], &j, sizeof(int));
1233
1234         i += sizeof(int);
1235
1236         buf[i] = (char) NETDB_EX_HOPS;
1237         ++i;
1238
1239         j = htonl((int) (n->hops * 1000));
1240
1241         memcpy(&buf[i], &j, sizeof(int));
1242
1243         i += sizeof(int);
1244
1245         if (i + rec_sz > 4096) {
1246             s->append(buf, i);
1247             i = 0;
1248         }
1249     }
1250
1251     if (i > 0) {
1252         s->append(buf, i);
1253         i = 0;
1254     }
1255
1256     assert(0 == i);
1257     s->flush();
1258     memFree(buf, MEM_4K_BUF);
1259 #else
1260
1261     reply->setHeaders(Http::scBadRequest, "Bad Request", NULL, -1, squid_curtime, -2);
1262     s->replaceHttpReply(reply);
1263     storeAppendPrintf(s, "NETDB support not compiled into this Squid cache.\n");
1264 #endif
1265
1266     s->complete();
1267 }
1268
1269 #if USE_ICMP
1270 CBDATA_TYPE(netdbExchangeState);
1271 #endif
1272
1273 void
1274 netdbExchangeStart(void *data)
1275 {
1276 #if USE_ICMP
1277     CachePeer *p = (CachePeer *)data;
1278     char *uri;
1279     netdbExchangeState *ex;
1280     StoreIOBuffer tempBuffer;
1281     CBDATA_INIT_TYPE(netdbExchangeState);
1282     ex = cbdataAlloc(netdbExchangeState);
1283     ex->p = cbdataReference(p);
1284     uri = internalRemoteUri(p->host, p->http_port, "/squid-internal-dynamic/", "netdb");
1285     debugs(38, 3, "netdbExchangeStart: Requesting '" << uri << "'");
1286     assert(NULL != uri);
1287     ex->r = HttpRequest::CreateFromUrl(uri);
1288
1289     if (NULL == ex->r) {
1290         debugs(38, DBG_IMPORTANT, "netdbExchangeStart: Bad URI " << uri);
1291         return;
1292     }
1293
1294     HTTPMSGLOCK(ex->r);
1295     assert(NULL != ex->r);
1296     ex->r->http_ver = Http::ProtocolVersion();
1297     ex->connstate = STATE_HEADER;
1298     ex->e = storeCreateEntry(uri, uri, RequestFlags(), Http::METHOD_GET);
1299     ex->buf_sz = NETDB_REQBUF_SZ;
1300     assert(NULL != ex->e);
1301     ex->sc = storeClientListAdd(ex->e, ex);
1302     tempBuffer.offset = 0;
1303     tempBuffer.length = ex->buf_sz;
1304     tempBuffer.data = ex->buf;
1305     storeClientCopy(ex->sc, ex->e, tempBuffer,
1306                     netdbExchangeHandleReply, ex);
1307     ex->r->flags.loopDetected = true;   /* cheat! -- force direct */
1308
1309     // XXX: send as Proxy-Authenticate instead
1310     if (p->login)
1311         ex->r->url.userInfo(SBuf(p->login));
1312
1313     urlCanonical(ex->r);
1314
1315     FwdState::fwdStart(Comm::ConnectionPointer(), ex->e, ex->r);
1316
1317 #endif
1318 }
1319
1320 CachePeer *
1321 netdbClosestParent(HttpRequest * request)
1322 {
1323 #if USE_ICMP
1324     CachePeer *p = NULL;
1325     netdbEntry *n;
1326     const ipcache_addrs *ia;
1327     net_db_peer *h;
1328     int i;
1329     n = netdbLookupHost(request->GetHost());
1330
1331     if (NULL == n) {
1332         /* try IP addr */
1333         ia = ipcache_gethostbyname(request->GetHost(), 0);
1334
1335         if (NULL != ia)
1336             n = netdbLookupAddr(ia->in_addrs[ia->cur]);
1337     }
1338
1339     if (NULL == n)
1340         return NULL;
1341
1342     if (0 == n->n_peers)
1343         return NULL;
1344
1345     n->last_use_time = squid_curtime;
1346
1347     /*
1348      * Find the parent with the least RTT to the origin server.
1349      * Make sure we don't return a parent who is farther away than
1350      * we are.  Note, the n->peers list is pre-sorted by RTT.
1351      */
1352     for (i = 0; i < n->n_peers; ++i) {
1353         h = &n->peers[i];
1354
1355         if (n->rtt > 0)
1356             if (n->rtt < h->rtt)
1357                 break;
1358
1359         p = peerFindByName(h->peername);
1360
1361         if (NULL == p)      /* not found */
1362             continue;
1363
1364         if (neighborType(p, request) != PEER_PARENT)
1365             continue;
1366
1367         if (!peerHTTPOkay(p, request))  /* not allowed */
1368             continue;
1369
1370         return p;
1371     }
1372
1373 #endif
1374     return NULL;
1375 }
1376