src/icmp/net_db.cc

   1 /*
   2  * Copyright (C) 1996-2014 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 38    Network Measurement Database */
  10
  11 /*
  12  * XXX XXX XXX
  13  *
  14  * This code may be slightly broken now. If you're getting consistent
  15  * (sometimes working) corrupt data exchanges, please contact adrian
  16  * (adrian@squid-cache.org) to sort them out.
  17  */
  18
  19 #include "squid.h"
  20 #include "CachePeer.h"
  21 #include "cbdata.h"
  22 #include "disk.h"
  23 #include "event.h"
  24 #include "fde.h"
  25 #include "FwdState.h"
  26 #include "HttpReply.h"
  27 #include "HttpRequest.h"
  28 #include "icmp/net_db.h"
  29 #include "internal.h"
  30 #include "ip/Address.h"
  31 #include "log/File.h"
  32 #include "Mem.h"
  33 #include "MemObject.h"
  34 #include "mgr/Registration.h"
  35 #include "mime_header.h"
  36 #include "neighbors.h"
  37 #include "SquidTime.h"
  38 #include "Store.h"
  39 #include "StoreClient.h"
  40 #include "SwapDir.h"
  41 #include "tools.h"
  42 #include "URL.h"
  43 #include "wordlist.h"
  44
  45 #if HAVE_SYS_STAT_H
  46 #include <sys/stat.h>
  47 #endif
  48
  49 #if USE_ICMP
  50 #include "icmp/IcmpSquid.h"
  51 #include "ipcache.h"
  52 #include "StoreClient.h"
  53
  54 #define NETDB_REQBUF_SZ 4096
  55
  56 typedef enum {
  57     STATE_NONE,
  58     STATE_HEADER,
  59     STATE_BODY
  60 } netdb_conn_state_t;
  61
  62 typedef struct {
  63     CachePeer *p;
  64     StoreEntry *e;
  65     store_client *sc;
  66     HttpRequest *r;
  67     int64_t used;
  68     size_t buf_sz;
  69     char buf[NETDB_REQBUF_SZ];
  70     int buf_ofs;
  71     netdb_conn_state_t connstate;
  72 } netdbExchangeState;
  73
  74 static hash_table *addr_table = NULL;
  75 static hash_table *host_table = NULL;
  76
  77 Ip::Address networkFromInaddr(const Ip::Address &a);
  78 static void netdbRelease(netdbEntry * n);
  79
  80 static void netdbHashInsert(netdbEntry * n, Ip::Address &addr);
  81 static void netdbHashDelete(const char *key);
  82 static void netdbHostInsert(netdbEntry * n, const char *hostname);
  83 static void netdbHostDelete(const net_db_name * x);
  84 static void netdbPurgeLRU(void);
  85 static netdbEntry *netdbLookupHost(const char *key);
  86 static net_db_peer *netdbPeerByName(const netdbEntry * n, const char *);
  87 static net_db_peer *netdbPeerAdd(netdbEntry * n, CachePeer * e);
  88 static const char *netdbPeerName(const char *name);
  89 static IPH netdbSendPing;
  90 static QS sortPeerByRtt;
  91 static QS sortByRtt;
  92 static QS netdbLRU;
  93 static FREE netdbFreeNameEntry;
  94 static FREE netdbFreeNetdbEntry;
  95 static STCB netdbExchangeHandleReply;
  96 static void netdbExchangeDone(void *);
  97
  98 /* We have to keep a local list of CachePeer names.  The Peers structure
  99  * gets freed during a reconfigure.  We want this database to
 100  * remain persisitent, so _net_db_peer->peername points into this
 101  * linked list */
 102 static wordlist *peer_names = NULL;
 103
 104 static void
 105 netdbHashInsert(netdbEntry * n, Ip::Address &addr)
 106 {
 107     networkFromInaddr(addr).toStr(n->network, MAX_IPSTRLEN);
 108     n->hash.key = n->network;
 109     assert(hash_lookup(addr_table, n->network) == NULL);
 110     hash_join(addr_table, &n->hash);
 111 }
 112
 113 static void
 114 netdbHashDelete(const char *key)
 115 {
 116     hash_link *hptr = (hash_link *)hash_lookup(addr_table, key);
 117
 118     if (hptr == NULL) {
 119         debug_trap("netdbHashDelete: key not found");
 120         return;
 121     }
 122
 123     hash_remove_link(addr_table, hptr);
 124 }
 125
 126 static void
 127 netdbHostInsert(netdbEntry * n, const char *hostname)
 128 {
 129     net_db_name *x = (net_db_name *)memAllocate(MEM_NET_DB_NAME);
 130     x->hash.key = xstrdup(hostname);
 131     x->next = n->hosts;
 132     n->hosts = x;
 133     x->net_db_entry = n;
 134     assert(hash_lookup(host_table, hostname) == NULL);
 135     hash_join(host_table, &x->hash);
 136     ++ n->link_count;
 137 }
 138
 139 static void
 140 netdbHostDelete(const net_db_name * x)
 141 {
 142     netdbEntry *n;
 143     net_db_name **X;
 144     assert(x != NULL);
 145     assert(x->net_db_entry != NULL);
 146     n = x->net_db_entry;
 147     -- n->link_count;
 148
 149     for (X = &n->hosts; *X; X = &(*X)->next) {
 150         if (*X == x) {
 151             *X = x->next;
 152             break;
 153         }
 154     }
 155
 156     hash_remove_link(host_table, (hash_link *) x);
 157     xfree(x->hash.key);
 158     memFree((void *) x, MEM_NET_DB_NAME);
 159 }
 160
 161 static netdbEntry *
 162 netdbLookupHost(const char *key)
 163 {
 164     net_db_name *x = (net_db_name *) hash_lookup(host_table, key);
 165     return x ? x->net_db_entry : NULL;
 166 }
 167
 168 static void
 169 netdbRelease(netdbEntry * n)
 170 {
 171     net_db_name *x;
 172     net_db_name *next;
 173
 174     for (x = n->hosts; x; x = next) {
 175         next = x->next;
 176         netdbHostDelete(x);
 177     }
 178
 179     n->hosts = NULL;
 180     safe_free(n->peers);
 181     n->peers = NULL;
 182     n->n_peers = 0;
 183     n->n_peers_alloc = 0;
 184
 185     if (n->link_count == 0) {
 186         netdbHashDelete(n->network);
 187         memFree(n, MEM_NETDBENTRY);
 188     }
 189 }
 190
 191 static int
 192 netdbLRU(const void *A, const void *B)
 193 {
 194     const netdbEntry *const *n1 = (const netdbEntry *const *)A;
 195     const netdbEntry *const *n2 = (const netdbEntry *const *)B;
 196
 197     if ((*n1)->last_use_time > (*n2)->last_use_time)
 198         return (1);
 199
 200     if ((*n1)->last_use_time < (*n2)->last_use_time)
 201         return (-1);
 202
 203     return (0);
 204 }
 205
 206 static void
 207 netdbPurgeLRU(void)
 208 {
 209     netdbEntry *n;
 210     netdbEntry **list;
 211     int k = 0;
 212     int list_count = 0;
 213     int removed = 0;
 214     list = (netdbEntry **)xcalloc(memInUse(MEM_NETDBENTRY), sizeof(netdbEntry *));
 215     hash_first(addr_table);
 216
 217     while ((n = (netdbEntry *) hash_next(addr_table))) {
 218         assert(list_count < memInUse(MEM_NETDBENTRY));
 219         *(list + list_count) = n;
 220         ++list_count;
 221     }
 222
 223     qsort((char *) list,
 224           list_count,
 225           sizeof(netdbEntry *),
 226           netdbLRU);
 227
 228     for (k = 0; k < list_count; ++k) {
 229         if (memInUse(MEM_NETDBENTRY) < Config.Netdb.low)
 230             break;
 231
 232         netdbRelease(*(list + k));
 233
 234         ++removed;
 235     }
 236
 237     xfree(list);
 238 }
 239
 240 static netdbEntry *
 241 netdbLookupAddr(const Ip::Address &addr)
 242 {
 243     netdbEntry *n;
 244     char *key = new char[MAX_IPSTRLEN];
 245     networkFromInaddr(addr).toStr(key,MAX_IPSTRLEN);
 246     n = (netdbEntry *) hash_lookup(addr_table, key);
 247     delete[] key;
 248     return n;
 249 }
 250
 251 static netdbEntry *
 252 netdbAdd(Ip::Address &addr)
 253 {
 254     netdbEntry *n;
 255
 256     if (memInUse(MEM_NETDBENTRY) > Config.Netdb.high)
 257         netdbPurgeLRU();
 258
 259     if ((n = netdbLookupAddr(addr)) == NULL) {
 260         n = (netdbEntry *)memAllocate(MEM_NETDBENTRY);
 261         netdbHashInsert(n, addr);
 262     }
 263
 264     return n;
 265 }
 266
 267 static void
 268 netdbSendPing(const ipcache_addrs *ia, const DnsLookupDetails &, void *data)
 269 {
 270     Ip::Address addr;
 271     char *hostname = NULL;
 272     static_cast<generic_cbdata *>(data)->unwrap(&hostname);
 273     netdbEntry *n;
 274     netdbEntry *na;
 275     net_db_name *x;
 276     net_db_name **X;
 277
 278     if (ia == NULL) {
 279         xfree(hostname);
 280         return;
 281     }
 282
 283     addr = ia->in_addrs[ia->cur];
 284
 285     if ((n = netdbLookupHost(hostname)) == NULL) {
 286         n = netdbAdd(addr);
 287         netdbHostInsert(n, hostname);
 288     } else if ((na = netdbLookupAddr(addr)) != n) {
 289         /*
 290          *hostname moved from 'network n' to 'network na'!
 291          */
 292
 293         if (na == NULL)
 294             na = netdbAdd(addr);
 295
 296         debugs(38, 3, "netdbSendPing: " << hostname << " moved from " << n->network << " to " << na->network);
 297
 298         x = (net_db_name *) hash_lookup(host_table, hostname);
 299
 300         if (x == NULL) {
 301             debugs(38, DBG_IMPORTANT, "netdbSendPing: net_db_name list bug: " << hostname << " not found");
 302             xfree(hostname);
 303             return;
 304         }
 305
 306         /* remove net_db_name from 'network n' linked list */
 307         for (X = &n->hosts; *X; X = &(*X)->next) {
 308             if (*X == x) {
 309                 *X = x->next;
 310                 break;
 311             }
 312         }
 313
 314         -- n->link_count;
 315         /* point to 'network na' from host entry */
 316         x->net_db_entry = na;
 317         /* link net_db_name to 'network na' */
 318         x->next = na->hosts;
 319         na->hosts = x;
 320         ++ na->link_count;
 321         n = na;
 322     }
 323
 324     if (n->next_ping_time <= squid_curtime) {
 325         debugs(38, 3, "netdbSendPing: pinging " << hostname);
 326         icmpEngine.DomainPing(addr, hostname);
 327         ++ n->pings_sent;
 328         n->next_ping_time = squid_curtime + Config.Netdb.period;
 329         n->last_use_time = squid_curtime;
 330     }
 331
 332     xfree(hostname);
 333 }
 334
 335 Ip::Address
 336 networkFromInaddr(const Ip::Address &in)
 337 {
 338     Ip::Address out;
 339
 340     out = in;
 341
 342     /* in IPv6 the 'network' should be the routing section. */
 343     if ( in.isIPv6() ) {
 344         out.applyMask(64, AF_INET6);
 345         debugs(14, 5, "networkFromInaddr : Masked IPv6 Address to " << in << "/64 routing part.");
 346         return out;
 347     }
 348
 349 #if USE_CLASSFUL
 350     struct in_addr b;
 351
 352     in.getInAddr(b);
 353
 354     if (IN_CLASSC(b.s_addr))
 355         b.s_addr &= IN_CLASSC_NET;
 356     else if (IN_CLASSB(b.s_addr))
 357         b.s_addr &= IN_CLASSB_NET;
 358     else if (IN_CLASSA(b.s_addr))
 359         b.s_addr &= IN_CLASSA_NET;
 360
 361     out = b;
 362
 363 #endif
 364
 365     debugs(14, 5, "networkFromInaddr : Masked IPv4 Address to " << out << "/24.");
 366
 367     /* use /24 for everything under IPv4 */
 368     out.applyMask(24, AF_INET);
 369     debugs(14, 5, "networkFromInaddr : Masked IPv4 Address to " << in << "/24.");
 370
 371     return out;
 372 }
 373
 374 static int
 375 sortByRtt(const void *A, const void *B)
 376 {
 377     const netdbEntry *const *n1 = (const netdbEntry *const *)A;
 378     const netdbEntry *const *n2 = (const netdbEntry *const *)B;
 379
 380     if ((*n1)->rtt > (*n2)->rtt)
 381         return 1;
 382     else if ((*n1)->rtt < (*n2)->rtt)
 383         return -1;
 384     else
 385         return 0;
 386 }
 387
 388 static net_db_peer *
 389 netdbPeerByName(const netdbEntry * n, const char *peername)
 390 {
 391     int i;
 392     net_db_peer *p = n->peers;
 393
 394     for (i = 0; i < n->n_peers; ++i, ++p) {
 395         if (!strcmp(p->peername, peername))
 396             return p;
 397     }
 398
 399     return NULL;
 400 }
 401
 402 static net_db_peer *
 403 netdbPeerAdd(netdbEntry * n, CachePeer * e)
 404 {
 405     net_db_peer *p;
 406     net_db_peer *o;
 407     int osize;
 408     int i;
 409
 410     if (n->n_peers == n->n_peers_alloc) {
 411         o = n->peers;
 412         osize = n->n_peers_alloc;
 413
 414         if (n->n_peers_alloc == 0)
 415             n->n_peers_alloc = 2;
 416         else
 417             n->n_peers_alloc <<= 1;
 418
 419         debugs(38, 3, "netdbPeerAdd: Growing peer list for '" << n->network << "' to " << n->n_peers_alloc);
 420
 421         n->peers = (net_db_peer *)xcalloc(n->n_peers_alloc, sizeof(net_db_peer));
 422
 423         for (i = 0; i < osize; ++i)
 424             *(n->peers + i) = *(o + i);
 425
 426         if (osize) {
 427             safe_free(o);
 428         }
 429     }
 430
 431     p = n->peers + n->n_peers;
 432     p->peername = netdbPeerName(e->host);
 433     ++ n->n_peers;
 434     return p;
 435 }
 436
 437 static int
 438 sortPeerByRtt(const void *A, const void *B)
 439 {
 440     const net_db_peer *p1 = (net_db_peer *)A;
 441     const net_db_peer *p2 = (net_db_peer *)B;
 442
 443     if (p1->rtt > p2->rtt)
 444         return 1;
 445     else if (p1->rtt < p2->rtt)
 446         return -1;
 447     else
 448         return 0;
 449 }
 450
 451 static void
 452 netdbSaveState(void *foo)
 453 {
 454     if (strcmp(Config.netdbFilename, "none") == 0)
 455         return;
 456
 457     Logfile *lf;
 458     netdbEntry *n;
 459     net_db_name *x;
 460
 461     struct timeval start = current_time;
 462     int count = 0;
 463     /*
 464      * This was nicer when we were using stdio, but thanks to
 465      * Solaris bugs, its a bad idea.  fopen can fail if more than
 466      * 256 FDs are open.
 467      */
 468     /*
 469      * unlink() is here because there is currently no way to make
 470      * logfileOpen() use O_TRUNC.
 471      */
 472     unlink(Config.netdbFilename);
 473     lf = logfileOpen(Config.netdbFilename, 4096, 0);
 474
 475     if (NULL == lf) {
 476         debugs(50, DBG_IMPORTANT, "netdbSaveState: " << Config.netdbFilename << ": " << xstrerror());
 477         return;
 478     }
 479
 480     hash_first(addr_table);
 481
 482     while ((n = (netdbEntry *) hash_next(addr_table))) {
 483         if (n->pings_recv == 0)
 484             continue;
 485
 486         logfilePrintf(lf, "%s %d %d %10.5f %10.5f %d %d",
 487                       n->network,
 488                       n->pings_sent,
 489                       n->pings_recv,
 490                       n->hops,
 491                       n->rtt,
 492                       (int) n->next_ping_time,
 493                       (int) n->last_use_time);
 494
 495         for (x = n->hosts; x; x = x->next)
 496             logfilePrintf(lf, " %s", hashKeyStr(&x->hash));
 497
 498         logfilePrintf(lf, "\n");
 499
 500         ++count;
 501
 502 #undef RBUF_SZ
 503
 504     }
 505
 506     logfileClose(lf);
 507     getCurrentTime();
 508     debugs(38, DBG_IMPORTANT, "NETDB state saved; " <<
 509            count << " entries, " <<
 510            tvSubMsec(start, current_time) << " msec" );
 511     eventAddIsh("netdbSaveState", netdbSaveState, NULL, 3600.0, 1);
 512 }
 513
 514 static void
 515 netdbReloadState(void)
 516 {
 517     if (strcmp(Config.netdbFilename, "none") == 0)
 518         return;
 519
 520     char *s;
 521     int fd;
 522     int l;
 523
 524     struct stat sb;
 525     netdbEntry *n;
 526     netdbEntry N;
 527
 528     Ip::Address addr;
 529     int count = 0;
 530
 531     struct timeval start = current_time;
 532     /*
 533      * This was nicer when we were using stdio, but thanks to
 534      * Solaris bugs, its a bad idea.  fopen can fail if more than
 535      * 256 FDs are open.
 536      */
 537     fd = file_open(Config.netdbFilename, O_RDONLY | O_BINARY);
 538
 539     if (fd < 0)
 540         return;
 541
 542     if (fstat(fd, &sb) < 0) {
 543         file_close(fd);
 544         return;
 545     }
 546
 547     char *t;
 548     char *buf = (char *)xcalloc(1, sb.st_size + 1);
 549     t = buf;
 550     l = FD_READ_METHOD(fd, buf, sb.st_size);
 551     file_close(fd);
 552
 553     if (l <= 0) {
 554         safe_free (buf);
 555         return;
 556     };
 557
 558     while ((s = strchr(t, '\n'))) {
 559         char *q;
 560         assert(s - buf < l);
 561         *s = '\0';
 562         memset(&N, '\0', sizeof(netdbEntry));
 563         q = strtok(t, w_space);
 564         t = s + 1;
 565
 566         if (NULL == q)
 567             continue;
 568
 569         if (! (addr = q) )
 570             continue;
 571
 572         if (netdbLookupAddr(addr) != NULL)      /* no dups! */
 573             continue;
 574
 575         if ((q = strtok(NULL, w_space)) == NULL)
 576             continue;
 577
 578         N.pings_sent = atoi(q);
 579
 580         if ((q = strtok(NULL, w_space)) == NULL)
 581             continue;
 582
 583         N.pings_recv = atoi(q);
 584
 585         if (N.pings_recv == 0)
 586             continue;
 587
 588         /* give this measurement low weight */
 589         N.pings_sent = 1;
 590
 591         N.pings_recv = 1;
 592
 593         if ((q = strtok(NULL, w_space)) == NULL)
 594             continue;
 595
 596         N.hops = atof(q);
 597
 598         if ((q = strtok(NULL, w_space)) == NULL)
 599             continue;
 600
 601         N.rtt = atof(q);
 602
 603         if ((q = strtok(NULL, w_space)) == NULL)
 604             continue;
 605
 606         N.next_ping_time = (time_t) atoi(q);
 607
 608         if ((q = strtok(NULL, w_space)) == NULL)
 609             continue;
 610
 611         N.last_use_time = (time_t) atoi(q);
 612
 613         n = (netdbEntry *)memAllocate(MEM_NETDBENTRY);
 614
 615         memcpy(n, &N, sizeof(netdbEntry));
 616
 617         netdbHashInsert(n, addr);
 618
 619         while ((q = strtok(NULL, w_space)) != NULL) {
 620             if (netdbLookupHost(q) != NULL)     /* no dups! */
 621                 continue;
 622
 623             netdbHostInsert(n, q);
 624         }
 625
 626         ++count;
 627     }
 628
 629     xfree(buf);
 630     getCurrentTime();
 631     debugs(38, DBG_IMPORTANT, "NETDB state reloaded; " <<
 632            count << " entries, " <<
 633            tvSubMsec(start, current_time) << " msec" );
 634 }
 635
 636 static const char *
 637 netdbPeerName(const char *name)
 638 {
 639     const wordlist *w;
 640
 641     for (w = peer_names; w; w = w->next) {
 642         if (!strcmp(w->key, name))
 643             return w->key;
 644     }
 645
 646     return wordlistAdd(&peer_names, name);
 647 }
 648
 649 static void
 650 netdbFreeNetdbEntry(void *data)
 651 {
 652     netdbEntry *n = (netdbEntry *)data;
 653     safe_free(n->peers);
 654     memFree(n, MEM_NETDBENTRY);
 655 }
 656
 657 static void
 658 netdbFreeNameEntry(void *data)
 659 {
 660     net_db_name *x = (net_db_name *)data;
 661     xfree(x->hash.key);
 662     memFree(x, MEM_NET_DB_NAME);
 663 }
 664
 665 static void
 666 netdbExchangeHandleReply(void *data, StoreIOBuffer receivedData)
 667 {
 668     Ip::Address addr;
 669
 670     netdbExchangeState *ex = (netdbExchangeState *)data;
 671     int rec_sz = 0;
 672     int o;
 673
 674     struct in_addr line_addr;
 675     double rtt;
 676     double hops;
 677     char *p;
 678     int j;
 679     HttpReply const *rep;
 680     size_t hdr_sz;
 681     int nused = 0;
 682     int size;
 683     int oldbufofs = ex->buf_ofs;
 684
 685     rec_sz = 0;
 686     rec_sz += 1 + sizeof(struct in_addr);
 687     rec_sz += 1 + sizeof(int);
 688     rec_sz += 1 + sizeof(int);
 689     debugs(38, 3, "netdbExchangeHandleReply: " << receivedData.length << " read bytes");
 690
 691     if (!cbdataReferenceValid(ex->p)) {
 692         debugs(38, 3, "netdbExchangeHandleReply: Peer became invalid");
 693         netdbExchangeDone(ex);
 694         return;
 695     }
 696
 697     debugs(38, 3, "netdbExchangeHandleReply: for '" << ex->p->host << ":" << ex->p->http_port << "'");
 698
 699     if (receivedData.length == 0 &&
 700             !receivedData.flags.error) {
 701         debugs(38, 3, "netdbExchangeHandleReply: Done");
 702         netdbExchangeDone(ex);
 703         return;
 704     }
 705
 706     p = ex->buf;
 707
 708     /* Get the size of the buffer now */
 709     size = ex->buf_ofs + receivedData.length;
 710     debugs(38, 3, "netdbExchangeHandleReply: " << size << " bytes buf");
 711
 712     /* Check if we're still doing headers */
 713
 714     if (ex->connstate == STATE_HEADER) {
 715
 716         ex->buf_ofs += receivedData.length;
 717
 718         /* skip reply headers */
 719
 720         if ((hdr_sz = headersEnd(p, ex->buf_ofs))) {
 721             debugs(38, 5, "netdbExchangeHandleReply: hdr_sz = " << hdr_sz);
 722             rep = ex->e->getReply();
 723             assert(rep->sline.status() != Http::scNone);
 724             debugs(38, 3, "netdbExchangeHandleReply: reply status " << rep->sline.status());
 725
 726             if (rep->sline.status() != Http::scOkay) {
 727                 netdbExchangeDone(ex);
 728                 return;
 729             }
 730
 731             assert((size_t)ex->buf_ofs >= hdr_sz);
 732
 733             /*
 734              * Now, point p to the part of the buffer where the data
 735              * starts, and update the size accordingly
 736              */
 737             assert(ex->used == 0);
 738             ex->used = hdr_sz;
 739             size = ex->buf_ofs - hdr_sz;
 740             p += hdr_sz;
 741
 742             /* Finally, set the conn state mode to STATE_BODY */
 743             ex->connstate = STATE_BODY;
 744         } else {
 745             StoreIOBuffer tempBuffer;
 746             tempBuffer.offset = ex->buf_ofs;
 747             tempBuffer.length = ex->buf_sz - ex->buf_ofs;
 748             tempBuffer.data = ex->buf + ex->buf_ofs;
 749             /* Have more headers .. */
 750             storeClientCopy(ex->sc, ex->e, tempBuffer,
 751                             netdbExchangeHandleReply, ex);
 752             return;
 753         }
 754     }
 755
 756     assert(ex->connstate == STATE_BODY);
 757
 758     /* If we get here, we have some body to parse .. */
 759     debugs(38, 5, "netdbExchangeHandleReply: start parsing loop, size = " << size);
 760
 761     while (size >= rec_sz) {
 762         debugs(38, 5, "netdbExchangeHandleReply: in parsing loop, size = " << size);
 763         addr.setAnyAddr();
 764         hops = rtt = 0.0;
 765
 766         for (o = 0; o < rec_sz;) {
 767             switch ((int) *(p + o)) {
 768
 769             case NETDB_EX_NETWORK:
 770                 ++o;
 771                 /* FIXME INET6 : NetDB can still ony send IPv4 */
 772                 memcpy(&line_addr, p + o, sizeof(struct in_addr));
 773                 addr = line_addr;
 774                 o += sizeof(struct in_addr);
 775                 break;
 776
 777             case NETDB_EX_RTT:
 778                 ++o;
 779                 memcpy(&j, p + o, sizeof(int));
 780                 o += sizeof(int);
 781                 rtt = (double) ntohl(j) / 1000.0;
 782                 break;
 783
 784             case NETDB_EX_HOPS:
 785                 ++o;
 786                 memcpy(&j, p + o, sizeof(int));
 787                 o += sizeof(int);
 788                 hops = (double) ntohl(j) / 1000.0;
 789                 break;
 790
 791             default:
 792                 debugs(38, DBG_IMPORTANT, "netdbExchangeHandleReply: corrupt data, aborting");
 793                 netdbExchangeDone(ex);
 794                 return;
 795             }
 796         }
 797
 798         if (!addr.isAnyAddr() && rtt > 0)
 799             netdbExchangeUpdatePeer(addr, ex->p, rtt, hops);
 800
 801         assert(o == rec_sz);
 802
 803         ex->used += rec_sz;
 804
 805         size -= rec_sz;
 806
 807         p += rec_sz;
 808
 809         ++nused;
 810     }
 811
 812     /*
 813      * Copy anything that is left over to the beginning of the buffer,
 814      * and adjust buf_ofs accordingly
 815      */
 816
 817     /*
 818      * Evilly, size refers to the buf size left now,
 819      * ex->buf_ofs is the original buffer size, so just copy that
 820      * much data over
 821      */
 822     memmove(ex->buf, ex->buf + (ex->buf_ofs - size), size);
 823
 824     ex->buf_ofs = size;
 825
 826     /*
 827      * And don't re-copy the remaining data ..
 828      */
 829     ex->used += size;
 830
 831     /*
 832      * Now the tricky bit - size _included_ the leftover bit from the _last_
 833      * storeClientCopy. We don't want to include that, or our offset will be wrong.
 834      * So, don't count the size of the leftover buffer we began with.
 835      * This can _disappear_ when we're not tracking offsets ..
 836      */
 837     ex->used -= oldbufofs;
 838
 839     debugs(38, 3, "netdbExchangeHandleReply: size left over in this buffer: " << size << " bytes");
 840
 841     debugs(38, 3, "netdbExchangeHandleReply: used " << nused <<
 842            " entries, (x " << rec_sz << " bytes) == " << nused * rec_sz <<
 843            " bytes total");
 844
 845     debugs(38, 3, "netdbExchangeHandleReply: used " << ex->used);
 846
 847     if (EBIT_TEST(ex->e->flags, ENTRY_ABORTED)) {
 848         debugs(38, 3, "netdbExchangeHandleReply: ENTRY_ABORTED");
 849         netdbExchangeDone(ex);
 850     } else if (ex->e->store_status == STORE_PENDING) {
 851         StoreIOBuffer tempBuffer;
 852         tempBuffer.offset = ex->used;
 853         tempBuffer.length = ex->buf_sz - ex->buf_ofs;
 854         tempBuffer.data = ex->buf + ex->buf_ofs;
 855         debugs(38, 3, "netdbExchangeHandleReply: EOF not received");
 856         storeClientCopy(ex->sc, ex->e, tempBuffer,
 857                         netdbExchangeHandleReply, ex);
 858     }
 859 }
 860
 861 static void
 862 netdbExchangeDone(void *data)
 863 {
 864     netdbExchangeState *ex = (netdbExchangeState *)data;
 865     debugs(38, 3, "netdbExchangeDone: " << ex->e->url()  );
 866     HTTPMSGUNLOCK(ex->r);
 867     storeUnregister(ex->sc, ex->e, ex);
 868     ex->e->unlock("netdbExchangeDone");
 869     cbdataReferenceDone(ex->p);
 870     cbdataFree(ex);
 871 }
 872
 873 static void
 874 netdbRegisterWithCacheManager(void)
 875 {
 876     Mgr::RegisterAction("netdb", "Network Measurement Database", netdbDump, 0, 1);
 877 }
 878
 879 #endif /* USE_ICMP */
 880
 881 /* PUBLIC FUNCTIONS */
 882
 883 void
 884 netdbInit(void)
 885 {
 886 #if USE_ICMP
 887     int n;
 888
 889     netdbRegisterWithCacheManager();
 890
 891     if (addr_table)
 892         return;
 893
 894     n = hashPrime(Config.Netdb.high / 4);
 895
 896     addr_table = hash_create((HASHCMP *) strcmp, n, hash_string);
 897
 898     n = hashPrime(3 * Config.Netdb.high / 4);
 899
 900     host_table = hash_create((HASHCMP *) strcmp, n, hash_string);
 901
 902     eventAddIsh("netdbSaveState", netdbSaveState, NULL, 3600.0, 1);
 903
 904     netdbReloadState();
 905
 906 #endif
 907 }
 908
 909 void
 910 netdbPingSite(const char *hostname)
 911 {
 912 #if USE_ICMP
 913     netdbEntry *n;
 914
 915     if ((n = netdbLookupHost(hostname)) != NULL)
 916         if (n->next_ping_time > squid_curtime)
 917             return;
 918
 919     ipcache_nbgethostbyname(hostname, netdbSendPing,
 920                             new generic_cbdata(xstrdup(hostname)));
 921
 922 #endif
 923 }
 924
 925 void
 926 netdbHandlePingReply(const Ip::Address &from, int hops, int rtt)
 927 {
 928 #if USE_ICMP
 929     netdbEntry *n;
 930     int N;
 931     debugs(38, 3, "netdbHandlePingReply: from " << from);
 932
 933     if ((n = netdbLookupAddr(from)) == NULL)
 934         return;
 935
 936     N = ++n->pings_recv;
 937
 938     if (N > 5)
 939         N = 5;
 940
 941     if (rtt < 1)
 942         rtt = 1;
 943
 944     n->hops = ((n->hops * (N - 1)) + hops) / N;
 945
 946     n->rtt = ((n->rtt * (N - 1)) + rtt) / N;
 947
 948     debugs(38, 3, "netdbHandlePingReply: " << n->network  << "; rtt="<<
 949            std::setw(5)<< std::setprecision(2) << n->rtt << "  hops="<<
 950            std::setw(4) << n->hops);
 951
 952 #endif
 953 }
 954
 955 void
 956 netdbFreeMemory(void)
 957 {
 958 #if USE_ICMP
 959     hashFreeItems(addr_table, netdbFreeNetdbEntry);
 960     hashFreeMemory(addr_table);
 961     addr_table = NULL;
 962     hashFreeItems(host_table, netdbFreeNameEntry);
 963     hashFreeMemory(host_table);
 964     host_table = NULL;
 965     wordlistDestroy(&peer_names);
 966     peer_names = NULL;
 967 #endif
 968 }
 969
 970 void
 971 netdbDump(StoreEntry * sentry)
 972 {
 973 #if USE_ICMP
 974     netdbEntry *n;
 975     netdbEntry **list;
 976     net_db_name *x;
 977     int k;
 978     int i;
 979     int j;
 980     net_db_peer *p;
 981     storeAppendPrintf(sentry, "Network DB Statistics:\n");
 982     storeAppendPrintf(sentry, "%-46.46s %9s %7s %5s %s\n",  /* Max between 16 (IPv4) or 46 (IPv6)   */
 983                       "Network",
 984                       "recv/sent",
 985                       "RTT",
 986                       "Hops",
 987                       "Hostnames");
 988     list = (netdbEntry **)xcalloc(memInUse(MEM_NETDBENTRY), sizeof(netdbEntry *));
 989     i = 0;
 990     hash_first(addr_table);
 991
 992     while ((n = (netdbEntry *) hash_next(addr_table))) {
 993         *(list + i) = n;
 994         ++i;
 995     }
 996
 997     if (i != memInUse(MEM_NETDBENTRY))
 998         debugs(38, DBG_CRITICAL, "WARNING: netdb_addrs count off, found " << i <<
 999                ", expected " << memInUse(MEM_NETDBENTRY));
1000
1001     qsort((char *) list,
1002           i,
1003           sizeof(netdbEntry *),
1004           sortByRtt);
1005
1006     for (k = 0; k < i; ++k) {
1007         n = *(list + k);
1008         storeAppendPrintf(sentry, "%-46.46s %4d/%4d %7.1f %5.1f", /* Max between 16 (IPv4) or 46 (IPv6)   */
1009                           n->network,
1010                           n->pings_recv,
1011                           n->pings_sent,
1012                           n->rtt,
1013                           n->hops);
1014
1015         for (x = n->hosts; x; x = x->next)
1016             storeAppendPrintf(sentry, " %s", hashKeyStr(&x->hash));
1017
1018         storeAppendPrintf(sentry, "\n");
1019
1020         p = n->peers;
1021
1022         for (j = 0; j < n->n_peers; ++j, ++p) {
1023             storeAppendPrintf(sentry, "    %-22.22s %7.1f %5.1f\n",
1024                               p->peername,
1025                               p->rtt,
1026                               p->hops);
1027         }
1028     }
1029
1030     xfree(list);
1031 #else
1032
1033     storeAppendPrintf(sentry,"NETDB support not compiled into this Squid cache.\n");
1034 #endif
1035 }
1036
1037 int
1038 netdbHostHops(const char *host)
1039 {
1040 #if USE_ICMP
1041     netdbEntry *n = netdbLookupHost(host);
1042
1043     if (n) {
1044         n->last_use_time = squid_curtime;
1045         return (int) (n->hops + 0.5);
1046     }
1047
1048 #endif
1049     return 0;
1050 }
1051
1052 int
1053 netdbHostRtt(const char *host)
1054 {
1055 #if USE_ICMP
1056     netdbEntry *n = netdbLookupHost(host);
1057
1058     if (n) {
1059         n->last_use_time = squid_curtime;
1060         return (int) (n->rtt + 0.5);
1061     }
1062
1063 #endif
1064     return 0;
1065 }
1066
1067 void
1068 netdbHostData(const char *host, int *samp, int *rtt, int *hops)
1069 {
1070 #if USE_ICMP
1071     netdbEntry *n = netdbLookupHost(host);
1072
1073     if (n == NULL)
1074         return;
1075
1076     *samp = n->pings_recv;
1077
1078     *rtt = (int) (n->rtt + 0.5);
1079
1080     *hops = (int) (n->hops + 0.5);
1081
1082     n->last_use_time = squid_curtime;
1083
1084 #endif
1085 }
1086
1087 void
1088 netdbUpdatePeer(HttpRequest * r, CachePeer * e, int irtt, int ihops)
1089 {
1090 #if USE_ICMP
1091     netdbEntry *n;
1092     double rtt = (double) irtt;
1093     double hops = (double) ihops;
1094     net_db_peer *p;
1095     debugs(38, 3, "netdbUpdatePeer: '" << r->GetHost() << "', " << ihops << " hops, " << irtt << " rtt");
1096     n = netdbLookupHost(r->GetHost());
1097
1098     if (n == NULL) {
1099         debugs(38, 3, "netdbUpdatePeer: host '" << r->GetHost() << "' not found");
1100         return;
1101     }
1102
1103     if ((p = netdbPeerByName(n, e->host)) == NULL)
1104         p = netdbPeerAdd(n, e);
1105
1106     p->rtt = rtt;
1107
1108     p->hops = hops;
1109
1110     p->expires = squid_curtime + 3600;
1111
1112     if (n->n_peers < 2)
1113         return;
1114
1115     qsort((char *) n->peers,
1116           n->n_peers,
1117           sizeof(net_db_peer),
1118           sortPeerByRtt);
1119
1120 #endif
1121 }
1122
1123 void
1124 netdbExchangeUpdatePeer(Ip::Address &addr, CachePeer * e, double rtt, double hops)
1125 {
1126 #if USE_ICMP
1127     netdbEntry *n;
1128     net_db_peer *p;
1129     debugs(38, 5, "netdbExchangeUpdatePeer: '" << addr << "', "<<
1130            std::setfill('0')<< std::setprecision(2) << hops << " hops, " <<
1131            rtt << " rtt");
1132
1133     if ( !addr.isIPv4() ) {
1134         debugs(38, 5, "netdbExchangeUpdatePeer: Aborting peer update for '" << addr << "', NetDB cannot handle IPv6.");
1135         return;
1136     }
1137
1138     n = netdbLookupAddr(addr);
1139
1140     if (n == NULL)
1141         n = netdbAdd(addr);
1142
1143     assert(NULL != n);
1144
1145     if ((p = netdbPeerByName(n, e->host)) == NULL)
1146         p = netdbPeerAdd(n, e);
1147
1148     p->rtt = rtt;
1149
1150     p->hops = hops;
1151
1152     p->expires = squid_curtime + 3600;  /* XXX ? */
1153
1154     if (n->n_peers < 2)
1155         return;
1156
1157     qsort((char *) n->peers,
1158           n->n_peers,
1159           sizeof(net_db_peer),
1160           sortPeerByRtt);
1161
1162 #endif
1163 }
1164
1165 void
1166 netdbDeleteAddrNetwork(Ip::Address &addr)
1167 {
1168 #if USE_ICMP
1169     netdbEntry *n = netdbLookupAddr(addr);
1170
1171     if (n == NULL)
1172         return;
1173
1174     debugs(38, 3, "netdbDeleteAddrNetwork: " << n->network);
1175
1176     netdbRelease(n);
1177 #endif
1178 }
1179
1180 void
1181 netdbBinaryExchange(StoreEntry * s)
1182 {
1183     HttpReply *reply = new HttpReply;
1184 #if USE_ICMP
1185
1186     Ip::Address addr;
1187
1188     netdbEntry *n;
1189     int i;
1190     int j;
1191     int rec_sz;
1192     char *buf;
1193
1194     struct in_addr line_addr;
1195     s->buffer();
1196     reply->setHeaders(Http::scOkay, "OK", NULL, -1, squid_curtime, -2);
1197     s->replaceHttpReply(reply);
1198     rec_sz = 0;
1199     rec_sz += 1 + sizeof(struct in_addr);
1200     rec_sz += 1 + sizeof(int);
1201     rec_sz += 1 + sizeof(int);
1202     buf = (char *)memAllocate(MEM_4K_BUF);
1203     i = 0;
1204     hash_first(addr_table);
1205
1206     while ((n = (netdbEntry *) hash_next(addr_table))) {
1207         if (0.0 == n->rtt)
1208             continue;
1209
1210         if (n->rtt > 60000)     /* RTT > 1 MIN probably bogus */
1211             continue;
1212
1213         if (! (addr = n->network) )
1214             continue;
1215
1216         /* FIXME INET6 : NetDB cannot yet handle IPv6 addresses. Ensure only IPv4 get sent. */
1217         if ( !addr.isIPv4() )
1218             continue;
1219
1220         buf[i] = (char) NETDB_EX_NETWORK;
1221         ++i;
1222
1223         addr.getInAddr(line_addr);
1224         memcpy(&buf[i], &line_addr, sizeof(struct in_addr));
1225
1226         i += sizeof(struct in_addr);
1227
1228         buf[i] = (char) NETDB_EX_RTT;
1229         ++i;
1230
1231         j = htonl((int) (n->rtt * 1000));
1232
1233         memcpy(&buf[i], &j, sizeof(int));
1234
1235         i += sizeof(int);
1236
1237         buf[i] = (char) NETDB_EX_HOPS;
1238         ++i;
1239
1240         j = htonl((int) (n->hops * 1000));
1241
1242         memcpy(&buf[i], &j, sizeof(int));
1243
1244         i += sizeof(int);
1245
1246         if (i + rec_sz > 4096) {
1247             s->append(buf, i);
1248             i = 0;
1249         }
1250     }
1251
1252     if (i > 0) {
1253         s->append(buf, i);
1254         i = 0;
1255     }
1256
1257     assert(0 == i);
1258     s->flush();
1259     memFree(buf, MEM_4K_BUF);
1260 #else
1261
1262     reply->setHeaders(Http::scBadRequest, "Bad Request", NULL, -1, squid_curtime, -2);
1263     s->replaceHttpReply(reply);
1264     storeAppendPrintf(s, "NETDB support not compiled into this Squid cache.\n");
1265 #endif
1266
1267     s->complete();
1268 }
1269
1270 #if USE_ICMP
1271 CBDATA_TYPE(netdbExchangeState);
1272 #endif
1273
1274 void
1275 netdbExchangeStart(void *data)
1276 {
1277 #if USE_ICMP
1278     CachePeer *p = (CachePeer *)data;
1279     char *uri;
1280     netdbExchangeState *ex;
1281     StoreIOBuffer tempBuffer;
1282     CBDATA_INIT_TYPE(netdbExchangeState);
1283     ex = cbdataAlloc(netdbExchangeState);
1284     ex->p = cbdataReference(p);
1285     uri = internalRemoteUri(p->host, p->http_port, "/squid-internal-dynamic/", "netdb");
1286     debugs(38, 3, "netdbExchangeStart: Requesting '" << uri << "'");
1287     assert(NULL != uri);
1288     ex->r = HttpRequest::CreateFromUrl(uri);
1289
1290     if (NULL == ex->r) {
1291         debugs(38, DBG_IMPORTANT, "netdbExchangeStart: Bad URI " << uri);
1292         return;
1293     }
1294
1295     HTTPMSGLOCK(ex->r);
1296     assert(NULL != ex->r);
1297     ex->r->http_ver = Http::ProtocolVersion(1,1);
1298     ex->connstate = STATE_HEADER;
1299     ex->e = storeCreateEntry(uri, uri, RequestFlags(), Http::METHOD_GET);
1300     ex->buf_sz = NETDB_REQBUF_SZ;
1301     assert(NULL != ex->e);
1302     ex->sc = storeClientListAdd(ex->e, ex);
1303     tempBuffer.offset = 0;
1304     tempBuffer.length = ex->buf_sz;
1305     tempBuffer.data = ex->buf;
1306     storeClientCopy(ex->sc, ex->e, tempBuffer,
1307                     netdbExchangeHandleReply, ex);
1308     ex->r->flags.loopDetected = true;   /* cheat! -- force direct */
1309
1310     if (p->login)
1311         xstrncpy(ex->r->login, p->login, MAX_LOGIN_SZ);
1312
1313     urlCanonical(ex->r);
1314
1315     FwdState::fwdStart(Comm::ConnectionPointer(), ex->e, ex->r);
1316
1317 #endif
1318 }
1319
1320 CachePeer *
1321 netdbClosestParent(HttpRequest * request)
1322 {
1323 #if USE_ICMP
1324     CachePeer *p = NULL;
1325     netdbEntry *n;
1326     const ipcache_addrs *ia;
1327     net_db_peer *h;
1328     int i;
1329     n = netdbLookupHost(request->GetHost());
1330
1331     if (NULL == n) {
1332         /* try IP addr */
1333         ia = ipcache_gethostbyname(request->GetHost(), 0);
1334
1335         if (NULL != ia)
1336             n = netdbLookupAddr(ia->in_addrs[ia->cur]);
1337     }
1338
1339     if (NULL == n)
1340         return NULL;
1341
1342     if (0 == n->n_peers)
1343         return NULL;
1344
1345     n->last_use_time = squid_curtime;
1346
1347     /*
1348      * Find the parent with the least RTT to the origin server.
1349      * Make sure we don't return a parent who is farther away than
1350      * we are.  Note, the n->peers list is pre-sorted by RTT.
1351      */
1352     for (i = 0; i < n->n_peers; ++i) {
1353         h = &n->peers[i];
1354
1355         if (n->rtt > 0)
1356             if (n->rtt < h->rtt)
1357                 break;
1358
1359         p = peerFindByName(h->peername);
1360
1361         if (NULL == p)          /* not found */
1362             continue;
1363
1364         if (neighborType(p, request) != PEER_PARENT)
1365             continue;
1366
1367         if (!peerHTTPOkay(p, request))  /* not allowed */
1368             continue;
1369
1370         return p;
1371     }
1372
1373 #endif
1374     return NULL;
1375 }