src/icmp/net_db.cc

   1 /*
   2  * Copyright (C) 1996-2016 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 38    Network Measurement Database */
  10
  11 /*
  12  * XXX XXX XXX
  13  *
  14  * This code may be slightly broken now. If you're getting consistent
  15  * (sometimes working) corrupt data exchanges, please contact adrian
  16  * (adrian@squid-cache.org) to sort them out.
  17  */
  18
  19 #include "squid.h"
  20 #include "CachePeer.h"
  21 #include "cbdata.h"
  22 #include "event.h"
  23 #include "fde.h"
  24 #include "fs_io.h"
  25 #include "FwdState.h"
  26 #include "HttpReply.h"
  27 #include "HttpRequest.h"
  28 #include "icmp/net_db.h"
  29 #include "internal.h"
  30 #include "ip/Address.h"
  31 #include "log/File.h"
  32 #include "MemObject.h"
  33 #include "mgr/Registration.h"
  34 #include "mime_header.h"
  35 #include "neighbors.h"
  36 #include "SquidConfig.h"
  37 #include "SquidTime.h"
  38 #include "Store.h"
  39 #include "StoreClient.h"
  40 #include "tools.h"
  41 #include "URL.h"
  42 #include "wordlist.h"
  43
  44 #if HAVE_SYS_STAT_H
  45 #include <sys/stat.h>
  46 #endif
  47
  48 #if USE_ICMP
  49 #include "icmp/IcmpSquid.h"
  50 #include "ipcache.h"
  51 #include "StoreClient.h"
  52
  53 #define NETDB_REQBUF_SZ 4096
  54
  55 typedef enum {
  56     STATE_NONE,
  57     STATE_HEADER,
  58     STATE_BODY
  59 } netdb_conn_state_t;
  60
  61 class netdbExchangeState
  62 {
  63     CBDATA_CLASS(netdbExchangeState);
  64
  65 public:
  66     netdbExchangeState(CachePeer *aPeer, HttpRequest *theReq) :
  67         p(cbdataReference(aPeer)),
  68         e(NULL),
  69         sc(NULL),
  70         r(theReq),
  71         used(0),
  72         buf_sz(NETDB_REQBUF_SZ),
  73         buf_ofs(0),
  74         connstate(STATE_HEADER)
  75     {
  76         *buf = 0;
  77
  78         assert(NULL != r);
  79         HTTPMSGLOCK(r);
  80         // TODO: check if we actually need to do this. should be implicit
  81         r->http_ver = Http::ProtocolVersion();
  82     }
  83
  84     ~netdbExchangeState() {
  85         debugs(38, 3, e->url());
  86         storeUnregister(sc, e, this);
  87         e->unlock("netdbExchangeDone");
  88         HTTPMSGUNLOCK(r);
  89         cbdataReferenceDone(p);
  90     }
  91
  92     CachePeer *p;
  93     StoreEntry *e;
  94     store_client *sc;
  95     HttpRequest *r;
  96     int64_t used;
  97     size_t buf_sz;
  98     char buf[NETDB_REQBUF_SZ];
  99     int buf_ofs;
 100     netdb_conn_state_t connstate;
 101 };
 102
 103 CBDATA_CLASS_INIT(netdbExchangeState);
 104
 105 static hash_table *addr_table = NULL;
 106 static hash_table *host_table = NULL;
 107
 108 Ip::Address networkFromInaddr(const Ip::Address &a);
 109 static void netdbRelease(netdbEntry * n);
 110
 111 static void netdbHashInsert(netdbEntry * n, Ip::Address &addr);
 112 static void netdbHashDelete(const char *key);
 113 static void netdbHostInsert(netdbEntry * n, const char *hostname);
 114 static void netdbHostDelete(const net_db_name * x);
 115 static void netdbPurgeLRU(void);
 116 static netdbEntry *netdbLookupHost(const char *key);
 117 static net_db_peer *netdbPeerByName(const netdbEntry * n, const char *);
 118 static net_db_peer *netdbPeerAdd(netdbEntry * n, CachePeer * e);
 119 static const char *netdbPeerName(const char *name);
 120 static IPH netdbSendPing;
 121 static FREE netdbFreeNameEntry;
 122 static FREE netdbFreeNetdbEntry;
 123 static STCB netdbExchangeHandleReply;
 124
 125 /* We have to keep a local list of CachePeer names.  The Peers structure
 126  * gets freed during a reconfigure.  We want this database to
 127  * remain persisitent, so _net_db_peer->peername points into this
 128  * linked list */
 129 static wordlist *peer_names = NULL;
 130
 131 static void
 132 netdbHashInsert(netdbEntry * n, Ip::Address &addr)
 133 {
 134     networkFromInaddr(addr).toStr(n->network, MAX_IPSTRLEN);
 135     n->hash.key = n->network;
 136     assert(hash_lookup(addr_table, n->network) == NULL);
 137     hash_join(addr_table, &n->hash);
 138 }
 139
 140 static void
 141 netdbHashDelete(const char *key)
 142 {
 143     hash_link *hptr = (hash_link *)hash_lookup(addr_table, key);
 144
 145     if (hptr == NULL) {
 146         debug_trap("netdbHashDelete: key not found");
 147         return;
 148     }
 149
 150     hash_remove_link(addr_table, hptr);
 151 }
 152
 153 net_db_name::net_db_name(const char *hostname, netdbEntry *e) :
 154     next(e ? e->hosts : nullptr),
 155     net_db_entry(e)
 156 {
 157     hash.key = xstrdup(hostname);
 158     if (e) {
 159         e->hosts = this;
 160         ++ e->link_count;
 161     }
 162 }
 163
 164 static void
 165 netdbHostInsert(netdbEntry * n, const char *hostname)
 166 {
 167     net_db_name *x = new net_db_name(hostname, n);
 168     assert(hash_lookup(host_table, hostname) == NULL);
 169     hash_join(host_table, &x->hash);
 170 }
 171
 172 static void
 173 netdbHostDelete(const net_db_name * x)
 174 {
 175     assert(x != NULL);
 176     assert(x->net_db_entry != NULL);
 177
 178     netdbEntry *n = x->net_db_entry;
 179     -- n->link_count;
 180
 181     for (auto **X = &n->hosts; *X; X = &(*X)->next) {
 182         if (*X == x) {
 183             *X = x->next;
 184             break;
 185         }
 186     }
 187
 188     hash_remove_link(host_table, (hash_link *) x);
 189     delete x;
 190 }
 191
 192 static netdbEntry *
 193 netdbLookupHost(const char *key)
 194 {
 195     net_db_name *x = (net_db_name *) hash_lookup(host_table, key);
 196     return x ? x->net_db_entry : NULL;
 197 }
 198
 199 static void
 200 netdbRelease(netdbEntry * n)
 201 {
 202     net_db_name *x;
 203     net_db_name *next;
 204
 205     for (x = n->hosts; x; x = next) {
 206         next = x->next;
 207         netdbHostDelete(x);
 208     }
 209
 210     n->hosts = NULL;
 211     safe_free(n->peers);
 212     n->peers = NULL;
 213     n->n_peers = 0;
 214     n->n_peers_alloc = 0;
 215
 216     if (n->link_count == 0) {
 217         netdbHashDelete(n->network);
 218         memFree(n, MEM_NETDBENTRY);
 219     }
 220 }
 221
 222 static int
 223 netdbLRU(const void *A, const void *B)
 224 {
 225     const netdbEntry *const *n1 = (const netdbEntry *const *)A;
 226     const netdbEntry *const *n2 = (const netdbEntry *const *)B;
 227
 228     if ((*n1)->last_use_time > (*n2)->last_use_time)
 229         return (1);
 230
 231     if ((*n1)->last_use_time < (*n2)->last_use_time)
 232         return (-1);
 233
 234     return (0);
 235 }
 236
 237 static void
 238 netdbPurgeLRU(void)
 239 {
 240     netdbEntry *n;
 241     netdbEntry **list;
 242     int k = 0;
 243     int list_count = 0;
 244     int removed = 0;
 245     list = (netdbEntry **)xcalloc(memInUse(MEM_NETDBENTRY), sizeof(netdbEntry *));
 246     hash_first(addr_table);
 247
 248     while ((n = (netdbEntry *) hash_next(addr_table))) {
 249         assert(list_count < memInUse(MEM_NETDBENTRY));
 250         *(list + list_count) = n;
 251         ++list_count;
 252     }
 253
 254     qsort((char *) list,
 255           list_count,
 256           sizeof(netdbEntry *),
 257           netdbLRU);
 258
 259     for (k = 0; k < list_count; ++k) {
 260         if (memInUse(MEM_NETDBENTRY) < Config.Netdb.low)
 261             break;
 262
 263         netdbRelease(*(list + k));
 264
 265         ++removed;
 266     }
 267
 268     xfree(list);
 269 }
 270
 271 static netdbEntry *
 272 netdbLookupAddr(const Ip::Address &addr)
 273 {
 274     netdbEntry *n;
 275     char *key = new char[MAX_IPSTRLEN];
 276     networkFromInaddr(addr).toStr(key,MAX_IPSTRLEN);
 277     n = (netdbEntry *) hash_lookup(addr_table, key);
 278     delete[] key;
 279     return n;
 280 }
 281
 282 static netdbEntry *
 283 netdbAdd(Ip::Address &addr)
 284 {
 285     netdbEntry *n;
 286
 287     if (memInUse(MEM_NETDBENTRY) > Config.Netdb.high)
 288         netdbPurgeLRU();
 289
 290     if ((n = netdbLookupAddr(addr)) == NULL) {
 291         n = (netdbEntry *)memAllocate(MEM_NETDBENTRY);
 292         netdbHashInsert(n, addr);
 293     }
 294
 295     return n;
 296 }
 297
 298 static void
 299 netdbSendPing(const ipcache_addrs *ia, const Dns::LookupDetails &, void *data)
 300 {
 301     Ip::Address addr;
 302     char *hostname = NULL;
 303     static_cast<generic_cbdata *>(data)->unwrap(&hostname);
 304     netdbEntry *n;
 305     netdbEntry *na;
 306     net_db_name *x;
 307     net_db_name **X;
 308
 309     if (ia == NULL) {
 310         xfree(hostname);
 311         return;
 312     }
 313
 314     addr = ia->in_addrs[ia->cur];
 315
 316     if ((n = netdbLookupHost(hostname)) == NULL) {
 317         n = netdbAdd(addr);
 318         netdbHostInsert(n, hostname);
 319     } else if ((na = netdbLookupAddr(addr)) != n) {
 320         /*
 321          *hostname moved from 'network n' to 'network na'!
 322          */
 323
 324         if (na == NULL)
 325             na = netdbAdd(addr);
 326
 327         debugs(38, 3, "netdbSendPing: " << hostname << " moved from " << n->network << " to " << na->network);
 328
 329         x = (net_db_name *) hash_lookup(host_table, hostname);
 330
 331         if (x == NULL) {
 332             debugs(38, DBG_IMPORTANT, "netdbSendPing: net_db_name list bug: " << hostname << " not found");
 333             xfree(hostname);
 334             return;
 335         }
 336
 337         /* remove net_db_name from 'network n' linked list */
 338         for (X = &n->hosts; *X; X = &(*X)->next) {
 339             if (*X == x) {
 340                 *X = x->next;
 341                 break;
 342             }
 343         }
 344
 345         -- n->link_count;
 346         /* point to 'network na' from host entry */
 347         x->net_db_entry = na;
 348         /* link net_db_name to 'network na' */
 349         x->next = na->hosts;
 350         na->hosts = x;
 351         ++ na->link_count;
 352         n = na;
 353     }
 354
 355     if (n->next_ping_time <= squid_curtime) {
 356         debugs(38, 3, "netdbSendPing: pinging " << hostname);
 357         icmpEngine.DomainPing(addr, hostname);
 358         ++ n->pings_sent;
 359         n->next_ping_time = squid_curtime + Config.Netdb.period;
 360         n->last_use_time = squid_curtime;
 361     }
 362
 363     xfree(hostname);
 364 }
 365
 366 Ip::Address
 367 networkFromInaddr(const Ip::Address &in)
 368 {
 369     Ip::Address out;
 370
 371     out = in;
 372
 373     /* in IPv6 the 'network' should be the routing section. */
 374     if ( in.isIPv6() ) {
 375         out.applyMask(64, AF_INET6);
 376         debugs(14, 5, "networkFromInaddr : Masked IPv6 Address to " << in << "/64 routing part.");
 377         return out;
 378     }
 379
 380 #if USE_CLASSFUL
 381     struct in_addr b;
 382
 383     in.getInAddr(b);
 384
 385     if (IN_CLASSC(b.s_addr))
 386         b.s_addr &= IN_CLASSC_NET;
 387     else if (IN_CLASSB(b.s_addr))
 388         b.s_addr &= IN_CLASSB_NET;
 389     else if (IN_CLASSA(b.s_addr))
 390         b.s_addr &= IN_CLASSA_NET;
 391
 392     out = b;
 393
 394 #endif
 395
 396     debugs(14, 5, "networkFromInaddr : Masked IPv4 Address to " << out << "/24.");
 397
 398     /* use /24 for everything under IPv4 */
 399     out.applyMask(24, AF_INET);
 400     debugs(14, 5, "networkFromInaddr : Masked IPv4 Address to " << in << "/24.");
 401
 402     return out;
 403 }
 404
 405 static int
 406 sortByRtt(const void *A, const void *B)
 407 {
 408     const netdbEntry *const *n1 = (const netdbEntry *const *)A;
 409     const netdbEntry *const *n2 = (const netdbEntry *const *)B;
 410
 411     if ((*n1)->rtt > (*n2)->rtt)
 412         return 1;
 413     else if ((*n1)->rtt < (*n2)->rtt)
 414         return -1;
 415     else
 416         return 0;
 417 }
 418
 419 static net_db_peer *
 420 netdbPeerByName(const netdbEntry * n, const char *peername)
 421 {
 422     int i;
 423     net_db_peer *p = n->peers;
 424
 425     for (i = 0; i < n->n_peers; ++i, ++p) {
 426         if (!strcmp(p->peername, peername))
 427             return p;
 428     }
 429
 430     return NULL;
 431 }
 432
 433 static net_db_peer *
 434 netdbPeerAdd(netdbEntry * n, CachePeer * e)
 435 {
 436     net_db_peer *p;
 437     net_db_peer *o;
 438     int osize;
 439     int i;
 440
 441     if (n->n_peers == n->n_peers_alloc) {
 442         o = n->peers;
 443         osize = n->n_peers_alloc;
 444
 445         if (n->n_peers_alloc == 0)
 446             n->n_peers_alloc = 2;
 447         else
 448             n->n_peers_alloc <<= 1;
 449
 450         debugs(38, 3, "netdbPeerAdd: Growing peer list for '" << n->network << "' to " << n->n_peers_alloc);
 451
 452         n->peers = (net_db_peer *)xcalloc(n->n_peers_alloc, sizeof(net_db_peer));
 453
 454         for (i = 0; i < osize; ++i)
 455             *(n->peers + i) = *(o + i);
 456
 457         if (osize) {
 458             safe_free(o);
 459         }
 460     }
 461
 462     p = n->peers + n->n_peers;
 463     p->peername = netdbPeerName(e->host);
 464     ++ n->n_peers;
 465     return p;
 466 }
 467
 468 static int
 469 sortPeerByRtt(const void *A, const void *B)
 470 {
 471     const net_db_peer *p1 = (net_db_peer *)A;
 472     const net_db_peer *p2 = (net_db_peer *)B;
 473
 474     if (p1->rtt > p2->rtt)
 475         return 1;
 476     else if (p1->rtt < p2->rtt)
 477         return -1;
 478     else
 479         return 0;
 480 }
 481
 482 static void
 483 netdbSaveState(void *foo)
 484 {
 485     if (strcmp(Config.netdbFilename, "none") == 0)
 486         return;
 487
 488     Logfile *lf;
 489     netdbEntry *n;
 490     net_db_name *x;
 491
 492     struct timeval start = current_time;
 493     int count = 0;
 494     /*
 495      * This was nicer when we were using stdio, but thanks to
 496      * Solaris bugs, its a bad idea.  fopen can fail if more than
 497      * 256 FDs are open.
 498      */
 499     /*
 500      * unlink() is here because there is currently no way to make
 501      * logfileOpen() use O_TRUNC.
 502      */
 503     unlink(Config.netdbFilename);
 504     lf = logfileOpen(Config.netdbFilename, 4096, 0);
 505
 506     if (lf) {
 507         int xerrno = errno;
 508         debugs(50, DBG_IMPORTANT, MYNAME << Config.netdbFilename << ": " << xstrerr(xerrno));
 509         return;
 510     }
 511
 512     hash_first(addr_table);
 513
 514     while ((n = (netdbEntry *) hash_next(addr_table))) {
 515         if (n->pings_recv == 0)
 516             continue;
 517
 518         logfilePrintf(lf, "%s %d %d %10.5f %10.5f %d %d",
 519                       n->network,
 520                       n->pings_sent,
 521                       n->pings_recv,
 522                       n->hops,
 523                       n->rtt,
 524                       (int) n->next_ping_time,
 525                       (int) n->last_use_time);
 526
 527         for (x = n->hosts; x; x = x->next)
 528             logfilePrintf(lf, " %s", hashKeyStr(&x->hash));
 529
 530         logfilePrintf(lf, "\n");
 531
 532         ++count;
 533
 534 #undef RBUF_SZ
 535
 536     }
 537
 538     logfileClose(lf);
 539     getCurrentTime();
 540     debugs(38, DBG_IMPORTANT, "NETDB state saved; " <<
 541            count << " entries, " <<
 542            tvSubMsec(start, current_time) << " msec" );
 543     eventAddIsh("netdbSaveState", netdbSaveState, NULL, 3600.0, 1);
 544 }
 545
 546 static void
 547 netdbReloadState(void)
 548 {
 549     if (strcmp(Config.netdbFilename, "none") == 0)
 550         return;
 551
 552     char *s;
 553     int fd;
 554     int l;
 555
 556     struct stat sb;
 557     netdbEntry *n;
 558     netdbEntry N;
 559
 560     Ip::Address addr;
 561     int count = 0;
 562
 563     struct timeval start = current_time;
 564     /*
 565      * This was nicer when we were using stdio, but thanks to
 566      * Solaris bugs, its a bad idea.  fopen can fail if more than
 567      * 256 FDs are open.
 568      */
 569     fd = file_open(Config.netdbFilename, O_RDONLY | O_BINARY);
 570
 571     if (fd < 0)
 572         return;
 573
 574     if (fstat(fd, &sb) < 0) {
 575         file_close(fd);
 576         return;
 577     }
 578
 579     char *t;
 580     char *buf = (char *)xcalloc(1, sb.st_size + 1);
 581     t = buf;
 582     l = FD_READ_METHOD(fd, buf, sb.st_size);
 583     file_close(fd);
 584
 585     if (l <= 0) {
 586         safe_free (buf);
 587         return;
 588     };
 589
 590     while ((s = strchr(t, '\n'))) {
 591         char *q;
 592         assert(s - buf < l);
 593         *s = '\0';
 594         memset(&N, '\0', sizeof(netdbEntry));
 595         q = strtok(t, w_space);
 596         t = s + 1;
 597
 598         if (NULL == q)
 599             continue;
 600
 601         if (! (addr = q) )
 602             continue;
 603
 604         if (netdbLookupAddr(addr) != NULL)  /* no dups! */
 605             continue;
 606
 607         if ((q = strtok(NULL, w_space)) == NULL)
 608             continue;
 609
 610         N.pings_sent = atoi(q);
 611
 612         if ((q = strtok(NULL, w_space)) == NULL)
 613             continue;
 614
 615         N.pings_recv = atoi(q);
 616
 617         if (N.pings_recv == 0)
 618             continue;
 619
 620         /* give this measurement low weight */
 621         N.pings_sent = 1;
 622
 623         N.pings_recv = 1;
 624
 625         if ((q = strtok(NULL, w_space)) == NULL)
 626             continue;
 627
 628         N.hops = atof(q);
 629
 630         if ((q = strtok(NULL, w_space)) == NULL)
 631             continue;
 632
 633         N.rtt = atof(q);
 634
 635         if ((q = strtok(NULL, w_space)) == NULL)
 636             continue;
 637
 638         N.next_ping_time = (time_t) atoi(q);
 639
 640         if ((q = strtok(NULL, w_space)) == NULL)
 641             continue;
 642
 643         N.last_use_time = (time_t) atoi(q);
 644
 645         n = (netdbEntry *)memAllocate(MEM_NETDBENTRY);
 646
 647         memcpy(n, &N, sizeof(netdbEntry));
 648
 649         netdbHashInsert(n, addr);
 650
 651         while ((q = strtok(NULL, w_space)) != NULL) {
 652             if (netdbLookupHost(q) != NULL) /* no dups! */
 653                 continue;
 654
 655             netdbHostInsert(n, q);
 656         }
 657
 658         ++count;
 659     }
 660
 661     xfree(buf);
 662     getCurrentTime();
 663     debugs(38, DBG_IMPORTANT, "NETDB state reloaded; " <<
 664            count << " entries, " <<
 665            tvSubMsec(start, current_time) << " msec" );
 666 }
 667
 668 static const char *
 669 netdbPeerName(const char *name)
 670 {
 671     const wordlist *w;
 672
 673     for (w = peer_names; w; w = w->next) {
 674         if (!strcmp(w->key, name))
 675             return w->key;
 676     }
 677
 678     return wordlistAdd(&peer_names, name);
 679 }
 680
 681 static void
 682 netdbFreeNetdbEntry(void *data)
 683 {
 684     netdbEntry *n = (netdbEntry *)data;
 685     safe_free(n->peers);
 686     memFree(n, MEM_NETDBENTRY);
 687 }
 688
 689 static void
 690 netdbFreeNameEntry(void *data)
 691 {
 692     net_db_name *x = (net_db_name *)data;
 693     delete x;
 694 }
 695
 696 static void
 697 netdbExchangeHandleReply(void *data, StoreIOBuffer receivedData)
 698 {
 699     Ip::Address addr;
 700
 701     netdbExchangeState *ex = (netdbExchangeState *)data;
 702     int rec_sz = 0;
 703     int o;
 704
 705     struct in_addr line_addr;
 706     double rtt;
 707     double hops;
 708     char *p;
 709     int j;
 710     HttpReply const *rep;
 711     size_t hdr_sz;
 712     int nused = 0;
 713     int size;
 714     int oldbufofs = ex->buf_ofs;
 715
 716     rec_sz = 0;
 717     rec_sz += 1 + sizeof(struct in_addr);
 718     rec_sz += 1 + sizeof(int);
 719     rec_sz += 1 + sizeof(int);
 720     debugs(38, 3, "netdbExchangeHandleReply: " << receivedData.length << " read bytes");
 721
 722     if (!cbdataReferenceValid(ex->p)) {
 723         debugs(38, 3, "netdbExchangeHandleReply: Peer became invalid");
 724         delete ex;
 725         return;
 726     }
 727
 728     debugs(38, 3, "netdbExchangeHandleReply: for '" << ex->p->host << ":" << ex->p->http_port << "'");
 729
 730     if (receivedData.length == 0 && !receivedData.flags.error) {
 731         debugs(38, 3, "netdbExchangeHandleReply: Done");
 732         delete ex;
 733         return;
 734     }
 735
 736     p = ex->buf;
 737
 738     /* Get the size of the buffer now */
 739     size = ex->buf_ofs + receivedData.length;
 740     debugs(38, 3, "netdbExchangeHandleReply: " << size << " bytes buf");
 741
 742     /* Check if we're still doing headers */
 743
 744     if (ex->connstate == STATE_HEADER) {
 745
 746         ex->buf_ofs += receivedData.length;
 747
 748         /* skip reply headers */
 749
 750         if ((hdr_sz = headersEnd(p, ex->buf_ofs))) {
 751             debugs(38, 5, "netdbExchangeHandleReply: hdr_sz = " << hdr_sz);
 752             rep = ex->e->getReply();
 753             assert(rep->sline.status() != Http::scNone);
 754             debugs(38, 3, "netdbExchangeHandleReply: reply status " << rep->sline.status());
 755
 756             if (rep->sline.status() != Http::scOkay) {
 757                 delete ex;
 758                 return;
 759             }
 760
 761             assert((size_t)ex->buf_ofs >= hdr_sz);
 762
 763             /*
 764              * Now, point p to the part of the buffer where the data
 765              * starts, and update the size accordingly
 766              */
 767             assert(ex->used == 0);
 768             ex->used = hdr_sz;
 769             size = ex->buf_ofs - hdr_sz;
 770             p += hdr_sz;
 771
 772             /* Finally, set the conn state mode to STATE_BODY */
 773             ex->connstate = STATE_BODY;
 774         } else {
 775             StoreIOBuffer tempBuffer;
 776             tempBuffer.offset = ex->buf_ofs;
 777             tempBuffer.length = ex->buf_sz - ex->buf_ofs;
 778             tempBuffer.data = ex->buf + ex->buf_ofs;
 779             /* Have more headers .. */
 780             storeClientCopy(ex->sc, ex->e, tempBuffer,
 781                             netdbExchangeHandleReply, ex);
 782             return;
 783         }
 784     }
 785
 786     assert(ex->connstate == STATE_BODY);
 787
 788     /* If we get here, we have some body to parse .. */
 789     debugs(38, 5, "netdbExchangeHandleReply: start parsing loop, size = " << size);
 790
 791     while (size >= rec_sz) {
 792         debugs(38, 5, "netdbExchangeHandleReply: in parsing loop, size = " << size);
 793         addr.setAnyAddr();
 794         hops = rtt = 0.0;
 795
 796         for (o = 0; o < rec_sz;) {
 797             switch ((int) *(p + o)) {
 798
 799             case NETDB_EX_NETWORK:
 800                 ++o;
 801                 /* FIXME INET6 : NetDB can still ony send IPv4 */
 802                 memcpy(&line_addr, p + o, sizeof(struct in_addr));
 803                 addr = line_addr;
 804                 o += sizeof(struct in_addr);
 805                 break;
 806
 807             case NETDB_EX_RTT:
 808                 ++o;
 809                 memcpy(&j, p + o, sizeof(int));
 810                 o += sizeof(int);
 811                 rtt = (double) ntohl(j) / 1000.0;
 812                 break;
 813
 814             case NETDB_EX_HOPS:
 815                 ++o;
 816                 memcpy(&j, p + o, sizeof(int));
 817                 o += sizeof(int);
 818                 hops = (double) ntohl(j) / 1000.0;
 819                 break;
 820
 821             default:
 822                 debugs(38, DBG_IMPORTANT, "netdbExchangeHandleReply: corrupt data, aborting");
 823                 delete ex;
 824                 return;
 825             }
 826         }
 827
 828         if (!addr.isAnyAddr() && rtt > 0)
 829             netdbExchangeUpdatePeer(addr, ex->p, rtt, hops);
 830
 831         assert(o == rec_sz);
 832
 833         ex->used += rec_sz;
 834
 835         size -= rec_sz;
 836
 837         p += rec_sz;
 838
 839         ++nused;
 840     }
 841
 842     /*
 843      * Copy anything that is left over to the beginning of the buffer,
 844      * and adjust buf_ofs accordingly
 845      */
 846
 847     /*
 848      * Evilly, size refers to the buf size left now,
 849      * ex->buf_ofs is the original buffer size, so just copy that
 850      * much data over
 851      */
 852     memmove(ex->buf, ex->buf + (ex->buf_ofs - size), size);
 853
 854     ex->buf_ofs = size;
 855
 856     /*
 857      * And don't re-copy the remaining data ..
 858      */
 859     ex->used += size;
 860
 861     /*
 862      * Now the tricky bit - size _included_ the leftover bit from the _last_
 863      * storeClientCopy. We don't want to include that, or our offset will be wrong.
 864      * So, don't count the size of the leftover buffer we began with.
 865      * This can _disappear_ when we're not tracking offsets ..
 866      */
 867     ex->used -= oldbufofs;
 868
 869     debugs(38, 3, "netdbExchangeHandleReply: size left over in this buffer: " << size << " bytes");
 870
 871     debugs(38, 3, "netdbExchangeHandleReply: used " << nused <<
 872            " entries, (x " << rec_sz << " bytes) == " << nused * rec_sz <<
 873            " bytes total");
 874
 875     debugs(38, 3, "netdbExchangeHandleReply: used " << ex->used);
 876
 877     if (EBIT_TEST(ex->e->flags, ENTRY_ABORTED)) {
 878         debugs(38, 3, "netdbExchangeHandleReply: ENTRY_ABORTED");
 879         delete ex;
 880     } else if (ex->e->store_status == STORE_PENDING) {
 881         StoreIOBuffer tempBuffer;
 882         tempBuffer.offset = ex->used;
 883         tempBuffer.length = ex->buf_sz - ex->buf_ofs;
 884         tempBuffer.data = ex->buf + ex->buf_ofs;
 885         debugs(38, 3, "netdbExchangeHandleReply: EOF not received");
 886         storeClientCopy(ex->sc, ex->e, tempBuffer,
 887                         netdbExchangeHandleReply, ex);
 888     }
 889 }
 890
 891 static void
 892 netdbRegisterWithCacheManager(void)
 893 {
 894     Mgr::RegisterAction("netdb", "Network Measurement Database", netdbDump, 0, 1);
 895 }
 896
 897 #endif /* USE_ICMP */
 898
 899 /* PUBLIC FUNCTIONS */
 900
 901 void
 902 netdbInit(void)
 903 {
 904 #if USE_ICMP
 905     int n;
 906
 907     netdbRegisterWithCacheManager();
 908
 909     if (addr_table)
 910         return;
 911
 912     n = hashPrime(Config.Netdb.high / 4);
 913
 914     addr_table = hash_create((HASHCMP *) strcmp, n, hash_string);
 915
 916     n = hashPrime(3 * Config.Netdb.high / 4);
 917
 918     host_table = hash_create((HASHCMP *) strcmp, n, hash_string);
 919
 920     eventAddIsh("netdbSaveState", netdbSaveState, NULL, 3600.0, 1);
 921
 922     netdbReloadState();
 923
 924 #endif
 925 }
 926
 927 void
 928 netdbPingSite(const char *hostname)
 929 {
 930 #if USE_ICMP
 931     netdbEntry *n;
 932
 933     if ((n = netdbLookupHost(hostname)) != NULL)
 934         if (n->next_ping_time > squid_curtime)
 935             return;
 936
 937     ipcache_nbgethostbyname(hostname, netdbSendPing,
 938                             new generic_cbdata(xstrdup(hostname)));
 939
 940 #endif
 941 }
 942
 943 void
 944 netdbHandlePingReply(const Ip::Address &from, int hops, int rtt)
 945 {
 946 #if USE_ICMP
 947     netdbEntry *n;
 948     int N;
 949     debugs(38, 3, "netdbHandlePingReply: from " << from);
 950
 951     if ((n = netdbLookupAddr(from)) == NULL)
 952         return;
 953
 954     N = ++n->pings_recv;
 955
 956     if (N > 5)
 957         N = 5;
 958
 959     if (rtt < 1)
 960         rtt = 1;
 961
 962     n->hops = ((n->hops * (N - 1)) + hops) / N;
 963
 964     n->rtt = ((n->rtt * (N - 1)) + rtt) / N;
 965
 966     debugs(38, 3, "netdbHandlePingReply: " << n->network  << "; rtt="<<
 967            std::setw(5)<< std::setprecision(2) << n->rtt << "  hops="<<
 968            std::setw(4) << n->hops);
 969
 970 #endif
 971 }
 972
 973 void
 974 netdbFreeMemory(void)
 975 {
 976 #if USE_ICMP
 977     hashFreeItems(addr_table, netdbFreeNetdbEntry);
 978     hashFreeMemory(addr_table);
 979     addr_table = NULL;
 980     hashFreeItems(host_table, netdbFreeNameEntry);
 981     hashFreeMemory(host_table);
 982     host_table = NULL;
 983     wordlistDestroy(&peer_names);
 984     peer_names = NULL;
 985 #endif
 986 }
 987
 988 void
 989 netdbDump(StoreEntry * sentry)
 990 {
 991 #if USE_ICMP
 992     netdbEntry *n;
 993     netdbEntry **list;
 994     net_db_name *x;
 995     int k;
 996     int i;
 997     int j;
 998     net_db_peer *p;
 999     storeAppendPrintf(sentry, "Network DB Statistics:\n");
1000     storeAppendPrintf(sentry, "%-46.46s %9s %7s %5s %s\n",  /* Max between 16 (IPv4) or 46 (IPv6)   */
1001                       "Network",
1002                       "recv/sent",
1003                       "RTT",
1004                       "Hops",
1005                       "Hostnames");
1006     list = (netdbEntry **)xcalloc(memInUse(MEM_NETDBENTRY), sizeof(netdbEntry *));
1007     i = 0;
1008     hash_first(addr_table);
1009
1010     while ((n = (netdbEntry *) hash_next(addr_table))) {
1011         *(list + i) = n;
1012         ++i;
1013     }
1014
1015     if (i != memInUse(MEM_NETDBENTRY))
1016         debugs(38, DBG_CRITICAL, "WARNING: netdb_addrs count off, found " << i <<
1017                ", expected " << memInUse(MEM_NETDBENTRY));
1018
1019     qsort((char *) list,
1020           i,
1021           sizeof(netdbEntry *),
1022           sortByRtt);
1023
1024     for (k = 0; k < i; ++k) {
1025         n = *(list + k);
1026         storeAppendPrintf(sentry, "%-46.46s %4d/%4d %7.1f %5.1f", /* Max between 16 (IPv4) or 46 (IPv6)   */
1027                           n->network,
1028                           n->pings_recv,
1029                           n->pings_sent,
1030                           n->rtt,
1031                           n->hops);
1032
1033         for (x = n->hosts; x; x = x->next)
1034             storeAppendPrintf(sentry, " %s", hashKeyStr(&x->hash));
1035
1036         storeAppendPrintf(sentry, "\n");
1037
1038         p = n->peers;
1039
1040         for (j = 0; j < n->n_peers; ++j, ++p) {
1041             storeAppendPrintf(sentry, "    %-22.22s %7.1f %5.1f\n",
1042                               p->peername,
1043                               p->rtt,
1044                               p->hops);
1045         }
1046     }
1047
1048     xfree(list);
1049 #else
1050
1051     storeAppendPrintf(sentry,"NETDB support not compiled into this Squid cache.\n");
1052 #endif
1053 }
1054
1055 int
1056 netdbHostHops(const char *host)
1057 {
1058 #if USE_ICMP
1059     netdbEntry *n = netdbLookupHost(host);
1060
1061     if (n) {
1062         n->last_use_time = squid_curtime;
1063         return (int) (n->hops + 0.5);
1064     }
1065
1066 #endif
1067     return 0;
1068 }
1069
1070 int
1071 netdbHostRtt(const char *host)
1072 {
1073 #if USE_ICMP
1074     netdbEntry *n = netdbLookupHost(host);
1075
1076     if (n) {
1077         n->last_use_time = squid_curtime;
1078         return (int) (n->rtt + 0.5);
1079     }
1080
1081 #endif
1082     return 0;
1083 }
1084
1085 void
1086 netdbHostData(const char *host, int *samp, int *rtt, int *hops)
1087 {
1088 #if USE_ICMP
1089     netdbEntry *n = netdbLookupHost(host);
1090
1091     if (n == NULL)
1092         return;
1093
1094     *samp = n->pings_recv;
1095
1096     *rtt = (int) (n->rtt + 0.5);
1097
1098     *hops = (int) (n->hops + 0.5);
1099
1100     n->last_use_time = squid_curtime;
1101
1102 #endif
1103 }
1104
1105 void
1106 netdbUpdatePeer(const URL &url, CachePeer * e, int irtt, int ihops)
1107 {
1108 #if USE_ICMP
1109     netdbEntry *n;
1110     double rtt = (double) irtt;
1111     double hops = (double) ihops;
1112     net_db_peer *p;
1113     debugs(38, 3, url.host() << ", " << ihops << " hops, " << irtt << " rtt");
1114     n = netdbLookupHost(url.host());
1115
1116     if (n == NULL) {
1117         debugs(38, 3, "host " << url.host() << " not found");
1118         return;
1119     }
1120
1121     if ((p = netdbPeerByName(n, e->host)) == NULL)
1122         p = netdbPeerAdd(n, e);
1123
1124     p->rtt = rtt;
1125
1126     p->hops = hops;
1127
1128     p->expires = squid_curtime + 3600;
1129
1130     if (n->n_peers < 2)
1131         return;
1132
1133     qsort((char *) n->peers,
1134           n->n_peers,
1135           sizeof(net_db_peer),
1136           sortPeerByRtt);
1137
1138 #endif
1139 }
1140
1141 void
1142 netdbExchangeUpdatePeer(Ip::Address &addr, CachePeer * e, double rtt, double hops)
1143 {
1144 #if USE_ICMP
1145     netdbEntry *n;
1146     net_db_peer *p;
1147     debugs(38, 5, "netdbExchangeUpdatePeer: '" << addr << "', "<<
1148            std::setfill('0')<< std::setprecision(2) << hops << " hops, " <<
1149            rtt << " rtt");
1150
1151     if ( !addr.isIPv4() ) {
1152         debugs(38, 5, "netdbExchangeUpdatePeer: Aborting peer update for '" << addr << "', NetDB cannot handle IPv6.");
1153         return;
1154     }
1155
1156     n = netdbLookupAddr(addr);
1157
1158     if (n == NULL)
1159         n = netdbAdd(addr);
1160
1161     assert(NULL != n);
1162
1163     if ((p = netdbPeerByName(n, e->host)) == NULL)
1164         p = netdbPeerAdd(n, e);
1165
1166     p->rtt = rtt;
1167
1168     p->hops = hops;
1169
1170     p->expires = squid_curtime + 3600;  /* XXX ? */
1171
1172     if (n->n_peers < 2)
1173         return;
1174
1175     qsort((char *) n->peers,
1176           n->n_peers,
1177           sizeof(net_db_peer),
1178           sortPeerByRtt);
1179
1180 #endif
1181 }
1182
1183 void
1184 netdbDeleteAddrNetwork(Ip::Address &addr)
1185 {
1186 #if USE_ICMP
1187     netdbEntry *n = netdbLookupAddr(addr);
1188
1189     if (n == NULL)
1190         return;
1191
1192     debugs(38, 3, "netdbDeleteAddrNetwork: " << n->network);
1193
1194     netdbRelease(n);
1195 #endif
1196 }
1197
1198 void
1199 netdbBinaryExchange(StoreEntry * s)
1200 {
1201     HttpReply *reply = new HttpReply;
1202 #if USE_ICMP
1203
1204     Ip::Address addr;
1205
1206     netdbEntry *n;
1207     int i;
1208     int j;
1209     int rec_sz;
1210     char *buf;
1211
1212     struct in_addr line_addr;
1213     s->buffer();
1214     reply->setHeaders(Http::scOkay, "OK", NULL, -1, squid_curtime, -2);
1215     s->replaceHttpReply(reply);
1216     rec_sz = 0;
1217     rec_sz += 1 + sizeof(struct in_addr);
1218     rec_sz += 1 + sizeof(int);
1219     rec_sz += 1 + sizeof(int);
1220     buf = (char *)memAllocate(MEM_4K_BUF);
1221     i = 0;
1222     hash_first(addr_table);
1223
1224     while ((n = (netdbEntry *) hash_next(addr_table))) {
1225         if (0.0 == n->rtt)
1226             continue;
1227
1228         if (n->rtt > 60000) /* RTT > 1 MIN probably bogus */
1229             continue;
1230
1231         if (! (addr = n->network) )
1232             continue;
1233
1234         /* FIXME INET6 : NetDB cannot yet handle IPv6 addresses. Ensure only IPv4 get sent. */
1235         if ( !addr.isIPv4() )
1236             continue;
1237
1238         buf[i] = (char) NETDB_EX_NETWORK;
1239         ++i;
1240
1241         addr.getInAddr(line_addr);
1242         memcpy(&buf[i], &line_addr, sizeof(struct in_addr));
1243
1244         i += sizeof(struct in_addr);
1245
1246         buf[i] = (char) NETDB_EX_RTT;
1247         ++i;
1248
1249         j = htonl((int) (n->rtt * 1000));
1250
1251         memcpy(&buf[i], &j, sizeof(int));
1252
1253         i += sizeof(int);
1254
1255         buf[i] = (char) NETDB_EX_HOPS;
1256         ++i;
1257
1258         j = htonl((int) (n->hops * 1000));
1259
1260         memcpy(&buf[i], &j, sizeof(int));
1261
1262         i += sizeof(int);
1263
1264         if (i + rec_sz > 4096) {
1265             s->append(buf, i);
1266             i = 0;
1267         }
1268     }
1269
1270     if (i > 0) {
1271         s->append(buf, i);
1272         i = 0;
1273     }
1274
1275     assert(0 == i);
1276     s->flush();
1277     memFree(buf, MEM_4K_BUF);
1278 #else
1279
1280     reply->setHeaders(Http::scBadRequest, "Bad Request", NULL, -1, squid_curtime, -2);
1281     s->replaceHttpReply(reply);
1282     storeAppendPrintf(s, "NETDB support not compiled into this Squid cache.\n");
1283 #endif
1284
1285     s->complete();
1286 }
1287
1288 void
1289 netdbExchangeStart(void *data)
1290 {
1291 #if USE_ICMP
1292     CachePeer *p = (CachePeer *)data;
1293     static const SBuf netDB("netdb");
1294     char *uri = internalRemoteUri(p->host, p->http_port, "/squid-internal-dynamic/", netDB);
1295     debugs(38, 3, "netdbExchangeStart: Requesting '" << uri << "'");
1296     assert(NULL != uri);
1297     HttpRequest *req = HttpRequest::CreateFromUrl(uri);
1298
1299     if (req == NULL) {
1300         debugs(38, DBG_IMPORTANT, "netdbExchangeStart: Bad URI " << uri);
1301         return;
1302     }
1303
1304     netdbExchangeState *ex = new netdbExchangeState(p, req);
1305     ex->e = storeCreateEntry(uri, uri, RequestFlags(), Http::METHOD_GET);
1306     assert(NULL != ex->e);
1307
1308     StoreIOBuffer tempBuffer;
1309     tempBuffer.length = ex->buf_sz;
1310     tempBuffer.data = ex->buf;
1311
1312     ex->sc = storeClientListAdd(ex->e, ex);
1313
1314     storeClientCopy(ex->sc, ex->e, tempBuffer,
1315                     netdbExchangeHandleReply, ex);
1316     ex->r->flags.loopDetected = true;   /* cheat! -- force direct */
1317
1318     // XXX: send as Proxy-Authenticate instead
1319     if (p->login)
1320         ex->r->url.userInfo(SBuf(p->login));
1321
1322     FwdState::fwdStart(Comm::ConnectionPointer(), ex->e, ex->r);
1323
1324 #endif
1325 }
1326
1327 CachePeer *
1328 netdbClosestParent(HttpRequest * request)
1329 {
1330 #if USE_ICMP
1331     CachePeer *p = NULL;
1332     netdbEntry *n;
1333     const ipcache_addrs *ia;
1334     net_db_peer *h;
1335     int i;
1336     n = netdbLookupHost(request->url.host());
1337
1338     if (NULL == n) {
1339         /* try IP addr */
1340         ia = ipcache_gethostbyname(request->url.host(), 0);
1341
1342         if (NULL != ia)
1343             n = netdbLookupAddr(ia->in_addrs[ia->cur]);
1344     }
1345
1346     if (NULL == n)
1347         return NULL;
1348
1349     if (0 == n->n_peers)
1350         return NULL;
1351
1352     n->last_use_time = squid_curtime;
1353
1354     /*
1355      * Find the parent with the least RTT to the origin server.
1356      * Make sure we don't return a parent who is farther away than
1357      * we are.  Note, the n->peers list is pre-sorted by RTT.
1358      */
1359     for (i = 0; i < n->n_peers; ++i) {
1360         h = &n->peers[i];
1361
1362         if (n->rtt > 0)
1363             if (n->rtt < h->rtt)
1364                 break;
1365
1366         p = peerFindByName(h->peername);
1367
1368         if (NULL == p)      /* not found */
1369             continue;
1370
1371         if (neighborType(p, request->url) != PEER_PARENT)
1372             continue;
1373
1374         if (!peerHTTPOkay(p, request))  /* not allowed */
1375             continue;
1376
1377         return p;
1378     }
1379
1380 #endif
1381     return NULL;
1382 }
1383