src/icmp/net_db.cc

   1 /*
   2  * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 38    Network Measurement Database */
  10
  11 /*
  12  * XXX XXX XXX
  13  *
  14  * This code may be slightly broken now. If you're getting consistent
  15  * (sometimes working) corrupt data exchanges, please contact adrian
  16  * (adrian@squid-cache.org) to sort them out.
  17  */
  18
  19 #include "squid.h"
  20 #include "CachePeer.h"
  21 #include "cbdata.h"
  22 #include "fs_io.h"
  23 #include "event.h"
  24 #include "fde.h"
  25 #include "FwdState.h"
  26 #include "HttpReply.h"
  27 #include "HttpRequest.h"
  28 #include "icmp/net_db.h"
  29 #include "internal.h"
  30 #include "ip/Address.h"
  31 #include "log/File.h"
  32 #include "MemObject.h"
  33 #include "mgr/Registration.h"
  34 #include "mime_header.h"
  35 #include "neighbors.h"
  36 #include "SquidTime.h"
  37 #include "Store.h"
  38 #include "StoreClient.h"
  39 #include "tools.h"
  40 #include "URL.h"
  41 #include "wordlist.h"
  42
  43 #if HAVE_SYS_STAT_H
  44 #include <sys/stat.h>
  45 #endif
  46
  47 #if USE_ICMP
  48 #include "icmp/IcmpSquid.h"
  49 #include "ipcache.h"
  50 #include "StoreClient.h"
  51
  52 #define NETDB_REQBUF_SZ 4096
  53
  54 typedef enum {
  55     STATE_NONE,
  56     STATE_HEADER,
  57     STATE_BODY
  58 } netdb_conn_state_t;
  59
  60 class netdbExchangeState
  61 {
  62     CBDATA_CLASS(netdbExchangeState);
  63
  64 public:
  65     netdbExchangeState(CachePeer *aPeer, HttpRequest *theReq) :
  66         p(cbdataReference(aPeer)),
  67         e(NULL),
  68         sc(NULL),
  69         r(theReq),
  70         used(0),
  71         buf_sz(NETDB_REQBUF_SZ),
  72         buf_ofs(0),
  73         connstate(STATE_HEADER)
  74     {
  75         *buf = 0;
  76
  77         assert(NULL != r);
  78         HTTPMSGLOCK(r);
  79         // TODO: check if we actually need to do this. should be implicit
  80         r->http_ver = Http::ProtocolVersion();
  81     }
  82
  83     ~netdbExchangeState() {
  84         debugs(38, 3, e->url());
  85         storeUnregister(sc, e, this);
  86         e->unlock("netdbExchangeDone");
  87         HTTPMSGUNLOCK(r);
  88         cbdataReferenceDone(p);
  89     }
  90
  91     CachePeer *p;
  92     StoreEntry *e;
  93     store_client *sc;
  94     HttpRequest *r;
  95     int64_t used;
  96     size_t buf_sz;
  97     char buf[NETDB_REQBUF_SZ];
  98     int buf_ofs;
  99     netdb_conn_state_t connstate;
 100 };
 101
 102 CBDATA_CLASS_INIT(netdbExchangeState);
 103
 104 static hash_table *addr_table = NULL;
 105 static hash_table *host_table = NULL;
 106
 107 Ip::Address networkFromInaddr(const Ip::Address &a);
 108 static void netdbRelease(netdbEntry * n);
 109
 110 static void netdbHashInsert(netdbEntry * n, Ip::Address &addr);
 111 static void netdbHashDelete(const char *key);
 112 static void netdbHostInsert(netdbEntry * n, const char *hostname);
 113 static void netdbHostDelete(const net_db_name * x);
 114 static void netdbPurgeLRU(void);
 115 static netdbEntry *netdbLookupHost(const char *key);
 116 static net_db_peer *netdbPeerByName(const netdbEntry * n, const char *);
 117 static net_db_peer *netdbPeerAdd(netdbEntry * n, CachePeer * e);
 118 static const char *netdbPeerName(const char *name);
 119 static IPH netdbSendPing;
 120 static FREE netdbFreeNameEntry;
 121 static FREE netdbFreeNetdbEntry;
 122 static STCB netdbExchangeHandleReply;
 123
 124 /* We have to keep a local list of CachePeer names.  The Peers structure
 125  * gets freed during a reconfigure.  We want this database to
 126  * remain persisitent, so _net_db_peer->peername points into this
 127  * linked list */
 128 static wordlist *peer_names = NULL;
 129
 130 static void
 131 netdbHashInsert(netdbEntry * n, Ip::Address &addr)
 132 {
 133     networkFromInaddr(addr).toStr(n->network, MAX_IPSTRLEN);
 134     n->hash.key = n->network;
 135     assert(hash_lookup(addr_table, n->network) == NULL);
 136     hash_join(addr_table, &n->hash);
 137 }
 138
 139 static void
 140 netdbHashDelete(const char *key)
 141 {
 142     hash_link *hptr = (hash_link *)hash_lookup(addr_table, key);
 143
 144     if (hptr == NULL) {
 145         debug_trap("netdbHashDelete: key not found");
 146         return;
 147     }
 148
 149     hash_remove_link(addr_table, hptr);
 150 }
 151
 152 static void
 153 netdbHostInsert(netdbEntry * n, const char *hostname)
 154 {
 155     net_db_name *x = (net_db_name *)memAllocate(MEM_NET_DB_NAME);
 156     x->hash.key = xstrdup(hostname);
 157     x->next = n->hosts;
 158     n->hosts = x;
 159     x->net_db_entry = n;
 160     assert(hash_lookup(host_table, hostname) == NULL);
 161     hash_join(host_table, &x->hash);
 162     ++ n->link_count;
 163 }
 164
 165 static void
 166 netdbHostDelete(const net_db_name * x)
 167 {
 168     netdbEntry *n;
 169     net_db_name **X;
 170     assert(x != NULL);
 171     assert(x->net_db_entry != NULL);
 172     n = x->net_db_entry;
 173     -- n->link_count;
 174
 175     for (X = &n->hosts; *X; X = &(*X)->next) {
 176         if (*X == x) {
 177             *X = x->next;
 178             break;
 179         }
 180     }
 181
 182     hash_remove_link(host_table, (hash_link *) x);
 183     xfree(x->hash.key);
 184     memFree((void *) x, MEM_NET_DB_NAME);
 185 }
 186
 187 static netdbEntry *
 188 netdbLookupHost(const char *key)
 189 {
 190     net_db_name *x = (net_db_name *) hash_lookup(host_table, key);
 191     return x ? x->net_db_entry : NULL;
 192 }
 193
 194 static void
 195 netdbRelease(netdbEntry * n)
 196 {
 197     net_db_name *x;
 198     net_db_name *next;
 199
 200     for (x = n->hosts; x; x = next) {
 201         next = x->next;
 202         netdbHostDelete(x);
 203     }
 204
 205     n->hosts = NULL;
 206     safe_free(n->peers);
 207     n->peers = NULL;
 208     n->n_peers = 0;
 209     n->n_peers_alloc = 0;
 210
 211     if (n->link_count == 0) {
 212         netdbHashDelete(n->network);
 213         memFree(n, MEM_NETDBENTRY);
 214     }
 215 }
 216
 217 static int
 218 netdbLRU(const void *A, const void *B)
 219 {
 220     const netdbEntry *const *n1 = (const netdbEntry *const *)A;
 221     const netdbEntry *const *n2 = (const netdbEntry *const *)B;
 222
 223     if ((*n1)->last_use_time > (*n2)->last_use_time)
 224         return (1);
 225
 226     if ((*n1)->last_use_time < (*n2)->last_use_time)
 227         return (-1);
 228
 229     return (0);
 230 }
 231
 232 static void
 233 netdbPurgeLRU(void)
 234 {
 235     netdbEntry *n;
 236     netdbEntry **list;
 237     int k = 0;
 238     int list_count = 0;
 239     int removed = 0;
 240     list = (netdbEntry **)xcalloc(memInUse(MEM_NETDBENTRY), sizeof(netdbEntry *));
 241     hash_first(addr_table);
 242
 243     while ((n = (netdbEntry *) hash_next(addr_table))) {
 244         assert(list_count < memInUse(MEM_NETDBENTRY));
 245         *(list + list_count) = n;
 246         ++list_count;
 247     }
 248
 249     qsort((char *) list,
 250           list_count,
 251           sizeof(netdbEntry *),
 252           netdbLRU);
 253
 254     for (k = 0; k < list_count; ++k) {
 255         if (memInUse(MEM_NETDBENTRY) < Config.Netdb.low)
 256             break;
 257
 258         netdbRelease(*(list + k));
 259
 260         ++removed;
 261     }
 262
 263     xfree(list);
 264 }
 265
 266 static netdbEntry *
 267 netdbLookupAddr(const Ip::Address &addr)
 268 {
 269     netdbEntry *n;
 270     char *key = new char[MAX_IPSTRLEN];
 271     networkFromInaddr(addr).toStr(key,MAX_IPSTRLEN);
 272     n = (netdbEntry *) hash_lookup(addr_table, key);
 273     delete[] key;
 274     return n;
 275 }
 276
 277 static netdbEntry *
 278 netdbAdd(Ip::Address &addr)
 279 {
 280     netdbEntry *n;
 281
 282     if (memInUse(MEM_NETDBENTRY) > Config.Netdb.high)
 283         netdbPurgeLRU();
 284
 285     if ((n = netdbLookupAddr(addr)) == NULL) {
 286         n = (netdbEntry *)memAllocate(MEM_NETDBENTRY);
 287         netdbHashInsert(n, addr);
 288     }
 289
 290     return n;
 291 }
 292
 293 static void
 294 netdbSendPing(const ipcache_addrs *ia, const Dns::LookupDetails &, void *data)
 295 {
 296     Ip::Address addr;
 297     char *hostname = NULL;
 298     static_cast<generic_cbdata *>(data)->unwrap(&hostname);
 299     netdbEntry *n;
 300     netdbEntry *na;
 301     net_db_name *x;
 302     net_db_name **X;
 303
 304     if (ia == NULL) {
 305         xfree(hostname);
 306         return;
 307     }
 308
 309     addr = ia->in_addrs[ia->cur];
 310
 311     if ((n = netdbLookupHost(hostname)) == NULL) {
 312         n = netdbAdd(addr);
 313         netdbHostInsert(n, hostname);
 314     } else if ((na = netdbLookupAddr(addr)) != n) {
 315         /*
 316          *hostname moved from 'network n' to 'network na'!
 317          */
 318
 319         if (na == NULL)
 320             na = netdbAdd(addr);
 321
 322         debugs(38, 3, "netdbSendPing: " << hostname << " moved from " << n->network << " to " << na->network);
 323
 324         x = (net_db_name *) hash_lookup(host_table, hostname);
 325
 326         if (x == NULL) {
 327             debugs(38, DBG_IMPORTANT, "netdbSendPing: net_db_name list bug: " << hostname << " not found");
 328             xfree(hostname);
 329             return;
 330         }
 331
 332         /* remove net_db_name from 'network n' linked list */
 333         for (X = &n->hosts; *X; X = &(*X)->next) {
 334             if (*X == x) {
 335                 *X = x->next;
 336                 break;
 337             }
 338         }
 339
 340         -- n->link_count;
 341         /* point to 'network na' from host entry */
 342         x->net_db_entry = na;
 343         /* link net_db_name to 'network na' */
 344         x->next = na->hosts;
 345         na->hosts = x;
 346         ++ na->link_count;
 347         n = na;
 348     }
 349
 350     if (n->next_ping_time <= squid_curtime) {
 351         debugs(38, 3, "netdbSendPing: pinging " << hostname);
 352         icmpEngine.DomainPing(addr, hostname);
 353         ++ n->pings_sent;
 354         n->next_ping_time = squid_curtime + Config.Netdb.period;
 355         n->last_use_time = squid_curtime;
 356     }
 357
 358     xfree(hostname);
 359 }
 360
 361 Ip::Address
 362 networkFromInaddr(const Ip::Address &in)
 363 {
 364     Ip::Address out;
 365
 366     out = in;
 367
 368     /* in IPv6 the 'network' should be the routing section. */
 369     if ( in.isIPv6() ) {
 370         out.applyMask(64, AF_INET6);
 371         debugs(14, 5, "networkFromInaddr : Masked IPv6 Address to " << in << "/64 routing part.");
 372         return out;
 373     }
 374
 375 #if USE_CLASSFUL
 376     struct in_addr b;
 377
 378     in.getInAddr(b);
 379
 380     if (IN_CLASSC(b.s_addr))
 381         b.s_addr &= IN_CLASSC_NET;
 382     else if (IN_CLASSB(b.s_addr))
 383         b.s_addr &= IN_CLASSB_NET;
 384     else if (IN_CLASSA(b.s_addr))
 385         b.s_addr &= IN_CLASSA_NET;
 386
 387     out = b;
 388
 389 #endif
 390
 391     debugs(14, 5, "networkFromInaddr : Masked IPv4 Address to " << out << "/24.");
 392
 393     /* use /24 for everything under IPv4 */
 394     out.applyMask(24, AF_INET);
 395     debugs(14, 5, "networkFromInaddr : Masked IPv4 Address to " << in << "/24.");
 396
 397     return out;
 398 }
 399
 400 static int
 401 sortByRtt(const void *A, const void *B)
 402 {
 403     const netdbEntry *const *n1 = (const netdbEntry *const *)A;
 404     const netdbEntry *const *n2 = (const netdbEntry *const *)B;
 405
 406     if ((*n1)->rtt > (*n2)->rtt)
 407         return 1;
 408     else if ((*n1)->rtt < (*n2)->rtt)
 409         return -1;
 410     else
 411         return 0;
 412 }
 413
 414 static net_db_peer *
 415 netdbPeerByName(const netdbEntry * n, const char *peername)
 416 {
 417     int i;
 418     net_db_peer *p = n->peers;
 419
 420     for (i = 0; i < n->n_peers; ++i, ++p) {
 421         if (!strcmp(p->peername, peername))
 422             return p;
 423     }
 424
 425     return NULL;
 426 }
 427
 428 static net_db_peer *
 429 netdbPeerAdd(netdbEntry * n, CachePeer * e)
 430 {
 431     net_db_peer *p;
 432     net_db_peer *o;
 433     int osize;
 434     int i;
 435
 436     if (n->n_peers == n->n_peers_alloc) {
 437         o = n->peers;
 438         osize = n->n_peers_alloc;
 439
 440         if (n->n_peers_alloc == 0)
 441             n->n_peers_alloc = 2;
 442         else
 443             n->n_peers_alloc <<= 1;
 444
 445         debugs(38, 3, "netdbPeerAdd: Growing peer list for '" << n->network << "' to " << n->n_peers_alloc);
 446
 447         n->peers = (net_db_peer *)xcalloc(n->n_peers_alloc, sizeof(net_db_peer));
 448
 449         for (i = 0; i < osize; ++i)
 450             *(n->peers + i) = *(o + i);
 451
 452         if (osize) {
 453             safe_free(o);
 454         }
 455     }
 456
 457     p = n->peers + n->n_peers;
 458     p->peername = netdbPeerName(e->host);
 459     ++ n->n_peers;
 460     return p;
 461 }
 462
 463 static int
 464 sortPeerByRtt(const void *A, const void *B)
 465 {
 466     const net_db_peer *p1 = (net_db_peer *)A;
 467     const net_db_peer *p2 = (net_db_peer *)B;
 468
 469     if (p1->rtt > p2->rtt)
 470         return 1;
 471     else if (p1->rtt < p2->rtt)
 472         return -1;
 473     else
 474         return 0;
 475 }
 476
 477 static void
 478 netdbSaveState(void *foo)
 479 {
 480     if (strcmp(Config.netdbFilename, "none") == 0)
 481         return;
 482
 483     Logfile *lf;
 484     netdbEntry *n;
 485     net_db_name *x;
 486
 487     struct timeval start = current_time;
 488     int count = 0;
 489     /*
 490      * This was nicer when we were using stdio, but thanks to
 491      * Solaris bugs, its a bad idea.  fopen can fail if more than
 492      * 256 FDs are open.
 493      */
 494     /*
 495      * unlink() is here because there is currently no way to make
 496      * logfileOpen() use O_TRUNC.
 497      */
 498     unlink(Config.netdbFilename);
 499     lf = logfileOpen(Config.netdbFilename, 4096, 0);
 500
 501     if (NULL == lf) {
 502         debugs(50, DBG_IMPORTANT, "netdbSaveState: " << Config.netdbFilename << ": " << xstrerror());
 503         return;
 504     }
 505
 506     hash_first(addr_table);
 507
 508     while ((n = (netdbEntry *) hash_next(addr_table))) {
 509         if (n->pings_recv == 0)
 510             continue;
 511
 512         logfilePrintf(lf, "%s %d %d %10.5f %10.5f %d %d",
 513                       n->network,
 514                       n->pings_sent,
 515                       n->pings_recv,
 516                       n->hops,
 517                       n->rtt,
 518                       (int) n->next_ping_time,
 519                       (int) n->last_use_time);
 520
 521         for (x = n->hosts; x; x = x->next)
 522             logfilePrintf(lf, " %s", hashKeyStr(&x->hash));
 523
 524         logfilePrintf(lf, "\n");
 525
 526         ++count;
 527
 528 #undef RBUF_SZ
 529
 530     }
 531
 532     logfileClose(lf);
 533     getCurrentTime();
 534     debugs(38, DBG_IMPORTANT, "NETDB state saved; " <<
 535            count << " entries, " <<
 536            tvSubMsec(start, current_time) << " msec" );
 537     eventAddIsh("netdbSaveState", netdbSaveState, NULL, 3600.0, 1);
 538 }
 539
 540 static void
 541 netdbReloadState(void)
 542 {
 543     if (strcmp(Config.netdbFilename, "none") == 0)
 544         return;
 545
 546     char *s;
 547     int fd;
 548     int l;
 549
 550     struct stat sb;
 551     netdbEntry *n;
 552     netdbEntry N;
 553
 554     Ip::Address addr;
 555     int count = 0;
 556
 557     struct timeval start = current_time;
 558     /*
 559      * This was nicer when we were using stdio, but thanks to
 560      * Solaris bugs, its a bad idea.  fopen can fail if more than
 561      * 256 FDs are open.
 562      */
 563     fd = file_open(Config.netdbFilename, O_RDONLY | O_BINARY);
 564
 565     if (fd < 0)
 566         return;
 567
 568     if (fstat(fd, &sb) < 0) {
 569         file_close(fd);
 570         return;
 571     }
 572
 573     char *t;
 574     char *buf = (char *)xcalloc(1, sb.st_size + 1);
 575     t = buf;
 576     l = FD_READ_METHOD(fd, buf, sb.st_size);
 577     file_close(fd);
 578
 579     if (l <= 0) {
 580         safe_free (buf);
 581         return;
 582     };
 583
 584     while ((s = strchr(t, '\n'))) {
 585         char *q;
 586         assert(s - buf < l);
 587         *s = '\0';
 588         memset(&N, '\0', sizeof(netdbEntry));
 589         q = strtok(t, w_space);
 590         t = s + 1;
 591
 592         if (NULL == q)
 593             continue;
 594
 595         if (! (addr = q) )
 596             continue;
 597
 598         if (netdbLookupAddr(addr) != NULL)  /* no dups! */
 599             continue;
 600
 601         if ((q = strtok(NULL, w_space)) == NULL)
 602             continue;
 603
 604         N.pings_sent = atoi(q);
 605
 606         if ((q = strtok(NULL, w_space)) == NULL)
 607             continue;
 608
 609         N.pings_recv = atoi(q);
 610
 611         if (N.pings_recv == 0)
 612             continue;
 613
 614         /* give this measurement low weight */
 615         N.pings_sent = 1;
 616
 617         N.pings_recv = 1;
 618
 619         if ((q = strtok(NULL, w_space)) == NULL)
 620             continue;
 621
 622         N.hops = atof(q);
 623
 624         if ((q = strtok(NULL, w_space)) == NULL)
 625             continue;
 626
 627         N.rtt = atof(q);
 628
 629         if ((q = strtok(NULL, w_space)) == NULL)
 630             continue;
 631
 632         N.next_ping_time = (time_t) atoi(q);
 633
 634         if ((q = strtok(NULL, w_space)) == NULL)
 635             continue;
 636
 637         N.last_use_time = (time_t) atoi(q);
 638
 639         n = (netdbEntry *)memAllocate(MEM_NETDBENTRY);
 640
 641         memcpy(n, &N, sizeof(netdbEntry));
 642
 643         netdbHashInsert(n, addr);
 644
 645         while ((q = strtok(NULL, w_space)) != NULL) {
 646             if (netdbLookupHost(q) != NULL) /* no dups! */
 647                 continue;
 648
 649             netdbHostInsert(n, q);
 650         }
 651
 652         ++count;
 653     }
 654
 655     xfree(buf);
 656     getCurrentTime();
 657     debugs(38, DBG_IMPORTANT, "NETDB state reloaded; " <<
 658            count << " entries, " <<
 659            tvSubMsec(start, current_time) << " msec" );
 660 }
 661
 662 static const char *
 663 netdbPeerName(const char *name)
 664 {
 665     const wordlist *w;
 666
 667     for (w = peer_names; w; w = w->next) {
 668         if (!strcmp(w->key, name))
 669             return w->key;
 670     }
 671
 672     return wordlistAdd(&peer_names, name);
 673 }
 674
 675 static void
 676 netdbFreeNetdbEntry(void *data)
 677 {
 678     netdbEntry *n = (netdbEntry *)data;
 679     safe_free(n->peers);
 680     memFree(n, MEM_NETDBENTRY);
 681 }
 682
 683 static void
 684 netdbFreeNameEntry(void *data)
 685 {
 686     net_db_name *x = (net_db_name *)data;
 687     xfree(x->hash.key);
 688     memFree(x, MEM_NET_DB_NAME);
 689 }
 690
 691 static void
 692 netdbExchangeHandleReply(void *data, StoreIOBuffer receivedData)
 693 {
 694     Ip::Address addr;
 695
 696     netdbExchangeState *ex = (netdbExchangeState *)data;
 697     int rec_sz = 0;
 698     int o;
 699
 700     struct in_addr line_addr;
 701     double rtt;
 702     double hops;
 703     char *p;
 704     int j;
 705     HttpReply const *rep;
 706     size_t hdr_sz;
 707     int nused = 0;
 708     int size;
 709     int oldbufofs = ex->buf_ofs;
 710
 711     rec_sz = 0;
 712     rec_sz += 1 + sizeof(struct in_addr);
 713     rec_sz += 1 + sizeof(int);
 714     rec_sz += 1 + sizeof(int);
 715     debugs(38, 3, "netdbExchangeHandleReply: " << receivedData.length << " read bytes");
 716
 717     if (!cbdataReferenceValid(ex->p)) {
 718         debugs(38, 3, "netdbExchangeHandleReply: Peer became invalid");
 719         delete ex;
 720         return;
 721     }
 722
 723     debugs(38, 3, "netdbExchangeHandleReply: for '" << ex->p->host << ":" << ex->p->http_port << "'");
 724
 725     if (receivedData.length == 0 && !receivedData.flags.error) {
 726         debugs(38, 3, "netdbExchangeHandleReply: Done");
 727         delete ex;
 728         return;
 729     }
 730
 731     p = ex->buf;
 732
 733     /* Get the size of the buffer now */
 734     size = ex->buf_ofs + receivedData.length;
 735     debugs(38, 3, "netdbExchangeHandleReply: " << size << " bytes buf");
 736
 737     /* Check if we're still doing headers */
 738
 739     if (ex->connstate == STATE_HEADER) {
 740
 741         ex->buf_ofs += receivedData.length;
 742
 743         /* skip reply headers */
 744
 745         if ((hdr_sz = headersEnd(p, ex->buf_ofs))) {
 746             debugs(38, 5, "netdbExchangeHandleReply: hdr_sz = " << hdr_sz);
 747             rep = ex->e->getReply();
 748             assert(rep->sline.status() != Http::scNone);
 749             debugs(38, 3, "netdbExchangeHandleReply: reply status " << rep->sline.status());
 750
 751             if (rep->sline.status() != Http::scOkay) {
 752                 delete ex;
 753                 return;
 754             }
 755
 756             assert((size_t)ex->buf_ofs >= hdr_sz);
 757
 758             /*
 759              * Now, point p to the part of the buffer where the data
 760              * starts, and update the size accordingly
 761              */
 762             assert(ex->used == 0);
 763             ex->used = hdr_sz;
 764             size = ex->buf_ofs - hdr_sz;
 765             p += hdr_sz;
 766
 767             /* Finally, set the conn state mode to STATE_BODY */
 768             ex->connstate = STATE_BODY;
 769         } else {
 770             StoreIOBuffer tempBuffer;
 771             tempBuffer.offset = ex->buf_ofs;
 772             tempBuffer.length = ex->buf_sz - ex->buf_ofs;
 773             tempBuffer.data = ex->buf + ex->buf_ofs;
 774             /* Have more headers .. */
 775             storeClientCopy(ex->sc, ex->e, tempBuffer,
 776                             netdbExchangeHandleReply, ex);
 777             return;
 778         }
 779     }
 780
 781     assert(ex->connstate == STATE_BODY);
 782
 783     /* If we get here, we have some body to parse .. */
 784     debugs(38, 5, "netdbExchangeHandleReply: start parsing loop, size = " << size);
 785
 786     while (size >= rec_sz) {
 787         debugs(38, 5, "netdbExchangeHandleReply: in parsing loop, size = " << size);
 788         addr.setAnyAddr();
 789         hops = rtt = 0.0;
 790
 791         for (o = 0; o < rec_sz;) {
 792             switch ((int) *(p + o)) {
 793
 794             case NETDB_EX_NETWORK:
 795                 ++o;
 796                 /* FIXME INET6 : NetDB can still ony send IPv4 */
 797                 memcpy(&line_addr, p + o, sizeof(struct in_addr));
 798                 addr = line_addr;
 799                 o += sizeof(struct in_addr);
 800                 break;
 801
 802             case NETDB_EX_RTT:
 803                 ++o;
 804                 memcpy(&j, p + o, sizeof(int));
 805                 o += sizeof(int);
 806                 rtt = (double) ntohl(j) / 1000.0;
 807                 break;
 808
 809             case NETDB_EX_HOPS:
 810                 ++o;
 811                 memcpy(&j, p + o, sizeof(int));
 812                 o += sizeof(int);
 813                 hops = (double) ntohl(j) / 1000.0;
 814                 break;
 815
 816             default:
 817                 debugs(38, DBG_IMPORTANT, "netdbExchangeHandleReply: corrupt data, aborting");
 818                 delete ex;
 819                 return;
 820             }
 821         }
 822
 823         if (!addr.isAnyAddr() && rtt > 0)
 824             netdbExchangeUpdatePeer(addr, ex->p, rtt, hops);
 825
 826         assert(o == rec_sz);
 827
 828         ex->used += rec_sz;
 829
 830         size -= rec_sz;
 831
 832         p += rec_sz;
 833
 834         ++nused;
 835     }
 836
 837     /*
 838      * Copy anything that is left over to the beginning of the buffer,
 839      * and adjust buf_ofs accordingly
 840      */
 841
 842     /*
 843      * Evilly, size refers to the buf size left now,
 844      * ex->buf_ofs is the original buffer size, so just copy that
 845      * much data over
 846      */
 847     memmove(ex->buf, ex->buf + (ex->buf_ofs - size), size);
 848
 849     ex->buf_ofs = size;
 850
 851     /*
 852      * And don't re-copy the remaining data ..
 853      */
 854     ex->used += size;
 855
 856     /*
 857      * Now the tricky bit - size _included_ the leftover bit from the _last_
 858      * storeClientCopy. We don't want to include that, or our offset will be wrong.
 859      * So, don't count the size of the leftover buffer we began with.
 860      * This can _disappear_ when we're not tracking offsets ..
 861      */
 862     ex->used -= oldbufofs;
 863
 864     debugs(38, 3, "netdbExchangeHandleReply: size left over in this buffer: " << size << " bytes");
 865
 866     debugs(38, 3, "netdbExchangeHandleReply: used " << nused <<
 867            " entries, (x " << rec_sz << " bytes) == " << nused * rec_sz <<
 868            " bytes total");
 869
 870     debugs(38, 3, "netdbExchangeHandleReply: used " << ex->used);
 871
 872     if (EBIT_TEST(ex->e->flags, ENTRY_ABORTED)) {
 873         debugs(38, 3, "netdbExchangeHandleReply: ENTRY_ABORTED");
 874         delete ex;
 875     } else if (ex->e->store_status == STORE_PENDING) {
 876         StoreIOBuffer tempBuffer;
 877         tempBuffer.offset = ex->used;
 878         tempBuffer.length = ex->buf_sz - ex->buf_ofs;
 879         tempBuffer.data = ex->buf + ex->buf_ofs;
 880         debugs(38, 3, "netdbExchangeHandleReply: EOF not received");
 881         storeClientCopy(ex->sc, ex->e, tempBuffer,
 882                         netdbExchangeHandleReply, ex);
 883     }
 884 }
 885
 886 static void
 887 netdbRegisterWithCacheManager(void)
 888 {
 889     Mgr::RegisterAction("netdb", "Network Measurement Database", netdbDump, 0, 1);
 890 }
 891
 892 #endif /* USE_ICMP */
 893
 894 /* PUBLIC FUNCTIONS */
 895
 896 void
 897 netdbInit(void)
 898 {
 899 #if USE_ICMP
 900     int n;
 901
 902     netdbRegisterWithCacheManager();
 903
 904     if (addr_table)
 905         return;
 906
 907     n = hashPrime(Config.Netdb.high / 4);
 908
 909     addr_table = hash_create((HASHCMP *) strcmp, n, hash_string);
 910
 911     n = hashPrime(3 * Config.Netdb.high / 4);
 912
 913     host_table = hash_create((HASHCMP *) strcmp, n, hash_string);
 914
 915     eventAddIsh("netdbSaveState", netdbSaveState, NULL, 3600.0, 1);
 916
 917     netdbReloadState();
 918
 919 #endif
 920 }
 921
 922 void
 923 netdbPingSite(const char *hostname)
 924 {
 925 #if USE_ICMP
 926     netdbEntry *n;
 927
 928     if ((n = netdbLookupHost(hostname)) != NULL)
 929         if (n->next_ping_time > squid_curtime)
 930             return;
 931
 932     ipcache_nbgethostbyname(hostname, netdbSendPing,
 933                             new generic_cbdata(xstrdup(hostname)));
 934
 935 #endif
 936 }
 937
 938 void
 939 netdbHandlePingReply(const Ip::Address &from, int hops, int rtt)
 940 {
 941 #if USE_ICMP
 942     netdbEntry *n;
 943     int N;
 944     debugs(38, 3, "netdbHandlePingReply: from " << from);
 945
 946     if ((n = netdbLookupAddr(from)) == NULL)
 947         return;
 948
 949     N = ++n->pings_recv;
 950
 951     if (N > 5)
 952         N = 5;
 953
 954     if (rtt < 1)
 955         rtt = 1;
 956
 957     n->hops = ((n->hops * (N - 1)) + hops) / N;
 958
 959     n->rtt = ((n->rtt * (N - 1)) + rtt) / N;
 960
 961     debugs(38, 3, "netdbHandlePingReply: " << n->network  << "; rtt="<<
 962            std::setw(5)<< std::setprecision(2) << n->rtt << "  hops="<<
 963            std::setw(4) << n->hops);
 964
 965 #endif
 966 }
 967
 968 void
 969 netdbFreeMemory(void)
 970 {
 971 #if USE_ICMP
 972     hashFreeItems(addr_table, netdbFreeNetdbEntry);
 973     hashFreeMemory(addr_table);
 974     addr_table = NULL;
 975     hashFreeItems(host_table, netdbFreeNameEntry);
 976     hashFreeMemory(host_table);
 977     host_table = NULL;
 978     wordlistDestroy(&peer_names);
 979     peer_names = NULL;
 980 #endif
 981 }
 982
 983 void
 984 netdbDump(StoreEntry * sentry)
 985 {
 986 #if USE_ICMP
 987     netdbEntry *n;
 988     netdbEntry **list;
 989     net_db_name *x;
 990     int k;
 991     int i;
 992     int j;
 993     net_db_peer *p;
 994     storeAppendPrintf(sentry, "Network DB Statistics:\n");
 995     storeAppendPrintf(sentry, "%-46.46s %9s %7s %5s %s\n",  /* Max between 16 (IPv4) or 46 (IPv6)   */
 996                       "Network",
 997                       "recv/sent",
 998                       "RTT",
 999                       "Hops",
1000                       "Hostnames");
1001     list = (netdbEntry **)xcalloc(memInUse(MEM_NETDBENTRY), sizeof(netdbEntry *));
1002     i = 0;
1003     hash_first(addr_table);
1004
1005     while ((n = (netdbEntry *) hash_next(addr_table))) {
1006         *(list + i) = n;
1007         ++i;
1008     }
1009
1010     if (i != memInUse(MEM_NETDBENTRY))
1011         debugs(38, DBG_CRITICAL, "WARNING: netdb_addrs count off, found " << i <<
1012                ", expected " << memInUse(MEM_NETDBENTRY));
1013
1014     qsort((char *) list,
1015           i,
1016           sizeof(netdbEntry *),
1017           sortByRtt);
1018
1019     for (k = 0; k < i; ++k) {
1020         n = *(list + k);
1021         storeAppendPrintf(sentry, "%-46.46s %4d/%4d %7.1f %5.1f", /* Max between 16 (IPv4) or 46 (IPv6)   */
1022                           n->network,
1023                           n->pings_recv,
1024                           n->pings_sent,
1025                           n->rtt,
1026                           n->hops);
1027
1028         for (x = n->hosts; x; x = x->next)
1029             storeAppendPrintf(sentry, " %s", hashKeyStr(&x->hash));
1030
1031         storeAppendPrintf(sentry, "\n");
1032
1033         p = n->peers;
1034
1035         for (j = 0; j < n->n_peers; ++j, ++p) {
1036             storeAppendPrintf(sentry, "    %-22.22s %7.1f %5.1f\n",
1037                               p->peername,
1038                               p->rtt,
1039                               p->hops);
1040         }
1041     }
1042
1043     xfree(list);
1044 #else
1045
1046     storeAppendPrintf(sentry,"NETDB support not compiled into this Squid cache.\n");
1047 #endif
1048 }
1049
1050 int
1051 netdbHostHops(const char *host)
1052 {
1053 #if USE_ICMP
1054     netdbEntry *n = netdbLookupHost(host);
1055
1056     if (n) {
1057         n->last_use_time = squid_curtime;
1058         return (int) (n->hops + 0.5);
1059     }
1060
1061 #endif
1062     return 0;
1063 }
1064
1065 int
1066 netdbHostRtt(const char *host)
1067 {
1068 #if USE_ICMP
1069     netdbEntry *n = netdbLookupHost(host);
1070
1071     if (n) {
1072         n->last_use_time = squid_curtime;
1073         return (int) (n->rtt + 0.5);
1074     }
1075
1076 #endif
1077     return 0;
1078 }
1079
1080 void
1081 netdbHostData(const char *host, int *samp, int *rtt, int *hops)
1082 {
1083 #if USE_ICMP
1084     netdbEntry *n = netdbLookupHost(host);
1085
1086     if (n == NULL)
1087         return;
1088
1089     *samp = n->pings_recv;
1090
1091     *rtt = (int) (n->rtt + 0.5);
1092
1093     *hops = (int) (n->hops + 0.5);
1094
1095     n->last_use_time = squid_curtime;
1096
1097 #endif
1098 }
1099
1100 void
1101 netdbUpdatePeer(const URL &url, CachePeer * e, int irtt, int ihops)
1102 {
1103 #if USE_ICMP
1104     netdbEntry *n;
1105     double rtt = (double) irtt;
1106     double hops = (double) ihops;
1107     net_db_peer *p;
1108     debugs(38, 3, url.host() << ", " << ihops << " hops, " << irtt << " rtt");
1109     n = netdbLookupHost(url.host());
1110
1111     if (n == NULL) {
1112         debugs(38, 3, "host " << url.host() << " not found");
1113         return;
1114     }
1115
1116     if ((p = netdbPeerByName(n, e->host)) == NULL)
1117         p = netdbPeerAdd(n, e);
1118
1119     p->rtt = rtt;
1120
1121     p->hops = hops;
1122
1123     p->expires = squid_curtime + 3600;
1124
1125     if (n->n_peers < 2)
1126         return;
1127
1128     qsort((char *) n->peers,
1129           n->n_peers,
1130           sizeof(net_db_peer),
1131           sortPeerByRtt);
1132
1133 #endif
1134 }
1135
1136 void
1137 netdbExchangeUpdatePeer(Ip::Address &addr, CachePeer * e, double rtt, double hops)
1138 {
1139 #if USE_ICMP
1140     netdbEntry *n;
1141     net_db_peer *p;
1142     debugs(38, 5, "netdbExchangeUpdatePeer: '" << addr << "', "<<
1143            std::setfill('0')<< std::setprecision(2) << hops << " hops, " <<
1144            rtt << " rtt");
1145
1146     if ( !addr.isIPv4() ) {
1147         debugs(38, 5, "netdbExchangeUpdatePeer: Aborting peer update for '" << addr << "', NetDB cannot handle IPv6.");
1148         return;
1149     }
1150
1151     n = netdbLookupAddr(addr);
1152
1153     if (n == NULL)
1154         n = netdbAdd(addr);
1155
1156     assert(NULL != n);
1157
1158     if ((p = netdbPeerByName(n, e->host)) == NULL)
1159         p = netdbPeerAdd(n, e);
1160
1161     p->rtt = rtt;
1162
1163     p->hops = hops;
1164
1165     p->expires = squid_curtime + 3600;  /* XXX ? */
1166
1167     if (n->n_peers < 2)
1168         return;
1169
1170     qsort((char *) n->peers,
1171           n->n_peers,
1172           sizeof(net_db_peer),
1173           sortPeerByRtt);
1174
1175 #endif
1176 }
1177
1178 void
1179 netdbDeleteAddrNetwork(Ip::Address &addr)
1180 {
1181 #if USE_ICMP
1182     netdbEntry *n = netdbLookupAddr(addr);
1183
1184     if (n == NULL)
1185         return;
1186
1187     debugs(38, 3, "netdbDeleteAddrNetwork: " << n->network);
1188
1189     netdbRelease(n);
1190 #endif
1191 }
1192
1193 void
1194 netdbBinaryExchange(StoreEntry * s)
1195 {
1196     HttpReply *reply = new HttpReply;
1197 #if USE_ICMP
1198
1199     Ip::Address addr;
1200
1201     netdbEntry *n;
1202     int i;
1203     int j;
1204     int rec_sz;
1205     char *buf;
1206
1207     struct in_addr line_addr;
1208     s->buffer();
1209     reply->setHeaders(Http::scOkay, "OK", NULL, -1, squid_curtime, -2);
1210     s->replaceHttpReply(reply);
1211     rec_sz = 0;
1212     rec_sz += 1 + sizeof(struct in_addr);
1213     rec_sz += 1 + sizeof(int);
1214     rec_sz += 1 + sizeof(int);
1215     buf = (char *)memAllocate(MEM_4K_BUF);
1216     i = 0;
1217     hash_first(addr_table);
1218
1219     while ((n = (netdbEntry *) hash_next(addr_table))) {
1220         if (0.0 == n->rtt)
1221             continue;
1222
1223         if (n->rtt > 60000) /* RTT > 1 MIN probably bogus */
1224             continue;
1225
1226         if (! (addr = n->network) )
1227             continue;
1228
1229         /* FIXME INET6 : NetDB cannot yet handle IPv6 addresses. Ensure only IPv4 get sent. */
1230         if ( !addr.isIPv4() )
1231             continue;
1232
1233         buf[i] = (char) NETDB_EX_NETWORK;
1234         ++i;
1235
1236         addr.getInAddr(line_addr);
1237         memcpy(&buf[i], &line_addr, sizeof(struct in_addr));
1238
1239         i += sizeof(struct in_addr);
1240
1241         buf[i] = (char) NETDB_EX_RTT;
1242         ++i;
1243
1244         j = htonl((int) (n->rtt * 1000));
1245
1246         memcpy(&buf[i], &j, sizeof(int));
1247
1248         i += sizeof(int);
1249
1250         buf[i] = (char) NETDB_EX_HOPS;
1251         ++i;
1252
1253         j = htonl((int) (n->hops * 1000));
1254
1255         memcpy(&buf[i], &j, sizeof(int));
1256
1257         i += sizeof(int);
1258
1259         if (i + rec_sz > 4096) {
1260             s->append(buf, i);
1261             i = 0;
1262         }
1263     }
1264
1265     if (i > 0) {
1266         s->append(buf, i);
1267         i = 0;
1268     }
1269
1270     assert(0 == i);
1271     s->flush();
1272     memFree(buf, MEM_4K_BUF);
1273 #else
1274
1275     reply->setHeaders(Http::scBadRequest, "Bad Request", NULL, -1, squid_curtime, -2);
1276     s->replaceHttpReply(reply);
1277     storeAppendPrintf(s, "NETDB support not compiled into this Squid cache.\n");
1278 #endif
1279
1280     s->complete();
1281 }
1282
1283 void
1284 netdbExchangeStart(void *data)
1285 {
1286 #if USE_ICMP
1287     CachePeer *p = (CachePeer *)data;
1288     static const SBuf netDB("netdb");
1289     char *uri = internalRemoteUri(p->host, p->http_port, "/squid-internal-dynamic/", netDB);
1290     debugs(38, 3, "netdbExchangeStart: Requesting '" << uri << "'");
1291     assert(NULL != uri);
1292     HttpRequest *req = HttpRequest::CreateFromUrl(uri);
1293
1294     if (req == NULL) {
1295         debugs(38, DBG_IMPORTANT, "netdbExchangeStart: Bad URI " << uri);
1296         return;
1297     }
1298
1299     netdbExchangeState *ex = new netdbExchangeState(p, req);
1300     ex->e = storeCreateEntry(uri, uri, RequestFlags(), Http::METHOD_GET);
1301     assert(NULL != ex->e);
1302
1303     StoreIOBuffer tempBuffer;
1304     tempBuffer.length = ex->buf_sz;
1305     tempBuffer.data = ex->buf;
1306
1307     ex->sc = storeClientListAdd(ex->e, ex);
1308
1309     storeClientCopy(ex->sc, ex->e, tempBuffer,
1310                     netdbExchangeHandleReply, ex);
1311     ex->r->flags.loopDetected = true;   /* cheat! -- force direct */
1312
1313     // XXX: send as Proxy-Authenticate instead
1314     if (p->login)
1315         ex->r->url.userInfo(SBuf(p->login));
1316
1317     FwdState::fwdStart(Comm::ConnectionPointer(), ex->e, ex->r);
1318
1319 #endif
1320 }
1321
1322 CachePeer *
1323 netdbClosestParent(HttpRequest * request)
1324 {
1325 #if USE_ICMP
1326     CachePeer *p = NULL;
1327     netdbEntry *n;
1328     const ipcache_addrs *ia;
1329     net_db_peer *h;
1330     int i;
1331     n = netdbLookupHost(request->url.host());
1332
1333     if (NULL == n) {
1334         /* try IP addr */
1335         ia = ipcache_gethostbyname(request->url.host(), 0);
1336
1337         if (NULL != ia)
1338             n = netdbLookupAddr(ia->in_addrs[ia->cur]);
1339     }
1340
1341     if (NULL == n)
1342         return NULL;
1343
1344     if (0 == n->n_peers)
1345         return NULL;
1346
1347     n->last_use_time = squid_curtime;
1348
1349     /*
1350      * Find the parent with the least RTT to the origin server.
1351      * Make sure we don't return a parent who is farther away than
1352      * we are.  Note, the n->peers list is pre-sorted by RTT.
1353      */
1354     for (i = 0; i < n->n_peers; ++i) {
1355         h = &n->peers[i];
1356
1357         if (n->rtt > 0)
1358             if (n->rtt < h->rtt)
1359                 break;
1360
1361         p = peerFindByName(h->peername);
1362
1363         if (NULL == p)      /* not found */
1364             continue;
1365
1366         if (neighborType(p, request->url) != PEER_PARENT)
1367             continue;
1368
1369         if (!peerHTTPOkay(p, request))  /* not allowed */
1370             continue;
1371
1372         return p;
1373     }
1374
1375 #endif
1376     return NULL;
1377 }
1378