src/icmp/net_db.cc

   1 /*
   2  * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 38    Network Measurement Database */
  10
  11 /*
  12  * XXX XXX XXX
  13  *
  14  * This code may be slightly broken now. If you're getting consistent
  15  * (sometimes working) corrupt data exchanges, please contact adrian
  16  * (adrian@squid-cache.org) to sort them out.
  17  */
  18
  19 #include "squid.h"
  20 #include "CachePeer.h"
  21 #include "cbdata.h"
  22 #include "disk.h"
  23 #include "event.h"
  24 #include "fde.h"
  25 #include "FwdState.h"
  26 #include "HttpReply.h"
  27 #include "HttpRequest.h"
  28 #include "icmp/net_db.h"
  29 #include "internal.h"
  30 #include "ip/Address.h"
  31 #include "log/File.h"
  32 #include "MemObject.h"
  33 #include "mgr/Registration.h"
  34 #include "mime_header.h"
  35 #include "neighbors.h"
  36 #include "SquidTime.h"
  37 #include "Store.h"
  38 #include "StoreClient.h"
  39 #include "SwapDir.h"
  40 #include "tools.h"
  41 #include "URL.h"
  42 #include "wordlist.h"
  43
  44 #if HAVE_SYS_STAT_H
  45 #include <sys/stat.h>
  46 #endif
  47
  48 #if USE_ICMP
  49 #include "icmp/IcmpSquid.h"
  50 #include "ipcache.h"
  51 #include "StoreClient.h"
  52
  53 #define NETDB_REQBUF_SZ 4096
  54
  55 typedef enum {
  56     STATE_NONE,
  57     STATE_HEADER,
  58     STATE_BODY
  59 } netdb_conn_state_t;
  60
  61 class netdbExchangeState
  62 {
  63     CBDATA_CLASS(netdbExchangeState);
  64
  65 public:
  66     netdbExchangeState(CachePeer *aPeer, HttpRequest *theReq) :
  67         p(cbdataReference(aPeer)),
  68         e(NULL),
  69         sc(NULL),
  70         r(theReq),
  71         used(0),
  72         buf_sz(NETDB_REQBUF_SZ),
  73         buf_ofs(0),
  74         connstate(STATE_HEADER)
  75     {
  76         *buf = 0;
  77
  78         assert(NULL != r);
  79         HTTPMSGLOCK(r);
  80         // TODO: check if we actually need to do this. should be implicit
  81         r->http_ver = Http::ProtocolVersion();
  82     }
  83
  84     ~netdbExchangeState() {
  85         debugs(38, 3, e->url());
  86         storeUnregister(sc, e, this);
  87         e->unlock("netdbExchangeDone");
  88         HTTPMSGUNLOCK(r);
  89         cbdataReferenceDone(p);
  90     }
  91
  92     CachePeer *p;
  93     StoreEntry *e;
  94     store_client *sc;
  95     HttpRequest *r;
  96     int64_t used;
  97     size_t buf_sz;
  98     char buf[NETDB_REQBUF_SZ];
  99     int buf_ofs;
 100     netdb_conn_state_t connstate;
 101 };
 102
 103 CBDATA_CLASS_INIT(netdbExchangeState);
 104
 105 static hash_table *addr_table = NULL;
 106 static hash_table *host_table = NULL;
 107
 108 Ip::Address networkFromInaddr(const Ip::Address &a);
 109 static void netdbRelease(netdbEntry * n);
 110
 111 static void netdbHashInsert(netdbEntry * n, Ip::Address &addr);
 112 static void netdbHashDelete(const char *key);
 113 static void netdbHostInsert(netdbEntry * n, const char *hostname);
 114 static void netdbHostDelete(const net_db_name * x);
 115 static void netdbPurgeLRU(void);
 116 static netdbEntry *netdbLookupHost(const char *key);
 117 static net_db_peer *netdbPeerByName(const netdbEntry * n, const char *);
 118 static net_db_peer *netdbPeerAdd(netdbEntry * n, CachePeer * e);
 119 static const char *netdbPeerName(const char *name);
 120 static IPH netdbSendPing;
 121 static FREE netdbFreeNameEntry;
 122 static FREE netdbFreeNetdbEntry;
 123 static STCB netdbExchangeHandleReply;
 124
 125 /* We have to keep a local list of CachePeer names.  The Peers structure
 126  * gets freed during a reconfigure.  We want this database to
 127  * remain persisitent, so _net_db_peer->peername points into this
 128  * linked list */
 129 static wordlist *peer_names = NULL;
 130
 131 static void
 132 netdbHashInsert(netdbEntry * n, Ip::Address &addr)
 133 {
 134     networkFromInaddr(addr).toStr(n->network, MAX_IPSTRLEN);
 135     n->hash.key = n->network;
 136     assert(hash_lookup(addr_table, n->network) == NULL);
 137     hash_join(addr_table, &n->hash);
 138 }
 139
 140 static void
 141 netdbHashDelete(const char *key)
 142 {
 143     hash_link *hptr = (hash_link *)hash_lookup(addr_table, key);
 144
 145     if (hptr == NULL) {
 146         debug_trap("netdbHashDelete: key not found");
 147         return;
 148     }
 149
 150     hash_remove_link(addr_table, hptr);
 151 }
 152
 153 static void
 154 netdbHostInsert(netdbEntry * n, const char *hostname)
 155 {
 156     net_db_name *x = (net_db_name *)memAllocate(MEM_NET_DB_NAME);
 157     x->hash.key = xstrdup(hostname);
 158     x->next = n->hosts;
 159     n->hosts = x;
 160     x->net_db_entry = n;
 161     assert(hash_lookup(host_table, hostname) == NULL);
 162     hash_join(host_table, &x->hash);
 163     ++ n->link_count;
 164 }
 165
 166 static void
 167 netdbHostDelete(const net_db_name * x)
 168 {
 169     netdbEntry *n;
 170     net_db_name **X;
 171     assert(x != NULL);
 172     assert(x->net_db_entry != NULL);
 173     n = x->net_db_entry;
 174     -- n->link_count;
 175
 176     for (X = &n->hosts; *X; X = &(*X)->next) {
 177         if (*X == x) {
 178             *X = x->next;
 179             break;
 180         }
 181     }
 182
 183     hash_remove_link(host_table, (hash_link *) x);
 184     xfree(x->hash.key);
 185     memFree((void *) x, MEM_NET_DB_NAME);
 186 }
 187
 188 static netdbEntry *
 189 netdbLookupHost(const char *key)
 190 {
 191     net_db_name *x = (net_db_name *) hash_lookup(host_table, key);
 192     return x ? x->net_db_entry : NULL;
 193 }
 194
 195 static void
 196 netdbRelease(netdbEntry * n)
 197 {
 198     net_db_name *x;
 199     net_db_name *next;
 200
 201     for (x = n->hosts; x; x = next) {
 202         next = x->next;
 203         netdbHostDelete(x);
 204     }
 205
 206     n->hosts = NULL;
 207     safe_free(n->peers);
 208     n->peers = NULL;
 209     n->n_peers = 0;
 210     n->n_peers_alloc = 0;
 211
 212     if (n->link_count == 0) {
 213         netdbHashDelete(n->network);
 214         memFree(n, MEM_NETDBENTRY);
 215     }
 216 }
 217
 218 static int
 219 netdbLRU(const void *A, const void *B)
 220 {
 221     const netdbEntry *const *n1 = (const netdbEntry *const *)A;
 222     const netdbEntry *const *n2 = (const netdbEntry *const *)B;
 223
 224     if ((*n1)->last_use_time > (*n2)->last_use_time)
 225         return (1);
 226
 227     if ((*n1)->last_use_time < (*n2)->last_use_time)
 228         return (-1);
 229
 230     return (0);
 231 }
 232
 233 static void
 234 netdbPurgeLRU(void)
 235 {
 236     netdbEntry *n;
 237     netdbEntry **list;
 238     int k = 0;
 239     int list_count = 0;
 240     int removed = 0;
 241     list = (netdbEntry **)xcalloc(memInUse(MEM_NETDBENTRY), sizeof(netdbEntry *));
 242     hash_first(addr_table);
 243
 244     while ((n = (netdbEntry *) hash_next(addr_table))) {
 245         assert(list_count < memInUse(MEM_NETDBENTRY));
 246         *(list + list_count) = n;
 247         ++list_count;
 248     }
 249
 250     qsort((char *) list,
 251           list_count,
 252           sizeof(netdbEntry *),
 253           netdbLRU);
 254
 255     for (k = 0; k < list_count; ++k) {
 256         if (memInUse(MEM_NETDBENTRY) < Config.Netdb.low)
 257             break;
 258
 259         netdbRelease(*(list + k));
 260
 261         ++removed;
 262     }
 263
 264     xfree(list);
 265 }
 266
 267 static netdbEntry *
 268 netdbLookupAddr(const Ip::Address &addr)
 269 {
 270     netdbEntry *n;
 271     char *key = new char[MAX_IPSTRLEN];
 272     networkFromInaddr(addr).toStr(key,MAX_IPSTRLEN);
 273     n = (netdbEntry *) hash_lookup(addr_table, key);
 274     delete[] key;
 275     return n;
 276 }
 277
 278 static netdbEntry *
 279 netdbAdd(Ip::Address &addr)
 280 {
 281     netdbEntry *n;
 282
 283     if (memInUse(MEM_NETDBENTRY) > Config.Netdb.high)
 284         netdbPurgeLRU();
 285
 286     if ((n = netdbLookupAddr(addr)) == NULL) {
 287         n = (netdbEntry *)memAllocate(MEM_NETDBENTRY);
 288         netdbHashInsert(n, addr);
 289     }
 290
 291     return n;
 292 }
 293
 294 static void
 295 netdbSendPing(const ipcache_addrs *ia, const Dns::LookupDetails &, void *data)
 296 {
 297     Ip::Address addr;
 298     char *hostname = NULL;
 299     static_cast<generic_cbdata *>(data)->unwrap(&hostname);
 300     netdbEntry *n;
 301     netdbEntry *na;
 302     net_db_name *x;
 303     net_db_name **X;
 304
 305     if (ia == NULL) {
 306         xfree(hostname);
 307         return;
 308     }
 309
 310     addr = ia->in_addrs[ia->cur];
 311
 312     if ((n = netdbLookupHost(hostname)) == NULL) {
 313         n = netdbAdd(addr);
 314         netdbHostInsert(n, hostname);
 315     } else if ((na = netdbLookupAddr(addr)) != n) {
 316         /*
 317          *hostname moved from 'network n' to 'network na'!
 318          */
 319
 320         if (na == NULL)
 321             na = netdbAdd(addr);
 322
 323         debugs(38, 3, "netdbSendPing: " << hostname << " moved from " << n->network << " to " << na->network);
 324
 325         x = (net_db_name *) hash_lookup(host_table, hostname);
 326
 327         if (x == NULL) {
 328             debugs(38, DBG_IMPORTANT, "netdbSendPing: net_db_name list bug: " << hostname << " not found");
 329             xfree(hostname);
 330             return;
 331         }
 332
 333         /* remove net_db_name from 'network n' linked list */
 334         for (X = &n->hosts; *X; X = &(*X)->next) {
 335             if (*X == x) {
 336                 *X = x->next;
 337                 break;
 338             }
 339         }
 340
 341         -- n->link_count;
 342         /* point to 'network na' from host entry */
 343         x->net_db_entry = na;
 344         /* link net_db_name to 'network na' */
 345         x->next = na->hosts;
 346         na->hosts = x;
 347         ++ na->link_count;
 348         n = na;
 349     }
 350
 351     if (n->next_ping_time <= squid_curtime) {
 352         debugs(38, 3, "netdbSendPing: pinging " << hostname);
 353         icmpEngine.DomainPing(addr, hostname);
 354         ++ n->pings_sent;
 355         n->next_ping_time = squid_curtime + Config.Netdb.period;
 356         n->last_use_time = squid_curtime;
 357     }
 358
 359     xfree(hostname);
 360 }
 361
 362 Ip::Address
 363 networkFromInaddr(const Ip::Address &in)
 364 {
 365     Ip::Address out;
 366
 367     out = in;
 368
 369     /* in IPv6 the 'network' should be the routing section. */
 370     if ( in.isIPv6() ) {
 371         out.applyMask(64, AF_INET6);
 372         debugs(14, 5, "networkFromInaddr : Masked IPv6 Address to " << in << "/64 routing part.");
 373         return out;
 374     }
 375
 376 #if USE_CLASSFUL
 377     struct in_addr b;
 378
 379     in.getInAddr(b);
 380
 381     if (IN_CLASSC(b.s_addr))
 382         b.s_addr &= IN_CLASSC_NET;
 383     else if (IN_CLASSB(b.s_addr))
 384         b.s_addr &= IN_CLASSB_NET;
 385     else if (IN_CLASSA(b.s_addr))
 386         b.s_addr &= IN_CLASSA_NET;
 387
 388     out = b;
 389
 390 #endif
 391
 392     debugs(14, 5, "networkFromInaddr : Masked IPv4 Address to " << out << "/24.");
 393
 394     /* use /24 for everything under IPv4 */
 395     out.applyMask(24, AF_INET);
 396     debugs(14, 5, "networkFromInaddr : Masked IPv4 Address to " << in << "/24.");
 397
 398     return out;
 399 }
 400
 401 static int
 402 sortByRtt(const void *A, const void *B)
 403 {
 404     const netdbEntry *const *n1 = (const netdbEntry *const *)A;
 405     const netdbEntry *const *n2 = (const netdbEntry *const *)B;
 406
 407     if ((*n1)->rtt > (*n2)->rtt)
 408         return 1;
 409     else if ((*n1)->rtt < (*n2)->rtt)
 410         return -1;
 411     else
 412         return 0;
 413 }
 414
 415 static net_db_peer *
 416 netdbPeerByName(const netdbEntry * n, const char *peername)
 417 {
 418     int i;
 419     net_db_peer *p = n->peers;
 420
 421     for (i = 0; i < n->n_peers; ++i, ++p) {
 422         if (!strcmp(p->peername, peername))
 423             return p;
 424     }
 425
 426     return NULL;
 427 }
 428
 429 static net_db_peer *
 430 netdbPeerAdd(netdbEntry * n, CachePeer * e)
 431 {
 432     net_db_peer *p;
 433     net_db_peer *o;
 434     int osize;
 435     int i;
 436
 437     if (n->n_peers == n->n_peers_alloc) {
 438         o = n->peers;
 439         osize = n->n_peers_alloc;
 440
 441         if (n->n_peers_alloc == 0)
 442             n->n_peers_alloc = 2;
 443         else
 444             n->n_peers_alloc <<= 1;
 445
 446         debugs(38, 3, "netdbPeerAdd: Growing peer list for '" << n->network << "' to " << n->n_peers_alloc);
 447
 448         n->peers = (net_db_peer *)xcalloc(n->n_peers_alloc, sizeof(net_db_peer));
 449
 450         for (i = 0; i < osize; ++i)
 451             *(n->peers + i) = *(o + i);
 452
 453         if (osize) {
 454             safe_free(o);
 455         }
 456     }
 457
 458     p = n->peers + n->n_peers;
 459     p->peername = netdbPeerName(e->host);
 460     ++ n->n_peers;
 461     return p;
 462 }
 463
 464 static int
 465 sortPeerByRtt(const void *A, const void *B)
 466 {
 467     const net_db_peer *p1 = (net_db_peer *)A;
 468     const net_db_peer *p2 = (net_db_peer *)B;
 469
 470     if (p1->rtt > p2->rtt)
 471         return 1;
 472     else if (p1->rtt < p2->rtt)
 473         return -1;
 474     else
 475         return 0;
 476 }
 477
 478 static void
 479 netdbSaveState(void *foo)
 480 {
 481     if (strcmp(Config.netdbFilename, "none") == 0)
 482         return;
 483
 484     Logfile *lf;
 485     netdbEntry *n;
 486     net_db_name *x;
 487
 488     struct timeval start = current_time;
 489     int count = 0;
 490     /*
 491      * This was nicer when we were using stdio, but thanks to
 492      * Solaris bugs, its a bad idea.  fopen can fail if more than
 493      * 256 FDs are open.
 494      */
 495     /*
 496      * unlink() is here because there is currently no way to make
 497      * logfileOpen() use O_TRUNC.
 498      */
 499     unlink(Config.netdbFilename);
 500     lf = logfileOpen(Config.netdbFilename, 4096, 0);
 501
 502     if (NULL == lf) {
 503         debugs(50, DBG_IMPORTANT, "netdbSaveState: " << Config.netdbFilename << ": " << xstrerror());
 504         return;
 505     }
 506
 507     hash_first(addr_table);
 508
 509     while ((n = (netdbEntry *) hash_next(addr_table))) {
 510         if (n->pings_recv == 0)
 511             continue;
 512
 513         logfilePrintf(lf, "%s %d %d %10.5f %10.5f %d %d",
 514                       n->network,
 515                       n->pings_sent,
 516                       n->pings_recv,
 517                       n->hops,
 518                       n->rtt,
 519                       (int) n->next_ping_time,
 520                       (int) n->last_use_time);
 521
 522         for (x = n->hosts; x; x = x->next)
 523             logfilePrintf(lf, " %s", hashKeyStr(&x->hash));
 524
 525         logfilePrintf(lf, "\n");
 526
 527         ++count;
 528
 529 #undef RBUF_SZ
 530
 531     }
 532
 533     logfileClose(lf);
 534     getCurrentTime();
 535     debugs(38, DBG_IMPORTANT, "NETDB state saved; " <<
 536            count << " entries, " <<
 537            tvSubMsec(start, current_time) << " msec" );
 538     eventAddIsh("netdbSaveState", netdbSaveState, NULL, 3600.0, 1);
 539 }
 540
 541 static void
 542 netdbReloadState(void)
 543 {
 544     if (strcmp(Config.netdbFilename, "none") == 0)
 545         return;
 546
 547     char *s;
 548     int fd;
 549     int l;
 550
 551     struct stat sb;
 552     netdbEntry *n;
 553     netdbEntry N;
 554
 555     Ip::Address addr;
 556     int count = 0;
 557
 558     struct timeval start = current_time;
 559     /*
 560      * This was nicer when we were using stdio, but thanks to
 561      * Solaris bugs, its a bad idea.  fopen can fail if more than
 562      * 256 FDs are open.
 563      */
 564     fd = file_open(Config.netdbFilename, O_RDONLY | O_BINARY);
 565
 566     if (fd < 0)
 567         return;
 568
 569     if (fstat(fd, &sb) < 0) {
 570         file_close(fd);
 571         return;
 572     }
 573
 574     char *t;
 575     char *buf = (char *)xcalloc(1, sb.st_size + 1);
 576     t = buf;
 577     l = FD_READ_METHOD(fd, buf, sb.st_size);
 578     file_close(fd);
 579
 580     if (l <= 0) {
 581         safe_free (buf);
 582         return;
 583     };
 584
 585     while ((s = strchr(t, '\n'))) {
 586         char *q;
 587         assert(s - buf < l);
 588         *s = '\0';
 589         memset(&N, '\0', sizeof(netdbEntry));
 590         q = strtok(t, w_space);
 591         t = s + 1;
 592
 593         if (NULL == q)
 594             continue;
 595
 596         if (! (addr = q) )
 597             continue;
 598
 599         if (netdbLookupAddr(addr) != NULL)  /* no dups! */
 600             continue;
 601
 602         if ((q = strtok(NULL, w_space)) == NULL)
 603             continue;
 604
 605         N.pings_sent = atoi(q);
 606
 607         if ((q = strtok(NULL, w_space)) == NULL)
 608             continue;
 609
 610         N.pings_recv = atoi(q);
 611
 612         if (N.pings_recv == 0)
 613             continue;
 614
 615         /* give this measurement low weight */
 616         N.pings_sent = 1;
 617
 618         N.pings_recv = 1;
 619
 620         if ((q = strtok(NULL, w_space)) == NULL)
 621             continue;
 622
 623         N.hops = atof(q);
 624
 625         if ((q = strtok(NULL, w_space)) == NULL)
 626             continue;
 627
 628         N.rtt = atof(q);
 629
 630         if ((q = strtok(NULL, w_space)) == NULL)
 631             continue;
 632
 633         N.next_ping_time = (time_t) atoi(q);
 634
 635         if ((q = strtok(NULL, w_space)) == NULL)
 636             continue;
 637
 638         N.last_use_time = (time_t) atoi(q);
 639
 640         n = (netdbEntry *)memAllocate(MEM_NETDBENTRY);
 641
 642         memcpy(n, &N, sizeof(netdbEntry));
 643
 644         netdbHashInsert(n, addr);
 645
 646         while ((q = strtok(NULL, w_space)) != NULL) {
 647             if (netdbLookupHost(q) != NULL) /* no dups! */
 648                 continue;
 649
 650             netdbHostInsert(n, q);
 651         }
 652
 653         ++count;
 654     }
 655
 656     xfree(buf);
 657     getCurrentTime();
 658     debugs(38, DBG_IMPORTANT, "NETDB state reloaded; " <<
 659            count << " entries, " <<
 660            tvSubMsec(start, current_time) << " msec" );
 661 }
 662
 663 static const char *
 664 netdbPeerName(const char *name)
 665 {
 666     const wordlist *w;
 667
 668     for (w = peer_names; w; w = w->next) {
 669         if (!strcmp(w->key, name))
 670             return w->key;
 671     }
 672
 673     return wordlistAdd(&peer_names, name);
 674 }
 675
 676 static void
 677 netdbFreeNetdbEntry(void *data)
 678 {
 679     netdbEntry *n = (netdbEntry *)data;
 680     safe_free(n->peers);
 681     memFree(n, MEM_NETDBENTRY);
 682 }
 683
 684 static void
 685 netdbFreeNameEntry(void *data)
 686 {
 687     net_db_name *x = (net_db_name *)data;
 688     xfree(x->hash.key);
 689     memFree(x, MEM_NET_DB_NAME);
 690 }
 691
 692 static void
 693 netdbExchangeHandleReply(void *data, StoreIOBuffer receivedData)
 694 {
 695     Ip::Address addr;
 696
 697     netdbExchangeState *ex = (netdbExchangeState *)data;
 698     int rec_sz = 0;
 699     int o;
 700
 701     struct in_addr line_addr;
 702     double rtt;
 703     double hops;
 704     char *p;
 705     int j;
 706     HttpReply const *rep;
 707     size_t hdr_sz;
 708     int nused = 0;
 709     int size;
 710     int oldbufofs = ex->buf_ofs;
 711
 712     rec_sz = 0;
 713     rec_sz += 1 + sizeof(struct in_addr);
 714     rec_sz += 1 + sizeof(int);
 715     rec_sz += 1 + sizeof(int);
 716     debugs(38, 3, "netdbExchangeHandleReply: " << receivedData.length << " read bytes");
 717
 718     if (!cbdataReferenceValid(ex->p)) {
 719         debugs(38, 3, "netdbExchangeHandleReply: Peer became invalid");
 720         delete ex;
 721         return;
 722     }
 723
 724     debugs(38, 3, "netdbExchangeHandleReply: for '" << ex->p->host << ":" << ex->p->http_port << "'");
 725
 726     if (receivedData.length == 0 && !receivedData.flags.error) {
 727         debugs(38, 3, "netdbExchangeHandleReply: Done");
 728         delete ex;
 729         return;
 730     }
 731
 732     p = ex->buf;
 733
 734     /* Get the size of the buffer now */
 735     size = ex->buf_ofs + receivedData.length;
 736     debugs(38, 3, "netdbExchangeHandleReply: " << size << " bytes buf");
 737
 738     /* Check if we're still doing headers */
 739
 740     if (ex->connstate == STATE_HEADER) {
 741
 742         ex->buf_ofs += receivedData.length;
 743
 744         /* skip reply headers */
 745
 746         if ((hdr_sz = headersEnd(p, ex->buf_ofs))) {
 747             debugs(38, 5, "netdbExchangeHandleReply: hdr_sz = " << hdr_sz);
 748             rep = ex->e->getReply();
 749             assert(rep->sline.status() != Http::scNone);
 750             debugs(38, 3, "netdbExchangeHandleReply: reply status " << rep->sline.status());
 751
 752             if (rep->sline.status() != Http::scOkay) {
 753                 delete ex;
 754                 return;
 755             }
 756
 757             assert((size_t)ex->buf_ofs >= hdr_sz);
 758
 759             /*
 760              * Now, point p to the part of the buffer where the data
 761              * starts, and update the size accordingly
 762              */
 763             assert(ex->used == 0);
 764             ex->used = hdr_sz;
 765             size = ex->buf_ofs - hdr_sz;
 766             p += hdr_sz;
 767
 768             /* Finally, set the conn state mode to STATE_BODY */
 769             ex->connstate = STATE_BODY;
 770         } else {
 771             StoreIOBuffer tempBuffer;
 772             tempBuffer.offset = ex->buf_ofs;
 773             tempBuffer.length = ex->buf_sz - ex->buf_ofs;
 774             tempBuffer.data = ex->buf + ex->buf_ofs;
 775             /* Have more headers .. */
 776             storeClientCopy(ex->sc, ex->e, tempBuffer,
 777                             netdbExchangeHandleReply, ex);
 778             return;
 779         }
 780     }
 781
 782     assert(ex->connstate == STATE_BODY);
 783
 784     /* If we get here, we have some body to parse .. */
 785     debugs(38, 5, "netdbExchangeHandleReply: start parsing loop, size = " << size);
 786
 787     while (size >= rec_sz) {
 788         debugs(38, 5, "netdbExchangeHandleReply: in parsing loop, size = " << size);
 789         addr.setAnyAddr();
 790         hops = rtt = 0.0;
 791
 792         for (o = 0; o < rec_sz;) {
 793             switch ((int) *(p + o)) {
 794
 795             case NETDB_EX_NETWORK:
 796                 ++o;
 797                 /* FIXME INET6 : NetDB can still ony send IPv4 */
 798                 memcpy(&line_addr, p + o, sizeof(struct in_addr));
 799                 addr = line_addr;
 800                 o += sizeof(struct in_addr);
 801                 break;
 802
 803             case NETDB_EX_RTT:
 804                 ++o;
 805                 memcpy(&j, p + o, sizeof(int));
 806                 o += sizeof(int);
 807                 rtt = (double) ntohl(j) / 1000.0;
 808                 break;
 809
 810             case NETDB_EX_HOPS:
 811                 ++o;
 812                 memcpy(&j, p + o, sizeof(int));
 813                 o += sizeof(int);
 814                 hops = (double) ntohl(j) / 1000.0;
 815                 break;
 816
 817             default:
 818                 debugs(38, DBG_IMPORTANT, "netdbExchangeHandleReply: corrupt data, aborting");
 819                 delete ex;
 820                 return;
 821             }
 822         }
 823
 824         if (!addr.isAnyAddr() && rtt > 0)
 825             netdbExchangeUpdatePeer(addr, ex->p, rtt, hops);
 826
 827         assert(o == rec_sz);
 828
 829         ex->used += rec_sz;
 830
 831         size -= rec_sz;
 832
 833         p += rec_sz;
 834
 835         ++nused;
 836     }
 837
 838     /*
 839      * Copy anything that is left over to the beginning of the buffer,
 840      * and adjust buf_ofs accordingly
 841      */
 842
 843     /*
 844      * Evilly, size refers to the buf size left now,
 845      * ex->buf_ofs is the original buffer size, so just copy that
 846      * much data over
 847      */
 848     memmove(ex->buf, ex->buf + (ex->buf_ofs - size), size);
 849
 850     ex->buf_ofs = size;
 851
 852     /*
 853      * And don't re-copy the remaining data ..
 854      */
 855     ex->used += size;
 856
 857     /*
 858      * Now the tricky bit - size _included_ the leftover bit from the _last_
 859      * storeClientCopy. We don't want to include that, or our offset will be wrong.
 860      * So, don't count the size of the leftover buffer we began with.
 861      * This can _disappear_ when we're not tracking offsets ..
 862      */
 863     ex->used -= oldbufofs;
 864
 865     debugs(38, 3, "netdbExchangeHandleReply: size left over in this buffer: " << size << " bytes");
 866
 867     debugs(38, 3, "netdbExchangeHandleReply: used " << nused <<
 868            " entries, (x " << rec_sz << " bytes) == " << nused * rec_sz <<
 869            " bytes total");
 870
 871     debugs(38, 3, "netdbExchangeHandleReply: used " << ex->used);
 872
 873     if (EBIT_TEST(ex->e->flags, ENTRY_ABORTED)) {
 874         debugs(38, 3, "netdbExchangeHandleReply: ENTRY_ABORTED");
 875         delete ex;
 876     } else if (ex->e->store_status == STORE_PENDING) {
 877         StoreIOBuffer tempBuffer;
 878         tempBuffer.offset = ex->used;
 879         tempBuffer.length = ex->buf_sz - ex->buf_ofs;
 880         tempBuffer.data = ex->buf + ex->buf_ofs;
 881         debugs(38, 3, "netdbExchangeHandleReply: EOF not received");
 882         storeClientCopy(ex->sc, ex->e, tempBuffer,
 883                         netdbExchangeHandleReply, ex);
 884     }
 885 }
 886
 887 static void
 888 netdbRegisterWithCacheManager(void)
 889 {
 890     Mgr::RegisterAction("netdb", "Network Measurement Database", netdbDump, 0, 1);
 891 }
 892
 893 #endif /* USE_ICMP */
 894
 895 /* PUBLIC FUNCTIONS */
 896
 897 void
 898 netdbInit(void)
 899 {
 900 #if USE_ICMP
 901     int n;
 902
 903     netdbRegisterWithCacheManager();
 904
 905     if (addr_table)
 906         return;
 907
 908     n = hashPrime(Config.Netdb.high / 4);
 909
 910     addr_table = hash_create((HASHCMP *) strcmp, n, hash_string);
 911
 912     n = hashPrime(3 * Config.Netdb.high / 4);
 913
 914     host_table = hash_create((HASHCMP *) strcmp, n, hash_string);
 915
 916     eventAddIsh("netdbSaveState", netdbSaveState, NULL, 3600.0, 1);
 917
 918     netdbReloadState();
 919
 920 #endif
 921 }
 922
 923 void
 924 netdbPingSite(const char *hostname)
 925 {
 926 #if USE_ICMP
 927     netdbEntry *n;
 928
 929     if ((n = netdbLookupHost(hostname)) != NULL)
 930         if (n->next_ping_time > squid_curtime)
 931             return;
 932
 933     ipcache_nbgethostbyname(hostname, netdbSendPing,
 934                             new generic_cbdata(xstrdup(hostname)));
 935
 936 #endif
 937 }
 938
 939 void
 940 netdbHandlePingReply(const Ip::Address &from, int hops, int rtt)
 941 {
 942 #if USE_ICMP
 943     netdbEntry *n;
 944     int N;
 945     debugs(38, 3, "netdbHandlePingReply: from " << from);
 946
 947     if ((n = netdbLookupAddr(from)) == NULL)
 948         return;
 949
 950     N = ++n->pings_recv;
 951
 952     if (N > 5)
 953         N = 5;
 954
 955     if (rtt < 1)
 956         rtt = 1;
 957
 958     n->hops = ((n->hops * (N - 1)) + hops) / N;
 959
 960     n->rtt = ((n->rtt * (N - 1)) + rtt) / N;
 961
 962     debugs(38, 3, "netdbHandlePingReply: " << n->network  << "; rtt="<<
 963            std::setw(5)<< std::setprecision(2) << n->rtt << "  hops="<<
 964            std::setw(4) << n->hops);
 965
 966 #endif
 967 }
 968
 969 void
 970 netdbFreeMemory(void)
 971 {
 972 #if USE_ICMP
 973     hashFreeItems(addr_table, netdbFreeNetdbEntry);
 974     hashFreeMemory(addr_table);
 975     addr_table = NULL;
 976     hashFreeItems(host_table, netdbFreeNameEntry);
 977     hashFreeMemory(host_table);
 978     host_table = NULL;
 979     wordlistDestroy(&peer_names);
 980     peer_names = NULL;
 981 #endif
 982 }
 983
 984 void
 985 netdbDump(StoreEntry * sentry)
 986 {
 987 #if USE_ICMP
 988     netdbEntry *n;
 989     netdbEntry **list;
 990     net_db_name *x;
 991     int k;
 992     int i;
 993     int j;
 994     net_db_peer *p;
 995     storeAppendPrintf(sentry, "Network DB Statistics:\n");
 996     storeAppendPrintf(sentry, "%-46.46s %9s %7s %5s %s\n",  /* Max between 16 (IPv4) or 46 (IPv6)   */
 997                       "Network",
 998                       "recv/sent",
 999                       "RTT",
1000                       "Hops",
1001                       "Hostnames");
1002     list = (netdbEntry **)xcalloc(memInUse(MEM_NETDBENTRY), sizeof(netdbEntry *));
1003     i = 0;
1004     hash_first(addr_table);
1005
1006     while ((n = (netdbEntry *) hash_next(addr_table))) {
1007         *(list + i) = n;
1008         ++i;
1009     }
1010
1011     if (i != memInUse(MEM_NETDBENTRY))
1012         debugs(38, DBG_CRITICAL, "WARNING: netdb_addrs count off, found " << i <<
1013                ", expected " << memInUse(MEM_NETDBENTRY));
1014
1015     qsort((char *) list,
1016           i,
1017           sizeof(netdbEntry *),
1018           sortByRtt);
1019
1020     for (k = 0; k < i; ++k) {
1021         n = *(list + k);
1022         storeAppendPrintf(sentry, "%-46.46s %4d/%4d %7.1f %5.1f", /* Max between 16 (IPv4) or 46 (IPv6)   */
1023                           n->network,
1024                           n->pings_recv,
1025                           n->pings_sent,
1026                           n->rtt,
1027                           n->hops);
1028
1029         for (x = n->hosts; x; x = x->next)
1030             storeAppendPrintf(sentry, " %s", hashKeyStr(&x->hash));
1031
1032         storeAppendPrintf(sentry, "\n");
1033
1034         p = n->peers;
1035
1036         for (j = 0; j < n->n_peers; ++j, ++p) {
1037             storeAppendPrintf(sentry, "    %-22.22s %7.1f %5.1f\n",
1038                               p->peername,
1039                               p->rtt,
1040                               p->hops);
1041         }
1042     }
1043
1044     xfree(list);
1045 #else
1046
1047     storeAppendPrintf(sentry,"NETDB support not compiled into this Squid cache.\n");
1048 #endif
1049 }
1050
1051 int
1052 netdbHostHops(const char *host)
1053 {
1054 #if USE_ICMP
1055     netdbEntry *n = netdbLookupHost(host);
1056
1057     if (n) {
1058         n->last_use_time = squid_curtime;
1059         return (int) (n->hops + 0.5);
1060     }
1061
1062 #endif
1063     return 0;
1064 }
1065
1066 int
1067 netdbHostRtt(const char *host)
1068 {
1069 #if USE_ICMP
1070     netdbEntry *n = netdbLookupHost(host);
1071
1072     if (n) {
1073         n->last_use_time = squid_curtime;
1074         return (int) (n->rtt + 0.5);
1075     }
1076
1077 #endif
1078     return 0;
1079 }
1080
1081 void
1082 netdbHostData(const char *host, int *samp, int *rtt, int *hops)
1083 {
1084 #if USE_ICMP
1085     netdbEntry *n = netdbLookupHost(host);
1086
1087     if (n == NULL)
1088         return;
1089
1090     *samp = n->pings_recv;
1091
1092     *rtt = (int) (n->rtt + 0.5);
1093
1094     *hops = (int) (n->hops + 0.5);
1095
1096     n->last_use_time = squid_curtime;
1097
1098 #endif
1099 }
1100
1101 void
1102 netdbUpdatePeer(const URL &url, CachePeer * e, int irtt, int ihops)
1103 {
1104 #if USE_ICMP
1105     netdbEntry *n;
1106     double rtt = (double) irtt;
1107     double hops = (double) ihops;
1108     net_db_peer *p;
1109     debugs(38, 3, url.host() << ", " << ihops << " hops, " << irtt << " rtt");
1110     n = netdbLookupHost(url.host());
1111
1112     if (n == NULL) {
1113         debugs(38, 3, "host " << url.host() << " not found");
1114         return;
1115     }
1116
1117     if ((p = netdbPeerByName(n, e->host)) == NULL)
1118         p = netdbPeerAdd(n, e);
1119
1120     p->rtt = rtt;
1121
1122     p->hops = hops;
1123
1124     p->expires = squid_curtime + 3600;
1125
1126     if (n->n_peers < 2)
1127         return;
1128
1129     qsort((char *) n->peers,
1130           n->n_peers,
1131           sizeof(net_db_peer),
1132           sortPeerByRtt);
1133
1134 #endif
1135 }
1136
1137 void
1138 netdbExchangeUpdatePeer(Ip::Address &addr, CachePeer * e, double rtt, double hops)
1139 {
1140 #if USE_ICMP
1141     netdbEntry *n;
1142     net_db_peer *p;
1143     debugs(38, 5, "netdbExchangeUpdatePeer: '" << addr << "', "<<
1144            std::setfill('0')<< std::setprecision(2) << hops << " hops, " <<
1145            rtt << " rtt");
1146
1147     if ( !addr.isIPv4() ) {
1148         debugs(38, 5, "netdbExchangeUpdatePeer: Aborting peer update for '" << addr << "', NetDB cannot handle IPv6.");
1149         return;
1150     }
1151
1152     n = netdbLookupAddr(addr);
1153
1154     if (n == NULL)
1155         n = netdbAdd(addr);
1156
1157     assert(NULL != n);
1158
1159     if ((p = netdbPeerByName(n, e->host)) == NULL)
1160         p = netdbPeerAdd(n, e);
1161
1162     p->rtt = rtt;
1163
1164     p->hops = hops;
1165
1166     p->expires = squid_curtime + 3600;  /* XXX ? */
1167
1168     if (n->n_peers < 2)
1169         return;
1170
1171     qsort((char *) n->peers,
1172           n->n_peers,
1173           sizeof(net_db_peer),
1174           sortPeerByRtt);
1175
1176 #endif
1177 }
1178
1179 void
1180 netdbDeleteAddrNetwork(Ip::Address &addr)
1181 {
1182 #if USE_ICMP
1183     netdbEntry *n = netdbLookupAddr(addr);
1184
1185     if (n == NULL)
1186         return;
1187
1188     debugs(38, 3, "netdbDeleteAddrNetwork: " << n->network);
1189
1190     netdbRelease(n);
1191 #endif
1192 }
1193
1194 void
1195 netdbBinaryExchange(StoreEntry * s)
1196 {
1197     HttpReply *reply = new HttpReply;
1198 #if USE_ICMP
1199
1200     Ip::Address addr;
1201
1202     netdbEntry *n;
1203     int i;
1204     int j;
1205     int rec_sz;
1206     char *buf;
1207
1208     struct in_addr line_addr;
1209     s->buffer();
1210     reply->setHeaders(Http::scOkay, "OK", NULL, -1, squid_curtime, -2);
1211     s->replaceHttpReply(reply);
1212     rec_sz = 0;
1213     rec_sz += 1 + sizeof(struct in_addr);
1214     rec_sz += 1 + sizeof(int);
1215     rec_sz += 1 + sizeof(int);
1216     buf = (char *)memAllocate(MEM_4K_BUF);
1217     i = 0;
1218     hash_first(addr_table);
1219
1220     while ((n = (netdbEntry *) hash_next(addr_table))) {
1221         if (0.0 == n->rtt)
1222             continue;
1223
1224         if (n->rtt > 60000) /* RTT > 1 MIN probably bogus */
1225             continue;
1226
1227         if (! (addr = n->network) )
1228             continue;
1229
1230         /* FIXME INET6 : NetDB cannot yet handle IPv6 addresses. Ensure only IPv4 get sent. */
1231         if ( !addr.isIPv4() )
1232             continue;
1233
1234         buf[i] = (char) NETDB_EX_NETWORK;
1235         ++i;
1236
1237         addr.getInAddr(line_addr);
1238         memcpy(&buf[i], &line_addr, sizeof(struct in_addr));
1239
1240         i += sizeof(struct in_addr);
1241
1242         buf[i] = (char) NETDB_EX_RTT;
1243         ++i;
1244
1245         j = htonl((int) (n->rtt * 1000));
1246
1247         memcpy(&buf[i], &j, sizeof(int));
1248
1249         i += sizeof(int);
1250
1251         buf[i] = (char) NETDB_EX_HOPS;
1252         ++i;
1253
1254         j = htonl((int) (n->hops * 1000));
1255
1256         memcpy(&buf[i], &j, sizeof(int));
1257
1258         i += sizeof(int);
1259
1260         if (i + rec_sz > 4096) {
1261             s->append(buf, i);
1262             i = 0;
1263         }
1264     }
1265
1266     if (i > 0) {
1267         s->append(buf, i);
1268         i = 0;
1269     }
1270
1271     assert(0 == i);
1272     s->flush();
1273     memFree(buf, MEM_4K_BUF);
1274 #else
1275
1276     reply->setHeaders(Http::scBadRequest, "Bad Request", NULL, -1, squid_curtime, -2);
1277     s->replaceHttpReply(reply);
1278     storeAppendPrintf(s, "NETDB support not compiled into this Squid cache.\n");
1279 #endif
1280
1281     s->complete();
1282 }
1283
1284 void
1285 netdbExchangeStart(void *data)
1286 {
1287 #if USE_ICMP
1288     CachePeer *p = (CachePeer *)data;
1289     static const SBuf netDB("netdb");
1290     char *uri = internalRemoteUri(p->host, p->http_port, "/squid-internal-dynamic/", netDB);
1291     debugs(38, 3, "netdbExchangeStart: Requesting '" << uri << "'");
1292     assert(NULL != uri);
1293     HttpRequest *req = HttpRequest::CreateFromUrl(uri);
1294
1295     if (req == NULL) {
1296         debugs(38, DBG_IMPORTANT, "netdbExchangeStart: Bad URI " << uri);
1297         return;
1298     }
1299
1300     netdbExchangeState *ex = new netdbExchangeState(p, req);
1301     ex->e = storeCreateEntry(uri, uri, RequestFlags(), Http::METHOD_GET);
1302     assert(NULL != ex->e);
1303
1304     StoreIOBuffer tempBuffer;
1305     tempBuffer.length = ex->buf_sz;
1306     tempBuffer.data = ex->buf;
1307
1308     ex->sc = storeClientListAdd(ex->e, ex);
1309
1310     storeClientCopy(ex->sc, ex->e, tempBuffer,
1311                     netdbExchangeHandleReply, ex);
1312     ex->r->flags.loopDetected = true;   /* cheat! -- force direct */
1313
1314     // XXX: send as Proxy-Authenticate instead
1315     if (p->login)
1316         ex->r->url.userInfo(SBuf(p->login));
1317
1318     FwdState::fwdStart(Comm::ConnectionPointer(), ex->e, ex->r);
1319
1320 #endif
1321 }
1322
1323 CachePeer *
1324 netdbClosestParent(HttpRequest * request)
1325 {
1326 #if USE_ICMP
1327     CachePeer *p = NULL;
1328     netdbEntry *n;
1329     const ipcache_addrs *ia;
1330     net_db_peer *h;
1331     int i;
1332     n = netdbLookupHost(request->url.host());
1333
1334     if (NULL == n) {
1335         /* try IP addr */
1336         ia = ipcache_gethostbyname(request->url.host(), 0);
1337
1338         if (NULL != ia)
1339             n = netdbLookupAddr(ia->in_addrs[ia->cur]);
1340     }
1341
1342     if (NULL == n)
1343         return NULL;
1344
1345     if (0 == n->n_peers)
1346         return NULL;
1347
1348     n->last_use_time = squid_curtime;
1349
1350     /*
1351      * Find the parent with the least RTT to the origin server.
1352      * Make sure we don't return a parent who is farther away than
1353      * we are.  Note, the n->peers list is pre-sorted by RTT.
1354      */
1355     for (i = 0; i < n->n_peers; ++i) {
1356         h = &n->peers[i];
1357
1358         if (n->rtt > 0)
1359             if (n->rtt < h->rtt)
1360                 break;
1361
1362         p = peerFindByName(h->peername);
1363
1364         if (NULL == p)      /* not found */
1365             continue;
1366
1367         if (neighborType(p, request->url) != PEER_PARENT)
1368             continue;
1369
1370         if (!peerHTTPOkay(p, request))  /* not allowed */
1371             continue;
1372
1373         return p;
1374     }
1375
1376 #endif
1377     return NULL;
1378 }
1379