src/net_db.cc

   1 /*
   2  * $Id: net_db.cc,v 1.202 2008/01/07 17:12:28 hno Exp $
   3  *
   4  * DEBUG: section 38    Network Measurement Database
   5  * AUTHOR: Duane Wessels
   6  *
   7  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   8  * ----------------------------------------------------------
   9  *
  10  *  Squid is the result of efforts by numerous individuals from
  11  *  the Internet community; see the CONTRIBUTORS file for full
  12  *  details.   Many organizations have provided support for Squid's
  13  *  development; see the SPONSORS file for full details.  Squid is
  14  *  Copyrighted (C) 2001 by the Regents of the University of
  15  *  California; see the COPYRIGHT file for full details.  Squid
  16  *  incorporates software developed and/or copyrighted by other
  17  *  sources; see the CREDITS file for full details.
  18  *
  19  *  This program is free software; you can redistribute it and/or modify
  20  *  it under the terms of the GNU General Public License as published by
  21  *  the Free Software Foundation; either version 2 of the License, or
  22  *  (at your option) any later version.
  23  *
  24  *  This program is distributed in the hope that it will be useful,
  25  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  26  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  27  *  GNU General Public License for more details.
  28  *
  29  *  You should have received a copy of the GNU General Public License
  30  *  along with this program; if not, write to the Free Software
  31  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  32  *
  33  */
  34
  35 /*
  36  * XXX XXX XXX
  37  *
  38  * This code may be slightly broken now. If you're getting consistent
  39  * (sometimes working) corrupt data exchanges, please contact adrian
  40  * (adrian@squid-cache.org) to sort them out.
  41  */
  42
  43 #include "squid.h"
  44 #include "cbdata.h"
  45 #include "event.h"
  46 #include "CacheManager.h"
  47 #include "Store.h"
  48 #include "SwapDir.h"
  49 #include "HttpRequest.h"
  50 #include "HttpReply.h"
  51 #include "MemObject.h"
  52 #include "fde.h"
  53 #include "forward.h"
  54 #include "SquidTime.h"
  55 #include "wordlist.h"
  56 #include "IPAddress.h"
  57
  58 #if USE_ICMP
  59 #include "ICMPSquid.h"
  60 #include "StoreClient.h"
  61
  62 #define NETDB_REQBUF_SZ 4096
  63
  64 typedef enum {
  65     STATE_NONE,
  66     STATE_HEADER,
  67     STATE_BODY
  68 } netdb_conn_state_t;
  69
  70 typedef struct
  71 {
  72     peer *p;
  73     StoreEntry *e;
  74     store_client *sc;
  75     HttpRequest *r;
  76     int64_t used;
  77     size_t buf_sz;
  78     char buf[NETDB_REQBUF_SZ];
  79     int buf_ofs;
  80     netdb_conn_state_t connstate;
  81 } netdbExchangeState;
  82
  83 static hash_table *addr_table = NULL;
  84 static hash_table *host_table = NULL;
  85
  86 IPAddress networkFromInaddr(const IPAddress &a);
  87 static void netdbRelease(netdbEntry * n);
  88
  89 static void netdbHashInsert(netdbEntry * n, IPAddress &addr);
  90 static void netdbHashDelete(const char *key);
  91 static void netdbHostInsert(netdbEntry * n, const char *hostname);
  92 static void netdbHostDelete(const net_db_name * x);
  93 static void netdbPurgeLRU(void);
  94 static netdbEntry *netdbLookupHost(const char *key);
  95 static net_db_peer *netdbPeerByName(const netdbEntry * n, const char *);
  96 static net_db_peer *netdbPeerAdd(netdbEntry * n, peer * e);
  97 static const char *netdbPeerName(const char *name);
  98 static IPH netdbSendPing;
  99 static QS sortPeerByRtt;
 100 static QS sortByRtt;
 101 static QS netdbLRU;
 102 static FREE netdbFreeNameEntry;
 103 static FREE netdbFreeNetdbEntry;
 104 static STCB netdbExchangeHandleReply;
 105 static void netdbExchangeDone(void *);
 106
 107 /* We have to keep a local list of peer names.  The Peers structure
 108  * gets freed during a reconfigure.  We want this database to
 109  * remain persisitent, so _net_db_peer->peername points into this
 110  * linked list */
 111 static wordlist *peer_names = NULL;
 112
 113 static void
 114 netdbHashInsert(netdbEntry * n, IPAddress &addr)
 115 {
 116     networkFromInaddr(addr).NtoA(n->network, MAX_IPSTRLEN);
 117     n->hash.key = n->network;
 118     assert(hash_lookup(addr_table, n->network) == NULL);
 119     hash_join(addr_table, &n->hash);
 120 }
 121
 122 static void
 123 netdbHashDelete(const char *key)
 124 {
 125     hash_link *hptr = (hash_link *)hash_lookup(addr_table, key);
 126
 127     if (hptr == NULL) {
 128         debug_trap("netdbHashDelete: key not found");
 129         return;
 130     }
 131
 132     hash_remove_link(addr_table, hptr);
 133 }
 134
 135 static void
 136 netdbHostInsert(netdbEntry * n, const char *hostname)
 137 {
 138     net_db_name *x = (net_db_name *)memAllocate(MEM_NET_DB_NAME);
 139     x->hash.key = xstrdup(hostname);
 140     x->next = n->hosts;
 141     n->hosts = x;
 142     x->net_db_entry = n;
 143     assert(hash_lookup(host_table, hostname) == NULL);
 144     hash_join(host_table, &x->hash);
 145     n->link_count++;
 146 }
 147
 148 static void
 149 netdbHostDelete(const net_db_name * x)
 150 {
 151     netdbEntry *n;
 152     net_db_name **X;
 153     assert(x != NULL);
 154     assert(x->net_db_entry != NULL);
 155     n = x->net_db_entry;
 156     n->link_count--;
 157
 158     for (X = &n->hosts; *X; X = &(*X)->next) {
 159         if (*X == x) {
 160             *X = x->next;
 161             break;
 162         }
 163     }
 164
 165     hash_remove_link(host_table, (hash_link *) x);
 166     xfree(x->hash.key);
 167     memFree((void *) x, MEM_NET_DB_NAME);
 168 }
 169
 170 static netdbEntry *
 171 netdbLookupHost(const char *key)
 172 {
 173     net_db_name *x = (net_db_name *) hash_lookup(host_table, key);
 174     return x ? x->net_db_entry : NULL;
 175 }
 176
 177 static void
 178 netdbRelease(netdbEntry * n)
 179 {
 180     net_db_name *x;
 181     net_db_name *next;
 182
 183     for (x = n->hosts; x; x = next) {
 184         next = x->next;
 185         netdbHostDelete(x);
 186     }
 187
 188     n->hosts = NULL;
 189     safe_free(n->peers);
 190     n->peers = NULL;
 191     n->n_peers = 0;
 192     n->n_peers_alloc = 0;
 193
 194     if (n->link_count == 0) {
 195         netdbHashDelete(n->network);
 196         memFree(n, MEM_NETDBENTRY);
 197     }
 198 }
 199
 200 static int
 201 netdbLRU(const void *A, const void *B)
 202 {
 203     const netdbEntry *const *n1 = (const netdbEntry *const *)A;
 204     const netdbEntry *const *n2 = (const netdbEntry *const *)B;
 205
 206     if ((*n1)->last_use_time > (*n2)->last_use_time)
 207         return (1);
 208
 209     if ((*n1)->last_use_time < (*n2)->last_use_time)
 210         return (-1);
 211
 212     return (0);
 213 }
 214
 215 static void
 216 netdbPurgeLRU(void)
 217 {
 218     netdbEntry *n;
 219     netdbEntry **list;
 220     int k = 0;
 221     int list_count = 0;
 222     int removed = 0;
 223     list = (netdbEntry **)xcalloc(memInUse(MEM_NETDBENTRY), sizeof(netdbEntry *));
 224     hash_first(addr_table);
 225
 226     while ((n = (netdbEntry *) hash_next(addr_table))) {
 227         assert(list_count < memInUse(MEM_NETDBENTRY));
 228         *(list + list_count) = n;
 229         list_count++;
 230     }
 231
 232     qsort((char *) list,
 233           list_count,
 234           sizeof(netdbEntry *),
 235           netdbLRU);
 236
 237     for (k = 0; k < list_count; k++) {
 238         if (memInUse(MEM_NETDBENTRY) < Config.Netdb.low)
 239             break;
 240
 241         netdbRelease(*(list + k));
 242
 243         removed++;
 244     }
 245
 246     xfree(list);
 247 }
 248
 249 static netdbEntry *
 250 netdbLookupAddr(const IPAddress &addr)
 251 {
 252     netdbEntry *n;
 253     char *key = new char[MAX_IPSTRLEN];
 254     networkFromInaddr(addr).NtoA(key,MAX_IPSTRLEN);
 255     n = (netdbEntry *) hash_lookup(addr_table, key);
 256     return n;
 257 }
 258
 259 static netdbEntry *
 260 netdbAdd(IPAddress &addr)
 261 {
 262     netdbEntry *n;
 263
 264     if (memInUse(MEM_NETDBENTRY) > Config.Netdb.high)
 265         netdbPurgeLRU();
 266
 267     if ((n = netdbLookupAddr(addr)) == NULL)
 268     {
 269         n = (netdbEntry *)memAllocate(MEM_NETDBENTRY);
 270         netdbHashInsert(n, addr);
 271     }
 272
 273     return n;
 274 }
 275
 276 static void
 277 netdbSendPing(const ipcache_addrs * ia, void *data)
 278 {
 279     IPAddress addr;
 280     char *hostname = NULL;
 281     static_cast<generic_cbdata *>(data)->unwrap(&hostname);
 282     netdbEntry *n;
 283     netdbEntry *na;
 284     net_db_name *x;
 285     net_db_name **X;
 286
 287     if (ia == NULL) {
 288         xfree(hostname);
 289         return;
 290     }
 291
 292     addr = ia->in_addrs[ia->cur];
 293
 294     if ((n = netdbLookupHost(hostname)) == NULL) {
 295         n = netdbAdd(addr);
 296         netdbHostInsert(n, hostname);
 297     } else if ((na = netdbLookupAddr(addr)) != n) {
 298         /*
 299          *hostname moved from 'network n' to 'network na'!
 300          */
 301
 302         if (na == NULL)
 303             na = netdbAdd(addr);
 304
 305         debugs(38, 3, "netdbSendPing: " << hostname << " moved from " << n->network << " to " << na->network);
 306
 307         x = (net_db_name *) hash_lookup(host_table, hostname);
 308
 309         if (x == NULL) {
 310             debugs(38, 1, "netdbSendPing: net_db_name list bug: " << hostname << " not found");
 311             xfree(hostname);
 312             return;
 313         }
 314
 315         /* remove net_db_name from 'network n' linked list */
 316         for (X = &n->hosts; *X; X = &(*X)->next) {
 317             if (*X == x) {
 318                 *X = x->next;
 319                 break;
 320             }
 321         }
 322
 323         n->link_count--;
 324         /* point to 'network na' from host entry */
 325         x->net_db_entry = na;
 326         /* link net_db_name to 'network na' */
 327         x->next = na->hosts;
 328         na->hosts = x;
 329         na->link_count++;
 330         n = na;
 331     }
 332
 333     if (n->next_ping_time <= squid_curtime) {
 334         debugs(38, 3, "netdbSendPing: pinging " << hostname);
 335         icmpEngine.DomainPing(addr, hostname);
 336         n->pings_sent++;
 337         n->next_ping_time = squid_curtime + Config.Netdb.period;
 338         n->last_use_time = squid_curtime;
 339     }
 340
 341     xfree(hostname);
 342 }
 343
 344 IPAddress
 345 networkFromInaddr(const IPAddress &in)
 346 {
 347     IPAddress out;
 348
 349     out = in;
 350 #if USE_IPV6
 351
 352     /* in IPv6 the 'network' should be the routing section. */
 353
 354     if( in.IsIPv6() )
 355     {
 356         out.ApplyMask(64, AF_INET6);
 357         debugs(14, 5, "networkFromInaddr : Masked IPv6 Address to " << in << "/64 routing part.");
 358         return out;
 359     }
 360 #endif
 361
 362 #if USE_CLASSFUL
 363     struct in_addr b;
 364
 365     in.GetInAddr(b);
 366
 367     if (IN_CLASSC(b.s_addr))
 368         b.s_addr &= IN_CLASSC_NET;
 369     else if (IN_CLASSB(b.s_addr))
 370         b.s_addr &= IN_CLASSB_NET;
 371     else if (IN_CLASSA(b.s_addr))
 372         b.s_addr &= IN_CLASSA_NET;
 373
 374     out = b;
 375
 376 #endif
 377
 378     debugs(14, 5, "networkFromInaddr : Masked IPv4 Address to " << out << "/24.");
 379
 380     /* use /24 for everything under IPv4 */
 381     out.ApplyMask(24, AF_INET);
 382     debugs(14, 5, "networkFromInaddr : Masked IPv4 Address to " << in << "/24.");
 383
 384     return out;
 385 }
 386
 387 static int
 388 sortByRtt(const void *A, const void *B)
 389 {
 390     const netdbEntry *const *n1 = (const netdbEntry *const *)A;
 391     const netdbEntry *const *n2 = (const netdbEntry *const *)B;
 392
 393     if ((*n1)->rtt > (*n2)->rtt)
 394         return 1;
 395     else if ((*n1)->rtt < (*n2)->rtt)
 396         return -1;
 397     else
 398         return 0;
 399 }
 400
 401 static net_db_peer *
 402 netdbPeerByName(const netdbEntry * n, const char *peername)
 403 {
 404     int i;
 405     net_db_peer *p = n->peers;
 406
 407     for (i = 0; i < n->n_peers; i++, p++) {
 408         if (!strcmp(p->peername, peername))
 409             return p;
 410     }
 411
 412     return NULL;
 413 }
 414
 415 static net_db_peer *
 416 netdbPeerAdd(netdbEntry * n, peer * e)
 417 {
 418     net_db_peer *p;
 419     net_db_peer *o;
 420     int osize;
 421     int i;
 422
 423     if (n->n_peers == n->n_peers_alloc) {
 424         o = n->peers;
 425         osize = n->n_peers_alloc;
 426
 427         if (n->n_peers_alloc == 0)
 428             n->n_peers_alloc = 2;
 429         else
 430             n->n_peers_alloc <<= 1;
 431
 432         debugs(38, 3, "netdbPeerAdd: Growing peer list for '" << n->network << "' to " << n->n_peers_alloc);
 433
 434         n->peers = (net_db_peer *)xcalloc(n->n_peers_alloc, sizeof(net_db_peer));
 435
 436         for (i = 0; i < osize; i++)
 437             *(n->peers + i) = *(o + i);
 438
 439         if (osize) {
 440             safe_free(o);
 441         }
 442     }
 443
 444     p = n->peers + n->n_peers;
 445     p->peername = netdbPeerName(e->host);
 446     n->n_peers++;
 447     return p;
 448 }
 449
 450 static int
 451 sortPeerByRtt(const void *A, const void *B)
 452 {
 453     const net_db_peer *p1 = (net_db_peer *)A;
 454     const net_db_peer *p2 = (net_db_peer *)B;
 455
 456     if (p1->rtt > p2->rtt)
 457         return 1;
 458     else if (p1->rtt < p2->rtt)
 459         return -1;
 460     else
 461         return 0;
 462 }
 463
 464 static void
 465 netdbSaveState(void *foo)
 466 {
 467     if (strcmp(Config.netdbFilename, "none") == 0)
 468         return;
 469
 470     Logfile *lf;
 471     netdbEntry *n;
 472     net_db_name *x;
 473
 474     struct timeval start = current_time;
 475     int count = 0;
 476     /*
 477      * This was nicer when we were using stdio, but thanks to
 478      * Solaris bugs, its a bad idea.  fopen can fail if more than
 479      * 256 FDs are open.
 480      */
 481     /*
 482      * unlink() is here because there is currently no way to make
 483      * logfileOpen() use O_TRUNC.
 484      */
 485     unlink(Config.netdbFilename);
 486     lf = logfileOpen(Config.netdbFilename, 4096, 0);
 487
 488     if (NULL == lf) {
 489         debugs(50, 1, "netdbSaveState: " << Config.netdbFilename << ": " << xstrerror());
 490         return;
 491     }
 492
 493     hash_first(addr_table);
 494
 495     while ((n = (netdbEntry *) hash_next(addr_table))) {
 496         if (n->pings_recv == 0)
 497             continue;
 498
 499         logfilePrintf(lf, "%s %d %d %10.5f %10.5f %d %d",
 500                       n->network,
 501                       n->pings_sent,
 502                       n->pings_recv,
 503                       n->hops,
 504                       n->rtt,
 505                       (int) n->next_ping_time,
 506                       (int) n->last_use_time);
 507
 508         for (x = n->hosts; x; x = x->next)
 509             logfilePrintf(lf, " %s", hashKeyStr(&x->hash));
 510
 511         logfilePrintf(lf, "\n");
 512
 513         count++;
 514
 515 #undef RBUF_SZ
 516
 517     }
 518
 519     logfileClose(lf);
 520     getCurrentTime();
 521     debugs(38, 1, "NETDB state saved; " <<
 522                  count << " entries, " <<
 523                  tvSubMsec(start, current_time) << " msec" );
 524     eventAddIsh("netdbSaveState", netdbSaveState, NULL, 3600.0, 1);
 525 }
 526
 527 static void
 528 netdbReloadState(void)
 529 {
 530     if (strcmp(Config.netdbFilename, "none") == 0)
 531         return;
 532
 533     char *s;
 534     int fd;
 535     int l;
 536
 537     struct stat sb;
 538     netdbEntry *n;
 539     netdbEntry N;
 540
 541     IPAddress addr;
 542     int count = 0;
 543
 544     struct timeval start = current_time;
 545     /*
 546      * This was nicer when we were using stdio, but thanks to
 547      * Solaris bugs, its a bad idea.  fopen can fail if more than
 548      * 256 FDs are open.
 549      */
 550     fd = file_open(Config.netdbFilename, O_RDONLY | O_BINARY);
 551
 552     if (fd < 0)
 553         return;
 554
 555     if (fstat(fd, &sb) < 0) {
 556         file_close(fd);
 557         return;
 558     }
 559
 560     char *t;
 561     char *buf = (char *)xcalloc(1, sb.st_size + 1);
 562     t = buf;
 563     l = FD_READ_METHOD(fd, buf, sb.st_size);
 564     file_close(fd);
 565
 566     if (l <= 0) {
 567         safe_free (buf);
 568         return;
 569     };
 570
 571     while ((s = strchr(t, '\n'))) {
 572         char *q;
 573         assert(s - buf < l);
 574         *s = '\0';
 575         memset(&N, '\0', sizeof(netdbEntry));
 576         q = strtok(t, w_space);
 577         t = s + 1;
 578
 579         if (NULL == q)
 580             continue;
 581
 582         if (! (addr = q) )
 583             continue;
 584
 585         if (netdbLookupAddr(addr) != NULL)      /* no dups! */
 586             continue;
 587
 588         if ((q = strtok(NULL, w_space)) == NULL)
 589             continue;
 590
 591         N.pings_sent = atoi(q);
 592
 593         if ((q = strtok(NULL, w_space)) == NULL)
 594             continue;
 595
 596         N.pings_recv = atoi(q);
 597
 598         if (N.pings_recv == 0)
 599             continue;
 600
 601         /* give this measurement low weight */
 602         N.pings_sent = 1;
 603
 604         N.pings_recv = 1;
 605
 606         if ((q = strtok(NULL, w_space)) == NULL)
 607             continue;
 608
 609         N.hops = atof(q);
 610
 611         if ((q = strtok(NULL, w_space)) == NULL)
 612             continue;
 613
 614         N.rtt = atof(q);
 615
 616         if ((q = strtok(NULL, w_space)) == NULL)
 617             continue;
 618
 619         N.next_ping_time = (time_t) atoi(q);
 620
 621         if ((q = strtok(NULL, w_space)) == NULL)
 622             continue;
 623
 624         N.last_use_time = (time_t) atoi(q);
 625
 626         n = (netdbEntry *)memAllocate(MEM_NETDBENTRY);
 627
 628         xmemcpy(n, &N, sizeof(netdbEntry));
 629
 630         netdbHashInsert(n, addr);
 631
 632         while ((q = strtok(NULL, w_space)) != NULL) {
 633             if (netdbLookupHost(q) != NULL)     /* no dups! */
 634                 continue;
 635
 636             netdbHostInsert(n, q);
 637         }
 638
 639         count++;
 640     }
 641
 642     xfree(buf);
 643     getCurrentTime();
 644     debugs(38, 1, "NETDB state reloaded; " <<
 645                   count << " entries, " <<
 646                   tvSubMsec(start, current_time) << " msec" );
 647 }
 648
 649 static const char *
 650 netdbPeerName(const char *name)
 651 {
 652     const wordlist *w;
 653
 654     for (w = peer_names; w; w = w->next) {
 655         if (!strcmp(w->key, name))
 656             return w->key;
 657     }
 658
 659     return wordlistAdd(&peer_names, name);
 660 }
 661
 662 static void
 663 netdbFreeNetdbEntry(void *data)
 664 {
 665     netdbEntry *n = (netdbEntry *)data;
 666     safe_free(n->peers);
 667     memFree(n, MEM_NETDBENTRY);
 668 }
 669
 670 static void
 671 netdbFreeNameEntry(void *data)
 672 {
 673     net_db_name *x = (net_db_name *)data;
 674     xfree(x->hash.key);
 675     memFree(x, MEM_NET_DB_NAME);
 676 }
 677
 678
 679 static void
 680 netdbExchangeHandleReply(void *data, StoreIOBuffer receivedData)
 681 {
 682     IPAddress addr;
 683
 684     netdbExchangeState *ex = (netdbExchangeState *)data;
 685     int rec_sz = 0;
 686     int o;
 687
 688     struct in_addr line_addr;
 689     double rtt;
 690     double hops;
 691     char *p;
 692     int j;
 693     HttpReply const *rep;
 694     size_t hdr_sz;
 695     int nused = 0;
 696     int size;
 697     int oldbufofs = ex->buf_ofs;
 698
 699     rec_sz = 0;
 700     rec_sz += 1 + sizeof(struct in_addr);
 701     rec_sz += 1 + sizeof(int);
 702     rec_sz += 1 + sizeof(int);
 703     debugs(38, 3, "netdbExchangeHandleReply: " << receivedData.length << " read bytes");
 704
 705     if (!cbdataReferenceValid(ex->p)) {
 706         debugs(38, 3, "netdbExchangeHandleReply: Peer became invalid");
 707         netdbExchangeDone(ex);
 708         return;
 709     }
 710
 711     debugs(38, 3, "netdbExchangeHandleReply: for '" << ex->p->host << ":" << ex->p->http_port << "'");
 712
 713     if (receivedData.length == 0 &&
 714             !receivedData.flags.error) {
 715         debugs(38, 3, "netdbExchangeHandleReply: Done");
 716         netdbExchangeDone(ex);
 717         return;
 718     }
 719
 720     p = ex->buf;
 721
 722     /* Get the size of the buffer now */
 723     size = ex->buf_ofs + receivedData.length;
 724     debugs(38, 3, "netdbExchangeHandleReply: " << size << " bytes buf");
 725
 726     /* Check if we're still doing headers */
 727
 728     if (ex->connstate == STATE_HEADER) {
 729
 730         ex->buf_ofs += receivedData.length;
 731
 732         /* skip reply headers */
 733
 734         if ((hdr_sz = headersEnd(p, ex->buf_ofs))) {
 735             debugs(38, 5, "netdbExchangeHandleReply: hdr_sz = " << hdr_sz);
 736             rep = ex->e->getReply();
 737             assert (0 != rep->sline.status);
 738             debugs(38, 3, "netdbExchangeHandleReply: reply status " << rep->sline.status);
 739
 740             if (HTTP_OK != rep->sline.status) {
 741                 netdbExchangeDone(ex);
 742                 return;
 743             }
 744
 745             assert((size_t)ex->buf_ofs >= hdr_sz);
 746
 747             /*
 748              * Now, point p to the part of the buffer where the data
 749              * starts, and update the size accordingly
 750              */
 751             assert(ex->used == 0);
 752             ex->used = hdr_sz;
 753             size = ex->buf_ofs - hdr_sz;
 754             p += hdr_sz;
 755
 756             /* Finally, set the conn state mode to STATE_BODY */
 757             ex->connstate = STATE_BODY;
 758         } else {
 759             StoreIOBuffer tempBuffer;
 760             tempBuffer.offset = ex->buf_ofs;
 761             tempBuffer.length = ex->buf_sz - ex->buf_ofs;
 762             tempBuffer.data = ex->buf + ex->buf_ofs;
 763             /* Have more headers .. */
 764             storeClientCopy(ex->sc, ex->e, tempBuffer,
 765                             netdbExchangeHandleReply, ex);
 766             return;
 767         }
 768     }
 769
 770     assert(ex->connstate == STATE_BODY);
 771
 772     /* If we get here, we have some body to parse .. */
 773     debugs(38, 5, "netdbExchangeHandleReply: start parsing loop, size = " << size);
 774
 775     while (size >= rec_sz) {
 776         debugs(38, 5, "netdbExchangeHandleReply: in parsing loop, size = " << size);
 777         addr.SetAnyAddr();
 778         hops = rtt = 0.0;
 779
 780         for (o = 0; o < rec_sz;) {
 781             switch ((int) *(p + o)) {
 782
 783             case NETDB_EX_NETWORK:
 784                 o++;
 785                 /* FIXME INET6 : NetDB can still ony send IPv4 */
 786                 xmemcpy(&line_addr, p + o, sizeof(struct in_addr));
 787                 addr = line_addr;
 788                 o += sizeof(struct in_addr);
 789                 break;
 790
 791             case NETDB_EX_RTT:
 792                 o++;
 793                 xmemcpy(&j, p + o, sizeof(int));
 794                 o += sizeof(int);
 795                 rtt = (double) ntohl(j) / 1000.0;
 796                 break;
 797
 798             case NETDB_EX_HOPS:
 799                 o++;
 800                 xmemcpy(&j, p + o, sizeof(int));
 801                 o += sizeof(int);
 802                 hops = (double) ntohl(j) / 1000.0;
 803                 break;
 804
 805             default:
 806                 debugs(38, 1, "netdbExchangeHandleReply: corrupt data, aborting");
 807                 netdbExchangeDone(ex);
 808                 return;
 809             }
 810         }
 811
 812         if (!addr.IsAnyAddr() && rtt > 0)
 813             netdbExchangeUpdatePeer(addr, ex->p, rtt, hops);
 814
 815         assert(o == rec_sz);
 816
 817         ex->used += rec_sz;
 818
 819         size -= rec_sz;
 820
 821         p += rec_sz;
 822
 823         nused++;
 824     }
 825
 826     /*
 827      * Copy anything that is left over to the beginning of the buffer,
 828      * and adjust buf_ofs accordingly
 829      */
 830
 831     /*
 832      * Evilly, size refers to the buf size left now,
 833      * ex->buf_ofs is the original buffer size, so just copy that
 834      * much data over
 835      */
 836     memmove(ex->buf, ex->buf + (ex->buf_ofs - size), size);
 837
 838     ex->buf_ofs = size;
 839
 840     /*
 841      * And don't re-copy the remaining data ..
 842      */
 843     ex->used += size;
 844
 845     /*
 846      * Now the tricky bit - size _included_ the leftover bit from the _last_
 847      * storeClientCopy. We don't want to include that, or our offset will be wrong.
 848      * So, don't count the size of the leftover buffer we began with.
 849      * This can _disappear_ when we're not tracking offsets ..
 850      */
 851     ex->used -= oldbufofs;
 852
 853     debugs(38, 3, "netdbExchangeHandleReply: size left over in this buffer: " << size << " bytes");
 854
 855     debugs(38, 3, "netdbExchangeHandleReply: used " << nused <<
 856            " entries, (x " << rec_sz << " bytes) == " << nused * rec_sz <<
 857            " bytes total");
 858
 859     debugs(38, 3, "netdbExchangeHandleReply: used " << ex->used);
 860
 861     if (EBIT_TEST(ex->e->flags, ENTRY_ABORTED)) {
 862         debugs(38, 3, "netdbExchangeHandleReply: ENTRY_ABORTED");
 863         netdbExchangeDone(ex);
 864     } else if (ex->e->store_status == STORE_PENDING) {
 865         StoreIOBuffer tempBuffer;
 866         tempBuffer.offset = ex->used;
 867         tempBuffer.length = ex->buf_sz - ex->buf_ofs;
 868         tempBuffer.data = ex->buf + ex->buf_ofs;
 869         debugs(38, 3, "netdbExchangeHandleReply: EOF not received");
 870         storeClientCopy(ex->sc, ex->e, tempBuffer,
 871                         netdbExchangeHandleReply, ex);
 872     }
 873 }
 874
 875 static void
 876 netdbExchangeDone(void *data)
 877 {
 878     netdbExchangeState *ex = (netdbExchangeState *)data;
 879     debugs(38, 3, "netdbExchangeDone: " << ex->e->url()  );
 880     HTTPMSGUNLOCK(ex->r);
 881     storeUnregister(ex->sc, ex->e, ex);
 882     ex->e->unlock();
 883     cbdataReferenceDone(ex->p);
 884     cbdataFree(ex);
 885 }
 886
 887 static void
 888 netdbRegisterWithCacheManager(void)
 889 {
 890     CacheManager::GetInstance()->
 891         registerAction("netdb", "Network Measurement Database", netdbDump, 0, 1);
 892 }
 893
 894 #endif /* USE_ICMP */
 895
 896
 897 /* PUBLIC FUNCTIONS */
 898
 899 void
 900 netdbInit(void)
 901 {
 902 #if USE_ICMP
 903     int n;
 904
 905     netdbRegisterWithCacheManager();
 906
 907     if (addr_table)
 908         return;
 909
 910     n = hashPrime(Config.Netdb.high / 4);
 911
 912     addr_table = hash_create((HASHCMP *) strcmp, n, hash_string);
 913
 914     n = hashPrime(3 * Config.Netdb.high / 4);
 915
 916     host_table = hash_create((HASHCMP *) strcmp, n, hash_string);
 917
 918     eventAddIsh("netdbSaveState", netdbSaveState, NULL, 3600.0, 1);
 919
 920     netdbReloadState();
 921
 922 #endif
 923 }
 924
 925 void
 926 netdbPingSite(const char *hostname)
 927 {
 928 #if USE_ICMP
 929     netdbEntry *n;
 930
 931     if ((n = netdbLookupHost(hostname)) != NULL)
 932         if (n->next_ping_time > squid_curtime)
 933             return;
 934
 935     ipcache_nbgethostbyname(hostname, netdbSendPing,
 936                             new generic_cbdata(xstrdup(hostname)));
 937
 938 #endif
 939 }
 940
 941 void
 942 netdbHandlePingReply(const IPAddress &from, int hops, int rtt)
 943 {
 944 #if USE_ICMP
 945     netdbEntry *n;
 946     int N;
 947     debugs(38, 3, "netdbHandlePingReply: from " << from);
 948
 949     if ((n = netdbLookupAddr(from)) == NULL)
 950         return;
 951
 952     N = ++n->pings_recv;
 953
 954     if (N > 5)
 955         N = 5;
 956
 957     if (rtt < 1)
 958         rtt = 1;
 959
 960     n->hops = ((n->hops * (N - 1)) + hops) / N;
 961
 962     n->rtt = ((n->rtt * (N - 1)) + rtt) / N;
 963
 964     debugs(38, 3, "netdbHandlePingReply: " << n->network  << "; rtt="<<
 965            std::setw(5)<< std::setprecision(2) << n->rtt << "  hops="<<
 966            std::setw(4) << n->hops);
 967
 968 #endif
 969 }
 970
 971 void
 972 netdbFreeMemory(void)
 973 {
 974 #if USE_ICMP
 975     hashFreeItems(addr_table, netdbFreeNetdbEntry);
 976     hashFreeMemory(addr_table);
 977     addr_table = NULL;
 978     hashFreeItems(host_table, netdbFreeNameEntry);
 979     hashFreeMemory(host_table);
 980     host_table = NULL;
 981     wordlistDestroy(&peer_names);
 982     peer_names = NULL;
 983 #endif
 984 }
 985
 986
 987 #if 0 // AYJ: Looks to be unused code.
 988 int
 989 netdbHops(IPAddress &addr)
 990 {
 991 #if USE_ICMP
 992     netdbEntry *n = netdbLookupAddr(addr);
 993
 994     if (n && n->pings_recv)
 995     {
 996         n->last_use_time = squid_curtime;
 997         return (int) (n->hops + 0.5);
 998     }
 999
1000 #endif
1001     return 256;
1002 }
1003 #endif
1004
1005 void
1006 netdbDump(StoreEntry * sentry)
1007 {
1008 #if USE_ICMP
1009     netdbEntry *n;
1010     netdbEntry **list;
1011     net_db_name *x;
1012     int k;
1013     int i;
1014     int j;
1015     net_db_peer *p;
1016     storeAppendPrintf(sentry, "Network DB Statistics:\n");
1017     storeAppendPrintf(sentry, "%-46.46s %9s %7s %5s %s\n",  /* Max between 16 (IPv4) or 46 (IPv6)   */
1018                       "Network",
1019                       "recv/sent",
1020                       "RTT",
1021                       "Hops",
1022                       "Hostnames");
1023     list = (netdbEntry **)xcalloc(memInUse(MEM_NETDBENTRY), sizeof(netdbEntry *));
1024     i = 0;
1025     hash_first(addr_table);
1026
1027     while ((n = (netdbEntry *) hash_next(addr_table)))
1028         *(list + i++) = n;
1029
1030     if (i != memInUse(MEM_NETDBENTRY))
1031         debugs(38, 0, "WARNING: netdb_addrs count off, found " << i <<
1032                ", expected " << memInUse(MEM_NETDBENTRY));
1033
1034     qsort((char *) list,
1035           i,
1036           sizeof(netdbEntry *),
1037           sortByRtt);
1038
1039     for (k = 0; k < i; k++) {
1040         n = *(list + k);
1041         storeAppendPrintf(sentry, "%-46.46s %4d/%4d %7.1f %5.1f", /* Max between 16 (IPv4) or 46 (IPv6)   */
1042                           n->network,
1043                           n->pings_recv,
1044                           n->pings_sent,
1045                           n->rtt,
1046                           n->hops);
1047
1048         for (x = n->hosts; x; x = x->next)
1049             storeAppendPrintf(sentry, " %s", hashKeyStr(&x->hash));
1050
1051         storeAppendPrintf(sentry, "\n");
1052
1053         p = n->peers;
1054
1055         for (j = 0; j < n->n_peers; j++, p++) {
1056             storeAppendPrintf(sentry, "    %-22.22s %7.1f %5.1f\n",
1057                               p->peername,
1058                               p->rtt,
1059                               p->hops);
1060         }
1061     }
1062
1063     xfree(list);
1064 #else
1065
1066     storeAppendPrintf(sentry,
1067                       "NETDB support not compiled into this Squid cache.\n");
1068 #endif
1069 }
1070
1071 int
1072 netdbHostHops(const char *host)
1073 {
1074 #if USE_ICMP
1075     netdbEntry *n = netdbLookupHost(host);
1076
1077     if (n) {
1078         n->last_use_time = squid_curtime;
1079         return (int) (n->hops + 0.5);
1080     }
1081
1082 #endif
1083     return 0;
1084 }
1085
1086 int
1087 netdbHostRtt(const char *host)
1088 {
1089 #if USE_ICMP
1090     netdbEntry *n = netdbLookupHost(host);
1091
1092     if (n) {
1093         n->last_use_time = squid_curtime;
1094         return (int) (n->rtt + 0.5);
1095     }
1096
1097 #endif
1098     return 0;
1099 }
1100
1101 void
1102 netdbHostData(const char *host, int *samp, int *rtt, int *hops)
1103 {
1104 #if USE_ICMP
1105     netdbEntry *n = netdbLookupHost(host);
1106
1107     if (n == NULL)
1108         return;
1109
1110     *samp = n->pings_recv;
1111
1112     *rtt = (int) (n->rtt + 0.5);
1113
1114     *hops = (int) (n->hops + 0.5);
1115
1116     n->last_use_time = squid_curtime;
1117
1118 #endif
1119 }
1120
1121 void
1122 netdbUpdatePeer(HttpRequest * r, peer * e, int irtt, int ihops)
1123 {
1124 #if USE_ICMP
1125     netdbEntry *n;
1126     double rtt = (double) irtt;
1127     double hops = (double) ihops;
1128     net_db_peer *p;
1129     debugs(38, 3, "netdbUpdatePeer: '" << r->GetHost() << "', " << ihops << " hops, " << irtt << " rtt");
1130     n = netdbLookupHost(r->GetHost());
1131
1132     if (n == NULL) {
1133         debugs(38, 3, "netdbUpdatePeer: host '" << r->GetHost() << "' not found");
1134         return;
1135     }
1136
1137     if ((p = netdbPeerByName(n, e->host)) == NULL)
1138         p = netdbPeerAdd(n, e);
1139
1140     p->rtt = rtt;
1141
1142     p->hops = hops;
1143
1144     p->expires = squid_curtime + 3600;
1145
1146     if (n->n_peers < 2)
1147         return;
1148
1149     qsort((char *) n->peers,
1150           n->n_peers,
1151           sizeof(net_db_peer),
1152           sortPeerByRtt);
1153
1154 #endif
1155 }
1156
1157 void
1158 netdbExchangeUpdatePeer(IPAddress &addr, peer * e, double rtt, double hops)
1159 {
1160 #if USE_ICMP
1161     netdbEntry *n;
1162     net_db_peer *p;
1163     debugs(38, 5, "netdbExchangeUpdatePeer: '" << addr << "', "<<
1164            std::setfill('0')<< std::setprecision(2) << hops << " hops, " <<
1165            rtt << " rtt");
1166
1167     if( !addr.IsIPv4() ) {
1168         debugs(38, 5, "netdbExchangeUpdatePeer: Aborting peer update for '" << addr << "', NetDB cannot handle IPv6.");
1169         return;
1170     }
1171
1172     n = netdbLookupAddr(addr);
1173
1174     if (n == NULL)
1175         n = netdbAdd(addr);
1176
1177     assert(NULL != n);
1178
1179     if ((p = netdbPeerByName(n, e->host)) == NULL)
1180         p = netdbPeerAdd(n, e);
1181
1182     p->rtt = rtt;
1183
1184     p->hops = hops;
1185
1186     p->expires = squid_curtime + 3600;  /* XXX ? */
1187
1188     if (n->n_peers < 2)
1189         return;
1190
1191     qsort((char *) n->peers,
1192           n->n_peers,
1193           sizeof(net_db_peer),
1194           sortPeerByRtt);
1195
1196 #endif
1197 }
1198
1199 void
1200
1201 netdbDeleteAddrNetwork(IPAddress &addr)
1202 {
1203 #if USE_ICMP
1204     netdbEntry *n = netdbLookupAddr(addr);
1205
1206     if (n == NULL)
1207         return;
1208
1209     debugs(38, 3, "netdbDeleteAddrNetwork: " << n->network);
1210
1211     netdbRelease(n);
1212
1213 #endif
1214 }
1215
1216 void
1217 netdbBinaryExchange(StoreEntry * s)
1218 {
1219     HttpReply *reply = new HttpReply;
1220 #if USE_ICMP
1221
1222     IPAddress addr;
1223
1224     netdbEntry *n;
1225     int i;
1226     int j;
1227     int rec_sz;
1228     char *buf;
1229
1230     struct in_addr line_addr;
1231     s->buffer();
1232     HttpVersion version(1, 0);
1233     reply->setHeaders(version, HTTP_OK, "OK", NULL, -1, squid_curtime, -2);
1234     s->replaceHttpReply(reply);
1235     rec_sz = 0;
1236     rec_sz += 1 + sizeof(struct in_addr);
1237     rec_sz += 1 + sizeof(int);
1238     rec_sz += 1 + sizeof(int);
1239     buf = (char *)memAllocate(MEM_4K_BUF);
1240     i = 0;
1241     hash_first(addr_table);
1242
1243     while ((n = (netdbEntry *) hash_next(addr_table))) {
1244         if (0.0 == n->rtt)
1245             continue;
1246
1247         if (n->rtt > 60000)     /* RTT > 1 MIN probably bogus */
1248             continue;
1249
1250         if (! (addr = n->network) )
1251             continue;
1252
1253         /* FIXME INET6 : NetDB cannot yet handle IPv6 addresses. Ensure only IPv4 get sent. */
1254         if( !addr.IsIPv4() )
1255             continue;
1256
1257         buf[i++] = (char) NETDB_EX_NETWORK;
1258
1259         addr.GetInAddr(line_addr);
1260         xmemcpy(&buf[i], &line_addr, sizeof(struct in_addr));
1261
1262         i += sizeof(struct in_addr);
1263
1264         buf[i++] = (char) NETDB_EX_RTT;
1265
1266         j = htonl((int) (n->rtt * 1000));
1267
1268         xmemcpy(&buf[i], &j, sizeof(int));
1269
1270         i += sizeof(int);
1271
1272         buf[i++] = (char) NETDB_EX_HOPS;
1273
1274         j = htonl((int) (n->hops * 1000));
1275
1276         xmemcpy(&buf[i], &j, sizeof(int));
1277
1278         i += sizeof(int);
1279
1280         if (i + rec_sz > 4096) {
1281             s->append(buf, i);
1282             i = 0;
1283         }
1284     }
1285
1286     if (i > 0) {
1287         s->append(buf, i);
1288         i = 0;
1289     }
1290
1291     assert(0 == i);
1292     s->flush();
1293     memFree(buf, MEM_4K_BUF);
1294 #else
1295
1296     HttpVersion version(1,0);
1297     reply->setHeaders(version, HTTP_BAD_REQUEST, "Bad Request",
1298                       NULL, -1, squid_curtime, -2);
1299     s->replaceHttpReply(reply);
1300     storeAppendPrintf(s, "NETDB support not compiled into this Squid cache.\n");
1301 #endif
1302
1303     s->complete();
1304 }
1305
1306 #if USE_ICMP
1307 CBDATA_TYPE(netdbExchangeState);
1308 #endif
1309
1310 void
1311 netdbExchangeStart(void *data)
1312 {
1313 #if USE_ICMP
1314     peer *p = (peer *)data;
1315     char *uri;
1316     netdbExchangeState *ex;
1317     StoreIOBuffer tempBuffer;
1318     CBDATA_INIT_TYPE(netdbExchangeState);
1319     ex = cbdataAlloc(netdbExchangeState);
1320     ex->p = cbdataReference(p);
1321     uri = internalRemoteUri(p->host, p->http_port, "/squid-internal-dynamic/", "netdb");
1322     debugs(38, 3, "netdbExchangeStart: Requesting '" << uri << "'");
1323     assert(NULL != uri);
1324     ex->r = HttpRequest::CreateFromUrl(uri);
1325
1326     if (NULL == ex->r) {
1327         debugs(38, 1, "netdbExchangeStart: Bad URI " << uri);
1328         return;
1329     }
1330
1331     HTTPMSGLOCK(ex->r);
1332     assert(NULL != ex->r);
1333     ex->r->http_ver = HttpVersion(1,0);
1334     ex->connstate = STATE_HEADER;
1335     ex->e = storeCreateEntry(uri, uri, request_flags(), METHOD_GET);
1336     ex->buf_sz = NETDB_REQBUF_SZ;
1337     assert(NULL != ex->e);
1338     ex->sc = storeClientListAdd(ex->e, ex);
1339     tempBuffer.offset = 0;
1340     tempBuffer.length = ex->buf_sz;
1341     tempBuffer.data = ex->buf;
1342     storeClientCopy(ex->sc, ex->e, tempBuffer,
1343                     netdbExchangeHandleReply, ex);
1344     ex->r->flags.loopdetect = 1;        /* cheat! -- force direct */
1345
1346     if (p->login)
1347         xstrncpy(ex->r->login, p->login, MAX_LOGIN_SZ);
1348
1349     FwdState::fwdStart(-1, ex->e, ex->r);
1350
1351 #endif
1352 }
1353
1354 peer *
1355 netdbClosestParent(HttpRequest * request)
1356 {
1357 #if USE_ICMP
1358     peer *p = NULL;
1359     netdbEntry *n;
1360     const ipcache_addrs *ia;
1361     net_db_peer *h;
1362     int i;
1363     n = netdbLookupHost(request->GetHost());
1364
1365     if (NULL == n) {
1366         /* try IP addr */
1367         ia = ipcache_gethostbyname(request->GetHost(), 0);
1368
1369         if (NULL != ia)
1370             n = netdbLookupAddr(ia->in_addrs[ia->cur]);
1371     }
1372
1373     if (NULL == n)
1374         return NULL;
1375
1376     if (0 == n->n_peers)
1377         return NULL;
1378
1379     n->last_use_time = squid_curtime;
1380
1381     /*
1382      * Find the parent with the least RTT to the origin server.
1383      * Make sure we don't return a parent who is farther away than
1384      * we are.  Note, the n->peers list is pre-sorted by RTT.
1385      */
1386     for (i = 0; i < n->n_peers; i++) {
1387         h = &n->peers[i];
1388
1389         if (n->rtt > 0)
1390             if (n->rtt < h->rtt)
1391                 break;
1392
1393         p = peerFindByName(h->peername);
1394
1395         if (NULL == p)          /* not found */
1396             continue;
1397
1398         if (neighborType(p, request) != PEER_PARENT)
1399             continue;
1400
1401         if (!peerHTTPOkay(p, request))  /* not allowed */
1402             continue;
1403
1404         return p;
1405     }
1406
1407 #endif
1408     return NULL;
1409 }