src/neighbors.cc

   1 /*
   2  * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 15    Neighbor Routines */
  10
  11 #include "squid.h"
  12 #include "acl/FilledChecklist.h"
  13 #include "anyp/PortCfg.h"
  14 #include "CacheDigest.h"
  15 #include "CachePeer.h"
  16 #include "comm/Connection.h"
  17 #include "comm/ConnOpener.h"
  18 #include "event.h"
  19 #include "FwdState.h"
  20 #include "globals.h"
  21 #include "htcp.h"
  22 #include "HttpRequest.h"
  23 #include "icmp/net_db.h"
  24 #include "ICP.h"
  25 #include "int.h"
  26 #include "ip/Address.h"
  27 #include "ip/tools.h"
  28 #include "ipcache.h"
  29 #include "MemObject.h"
  30 #include "mgr/Registration.h"
  31 #include "multicast.h"
  32 #include "neighbors.h"
  33 #include "NeighborTypeDomainList.h"
  34 #include "pconn.h"
  35 #include "PeerDigest.h"
  36 #include "PeerPoolMgr.h"
  37 #include "PeerSelectState.h"
  38 #include "RequestFlags.h"
  39 #include "SquidConfig.h"
  40 #include "SquidMath.h"
  41 #include "SquidTime.h"
  42 #include "stat.h"
  43 #include "Store.h"
  44 #include "store_key_md5.h"
  45 #include "tools.h"
  46 #include "URL.h"
  47
  48 /* count mcast group peers every 15 minutes */
  49 #define MCAST_COUNT_RATE 900
  50
  51 bool peerAllowedToUse(const CachePeer *, HttpRequest *);
  52 static int peerWouldBePinged(const CachePeer *, HttpRequest *);
  53 static void neighborRemove(CachePeer *);
  54 static void neighborAlive(CachePeer *, const MemObject *, const icp_common_t *);
  55 #if USE_HTCP
  56 static void neighborAliveHtcp(CachePeer *, const MemObject *, const HtcpReplyData *);
  57 #endif
  58 static void neighborCountIgnored(CachePeer *);
  59 static void peerRefreshDNS(void *);
  60 static IPH peerDNSConfigure;
  61 static bool peerProbeConnect(CachePeer *);
  62 static CNCB peerProbeConnectDone;
  63 static void peerCountMcastPeersDone(void *data);
  64 static void peerCountMcastPeersStart(void *data);
  65 static void peerCountMcastPeersSchedule(CachePeer * p, time_t when);
  66 static IRCB peerCountHandleIcpReply;
  67
  68 static void neighborIgnoreNonPeer(const Ip::Address &, icp_opcode);
  69 static OBJH neighborDumpPeers;
  70 static OBJH neighborDumpNonPeers;
  71 static void dump_peers(StoreEntry * sentry, CachePeer * peers);
  72
  73 static unsigned short echo_port;
  74
  75 static int NLateReplies = 0;
  76 static CachePeer *first_ping = NULL;
  77
  78 const char *
  79 neighborTypeStr(const CachePeer * p)
  80 {
  81     if (p->type == PEER_NONE)
  82         return "Non-Peer";
  83
  84     if (p->type == PEER_SIBLING)
  85         return "Sibling";
  86
  87     if (p->type == PEER_MULTICAST)
  88         return "Multicast Group";
  89
  90     return "Parent";
  91 }
  92
  93 CachePeer *
  94 whichPeer(const Ip::Address &from)
  95 {
  96     int j;
  97
  98     CachePeer *p = NULL;
  99     debugs(15, 3, "whichPeer: from " << from);
 100
 101     for (p = Config.peers; p; p = p->next) {
 102         for (j = 0; j < p->n_addresses; ++j) {
 103             if (from == p->addresses[j] && from.port() == p->icp.port) {
 104                 return p;
 105             }
 106         }
 107     }
 108
 109     return NULL;
 110 }
 111
 112 peer_t
 113 neighborType(const CachePeer * p, const HttpRequest * request)
 114 {
 115
 116     const NeighborTypeDomainList *d = NULL;
 117
 118     for (d = p->typelist; d; d = d->next) {
 119         if (0 == matchDomainName(request->GetHost(), d->domain))
 120             if (d->type != PEER_NONE)
 121                 return d->type;
 122     }
 123 #if PEER_MULTICAST_SIBLINGS
 124     if (p->type == PEER_MULTICAST)
 125         if (p->options.mcast_siblings)
 126             return PEER_SIBLING;
 127 #endif
 128
 129     return p->type;
 130 }
 131
 132 /**
 133  * \return Whether it is appropriate to fetch REQUEST from PEER.
 134  */
 135 bool
 136 peerAllowedToUse(const CachePeer * p, HttpRequest * request)
 137 {
 138
 139     assert(request != NULL);
 140
 141     if (neighborType(p, request) == PEER_SIBLING) {
 142 #if PEER_MULTICAST_SIBLINGS
 143         if (p->type == PEER_MULTICAST && p->options.mcast_siblings &&
 144                 (request->flags.noCache || request->flags.refresh || request->flags.loopDetected || request->flags.needValidation))
 145             debugs(15, 2, "peerAllowedToUse(" << p->name << ", " << request->GetHost() << ") : multicast-siblings optimization match");
 146 #endif
 147         if (request->flags.noCache)
 148             return false;
 149
 150         if (request->flags.refresh)
 151             return false;
 152
 153         if (request->flags.loopDetected)
 154             return false;
 155
 156         if (request->flags.needValidation)
 157             return false;
 158     }
 159
 160     // CONNECT requests are proxy requests. Not to be forwarded to origin servers.
 161     // Unless the destination port matches, in which case we MAY perform a 'DIRECT' to this CachePeer.
 162     if (p->options.originserver && request->method == Http::METHOD_CONNECT && request->port != p->in_addr.port())
 163         return false;
 164
 165     if (p->access == NULL)
 166         return true;
 167
 168     ACLFilledChecklist checklist(p->access, request, NULL);
 169
 170     return (checklist.fastCheck() == ACCESS_ALLOWED);
 171 }
 172
 173 /* Return TRUE if it is okay to send an ICP request to this CachePeer.   */
 174 static int
 175 peerWouldBePinged(const CachePeer * p, HttpRequest * request)
 176 {
 177     if (p->icp.port == 0)
 178         return 0;
 179
 180     if (p->options.no_query)
 181         return 0;
 182
 183     if (p->options.mcast_responder)
 184         return 0;
 185
 186     if (p->n_addresses == 0)
 187         return 0;
 188
 189     if (p->options.background_ping && (squid_curtime - p->stats.last_query < Config.backgroundPingRate))
 190         return 0;
 191
 192     /* the case below seems strange, but can happen if the
 193      * URL host is on the other side of a firewall */
 194     if (p->type == PEER_SIBLING)
 195         if (!request->flags.hierarchical)
 196             return 0;
 197
 198     if (!peerAllowedToUse(p, request))
 199         return 0;
 200
 201     /* Ping dead peers every timeout interval */
 202     if (squid_curtime - p->stats.last_query > Config.Timeout.deadPeer)
 203         return 1;
 204
 205     if (!neighborUp(p))
 206         return 0;
 207
 208     return 1;
 209 }
 210
 211 bool
 212 peerCanOpenMore(const CachePeer *p)
 213 {
 214     const int effectiveLimit = p->max_conn <= 0 ? Squid_MaxFD : p->max_conn;
 215     const int remaining = effectiveLimit - p->stats.conn_open;
 216     debugs(15, 7, remaining << '=' << effectiveLimit << '-' << p->stats.conn_open);
 217     return remaining > 0;
 218 }
 219
 220 bool
 221 peerHasConnAvailable(const CachePeer *p)
 222 {
 223     // Standby connections can be used without opening new connections.
 224     const int standbys = p->standby.pool ? p->standby.pool->count() : 0;
 225
 226     // XXX: Some idle pconns can be used without opening new connections.
 227     // Complication: Idle pconns cannot be reused for some requests.
 228     const int usableIdles = 0;
 229
 230     const int available = standbys + usableIdles;
 231     debugs(15, 7, available << '=' << standbys << '+' << usableIdles);
 232     return available > 0;
 233 }
 234
 235 void
 236 peerConnClosed(CachePeer *p)
 237 {
 238     --p->stats.conn_open;
 239     if (p->standby.waitingForClose && peerCanOpenMore(p)) {
 240         p->standby.waitingForClose = false;
 241         PeerPoolMgr::Checkpoint(p->standby.mgr, "conn closed");
 242     }
 243 }
 244
 245 /* Return TRUE if it is okay to send an HTTP request to this CachePeer. */
 246 int
 247 peerHTTPOkay(const CachePeer * p, HttpRequest * request)
 248 {
 249     if (!peerCanOpenMore(p) && !peerHasConnAvailable(p))
 250         return 0;
 251
 252     if (!peerAllowedToUse(p, request))
 253         return 0;
 254
 255     if (!neighborUp(p))
 256         return 0;
 257
 258     return 1;
 259 }
 260
 261 int
 262 neighborsCount(HttpRequest * request)
 263 {
 264     CachePeer *p = NULL;
 265     int count = 0;
 266
 267     for (p = Config.peers; p; p = p->next)
 268         if (peerWouldBePinged(p, request))
 269             ++count;
 270
 271     debugs(15, 3, "neighborsCount: " << count);
 272
 273     return count;
 274 }
 275
 276 CachePeer *
 277 getFirstUpParent(HttpRequest * request)
 278 {
 279     CachePeer *p = NULL;
 280
 281     for (p = Config.peers; p; p = p->next) {
 282         if (!neighborUp(p))
 283             continue;
 284
 285         if (neighborType(p, request) != PEER_PARENT)
 286             continue;
 287
 288         if (!peerHTTPOkay(p, request))
 289             continue;
 290
 291         break;
 292     }
 293
 294     debugs(15, 3, "getFirstUpParent: returning " << (p ? p->host : "NULL"));
 295     return p;
 296 }
 297
 298 CachePeer *
 299 getRoundRobinParent(HttpRequest * request)
 300 {
 301     CachePeer *p;
 302     CachePeer *q = NULL;
 303
 304     for (p = Config.peers; p; p = p->next) {
 305         if (!p->options.roundrobin)
 306             continue;
 307
 308         if (neighborType(p, request) != PEER_PARENT)
 309             continue;
 310
 311         if (!peerHTTPOkay(p, request))
 312             continue;
 313
 314         if (p->weight == 0)
 315             continue;
 316
 317         if (q) {
 318             if (p->weight == q->weight) {
 319                 if (q->rr_count < p->rr_count)
 320                     continue;
 321             } else if ( ((double) q->rr_count / q->weight) < ((double) p->rr_count / p->weight)) {
 322                 continue;
 323             }
 324         }
 325
 326         q = p;
 327     }
 328
 329     if (q)
 330         ++ q->rr_count;
 331
 332     debugs(15, 3, HERE << "returning " << (q ? q->host : "NULL"));
 333
 334     return q;
 335 }
 336
 337 CachePeer *
 338 getWeightedRoundRobinParent(HttpRequest * request)
 339 {
 340     CachePeer *p;
 341     CachePeer *q = NULL;
 342     int weighted_rtt;
 343
 344     for (p = Config.peers; p; p = p->next) {
 345         if (!p->options.weighted_roundrobin)
 346             continue;
 347
 348         if (neighborType(p, request) != PEER_PARENT)
 349             continue;
 350
 351         if (!peerHTTPOkay(p, request))
 352             continue;
 353
 354         if (q && q->rr_count < p->rr_count)
 355             continue;
 356
 357         q = p;
 358     }
 359
 360     if (q && q->rr_count > 1000000)
 361         for (p = Config.peers; p; p = p->next) {
 362             if (!p->options.weighted_roundrobin)
 363                 continue;
 364
 365             if (neighborType(p, request) != PEER_PARENT)
 366                 continue;
 367
 368             p->rr_count = 0;
 369         }
 370
 371     if (q) {
 372         weighted_rtt = (q->stats.rtt - q->basetime) / q->weight;
 373
 374         if (weighted_rtt < 1)
 375             weighted_rtt = 1;
 376
 377         q->rr_count += weighted_rtt;
 378
 379         debugs(15, 3, "getWeightedRoundRobinParent: weighted_rtt " << weighted_rtt);
 380     }
 381
 382     debugs(15, 3, "getWeightedRoundRobinParent: returning " << (q ? q->host : "NULL"));
 383     return q;
 384 }
 385
 386 /**
 387  * This gets called every 5 minutes to clear the round-robin counter.
 388  * The exact timing is an arbitrary default, set on estimate timing of a
 389  * large number of requests in a high-performance environment during the
 390  * period. The larger the number of requests between cycled resets the
 391  * more balanced the operations.
 392  *
 393  \param data    unused.
 394  \todo Make the reset timing a selectable parameter in squid.conf
 395  */
 396 static void
 397 peerClearRRLoop(void *data)
 398 {
 399     peerClearRR();
 400     eventAdd("peerClearRR", peerClearRRLoop, data, 5 * 60.0, 0);
 401 }
 402
 403 /**
 404  * This gets called on startup and restart to kick off the CachePeer round-robin
 405  * maintenance event. It ensures that no matter how many times its called
 406  * no more than one event is scheduled.
 407  */
 408 void
 409 peerClearRRStart(void)
 410 {
 411     static bool event_added = false;
 412     if (!event_added) {
 413         peerClearRRLoop(NULL);
 414         event_added=true;
 415     }
 416 }
 417
 418 /**
 419  * Called whenever the round-robin counters need to be reset to a sane state.
 420  * So far those times are:
 421  *  - On startup and reconfigure - to set the counters to sane initial settings.
 422  *  -  When a CachePeer has revived from dead, to prevent the revived CachePeer being
 423  *     flooded with requests which it has 'missed' during the down period.
 424  */
 425 void
 426 peerClearRR()
 427 {
 428     CachePeer *p = NULL;
 429     for (p = Config.peers; p; p = p->next) {
 430         p->rr_count = 1;
 431     }
 432 }
 433
 434 /**
 435  * Perform all actions when a CachePeer is detected revived.
 436  */
 437 void
 438 peerAlive(CachePeer *p)
 439 {
 440     if (p->stats.logged_state == PEER_DEAD && p->tcp_up) {
 441         debugs(15, DBG_IMPORTANT, "Detected REVIVED " << neighborTypeStr(p) << ": " << p->name);
 442         p->stats.logged_state = PEER_ALIVE;
 443         peerClearRR();
 444         if (p->standby.mgr.valid())
 445             PeerPoolMgr::Checkpoint(p->standby.mgr, "revived peer");
 446     }
 447
 448     p->stats.last_reply = squid_curtime;
 449     p->stats.probe_start = 0;
 450 }
 451
 452 CachePeer *
 453 getDefaultParent(HttpRequest * request)
 454 {
 455     CachePeer *p = NULL;
 456
 457     for (p = Config.peers; p; p = p->next) {
 458         if (neighborType(p, request) != PEER_PARENT)
 459             continue;
 460
 461         if (!p->options.default_parent)
 462             continue;
 463
 464         if (!peerHTTPOkay(p, request))
 465             continue;
 466
 467         debugs(15, 3, "getDefaultParent: returning " << p->host);
 468
 469         return p;
 470     }
 471
 472     debugs(15, 3, "getDefaultParent: returning NULL");
 473     return NULL;
 474 }
 475
 476 CachePeer *
 477 getNextPeer(CachePeer * p)
 478 {
 479     return p->next;
 480 }
 481
 482 CachePeer *
 483 getFirstPeer(void)
 484 {
 485     return Config.peers;
 486 }
 487
 488 static void
 489 neighborRemove(CachePeer * target)
 490 {
 491     CachePeer *p = NULL;
 492     CachePeer **P = NULL;
 493     p = Config.peers;
 494     P = &Config.peers;
 495
 496     while (p) {
 497         if (target == p)
 498             break;
 499
 500         P = &p->next;
 501
 502         p = p->next;
 503     }
 504
 505     if (p) {
 506         *P = p->next;
 507         p->next = NULL;
 508         delete p;
 509         --Config.npeers;
 510     }
 511
 512     first_ping = Config.peers;
 513 }
 514
 515 static void
 516 neighborsRegisterWithCacheManager()
 517 {
 518     Mgr::RegisterAction("server_list",
 519                         "Peer Cache Statistics",
 520                         neighborDumpPeers, 0, 1);
 521
 522     if (Comm::IsConnOpen(icpIncomingConn)) {
 523         Mgr::RegisterAction("non_peers",
 524                             "List of Unknown sites sending ICP messages",
 525                             neighborDumpNonPeers, 0, 1);
 526     }
 527 }
 528
 529 void
 530 neighbors_init(void)
 531 {
 532     struct servent *sep = NULL;
 533     const char *me = getMyHostname();
 534     CachePeer *thisPeer = NULL;
 535     CachePeer *next = NULL;
 536
 537     neighborsRegisterWithCacheManager();
 538
 539     if (Comm::IsConnOpen(icpIncomingConn)) {
 540
 541         for (thisPeer = Config.peers; thisPeer; thisPeer = next) {
 542             next = thisPeer->next;
 543
 544             if (0 != strcmp(thisPeer->host, me))
 545                 continue;
 546
 547             for (AnyP::PortCfgPointer s = HttpPortList; s != NULL; s = s->next) {
 548                 if (thisPeer->http_port != s->s.port())
 549                     continue;
 550
 551                 debugs(15, DBG_IMPORTANT, "WARNING: Peer looks like this host");
 552
 553                 debugs(15, DBG_IMPORTANT, "         Ignoring " <<
 554                        neighborTypeStr(thisPeer) << " " << thisPeer->host <<
 555                        "/" << thisPeer->http_port << "/" <<
 556                        thisPeer->icp.port);
 557
 558                 neighborRemove(thisPeer);
 559             }
 560         }
 561     }
 562
 563     peerRefreshDNS((void *) 1);
 564
 565     sep = getservbyname("echo", "udp");
 566     echo_port = sep ? ntohs((unsigned short) sep->s_port) : 7;
 567
 568     first_ping = Config.peers;
 569 }
 570
 571 int
 572 neighborsUdpPing(HttpRequest * request,
 573                  StoreEntry * entry,
 574                  IRCB * callback,
 575                  void *callback_data,
 576                  int *exprep,
 577                  int *timeout)
 578 {
 579     const char *url = entry->url();
 580     MemObject *mem = entry->mem_obj;
 581     CachePeer *p = NULL;
 582     int i;
 583     int reqnum = 0;
 584     int flags;
 585     icp_common_t *query;
 586     int queries_sent = 0;
 587     int peers_pinged = 0;
 588     int parent_timeout = 0, parent_exprep = 0;
 589     int sibling_timeout = 0, sibling_exprep = 0;
 590     int mcast_timeout = 0, mcast_exprep = 0;
 591
 592     if (Config.peers == NULL)
 593         return 0;
 594
 595     assert(entry->swap_status == SWAPOUT_NONE);
 596
 597     mem->start_ping = current_time;
 598
 599     mem->ping_reply_callback = callback;
 600
 601     mem->ircb_data = callback_data;
 602
 603     reqnum = icpSetCacheKey((const cache_key *)entry->key);
 604
 605     for (i = 0, p = first_ping; i++ < Config.npeers; p = p->next) {
 606         if (p == NULL)
 607             p = Config.peers;
 608
 609         debugs(15, 5, "neighborsUdpPing: Peer " << p->host);
 610
 611         if (!peerWouldBePinged(p, request))
 612             continue;       /* next CachePeer */
 613
 614         ++peers_pinged;
 615
 616         debugs(15, 4, "neighborsUdpPing: pinging peer " << p->host << " for '" << url << "'");
 617
 618         debugs(15, 3, "neighborsUdpPing: key = '" << entry->getMD5Text() << "'");
 619
 620         debugs(15, 3, "neighborsUdpPing: reqnum = " << reqnum);
 621
 622 #if USE_HTCP
 623         if (p->options.htcp && !p->options.htcp_only_clr) {
 624             if (Config.Port.htcp <= 0) {
 625                 debugs(15, DBG_CRITICAL, "HTCP is disabled! Cannot send HTCP request to peer.");
 626                 continue;
 627             }
 628
 629             debugs(15, 3, "neighborsUdpPing: sending HTCP query");
 630             if (htcpQuery(entry, request, p) <= 0)
 631                 continue; // unable to send.
 632         } else
 633 #endif
 634         {
 635             if (Config.Port.icp <= 0 || !Comm::IsConnOpen(icpOutgoingConn)) {
 636                 debugs(15, DBG_CRITICAL, "ICP is disabled! Cannot send ICP request to peer.");
 637                 continue;
 638             } else {
 639
 640                 if (p->type == PEER_MULTICAST)
 641                     mcastSetTtl(icpOutgoingConn->fd, p->mcast.ttl);
 642
 643                 if (p->icp.port == echo_port) {
 644                     debugs(15, 4, "neighborsUdpPing: Looks like a dumb cache, send DECHO ping");
 645                     query = _icp_common_t::createMessage(ICP_DECHO, 0, url, reqnum, 0);
 646                     icpUdpSend(icpOutgoingConn->fd, p->in_addr, query, LOG_ICP_QUERY, 0);
 647                 } else {
 648                     flags = 0;
 649
 650                     if (Config.onoff.query_icmp)
 651                         if (p->icp.version == ICP_VERSION_2)
 652                             flags |= ICP_FLAG_SRC_RTT;
 653
 654                     query = _icp_common_t::createMessage(ICP_QUERY, flags, url, reqnum, 0);
 655
 656                     icpUdpSend(icpOutgoingConn->fd, p->in_addr, query, LOG_ICP_QUERY, 0.0);
 657                 }
 658             }
 659         }
 660
 661         ++queries_sent;
 662
 663         ++ p->stats.pings_sent;
 664
 665         if (p->type == PEER_MULTICAST) {
 666             mcast_exprep += p->mcast.n_replies_expected;
 667             mcast_timeout += (p->stats.rtt * p->mcast.n_replies_expected);
 668         } else if (neighborUp(p)) {
 669             /* its alive, expect a reply from it */
 670
 671             if (neighborType(p, request) == PEER_PARENT) {
 672                 ++parent_exprep;
 673                 parent_timeout += p->stats.rtt;
 674             } else {
 675                 ++sibling_exprep;
 676                 sibling_timeout += p->stats.rtt;
 677             }
 678         } else {
 679             /* Neighbor is dead; ping it anyway, but don't expect a reply */
 680             /* log it once at the threshold */
 681
 682             if (p->stats.logged_state == PEER_ALIVE) {
 683                 debugs(15, DBG_IMPORTANT, "Detected DEAD " << neighborTypeStr(p) << ": " << p->name);
 684                 p->stats.logged_state = PEER_DEAD;
 685             }
 686         }
 687
 688         p->stats.last_query = squid_curtime;
 689
 690         /*
 691          * keep probe_start == 0 for a multicast CachePeer,
 692          * so neighborUp() never says this CachePeer is dead.
 693          */
 694
 695         if ((p->type != PEER_MULTICAST) && (p->stats.probe_start == 0))
 696             p->stats.probe_start = squid_curtime;
 697     }
 698
 699     if ((first_ping = first_ping->next) == NULL)
 700         first_ping = Config.peers;
 701
 702     /*
 703      * How many replies to expect?
 704      */
 705     *exprep = parent_exprep + sibling_exprep + mcast_exprep;
 706
 707     /*
 708      * If there is a configured timeout, use it
 709      */
 710     if (Config.Timeout.icp_query)
 711         *timeout = Config.Timeout.icp_query;
 712     else {
 713         if (*exprep > 0) {
 714             if (parent_exprep)
 715                 *timeout = 2 * parent_timeout / parent_exprep;
 716             else if (mcast_exprep)
 717                 *timeout = 2 * mcast_timeout / mcast_exprep;
 718             else
 719                 *timeout = 2 * sibling_timeout / sibling_exprep;
 720         } else
 721             *timeout = 2000;    /* 2 seconds */
 722
 723         if (Config.Timeout.icp_query_max)
 724             if (*timeout > Config.Timeout.icp_query_max)
 725                 *timeout = Config.Timeout.icp_query_max;
 726
 727         if (*timeout < Config.Timeout.icp_query_min)
 728             *timeout = Config.Timeout.icp_query_min;
 729     }
 730
 731     return peers_pinged;
 732 }
 733
 734 /* lookup the digest of a given CachePeer */
 735 lookup_t
 736 peerDigestLookup(CachePeer * p, HttpRequest * request)
 737 {
 738 #if USE_CACHE_DIGESTS
 739     const cache_key *key = request ? storeKeyPublicByRequest(request) : NULL;
 740     assert(p);
 741     assert(request);
 742     debugs(15, 5, "peerDigestLookup: peer " << p->host);
 743     /* does the peeer have a valid digest? */
 744
 745     if (!p->digest) {
 746         debugs(15, 5, "peerDigestLookup: gone!");
 747         return LOOKUP_NONE;
 748     } else if (!peerHTTPOkay(p, request)) {
 749         debugs(15, 5, "peerDigestLookup: !peerHTTPOkay");
 750         return LOOKUP_NONE;
 751     } else if (!p->digest->flags.needed) {
 752         debugs(15, 5, "peerDigestLookup: note need");
 753         peerDigestNeeded(p->digest);
 754         return LOOKUP_NONE;
 755     } else if (!p->digest->flags.usable) {
 756         debugs(15, 5, "peerDigestLookup: !ready && " << (p->digest->flags.requested ? "" : "!") << "requested");
 757         return LOOKUP_NONE;
 758     }
 759
 760     debugs(15, 5, "peerDigestLookup: OK to lookup peer " << p->host);
 761     assert(p->digest->cd);
 762     /* does digest predict a hit? */
 763
 764     if (!cacheDigestTest(p->digest->cd, key))
 765         return LOOKUP_MISS;
 766
 767     debugs(15, 5, "peerDigestLookup: peer " << p->host << " says HIT!");
 768
 769     return LOOKUP_HIT;
 770
 771 #endif
 772
 773     return LOOKUP_NONE;
 774 }
 775
 776 /* select best CachePeer based on cache digests */
 777 CachePeer *
 778 neighborsDigestSelect(HttpRequest * request)
 779 {
 780     CachePeer *best_p = NULL;
 781 #if USE_CACHE_DIGESTS
 782
 783     int best_rtt = 0;
 784     int choice_count = 0;
 785     int ichoice_count = 0;
 786     CachePeer *p;
 787     int p_rtt;
 788     int i;
 789
 790     if (!request->flags.hierarchical)
 791         return NULL;
 792
 793     storeKeyPublicByRequest(request);
 794
 795     for (i = 0, p = first_ping; i++ < Config.npeers; p = p->next) {
 796         lookup_t lookup;
 797
 798         if (!p)
 799             p = Config.peers;
 800
 801         if (i == 1)
 802             first_ping = p;
 803
 804         lookup = peerDigestLookup(p, request);
 805
 806         if (lookup == LOOKUP_NONE)
 807             continue;
 808
 809         ++choice_count;
 810
 811         if (lookup == LOOKUP_MISS)
 812             continue;
 813
 814         p_rtt = netdbHostRtt(p->host);
 815
 816         debugs(15, 5, "neighborsDigestSelect: peer " << p->host << " rtt: " << p_rtt);
 817
 818         /* is this CachePeer better than others in terms of rtt ? */
 819         if (!best_p || (p_rtt && p_rtt < best_rtt)) {
 820             best_p = p;
 821             best_rtt = p_rtt;
 822
 823             if (p_rtt)      /* informative choice (aka educated guess) */
 824                 ++ichoice_count;
 825
 826             debugs(15, 4, "neighborsDigestSelect: peer " << p->host << " leads with rtt " << best_rtt);
 827         }
 828     }
 829
 830     debugs(15, 4, "neighborsDigestSelect: choices: " << choice_count << " (" << ichoice_count << ")");
 831     peerNoteDigestLookup(request, best_p,
 832                          best_p ? LOOKUP_HIT : (choice_count ? LOOKUP_MISS : LOOKUP_NONE));
 833     request->hier.n_choices = choice_count;
 834     request->hier.n_ichoices = ichoice_count;
 835 #endif
 836
 837     return best_p;
 838 }
 839
 840 void
 841 peerNoteDigestLookup(HttpRequest * request, CachePeer * p, lookup_t lookup)
 842 {
 843 #if USE_CACHE_DIGESTS
 844     if (p)
 845         strncpy(request->hier.cd_host, p->host, sizeof(request->hier.cd_host)-1);
 846     else
 847         *request->hier.cd_host = '\0';
 848
 849     request->hier.cd_lookup = lookup;
 850     debugs(15, 4, "peerNoteDigestLookup: peer " << (p? p->host : "<none>") << ", lookup: " << lookup_t_str[lookup]  );
 851 #endif
 852 }
 853
 854 static void
 855 neighborAlive(CachePeer * p, const MemObject *, const icp_common_t * header)
 856 {
 857     peerAlive(p);
 858     ++ p->stats.pings_acked;
 859
 860     if ((icp_opcode) header->opcode <= ICP_END)
 861         ++ p->icp.counts[header->opcode];
 862
 863     p->icp.version = (int) header->version;
 864 }
 865
 866 static void
 867 neighborUpdateRtt(CachePeer * p, MemObject * mem)
 868 {
 869     int rtt, rtt_av_factor;
 870
 871     if (!mem)
 872         return;
 873
 874     if (!mem->start_ping.tv_sec)
 875         return;
 876
 877     rtt = tvSubMsec(mem->start_ping, current_time);
 878
 879     if (rtt < 1 || rtt > 10000)
 880         return;
 881
 882     rtt_av_factor = RTT_AV_FACTOR;
 883
 884     if (p->options.weighted_roundrobin)
 885         rtt_av_factor = RTT_BACKGROUND_AV_FACTOR;
 886
 887     p->stats.rtt = Math::intAverage(p->stats.rtt, rtt, p->stats.pings_acked, rtt_av_factor);
 888 }
 889
 890 #if USE_HTCP
 891 static void
 892 neighborAliveHtcp(CachePeer * p, const MemObject *, const HtcpReplyData * htcp)
 893 {
 894     peerAlive(p);
 895     ++ p->stats.pings_acked;
 896     ++ p->htcp.counts[htcp->hit ? 1 : 0];
 897     p->htcp.version = htcp->version;
 898 }
 899
 900 #endif
 901
 902 static void
 903 neighborCountIgnored(CachePeer * p)
 904 {
 905     if (p == NULL)
 906         return;
 907
 908     ++ p->stats.ignored_replies;
 909
 910     ++NLateReplies;
 911 }
 912
 913 static CachePeer *non_peers = NULL;
 914
 915 static void
 916 neighborIgnoreNonPeer(const Ip::Address &from, icp_opcode opcode)
 917 {
 918     CachePeer *np;
 919
 920     for (np = non_peers; np; np = np->next) {
 921         if (np->in_addr != from)
 922             continue;
 923
 924         if (np->in_addr.port() != from.port())
 925             continue;
 926
 927         break;
 928     }
 929
 930     if (np == NULL) {
 931         np = new CachePeer;
 932         np->in_addr = from;
 933         np->icp.port = from.port();
 934         np->type = PEER_NONE;
 935         np->host = new char[MAX_IPSTRLEN];
 936         from.toStr(np->host,MAX_IPSTRLEN);
 937         np->next = non_peers;
 938         non_peers = np;
 939     }
 940
 941     ++ np->icp.counts[opcode];
 942
 943     if (isPowTen(++np->stats.ignored_replies))
 944         debugs(15, DBG_IMPORTANT, "WARNING: Ignored " << np->stats.ignored_replies << " replies from non-peer " << np->host);
 945 }
 946
 947 /* ignoreMulticastReply
 948  *
 949  * * We want to ignore replies from multicast peers if the
 950  * * cache_host_domain rules would normally prevent the CachePeer
 951  * * from being used
 952  */
 953 static int
 954 ignoreMulticastReply(CachePeer * p, MemObject * mem)
 955 {
 956     if (p == NULL)
 957         return 0;
 958
 959     if (!p->options.mcast_responder)
 960         return 0;
 961
 962     if (peerHTTPOkay(p, mem->request))
 963         return 0;
 964
 965     return 1;
 966 }
 967
 968 /**
 969  * I should attach these records to the entry.  We take the first
 970  * hit we get our wait until everyone misses.  The timeout handler
 971  * call needs to nip this shopping list or call one of the misses.
 972  *
 973  * If a hit process is already started, then sobeit
 974  */
 975 void
 976 neighborsUdpAck(const cache_key * key, icp_common_t * header, const Ip::Address &from)
 977 {
 978     CachePeer *p = NULL;
 979     StoreEntry *entry;
 980     MemObject *mem = NULL;
 981     peer_t ntype = PEER_NONE;
 982     icp_opcode opcode = (icp_opcode) header->opcode;
 983
 984     debugs(15, 6, "neighborsUdpAck: opcode " << opcode << " '" << storeKeyText(key) << "'");
 985
 986     if (NULL != (entry = Store::Root().get(key)))
 987         mem = entry->mem_obj;
 988
 989     if ((p = whichPeer(from)))
 990         neighborAlive(p, mem, header);
 991
 992     if (opcode > ICP_END)
 993         return;
 994
 995     const char *opcode_d = icp_opcode_str[opcode];
 996
 997     if (p)
 998         neighborUpdateRtt(p, mem);
 999
1000     /* Does the entry exist? */
1001     if (NULL == entry) {
1002         debugs(12, 3, "neighborsUdpAck: Cache key '" << storeKeyText(key) << "' not found");
1003         neighborCountIgnored(p);
1004         return;
1005     }
1006
1007     /* check if someone is already fetching it */
1008     if (EBIT_TEST(entry->flags, ENTRY_DISPATCHED)) {
1009         debugs(15, 3, "neighborsUdpAck: '" << storeKeyText(key) << "' already being fetched.");
1010         neighborCountIgnored(p);
1011         return;
1012     }
1013
1014     if (mem == NULL) {
1015         debugs(15, 2, "Ignoring " << opcode_d << " for missing mem_obj: " << storeKeyText(key));
1016         neighborCountIgnored(p);
1017         return;
1018     }
1019
1020     if (entry->ping_status != PING_WAITING) {
1021         debugs(15, 2, "neighborsUdpAck: Late " << opcode_d << " for " << storeKeyText(key));
1022         neighborCountIgnored(p);
1023         return;
1024     }
1025
1026     if (!entry->locked()) {
1027         // TODO: many entries are unlocked; why is this reported at level 1?
1028         debugs(12, DBG_IMPORTANT, "neighborsUdpAck: '" << storeKeyText(key) << "' has no locks");
1029         neighborCountIgnored(p);
1030         return;
1031     }
1032
1033     debugs(15, 3, "neighborsUdpAck: " << opcode_d << " for '" << storeKeyText(key) << "' from " << (p ? p->host : "source") << " ");
1034
1035     if (p) {
1036         ntype = neighborType(p, mem->request);
1037     }
1038
1039     if (ignoreMulticastReply(p, mem)) {
1040         neighborCountIgnored(p);
1041     } else if (opcode == ICP_MISS) {
1042         if (p == NULL) {
1043             neighborIgnoreNonPeer(from, opcode);
1044         } else {
1045             mem->ping_reply_callback(p, ntype, AnyP::PROTO_ICP, header, mem->ircb_data);
1046         }
1047     } else if (opcode == ICP_HIT) {
1048         if (p == NULL) {
1049             neighborIgnoreNonPeer(from, opcode);
1050         } else {
1051             header->opcode = ICP_HIT;
1052             mem->ping_reply_callback(p, ntype, AnyP::PROTO_ICP, header, mem->ircb_data);
1053         }
1054     } else if (opcode == ICP_DECHO) {
1055         if (p == NULL) {
1056             neighborIgnoreNonPeer(from, opcode);
1057         } else if (ntype == PEER_SIBLING) {
1058             debug_trap("neighborsUdpAck: Found non-ICP cache as SIBLING\n");
1059             debug_trap("neighborsUdpAck: non-ICP neighbors must be a PARENT\n");
1060         } else {
1061             mem->ping_reply_callback(p, ntype, AnyP::PROTO_ICP, header, mem->ircb_data);
1062         }
1063     } else if (opcode == ICP_SECHO) {
1064         if (p) {
1065             debugs(15, DBG_IMPORTANT, "Ignoring SECHO from neighbor " << p->host);
1066             neighborCountIgnored(p);
1067         } else {
1068             debugs(15, DBG_IMPORTANT, "Unsolicited SECHO from " << from);
1069         }
1070     } else if (opcode == ICP_DENIED) {
1071         if (p == NULL) {
1072             neighborIgnoreNonPeer(from, opcode);
1073         } else if (p->stats.pings_acked > 100) {
1074             if (100 * p->icp.counts[ICP_DENIED] / p->stats.pings_acked > 95) {
1075                 debugs(15, DBG_CRITICAL, "95%% of replies from '" << p->host << "' are UDP_DENIED");
1076                 debugs(15, DBG_CRITICAL, "Disabling '" << p->host << "', please check your configuration.");
1077                 neighborRemove(p);
1078                 p = NULL;
1079             } else {
1080                 neighborCountIgnored(p);
1081             }
1082         }
1083     } else if (opcode == ICP_MISS_NOFETCH) {
1084         mem->ping_reply_callback(p, ntype, AnyP::PROTO_ICP, header, mem->ircb_data);
1085     } else {
1086         debugs(15, DBG_CRITICAL, "neighborsUdpAck: Unexpected ICP reply: " << opcode_d);
1087     }
1088 }
1089
1090 CachePeer *
1091 peerFindByName(const char *name)
1092 {
1093     CachePeer *p = NULL;
1094
1095     for (p = Config.peers; p; p = p->next) {
1096         if (!strcasecmp(name, p->name))
1097             break;
1098     }
1099
1100     return p;
1101 }
1102
1103 CachePeer *
1104 peerFindByNameAndPort(const char *name, unsigned short port)
1105 {
1106     CachePeer *p = NULL;
1107
1108     for (p = Config.peers; p; p = p->next) {
1109         if (strcasecmp(name, p->name))
1110             continue;
1111
1112         if (port != p->http_port)
1113             continue;
1114
1115         break;
1116     }
1117
1118     return p;
1119 }
1120
1121 int
1122 neighborUp(const CachePeer * p)
1123 {
1124     if (!p->tcp_up) {
1125         if (!peerProbeConnect((CachePeer *) p)) {
1126             debugs(15, 8, "neighborUp: DOWN (probed): " << p->host << " (" << p->in_addr << ")");
1127             return 0;
1128         }
1129     }
1130
1131     /*
1132      * The CachePeer can not be UP if we don't have any IP addresses
1133      * for it.
1134      */
1135     if (0 == p->n_addresses) {
1136         debugs(15, 8, "neighborUp: DOWN (no-ip): " << p->host << " (" << p->in_addr << ")");
1137         return 0;
1138     }
1139
1140     if (p->options.no_query) {
1141         debugs(15, 8, "neighborUp: UP (no-query): " << p->host << " (" << p->in_addr << ")");
1142         return 1;
1143     }
1144
1145     if (p->stats.probe_start != 0 &&
1146             squid_curtime - p->stats.probe_start > Config.Timeout.deadPeer) {
1147         debugs(15, 8, "neighborUp: DOWN (dead): " << p->host << " (" << p->in_addr << ")");
1148         return 0;
1149     }
1150
1151     debugs(15, 8, "neighborUp: UP: " << p->host << " (" << p->in_addr << ")");
1152     return 1;
1153 }
1154
1155 void
1156 peerNoteDigestGone(CachePeer * p)
1157 {
1158 #if USE_CACHE_DIGESTS
1159     cbdataReferenceDone(p->digest);
1160 #endif
1161 }
1162
1163 static void
1164 peerDNSConfigure(const ipcache_addrs *ia, const Dns::LookupDetails &, void *data)
1165 {
1166     // TODO: connections to no-longer valid IP addresses should be
1167     // closed when we can detect such IP addresses.
1168
1169     CachePeer *p = (CachePeer *)data;
1170
1171     int j;
1172
1173     if (p->n_addresses == 0) {
1174         debugs(15, DBG_IMPORTANT, "Configuring " << neighborTypeStr(p) << " " << p->host << "/" << p->http_port << "/" << p->icp.port);
1175
1176         if (p->type == PEER_MULTICAST)
1177             debugs(15, DBG_IMPORTANT, "    Multicast TTL = " << p->mcast.ttl);
1178     }
1179
1180     p->n_addresses = 0;
1181
1182     if (ia == NULL) {
1183         debugs(0, DBG_CRITICAL, "WARNING: DNS lookup for '" << p->host << "' failed!");
1184         return;
1185     }
1186
1187     if ((int) ia->count < 1) {
1188         debugs(0, DBG_CRITICAL, "WARNING: No IP address found for '" << p->host << "'!");
1189         return;
1190     }
1191
1192     p->tcp_up = p->connect_fail_limit;
1193
1194     for (j = 0; j < (int) ia->count && j < PEER_MAX_ADDRESSES; ++j) {
1195         p->addresses[j] = ia->in_addrs[j];
1196         debugs(15, 2, "--> IP address #" << j << ": " << p->addresses[j]);
1197         ++ p->n_addresses;
1198     }
1199
1200     p->in_addr.setEmpty();
1201     p->in_addr = p->addresses[0];
1202     p->in_addr.port(p->icp.port);
1203
1204     if (p->type == PEER_MULTICAST)
1205         peerCountMcastPeersSchedule(p, 10);
1206
1207 #if USE_ICMP
1208     if (p->type != PEER_MULTICAST && IamWorkerProcess())
1209         if (!p->options.no_netdb_exchange)
1210             eventAddIsh("netdbExchangeStart", netdbExchangeStart, p, 30.0, 1);
1211 #endif
1212
1213     if (p->standby.mgr.valid())
1214         PeerPoolMgr::Checkpoint(p->standby.mgr, "resolved peer");
1215 }
1216
1217 static void
1218 peerRefreshDNS(void *data)
1219 {
1220     CachePeer *p = NULL;
1221
1222     if (eventFind(peerRefreshDNS, NULL))
1223         eventDelete(peerRefreshDNS, NULL);
1224
1225     if (!data && 0 == stat5minClientRequests()) {
1226         /* no recent client traffic, wait a bit */
1227         eventAddIsh("peerRefreshDNS", peerRefreshDNS, NULL, 180.0, 1);
1228         return;
1229     }
1230
1231     for (p = Config.peers; p; p = p->next)
1232         ipcache_nbgethostbyname(p->host, peerDNSConfigure, p);
1233
1234     /* Reconfigure the peers every hour */
1235     eventAddIsh("peerRefreshDNS", peerRefreshDNS, NULL, 3600.0, 1);
1236 }
1237
1238 static void
1239 peerConnectFailedSilent(CachePeer * p)
1240 {
1241     p->stats.last_connect_failure = squid_curtime;
1242
1243     if (!p->tcp_up) {
1244         debugs(15, 2, "TCP connection to " << p->host << "/" << p->http_port <<
1245                " dead");
1246         return;
1247     }
1248
1249     -- p->tcp_up;
1250
1251     if (!p->tcp_up) {
1252         debugs(15, DBG_IMPORTANT, "Detected DEAD " << neighborTypeStr(p) << ": " << p->name);
1253         p->stats.logged_state = PEER_DEAD;
1254     }
1255 }
1256
1257 void
1258 peerConnectFailed(CachePeer *p)
1259 {
1260     debugs(15, DBG_IMPORTANT, "TCP connection to " << p->host << "/" << p->http_port << " failed");
1261     peerConnectFailedSilent(p);
1262 }
1263
1264 void
1265 peerConnectSucceded(CachePeer * p)
1266 {
1267     if (!p->tcp_up) {
1268         debugs(15, 2, "TCP connection to " << p->host << "/" << p->http_port << " succeded");
1269         p->tcp_up = p->connect_fail_limit; // NP: so peerAlive(p) works properly.
1270         peerAlive(p);
1271         if (!p->n_addresses)
1272             ipcache_nbgethostbyname(p->host, peerDNSConfigure, p);
1273     } else
1274         p->tcp_up = p->connect_fail_limit;
1275 }
1276
1277 /*
1278 * peerProbeConnect will be called on dead peers by neighborUp
1279 */
1280 static bool
1281 peerProbeConnect(CachePeer * p)
1282 {
1283     time_t ctimeout = p->connect_timeout > 0 ? p->connect_timeout : Config.Timeout.peer_connect;
1284     bool ret = (squid_curtime - p->stats.last_connect_failure) > (ctimeout * 10);
1285
1286     if (p->testing_now > 0)
1287         return ret;/* probe already running */
1288
1289     if (squid_curtime - p->stats.last_connect_probe == 0)
1290         return ret;/* don't probe to often */
1291
1292     /* for each IP address of this CachePeer. find one that we can connect to and probe it. */
1293     for (int i = 0; i < p->n_addresses; ++i) {
1294         Comm::ConnectionPointer conn = new Comm::Connection;
1295         conn->remote = p->addresses[i];
1296         conn->remote.port(p->http_port);
1297         conn->setPeer(p);
1298         getOutgoingAddress(NULL, conn);
1299
1300         ++ p->testing_now;
1301
1302         AsyncCall::Pointer call = commCbCall(15,3, "peerProbeConnectDone", CommConnectCbPtrFun(peerProbeConnectDone, p));
1303         Comm::ConnOpener *cs = new Comm::ConnOpener(conn, call, ctimeout);
1304         cs->setHost(p->host);
1305         AsyncJob::Start(cs);
1306     }
1307
1308     p->stats.last_connect_probe = squid_curtime;
1309
1310     return ret;
1311 }
1312
1313 static void
1314 peerProbeConnectDone(const Comm::ConnectionPointer &conn, Comm::Flag status, int, void *data)
1315 {
1316     CachePeer *p = (CachePeer*)data;
1317
1318     if (status == Comm::OK) {
1319         peerConnectSucceded(p);
1320     } else {
1321         peerConnectFailedSilent(p);
1322     }
1323
1324     -- p->testing_now;
1325     conn->close();
1326     // TODO: log this traffic.
1327 }
1328
1329 static void
1330 peerCountMcastPeersSchedule(CachePeer * p, time_t when)
1331 {
1332     if (p->mcast.flags.count_event_pending)
1333         return;
1334
1335     eventAdd("peerCountMcastPeersStart",
1336              peerCountMcastPeersStart,
1337              p,
1338              (double) when, 1);
1339
1340     p->mcast.flags.count_event_pending = true;
1341 }
1342
1343 static void
1344 peerCountMcastPeersStart(void *data)
1345 {
1346     CachePeer *p = (CachePeer *)data;
1347     ps_state *psstate;
1348     StoreEntry *fake;
1349     MemObject *mem;
1350     icp_common_t *query;
1351     int reqnum;
1352     LOCAL_ARRAY(char, url, MAX_URL);
1353     assert(p->type == PEER_MULTICAST);
1354     p->mcast.flags.count_event_pending = false;
1355     snprintf(url, MAX_URL, "http://");
1356     p->in_addr.toUrl(url+7, MAX_URL -8 );
1357     strcat(url, "/");
1358     fake = storeCreateEntry(url, url, RequestFlags(), Http::METHOD_GET);
1359     HttpRequest *req = HttpRequest::CreateFromUrl(url);
1360     psstate = new ps_state;
1361     psstate->request = req;
1362     HTTPMSGLOCK(psstate->request);
1363     psstate->entry = fake;
1364     psstate->callback = NULL;
1365     psstate->callback_data = cbdataReference(p);
1366     psstate->ping.start = current_time;
1367     mem = fake->mem_obj;
1368     mem->request = psstate->request;
1369     HTTPMSGLOCK(mem->request);
1370     mem->start_ping = current_time;
1371     mem->ping_reply_callback = peerCountHandleIcpReply;
1372     mem->ircb_data = psstate;
1373     mcastSetTtl(icpOutgoingConn->fd, p->mcast.ttl);
1374     p->mcast.id = mem->id;
1375     reqnum = icpSetCacheKey((const cache_key *)fake->key);
1376     query = _icp_common_t::createMessage(ICP_QUERY, 0, url, reqnum, 0);
1377     icpUdpSend(icpOutgoingConn->fd, p->in_addr, query, LOG_ICP_QUERY, 0);
1378     fake->ping_status = PING_WAITING;
1379     eventAdd("peerCountMcastPeersDone",
1380              peerCountMcastPeersDone,
1381              psstate,
1382              Config.Timeout.mcast_icp_query / 1000.0, 1);
1383     p->mcast.flags.counting = true;
1384     peerCountMcastPeersSchedule(p, MCAST_COUNT_RATE);
1385 }
1386
1387 static void
1388 peerCountMcastPeersDone(void *data)
1389 {
1390     ps_state *psstate = (ps_state *)data;
1391     StoreEntry *fake = psstate->entry;
1392
1393     if (cbdataReferenceValid(psstate->callback_data)) {
1394         CachePeer *p = (CachePeer *)psstate->callback_data;
1395         p->mcast.flags.counting = false;
1396         p->mcast.avg_n_members = Math::doubleAverage(p->mcast.avg_n_members, (double) psstate->ping.n_recv, ++p->mcast.n_times_counted, 10);
1397         debugs(15, DBG_IMPORTANT, "Group " << p->host  << ": " << psstate->ping.n_recv  <<
1398                " replies, "<< std::setw(4)<< std::setprecision(2) <<
1399                p->mcast.avg_n_members <<" average, RTT " << p->stats.rtt);
1400         p->mcast.n_replies_expected = (int) p->mcast.avg_n_members;
1401     }
1402
1403     cbdataReferenceDone(psstate->callback_data);
1404
1405     fake->abort(); // sets ENTRY_ABORTED and initiates releated cleanup
1406     HTTPMSGUNLOCK(fake->mem_obj->request);
1407     fake->unlock("peerCountMcastPeersDone");
1408     delete psstate;
1409 }
1410
1411 static void
1412 peerCountHandleIcpReply(CachePeer * p, peer_t, AnyP::ProtocolType proto, void *, void *data)
1413 {
1414     ps_state *psstate = (ps_state *)data;
1415     StoreEntry *fake = psstate->entry;
1416     assert(fake);
1417     MemObject *mem = fake->mem_obj;
1418     assert(mem);
1419     int rtt = tvSubMsec(mem->start_ping, current_time);
1420     assert(proto == AnyP::PROTO_ICP);
1421     ++ psstate->ping.n_recv;
1422     int rtt_av_factor = RTT_AV_FACTOR;
1423
1424     if (p->options.weighted_roundrobin)
1425         rtt_av_factor = RTT_BACKGROUND_AV_FACTOR;
1426
1427     p->stats.rtt = Math::intAverage(p->stats.rtt, rtt, psstate->ping.n_recv, rtt_av_factor);
1428 }
1429
1430 static void
1431 neighborDumpPeers(StoreEntry * sentry)
1432 {
1433     dump_peers(sentry, Config.peers);
1434 }
1435
1436 static void
1437 neighborDumpNonPeers(StoreEntry * sentry)
1438 {
1439     dump_peers(sentry, non_peers);
1440 }
1441
1442 void
1443 dump_peer_options(StoreEntry * sentry, CachePeer * p)
1444 {
1445     if (p->options.proxy_only)
1446         storeAppendPrintf(sentry, " proxy-only");
1447
1448     if (p->options.no_query)
1449         storeAppendPrintf(sentry, " no-query");
1450
1451     if (p->options.background_ping)
1452         storeAppendPrintf(sentry, " background-ping");
1453
1454     if (p->options.no_digest)
1455         storeAppendPrintf(sentry, " no-digest");
1456
1457     if (p->options.default_parent)
1458         storeAppendPrintf(sentry, " default");
1459
1460     if (p->options.roundrobin)
1461         storeAppendPrintf(sentry, " round-robin");
1462
1463     if (p->options.carp)
1464         storeAppendPrintf(sentry, " carp");
1465
1466 #if USE_AUTH
1467     if (p->options.userhash)
1468         storeAppendPrintf(sentry, " userhash");
1469 #endif
1470
1471     if (p->options.sourcehash)
1472         storeAppendPrintf(sentry, " sourcehash");
1473
1474     if (p->options.weighted_roundrobin)
1475         storeAppendPrintf(sentry, " weighted-round-robin");
1476
1477     if (p->options.mcast_responder)
1478         storeAppendPrintf(sentry, " multicast-responder");
1479
1480 #if PEER_MULTICAST_SIBLINGS
1481     if (p->options.mcast_siblings)
1482         storeAppendPrintf(sentry, " multicast-siblings");
1483 #endif
1484
1485     if (p->weight != 1)
1486         storeAppendPrintf(sentry, " weight=%d", p->weight);
1487
1488     if (p->options.closest_only)
1489         storeAppendPrintf(sentry, " closest-only");
1490
1491 #if USE_HTCP
1492     if (p->options.htcp) {
1493         storeAppendPrintf(sentry, " htcp");
1494         if (p->options.htcp_oldsquid || p->options.htcp_no_clr || p->options.htcp_no_purge_clr || p->options.htcp_only_clr) {
1495             int doneopts=0;
1496             if (p->options.htcp_oldsquid)
1497                 storeAppendPrintf(sentry, "%soldsquid",(doneopts++>0?",":"="));
1498             if (p->options.htcp_no_clr)
1499                 storeAppendPrintf(sentry, "%sno-clr",(doneopts++>0?",":"="));
1500             if (p->options.htcp_no_purge_clr)
1501                 storeAppendPrintf(sentry, "%sno-purge-clr",(doneopts++>0?",":"="));
1502             if (p->options.htcp_only_clr)
1503                 storeAppendPrintf(sentry, "%sonly-clr",(doneopts++>0?",":"="));
1504         }
1505     }
1506 #endif
1507
1508     if (p->options.no_netdb_exchange)
1509         storeAppendPrintf(sentry, " no-netdb-exchange");
1510
1511 #if USE_DELAY_POOLS
1512     if (p->options.no_delay)
1513         storeAppendPrintf(sentry, " no-delay");
1514 #endif
1515
1516     if (p->login)
1517         storeAppendPrintf(sentry, " login=%s", p->login);
1518
1519     if (p->mcast.ttl > 0)
1520         storeAppendPrintf(sentry, " ttl=%d", p->mcast.ttl);
1521
1522     if (p->connect_timeout > 0)
1523         storeAppendPrintf(sentry, " connect-timeout=%d", (int) p->connect_timeout);
1524
1525     if (p->connect_fail_limit != PEER_TCP_MAGIC_COUNT)
1526         storeAppendPrintf(sentry, " connect-fail-limit=%d", p->connect_fail_limit);
1527
1528 #if USE_CACHE_DIGESTS
1529
1530     if (p->digest_url)
1531         storeAppendPrintf(sentry, " digest-url=%s", p->digest_url);
1532
1533 #endif
1534
1535     if (p->options.allow_miss)
1536         storeAppendPrintf(sentry, " allow-miss");
1537
1538     if (p->options.no_tproxy)
1539         storeAppendPrintf(sentry, " no-tproxy");
1540
1541     if (p->max_conn > 0)
1542         storeAppendPrintf(sentry, " max-conn=%d", p->max_conn);
1543     if (p->standby.limit > 0)
1544         storeAppendPrintf(sentry, " standby=%d", p->standby.limit);
1545
1546     if (p->options.originserver)
1547         storeAppendPrintf(sentry, " originserver");
1548
1549     if (p->domain)
1550         storeAppendPrintf(sentry, " forceddomain=%s", p->domain);
1551
1552     if (p->connection_auth == 0)
1553         storeAppendPrintf(sentry, " connection-auth=off");
1554     else if (p->connection_auth == 1)
1555         storeAppendPrintf(sentry, " connection-auth=on");
1556     else if (p->connection_auth == 2)
1557         storeAppendPrintf(sentry, " connection-auth=auto");
1558
1559     storeAppendPrintf(sentry, "\n");
1560 }
1561
1562 static void
1563 dump_peers(StoreEntry * sentry, CachePeer * peers)
1564 {
1565     char ntoabuf[MAX_IPSTRLEN];
1566     icp_opcode op;
1567     int i;
1568
1569     if (peers == NULL)
1570         storeAppendPrintf(sentry, "There are no neighbors installed.\n");
1571
1572     for (CachePeer *e = peers; e; e = e->next) {
1573         assert(e->host != NULL);
1574         storeAppendPrintf(sentry, "\n%-11.11s: %s\n",
1575                           neighborTypeStr(e),
1576                           e->name);
1577         storeAppendPrintf(sentry, "Host       : %s/%d/%d\n",
1578                           e->host,
1579                           e->http_port,
1580                           e->icp.port);
1581         storeAppendPrintf(sentry, "Flags      :");
1582         dump_peer_options(sentry, e);
1583
1584         for (i = 0; i < e->n_addresses; ++i) {
1585             storeAppendPrintf(sentry, "Address[%d] : %s\n", i,
1586                               e->addresses[i].toStr(ntoabuf,MAX_IPSTRLEN) );
1587         }
1588
1589         storeAppendPrintf(sentry, "Status     : %s\n",
1590                           neighborUp(e) ? "Up" : "Down");
1591         storeAppendPrintf(sentry, "FETCHES    : %d\n", e->stats.fetches);
1592         storeAppendPrintf(sentry, "OPEN CONNS : %d\n", e->stats.conn_open);
1593         storeAppendPrintf(sentry, "AVG RTT    : %d msec\n", e->stats.rtt);
1594
1595         if (!e->options.no_query) {
1596             storeAppendPrintf(sentry, "LAST QUERY : %8d seconds ago\n",
1597                               (int) (squid_curtime - e->stats.last_query));
1598
1599             if (e->stats.last_reply > 0)
1600                 storeAppendPrintf(sentry, "LAST REPLY : %8d seconds ago\n",
1601                                   (int) (squid_curtime - e->stats.last_reply));
1602             else
1603                 storeAppendPrintf(sentry, "LAST REPLY : none received\n");
1604
1605             storeAppendPrintf(sentry, "PINGS SENT : %8d\n", e->stats.pings_sent);
1606
1607             storeAppendPrintf(sentry, "PINGS ACKED: %8d %3d%%\n",
1608                               e->stats.pings_acked,
1609                               Math::intPercent(e->stats.pings_acked, e->stats.pings_sent));
1610         }
1611
1612         storeAppendPrintf(sentry, "IGNORED    : %8d %3d%%\n", e->stats.ignored_replies, Math::intPercent(e->stats.ignored_replies, e->stats.pings_acked));
1613
1614         if (!e->options.no_query) {
1615             storeAppendPrintf(sentry, "Histogram of PINGS ACKED:\n");
1616 #if USE_HTCP
1617
1618             if (e->options.htcp) {
1619                 storeAppendPrintf(sentry, "\tMisses\t%8d %3d%%\n",
1620                                   e->htcp.counts[0],
1621                                   Math::intPercent(e->htcp.counts[0], e->stats.pings_acked));
1622                 storeAppendPrintf(sentry, "\tHits\t%8d %3d%%\n",
1623                                   e->htcp.counts[1],
1624                                   Math::intPercent(e->htcp.counts[1], e->stats.pings_acked));
1625             } else {
1626 #endif
1627
1628                 for (op = ICP_INVALID; op < ICP_END; ++op) {
1629                     if (e->icp.counts[op] == 0)
1630                         continue;
1631
1632                     storeAppendPrintf(sentry, "    %12.12s : %8d %3d%%\n",
1633                                       icp_opcode_str[op],
1634                                       e->icp.counts[op],
1635                                       Math::intPercent(e->icp.counts[op], e->stats.pings_acked));
1636                 }
1637
1638 #if USE_HTCP
1639
1640             }
1641
1642 #endif
1643
1644         }
1645
1646         if (e->stats.last_connect_failure) {
1647             storeAppendPrintf(sentry, "Last failed connect() at: %s\n",
1648                               Time::FormatHttpd(e->stats.last_connect_failure));
1649         }
1650
1651         storeAppendPrintf(sentry, "keep-alive ratio: %d%%\n", Math::intPercent(e->stats.n_keepalives_recv, e->stats.n_keepalives_sent));
1652     }
1653 }
1654
1655 #if USE_HTCP
1656 void
1657 neighborsHtcpReply(const cache_key * key, HtcpReplyData * htcp, const Ip::Address &from)
1658 {
1659     StoreEntry *e = Store::Root().get(key);
1660     MemObject *mem = NULL;
1661     CachePeer *p;
1662     peer_t ntype = PEER_NONE;
1663     debugs(15, 6, "neighborsHtcpReply: " <<
1664            (htcp->hit ? "HIT" : "MISS") << " " <<
1665            storeKeyText(key)  );
1666
1667     if (NULL != e)
1668         mem = e->mem_obj;
1669
1670     if ((p = whichPeer(from)))
1671         neighborAliveHtcp(p, mem, htcp);
1672
1673     /* Does the entry exist? */
1674     if (NULL == e) {
1675         debugs(12, 3, "neighyborsHtcpReply: Cache key '" << storeKeyText(key) << "' not found");
1676         neighborCountIgnored(p);
1677         return;
1678     }
1679
1680     /* check if someone is already fetching it */
1681     if (EBIT_TEST(e->flags, ENTRY_DISPATCHED)) {
1682         debugs(15, 3, "neighborsUdpAck: '" << storeKeyText(key) << "' already being fetched.");
1683         neighborCountIgnored(p);
1684         return;
1685     }
1686
1687     if (mem == NULL) {
1688         debugs(15, 2, "Ignoring reply for missing mem_obj: " << storeKeyText(key));
1689         neighborCountIgnored(p);
1690         return;
1691     }
1692
1693     if (e->ping_status != PING_WAITING) {
1694         debugs(15, 2, "neighborsUdpAck: Entry " << storeKeyText(key) << " is not PING_WAITING");
1695         neighborCountIgnored(p);
1696         return;
1697     }
1698
1699     if (!e->locked()) {
1700         // TODO: many entries are unlocked; why is this reported at level 1?
1701         debugs(12, DBG_IMPORTANT, "neighborsUdpAck: '" << storeKeyText(key) << "' has no locks");
1702         neighborCountIgnored(p);
1703         return;
1704     }
1705
1706     if (p) {
1707         ntype = neighborType(p, mem->request);
1708         neighborUpdateRtt(p, mem);
1709     }
1710
1711     if (ignoreMulticastReply(p, mem)) {
1712         neighborCountIgnored(p);
1713         return;
1714     }
1715
1716     debugs(15, 3, "neighborsHtcpReply: e = " << e);
1717     mem->ping_reply_callback(p, ntype, AnyP::PROTO_HTCP, htcp, mem->ircb_data);
1718 }
1719
1720 /*
1721  * Send HTCP CLR messages to all peers configured to receive them.
1722  */
1723 void
1724 neighborsHtcpClear(StoreEntry * e, const char *uri, HttpRequest * req, const HttpRequestMethod &method, htcp_clr_reason reason)
1725 {
1726     CachePeer *p;
1727     char buf[128];
1728
1729     for (p = Config.peers; p; p = p->next) {
1730         if (!p->options.htcp) {
1731             continue;
1732         }
1733         if (p->options.htcp_no_clr) {
1734             continue;
1735         }
1736         if (p->options.htcp_no_purge_clr && reason == HTCP_CLR_PURGE) {
1737             continue;
1738         }
1739         debugs(15, 3, "neighborsHtcpClear: sending CLR to " << p->in_addr.toUrl(buf, 128));
1740         htcpClear(e, uri, req, method, p, reason);
1741     }
1742 }
1743
1744 #endif
1745