src/pconn.cc

   1 /*
   2  * Copyright (C) 1996-2014 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 48    Persistent Connections */
  10
  11 #include "squid.h"
  12 #include "CachePeer.h"
  13 #include "comm.h"
  14 #include "comm/Connection.h"
  15 #include "comm/Read.h"
  16 #include "fd.h"
  17 #include "fde.h"
  18 #include "globals.h"
  19 #include "mgr/Registration.h"
  20 #include "neighbors.h"
  21 #include "pconn.h"
  22 #include "PeerPoolMgr.h"
  23 #include "SquidConfig.h"
  24 #include "Store.h"
  25
  26 #define PCONN_FDS_SZ    8       /* pconn set size, increase for better memcache hit rate */
  27
  28 //TODO: re-attach to MemPools. WAS: static MemAllocator *pconn_fds_pool = NULL;
  29 PconnModule * PconnModule::instance = NULL;
  30 CBDATA_CLASS_INIT(IdleConnList);
  31
  32 /* ========== IdleConnList ============================================ */
  33
  34 IdleConnList::IdleConnList(const char *key, PconnPool *thePool) :
  35         capacity_(PCONN_FDS_SZ),
  36         size_(0),
  37         parent_(thePool)
  38 {
  39     hash.key = xstrdup(key);
  40     theList_ = new Comm::ConnectionPointer[capacity_];
  41 // TODO: re-attach to MemPools. WAS: theList = (?? *)pconn_fds_pool->alloc();
  42 }
  43
  44 IdleConnList::~IdleConnList()
  45 {
  46     if (parent_)
  47         parent_->unlinkList(this);
  48
  49     if (size_) {
  50         parent_ = NULL; // prevent reentrant notifications and deletions
  51         closeN(size_);
  52     }
  53
  54     delete[] theList_;
  55
  56     xfree(hash.key);
  57 }
  58
  59 /** Search the list. Matches by FD socket number.
  60  * Performed from the end of list where newest entries are.
  61  *
  62  * \retval <0   The connection is not listed
  63  * \retval >=0  The connection array index
  64  */
  65 int
  66 IdleConnList::findIndexOf(const Comm::ConnectionPointer &conn) const
  67 {
  68     for (int index = size_ - 1; index >= 0; --index) {
  69         if (conn->fd == theList_[index]->fd) {
  70             debugs(48, 3, HERE << "found " << conn << " at index " << index);
  71             return index;
  72         }
  73     }
  74
  75     debugs(48, 2, HERE << conn << " NOT FOUND!");
  76     return -1;
  77 }
  78
  79 /** Remove the entry at specified index.
  80  * May perform a shuffle of list entries to fill the gap.
  81  * \retval false The index is not an in-use entry.
  82  */
  83 bool
  84 IdleConnList::removeAt(int index)
  85 {
  86     if (index < 0 || index >= size_)
  87         return false;
  88
  89     // shuffle the remaining entries to fill the new gap.
  90     for (; index < size_ - 1; ++index)
  91         theList_[index] = theList_[index + 1];
  92     theList_[--size_] = NULL;
  93
  94     if (parent_) {
  95         parent_->noteConnectionRemoved();
  96         if (size_ == 0) {
  97             debugs(48, 3, HERE << "deleting " << hashKeyStr(&hash));
  98             delete this;
  99         }
 100     }
 101
 102     return true;
 103 }
 104
 105 // almost a duplicate of removeFD. But drops multiple entries.
 106 void
 107 IdleConnList::closeN(size_t n)
 108 {
 109     if (n < 1) {
 110         debugs(48, 2, HERE << "Nothing to do.");
 111         return;
 112     } else if (n >= (size_t)size_) {
 113         debugs(48, 2, HERE << "Closing all entries.");
 114         while (size_ > 0) {
 115             const Comm::ConnectionPointer conn = theList_[--size_];
 116             theList_[size_] = NULL;
 117             clearHandlers(conn);
 118             conn->close();
 119             if (parent_)
 120                 parent_->noteConnectionRemoved();
 121         }
 122     } else { //if (n < size_)
 123         debugs(48, 2, HERE << "Closing " << n << " of " << size_ << " entries.");
 124
 125         size_t index;
 126         // ensure the first N entries are closed
 127         for (index = 0; index < n; ++index) {
 128             const Comm::ConnectionPointer conn = theList_[index];
 129             theList_[index] = NULL;
 130             clearHandlers(conn);
 131             conn->close();
 132             if (parent_)
 133                 parent_->noteConnectionRemoved();
 134         }
 135         // shuffle the list N down.
 136         for (index = 0; index < (size_t)size_ - n; ++index) {
 137             theList_[index] = theList_[index + n];
 138         }
 139         // ensure the last N entries are unset
 140         while (index < ((size_t)size_)) {
 141             theList_[index] = NULL;
 142             ++index;
 143         }
 144         size_ -= n;
 145     }
 146
 147     if (parent_ && size_ == 0) {
 148         debugs(48, 3, HERE << "deleting " << hashKeyStr(&hash));
 149         delete this;
 150     }
 151 }
 152
 153 void
 154 IdleConnList::clearHandlers(const Comm::ConnectionPointer &conn)
 155 {
 156     debugs(48, 3, HERE << "removing close handler for " << conn);
 157     comm_read_cancel(conn->fd, IdleConnList::Read, this);
 158     commUnsetConnTimeout(conn);
 159 }
 160
 161 void
 162 IdleConnList::push(const Comm::ConnectionPointer &conn)
 163 {
 164     if (size_ == capacity_) {
 165         debugs(48, 3, HERE << "growing idle Connection array");
 166         capacity_ <<= 1;
 167         const Comm::ConnectionPointer *oldList = theList_;
 168         theList_ = new Comm::ConnectionPointer[capacity_];
 169         for (int index = 0; index < size_; ++index)
 170             theList_[index] = oldList[index];
 171
 172         delete[] oldList;
 173     }
 174
 175     if (parent_)
 176         parent_->noteConnectionAdded();
 177
 178     theList_[size_] = conn;
 179     ++size_;
 180     AsyncCall::Pointer readCall = commCbCall(5,4, "IdleConnList::Read",
 181                                   CommIoCbPtrFun(IdleConnList::Read, this));
 182     comm_read(conn, fakeReadBuf_, sizeof(fakeReadBuf_), readCall);
 183     AsyncCall::Pointer timeoutCall = commCbCall(5,4, "IdleConnList::Timeout",
 184                                      CommTimeoutCbPtrFun(IdleConnList::Timeout, this));
 185     commSetConnTimeout(conn, Config.Timeout.serverIdlePconn, timeoutCall);
 186 }
 187
 188 /// Determine whether an entry in the idle list is available for use.
 189 /// Returns false if the entry is unset, closed or closing.
 190 bool
 191 IdleConnList::isAvailable(int i) const
 192 {
 193     const Comm::ConnectionPointer &conn = theList_[i];
 194
 195     // connection already closed. useless.
 196     if (!Comm::IsConnOpen(conn))
 197         return false;
 198
 199     // our connection early-read/close handler is scheduled to run already. unsafe
 200     if (!COMMIO_FD_READCB(conn->fd)->active())
 201         return false;
 202
 203     return true;
 204 }
 205
 206 Comm::ConnectionPointer
 207 IdleConnList::pop()
 208 {
 209     for (int i=size_-1; i>=0; --i) {
 210
 211         if (!isAvailable(i))
 212             continue;
 213
 214         // our connection timeout handler is scheduled to run already. unsafe for now.
 215         // TODO: cancel the pending timeout callback and allow re-use of the conn.
 216         if (fd_table[theList_[i]->fd].timeoutHandler == NULL)
 217             continue;
 218
 219         // finally, a match. pop and return it.
 220         Comm::ConnectionPointer result = theList_[i];
 221         /* may delete this */
 222         removeAt(i);
 223         clearHandlers(result);
 224         return result;
 225     }
 226
 227     return Comm::ConnectionPointer();
 228 }
 229
 230 /*
 231  * XXX this routine isn't terribly efficient - if there's a pending
 232  * read event (which signifies the fd will close in the next IO loop!)
 233  * we ignore the FD and move onto the next one. This means, as an example,
 234  * if we have a lot of FDs open to a very popular server and we get a bunch
 235  * of requests JUST as they timeout (say, it shuts down) we'll be wasting
 236  * quite a bit of CPU. Just keep it in mind.
 237  */
 238 Comm::ConnectionPointer
 239 IdleConnList::findUseable(const Comm::ConnectionPointer &key)
 240 {
 241     assert(size_);
 242
 243     // small optimization: do the constant bool tests only once.
 244     const bool keyCheckAddr = !key->local.isAnyAddr();
 245     const bool keyCheckPort = key->local.port() > 0;
 246
 247     for (int i=size_-1; i>=0; --i) {
 248
 249         if (!isAvailable(i))
 250             continue;
 251
 252         // local end port is required, but dont match.
 253         if (keyCheckPort && key->local.port() != theList_[i]->local.port())
 254             continue;
 255
 256         // local address is required, but does not match.
 257         if (keyCheckAddr && key->local.matchIPAddr(theList_[i]->local) != 0)
 258             continue;
 259
 260         // our connection timeout handler is scheduled to run already. unsafe for now.
 261         // TODO: cancel the pending timeout callback and allow re-use of the conn.
 262         if (fd_table[theList_[i]->fd].timeoutHandler == NULL)
 263             continue;
 264
 265         // finally, a match. pop and return it.
 266         Comm::ConnectionPointer result = theList_[i];
 267         /* may delete this */
 268         removeAt(i);
 269         clearHandlers(result);
 270         return result;
 271     }
 272
 273     return Comm::ConnectionPointer();
 274 }
 275
 276 /* might delete list */
 277 void
 278 IdleConnList::findAndClose(const Comm::ConnectionPointer &conn)
 279 {
 280     const int index = findIndexOf(conn);
 281     if (index >= 0) {
 282         if (parent_)
 283             parent_->notifyManager("idle conn closure");
 284         /* might delete this */
 285         removeAt(index);
 286         clearHandlers(conn);
 287         conn->close();
 288     }
 289 }
 290
 291 void
 292 IdleConnList::Read(const Comm::ConnectionPointer &conn, char *buf, size_t len, Comm::Flag flag, int xerrno, void *data)
 293 {
 294     debugs(48, 3, HERE << len << " bytes from " << conn);
 295
 296     if (flag == Comm::ERR_CLOSING) {
 297         debugs(48, 3, HERE << "Comm::ERR_CLOSING from " << conn);
 298         /* Bail out on Comm::ERR_CLOSING - may happen when shutdown aborts our idle FD */
 299         return;
 300     }
 301
 302     IdleConnList *list = (IdleConnList *) data;
 303     /* may delete list/data */
 304     list->findAndClose(conn);
 305 }
 306
 307 void
 308 IdleConnList::Timeout(const CommTimeoutCbParams &io)
 309 {
 310     debugs(48, 3, HERE << io.conn);
 311     IdleConnList *list = static_cast<IdleConnList *>(io.data);
 312     /* may delete list/data */
 313     list->findAndClose(io.conn);
 314 }
 315
 316 /* ========== PconnPool PRIVATE FUNCTIONS ============================================ */
 317
 318 const char *
 319 PconnPool::key(const Comm::ConnectionPointer &destLink, const char *domain)
 320 {
 321     LOCAL_ARRAY(char, buf, SQUIDHOSTNAMELEN * 3 + 10);
 322
 323     destLink->remote.toUrl(buf, SQUIDHOSTNAMELEN * 3 + 10);
 324     if (domain) {
 325         const int used = strlen(buf);
 326         snprintf(buf+used, SQUIDHOSTNAMELEN * 3 + 10-used, "/%s", domain);
 327     }
 328
 329     debugs(48,6,"PconnPool::key(" << destLink << ", " << (domain?domain:"[no domain]") << ") is {" << buf << "}" );
 330     return buf;
 331 }
 332
 333 void
 334 PconnPool::dumpHist(StoreEntry * e) const
 335 {
 336     storeAppendPrintf(e,
 337                       "%s persistent connection counts:\n"
 338                       "\n"
 339                       "\t Requests\t Connection Count\n"
 340                       "\t --------\t ----------------\n",
 341                       descr);
 342
 343     for (int i = 0; i < PCONN_HIST_SZ; ++i) {
 344         if (hist[i] == 0)
 345             continue;
 346
 347         storeAppendPrintf(e, "\t%d\t%d\n", i, hist[i]);
 348     }
 349 }
 350
 351 void
 352 PconnPool::dumpHash(StoreEntry *e) const
 353 {
 354     hash_table *hid = table;
 355     hash_first(hid);
 356
 357     int i = 0;
 358     for (hash_link *walker = hash_next(hid); walker; walker = hash_next(hid)) {
 359         storeAppendPrintf(e, "\t item %d:\t%s\n", i, (char *)(walker->key));
 360         ++i;
 361     }
 362 }
 363
 364 /* ========== PconnPool PUBLIC FUNCTIONS ============================================ */
 365
 366 PconnPool::PconnPool(const char *aDescr, const CbcPointer<PeerPoolMgr> &aMgr):
 367         table(NULL), descr(aDescr),
 368         mgr(aMgr),
 369         theCount(0)
 370 {
 371     int i;
 372     table = hash_create((HASHCMP *) strcmp, 229, hash_string);
 373
 374     for (i = 0; i < PCONN_HIST_SZ; ++i)
 375         hist[i] = 0;
 376
 377     PconnModule::GetInstance()->add(this);
 378 }
 379
 380 static void
 381 DeleteIdleConnList(void *hashItem)
 382 {
 383     delete reinterpret_cast<IdleConnList*>(hashItem);
 384 }
 385
 386 PconnPool::~PconnPool()
 387 {
 388     PconnModule::GetInstance()->remove(this);
 389     hashFreeItems(table, &DeleteIdleConnList);
 390     hashFreeMemory(table);
 391     descr = NULL;
 392 }
 393
 394 void
 395 PconnPool::push(const Comm::ConnectionPointer &conn, const char *domain)
 396 {
 397     if (fdUsageHigh()) {
 398         debugs(48, 3, HERE << "Not many unused FDs");
 399         conn->close();
 400         return;
 401     } else if (shutting_down) {
 402         conn->close();
 403         debugs(48, 3, HERE << "Squid is shutting down. Refusing to do anything");
 404         return;
 405     }
 406     // TODO: also close used pconns if we exceed peer max-conn limit
 407
 408     const char *aKey = key(conn, domain);
 409     IdleConnList *list = (IdleConnList *) hash_lookup(table, aKey);
 410
 411     if (list == NULL) {
 412         list = new IdleConnList(aKey, this);
 413         debugs(48, 3, HERE << "new IdleConnList for {" << hashKeyStr(&list->hash) << "}" );
 414         hash_join(table, &list->hash);
 415     } else {
 416         debugs(48, 3, HERE << "found IdleConnList for {" << hashKeyStr(&list->hash) << "}" );
 417     }
 418
 419     list->push(conn);
 420     assert(!comm_has_incomplete_write(conn->fd));
 421
 422     LOCAL_ARRAY(char, desc, FD_DESC_SZ);
 423     snprintf(desc, FD_DESC_SZ, "Idle server: %s", aKey);
 424     fd_note(conn->fd, desc);
 425     debugs(48, 3, HERE << "pushed " << conn << " for " << aKey);
 426
 427     // successful push notifications resume multi-connection opening sequence
 428     notifyManager("push");
 429 }
 430
 431 Comm::ConnectionPointer
 432 PconnPool::pop(const Comm::ConnectionPointer &dest, const char *domain, bool keepOpen)
 433 {
 434
 435     const char * aKey = key(dest, domain);
 436
 437     IdleConnList *list = (IdleConnList *)hash_lookup(table, aKey);
 438     if (list == NULL) {
 439         debugs(48, 3, HERE << "lookup for key {" << aKey << "} failed.");
 440         // failure notifications resume standby conn creation after fdUsageHigh
 441         notifyManager("pop failure");
 442         return Comm::ConnectionPointer();
 443     } else {
 444         debugs(48, 3, HERE << "found " << hashKeyStr(&list->hash) <<
 445                (keepOpen ? " to use" : " to kill"));
 446     }
 447
 448     /* may delete list */
 449     Comm::ConnectionPointer popped = list->findUseable(dest);
 450     if (!keepOpen && Comm::IsConnOpen(popped))
 451         popped->close();
 452
 453     // successful pop notifications replenish standby connections pool
 454     notifyManager("pop");
 455     return popped;
 456 }
 457
 458 void
 459 PconnPool::notifyManager(const char *reason)
 460 {
 461     if (mgr.valid())
 462         PeerPoolMgr::Checkpoint(mgr, reason);
 463 }
 464
 465 void
 466 PconnPool::closeN(int n)
 467 {
 468     hash_table *hid = table;
 469     hash_first(hid);
 470
 471     // close N connections, one per list, to treat all lists "fairly"
 472     for (int i = 0; i < n && count(); ++i) {
 473
 474         hash_link *current = hash_next(hid);
 475         if (!current) {
 476             hash_first(hid);
 477             current = hash_next(hid);
 478             Must(current); // must have one because the count() was positive
 479         }
 480
 481         // may delete current
 482         reinterpret_cast<IdleConnList*>(current)->closeN(1);
 483     }
 484 }
 485
 486 void
 487 PconnPool::unlinkList(IdleConnList *list)
 488 {
 489     theCount -= list->count();
 490     assert(theCount >= 0);
 491     hash_remove_link(table, &list->hash);
 492 }
 493
 494 void
 495 PconnPool::noteUses(int uses)
 496 {
 497     if (uses >= PCONN_HIST_SZ)
 498         uses = PCONN_HIST_SZ - 1;
 499
 500     ++hist[uses];
 501 }
 502
 503 /* ========== PconnModule ============================================ */
 504
 505 /*
 506  * This simple class exists only for the cache manager
 507  */
 508
 509 PconnModule::PconnModule(): pools()
 510 {
 511     registerWithCacheManager();
 512 }
 513
 514 PconnModule *
 515 PconnModule::GetInstance()
 516 {
 517     if (instance == NULL)
 518         instance = new PconnModule;
 519
 520     return instance;
 521 }
 522
 523 void
 524 PconnModule::registerWithCacheManager(void)
 525 {
 526     Mgr::RegisterAction("pconn",
 527                         "Persistent Connection Utilization Histograms",
 528                         DumpWrapper, 0, 1);
 529 }
 530
 531 void
 532 PconnModule::add(PconnPool *aPool)
 533 {
 534     pools.insert(aPool);
 535 }
 536
 537 void
 538 PconnModule::remove(PconnPool *aPool)
 539 {
 540     pools.erase(aPool);
 541 }
 542
 543 void
 544 PconnModule::dump(StoreEntry *e)
 545 {
 546     typedef Pools::const_iterator PCI;
 547     int i = 0; // TODO: Why number pools if they all have names?
 548     for (PCI p = pools.begin(); p != pools.end(); ++p, ++i) {
 549         // TODO: Let each pool dump itself the way it wants to.
 550         storeAppendPrintf(e, "\n Pool %d Stats\n", i);
 551         (*p)->dumpHist(e);
 552         storeAppendPrintf(e, "\n Pool %d Hash Table\n",i);
 553         (*p)->dumpHash(e);
 554     }
 555 }
 556
 557 void
 558 PconnModule::DumpWrapper(StoreEntry *e)
 559 {
 560     PconnModule::GetInstance()->dump(e);
 561 }