src/pconn.cc

   1 /*
   2  * Copyright (C) 1996-2018 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 48    Persistent Connections */
  10
  11 #include "squid.h"
  12 #include "CachePeer.h"
  13 #include "comm.h"
  14 #include "comm/Connection.h"
  15 #include "comm/Read.h"
  16 #include "fd.h"
  17 #include "fde.h"
  18 #include "globals.h"
  19 #include "mgr/Registration.h"
  20 #include "neighbors.h"
  21 #include "pconn.h"
  22 #include "PeerPoolMgr.h"
  23 #include "SquidConfig.h"
  24 #include "Store.h"
  25
  26 #define PCONN_FDS_SZ    8   /* pconn set size, increase for better memcache hit rate */
  27
  28 //TODO: re-attach to MemPools. WAS: static MemAllocator *pconn_fds_pool = NULL;
  29 PconnModule * PconnModule::instance = NULL;
  30 CBDATA_CLASS_INIT(IdleConnList);
  31
  32 /* ========== IdleConnList ============================================ */
  33
  34 IdleConnList::IdleConnList(const char *aKey, PconnPool *thePool) :
  35     capacity_(PCONN_FDS_SZ),
  36     size_(0),
  37     parent_(thePool)
  38 {
  39     //Initialize hash_link members
  40     key = xstrdup(aKey);
  41     next = NULL;
  42
  43     theList_ = new Comm::ConnectionPointer[capacity_];
  44
  45     registerRunner();
  46
  47 // TODO: re-attach to MemPools. WAS: theList = (?? *)pconn_fds_pool->alloc();
  48 }
  49
  50 IdleConnList::~IdleConnList()
  51 {
  52     if (parent_)
  53         parent_->unlinkList(this);
  54
  55     if (size_) {
  56         parent_ = NULL; // prevent reentrant notifications and deletions
  57         closeN(size_);
  58     }
  59
  60     delete[] theList_;
  61
  62     xfree(key);
  63 }
  64
  65 /** Search the list. Matches by FD socket number.
  66  * Performed from the end of list where newest entries are.
  67  *
  68  * \retval <0   The connection is not listed
  69  * \retval >=0  The connection array index
  70  */
  71 int
  72 IdleConnList::findIndexOf(const Comm::ConnectionPointer &conn) const
  73 {
  74     for (int index = size_ - 1; index >= 0; --index) {
  75         if (conn->fd == theList_[index]->fd) {
  76             debugs(48, 3, HERE << "found " << conn << " at index " << index);
  77             return index;
  78         }
  79     }
  80
  81     debugs(48, 2, HERE << conn << " NOT FOUND!");
  82     return -1;
  83 }
  84
  85 /** Remove the entry at specified index.
  86  * May perform a shuffle of list entries to fill the gap.
  87  * \retval false The index is not an in-use entry.
  88  */
  89 bool
  90 IdleConnList::removeAt(int index)
  91 {
  92     if (index < 0 || index >= size_)
  93         return false;
  94
  95     // shuffle the remaining entries to fill the new gap.
  96     for (; index < size_ - 1; ++index)
  97         theList_[index] = theList_[index + 1];
  98     theList_[--size_] = NULL;
  99
 100     if (parent_) {
 101         parent_->noteConnectionRemoved();
 102         if (size_ == 0) {
 103             debugs(48, 3, "deleting " << hashKeyStr(this));
 104             delete this;
 105         }
 106     }
 107
 108     return true;
 109 }
 110
 111 // almost a duplicate of removeFD. But drops multiple entries.
 112 void
 113 IdleConnList::closeN(size_t n)
 114 {
 115     if (n < 1) {
 116         debugs(48, 2, HERE << "Nothing to do.");
 117         return;
 118     } else if (n >= (size_t)size_) {
 119         debugs(48, 2, HERE << "Closing all entries.");
 120         while (size_ > 0) {
 121             const Comm::ConnectionPointer conn = theList_[--size_];
 122             theList_[size_] = NULL;
 123             clearHandlers(conn);
 124             conn->close();
 125             if (parent_)
 126                 parent_->noteConnectionRemoved();
 127         }
 128     } else { //if (n < size_)
 129         debugs(48, 2, HERE << "Closing " << n << " of " << size_ << " entries.");
 130
 131         size_t index;
 132         // ensure the first N entries are closed
 133         for (index = 0; index < n; ++index) {
 134             const Comm::ConnectionPointer conn = theList_[index];
 135             theList_[index] = NULL;
 136             clearHandlers(conn);
 137             conn->close();
 138             if (parent_)
 139                 parent_->noteConnectionRemoved();
 140         }
 141         // shuffle the list N down.
 142         for (index = 0; index < (size_t)size_ - n; ++index) {
 143             theList_[index] = theList_[index + n];
 144         }
 145         // ensure the last N entries are unset
 146         while (index < ((size_t)size_)) {
 147             theList_[index] = NULL;
 148             ++index;
 149         }
 150         size_ -= n;
 151     }
 152
 153     if (parent_ && size_ == 0) {
 154         debugs(48, 3, "deleting " << hashKeyStr(this));
 155         delete this;
 156     }
 157 }
 158
 159 void
 160 IdleConnList::clearHandlers(const Comm::ConnectionPointer &conn)
 161 {
 162     debugs(48, 3, HERE << "removing close handler for " << conn);
 163     comm_read_cancel(conn->fd, IdleConnList::Read, this);
 164     commUnsetConnTimeout(conn);
 165 }
 166
 167 void
 168 IdleConnList::push(const Comm::ConnectionPointer &conn)
 169 {
 170     if (size_ == capacity_) {
 171         debugs(48, 3, HERE << "growing idle Connection array");
 172         capacity_ <<= 1;
 173         const Comm::ConnectionPointer *oldList = theList_;
 174         theList_ = new Comm::ConnectionPointer[capacity_];
 175         for (int index = 0; index < size_; ++index)
 176             theList_[index] = oldList[index];
 177
 178         delete[] oldList;
 179     }
 180
 181     if (parent_)
 182         parent_->noteConnectionAdded();
 183
 184     theList_[size_] = conn;
 185     ++size_;
 186     AsyncCall::Pointer readCall = commCbCall(5,4, "IdleConnList::Read",
 187                                   CommIoCbPtrFun(IdleConnList::Read, this));
 188     comm_read(conn, fakeReadBuf_, sizeof(fakeReadBuf_), readCall);
 189     AsyncCall::Pointer timeoutCall = commCbCall(5,4, "IdleConnList::Timeout",
 190                                      CommTimeoutCbPtrFun(IdleConnList::Timeout, this));
 191     commSetConnTimeout(conn, conn->timeLeft(Config.Timeout.serverIdlePconn), timeoutCall);
 192 }
 193
 194 /// Determine whether an entry in the idle list is available for use.
 195 /// Returns false if the entry is unset, closed or closing.
 196 bool
 197 IdleConnList::isAvailable(int i) const
 198 {
 199     const Comm::ConnectionPointer &conn = theList_[i];
 200
 201     // connection already closed. useless.
 202     if (!Comm::IsConnOpen(conn))
 203         return false;
 204
 205     // our connection early-read/close handler is scheduled to run already. unsafe
 206     if (!COMMIO_FD_READCB(conn->fd)->active())
 207         return false;
 208
 209     return true;
 210 }
 211
 212 Comm::ConnectionPointer
 213 IdleConnList::pop()
 214 {
 215     for (int i=size_-1; i>=0; --i) {
 216
 217         if (!isAvailable(i))
 218             continue;
 219
 220         // our connection timeout handler is scheduled to run already. unsafe for now.
 221         // TODO: cancel the pending timeout callback and allow re-use of the conn.
 222         if (fd_table[theList_[i]->fd].timeoutHandler == NULL)
 223             continue;
 224
 225         // finally, a match. pop and return it.
 226         Comm::ConnectionPointer result = theList_[i];
 227         clearHandlers(result);
 228         /* may delete this */
 229         removeAt(i);
 230         return result;
 231     }
 232
 233     return Comm::ConnectionPointer();
 234 }
 235
 236 /*
 237  * XXX this routine isn't terribly efficient - if there's a pending
 238  * read event (which signifies the fd will close in the next IO loop!)
 239  * we ignore the FD and move onto the next one. This means, as an example,
 240  * if we have a lot of FDs open to a very popular server and we get a bunch
 241  * of requests JUST as they timeout (say, it shuts down) we'll be wasting
 242  * quite a bit of CPU. Just keep it in mind.
 243  */
 244 Comm::ConnectionPointer
 245 IdleConnList::findUseable(const Comm::ConnectionPointer &aKey)
 246 {
 247     assert(size_);
 248
 249     // small optimization: do the constant bool tests only once.
 250     const bool keyCheckAddr = !aKey->local.isAnyAddr();
 251     const bool keyCheckPort = aKey->local.port() > 0;
 252
 253     for (int i=size_-1; i>=0; --i) {
 254
 255         if (!isAvailable(i))
 256             continue;
 257
 258         // local end port is required, but dont match.
 259         if (keyCheckPort && aKey->local.port() != theList_[i]->local.port())
 260             continue;
 261
 262         // local address is required, but does not match.
 263         if (keyCheckAddr && aKey->local.matchIPAddr(theList_[i]->local) != 0)
 264             continue;
 265
 266         // our connection timeout handler is scheduled to run already. unsafe for now.
 267         // TODO: cancel the pending timeout callback and allow re-use of the conn.
 268         if (fd_table[theList_[i]->fd].timeoutHandler == NULL)
 269             continue;
 270
 271         // finally, a match. pop and return it.
 272         Comm::ConnectionPointer result = theList_[i];
 273         clearHandlers(result);
 274         /* may delete this */
 275         removeAt(i);
 276         return result;
 277     }
 278
 279     return Comm::ConnectionPointer();
 280 }
 281
 282 /* might delete list */
 283 void
 284 IdleConnList::findAndClose(const Comm::ConnectionPointer &conn)
 285 {
 286     const int index = findIndexOf(conn);
 287     if (index >= 0) {
 288         if (parent_)
 289             parent_->notifyManager("idle conn closure");
 290         clearHandlers(conn);
 291         /* might delete this */
 292         removeAt(index);
 293         conn->close();
 294     }
 295 }
 296
 297 void
 298 IdleConnList::Read(const Comm::ConnectionPointer &conn, char *, size_t len, Comm::Flag flag, int, void *data)
 299 {
 300     debugs(48, 3, HERE << len << " bytes from " << conn);
 301
 302     if (flag == Comm::ERR_CLOSING) {
 303         debugs(48, 3, HERE << "Comm::ERR_CLOSING from " << conn);
 304         /* Bail out on Comm::ERR_CLOSING - may happen when shutdown aborts our idle FD */
 305         return;
 306     }
 307
 308     IdleConnList *list = (IdleConnList *) data;
 309     /* may delete list/data */
 310     list->findAndClose(conn);
 311 }
 312
 313 void
 314 IdleConnList::Timeout(const CommTimeoutCbParams &io)
 315 {
 316     debugs(48, 3, HERE << io.conn);
 317     IdleConnList *list = static_cast<IdleConnList *>(io.data);
 318     /* may delete list/data */
 319     list->findAndClose(io.conn);
 320 }
 321
 322 void
 323 IdleConnList::endingShutdown()
 324 {
 325     closeN(size_);
 326 }
 327
 328 /* ========== PconnPool PRIVATE FUNCTIONS ============================================ */
 329
 330 const char *
 331 PconnPool::key(const Comm::ConnectionPointer &destLink, const char *domain)
 332 {
 333     LOCAL_ARRAY(char, buf, SQUIDHOSTNAMELEN * 3 + 10);
 334
 335     destLink->remote.toUrl(buf, SQUIDHOSTNAMELEN * 3 + 10);
 336     if (domain) {
 337         const int used = strlen(buf);
 338         snprintf(buf+used, SQUIDHOSTNAMELEN * 3 + 10-used, "/%s", domain);
 339     }
 340
 341     debugs(48,6,"PconnPool::key(" << destLink << ", " << (domain?domain:"[no domain]") << ") is {" << buf << "}" );
 342     return buf;
 343 }
 344
 345 void
 346 PconnPool::dumpHist(StoreEntry * e) const
 347 {
 348     storeAppendPrintf(e,
 349                       "%s persistent connection counts:\n"
 350                       "\n"
 351                       "\t Requests\t Connection Count\n"
 352                       "\t --------\t ----------------\n",
 353                       descr);
 354
 355     for (int i = 0; i < PCONN_HIST_SZ; ++i) {
 356         if (hist[i] == 0)
 357             continue;
 358
 359         storeAppendPrintf(e, "\t%d\t%d\n", i, hist[i]);
 360     }
 361 }
 362
 363 void
 364 PconnPool::dumpHash(StoreEntry *e) const
 365 {
 366     hash_table *hid = table;
 367     hash_first(hid);
 368
 369     int i = 0;
 370     for (hash_link *walker = hash_next(hid); walker; walker = hash_next(hid)) {
 371         storeAppendPrintf(e, "\t item %d:\t%s\n", i, (char *)(walker->key));
 372         ++i;
 373     }
 374 }
 375
 376 /* ========== PconnPool PUBLIC FUNCTIONS ============================================ */
 377
 378 PconnPool::PconnPool(const char *aDescr, const CbcPointer<PeerPoolMgr> &aMgr):
 379     table(NULL), descr(aDescr),
 380     mgr(aMgr),
 381     theCount(0)
 382 {
 383     int i;
 384     table = hash_create((HASHCMP *) strcmp, 229, hash_string);
 385
 386     for (i = 0; i < PCONN_HIST_SZ; ++i)
 387         hist[i] = 0;
 388
 389     PconnModule::GetInstance()->add(this);
 390 }
 391
 392 static void
 393 DeleteIdleConnList(void *hashItem)
 394 {
 395     delete static_cast<IdleConnList*>(hashItem);
 396 }
 397
 398 PconnPool::~PconnPool()
 399 {
 400     PconnModule::GetInstance()->remove(this);
 401     hashFreeItems(table, &DeleteIdleConnList);
 402     hashFreeMemory(table);
 403     descr = NULL;
 404 }
 405
 406 void
 407 PconnPool::push(const Comm::ConnectionPointer &conn, const char *domain)
 408 {
 409     if (fdUsageHigh()) {
 410         debugs(48, 3, HERE << "Not many unused FDs");
 411         conn->close();
 412         return;
 413     } else if (shutting_down) {
 414         conn->close();
 415         debugs(48, 3, HERE << "Squid is shutting down. Refusing to do anything");
 416         return;
 417     }
 418     // TODO: also close used pconns if we exceed peer max-conn limit
 419
 420     const char *aKey = key(conn, domain);
 421     IdleConnList *list = (IdleConnList *) hash_lookup(table, aKey);
 422
 423     if (list == NULL) {
 424         list = new IdleConnList(aKey, this);
 425         debugs(48, 3, "new IdleConnList for {" << hashKeyStr(list) << "}" );
 426         hash_join(table, list);
 427     } else {
 428         debugs(48, 3, "found IdleConnList for {" << hashKeyStr(list) << "}" );
 429     }
 430
 431     list->push(conn);
 432     assert(!comm_has_incomplete_write(conn->fd));
 433
 434     LOCAL_ARRAY(char, desc, FD_DESC_SZ);
 435     snprintf(desc, FD_DESC_SZ, "Idle server: %s", aKey);
 436     fd_note(conn->fd, desc);
 437     debugs(48, 3, HERE << "pushed " << conn << " for " << aKey);
 438
 439     // successful push notifications resume multi-connection opening sequence
 440     notifyManager("push");
 441 }
 442
 443 Comm::ConnectionPointer
 444 PconnPool::pop(const Comm::ConnectionPointer &dest, const char *domain, bool keepOpen)
 445 {
 446
 447     const char * aKey = key(dest, domain);
 448
 449     IdleConnList *list = (IdleConnList *)hash_lookup(table, aKey);
 450     if (list == NULL) {
 451         debugs(48, 3, HERE << "lookup for key {" << aKey << "} failed.");
 452         // failure notifications resume standby conn creation after fdUsageHigh
 453         notifyManager("pop failure");
 454         return Comm::ConnectionPointer();
 455     } else {
 456         debugs(48, 3, "found " << hashKeyStr(list) <<
 457                (keepOpen ? " to use" : " to kill"));
 458     }
 459
 460     /* may delete list */
 461     Comm::ConnectionPointer popped = list->findUseable(dest);
 462     if (!keepOpen && Comm::IsConnOpen(popped))
 463         popped->close();
 464
 465     // successful pop notifications replenish standby connections pool
 466     notifyManager("pop");
 467     return popped;
 468 }
 469
 470 void
 471 PconnPool::notifyManager(const char *reason)
 472 {
 473     if (mgr.valid())
 474         PeerPoolMgr::Checkpoint(mgr, reason);
 475 }
 476
 477 void
 478 PconnPool::closeN(int n)
 479 {
 480     hash_table *hid = table;
 481     hash_first(hid);
 482
 483     // close N connections, one per list, to treat all lists "fairly"
 484     for (int i = 0; i < n && count(); ++i) {
 485
 486         hash_link *current = hash_next(hid);
 487         if (!current) {
 488             hash_first(hid);
 489             current = hash_next(hid);
 490             Must(current); // must have one because the count() was positive
 491         }
 492
 493         // may delete current
 494         static_cast<IdleConnList*>(current)->closeN(1);
 495     }
 496 }
 497
 498 void
 499 PconnPool::unlinkList(IdleConnList *list)
 500 {
 501     theCount -= list->count();
 502     assert(theCount >= 0);
 503     hash_remove_link(table, list);
 504 }
 505
 506 void
 507 PconnPool::noteUses(int uses)
 508 {
 509     if (uses >= PCONN_HIST_SZ)
 510         uses = PCONN_HIST_SZ - 1;
 511
 512     ++hist[uses];
 513 }
 514
 515 /* ========== PconnModule ============================================ */
 516
 517 /*
 518  * This simple class exists only for the cache manager
 519  */
 520
 521 PconnModule::PconnModule(): pools()
 522 {
 523     registerWithCacheManager();
 524 }
 525
 526 PconnModule *
 527 PconnModule::GetInstance()
 528 {
 529     if (instance == NULL)
 530         instance = new PconnModule;
 531
 532     return instance;
 533 }
 534
 535 void
 536 PconnModule::registerWithCacheManager(void)
 537 {
 538     Mgr::RegisterAction("pconn",
 539                         "Persistent Connection Utilization Histograms",
 540                         DumpWrapper, 0, 1);
 541 }
 542
 543 void
 544 PconnModule::add(PconnPool *aPool)
 545 {
 546     pools.insert(aPool);
 547 }
 548
 549 void
 550 PconnModule::remove(PconnPool *aPool)
 551 {
 552     pools.erase(aPool);
 553 }
 554
 555 void
 556 PconnModule::dump(StoreEntry *e)
 557 {
 558     typedef Pools::const_iterator PCI;
 559     int i = 0; // TODO: Why number pools if they all have names?
 560     for (PCI p = pools.begin(); p != pools.end(); ++p, ++i) {
 561         // TODO: Let each pool dump itself the way it wants to.
 562         storeAppendPrintf(e, "\n Pool %d Stats\n", i);
 563         (*p)->dumpHist(e);
 564         storeAppendPrintf(e, "\n Pool %d Hash Table\n",i);
 565         (*p)->dumpHash(e);
 566     }
 567 }
 568
 569 void
 570 PconnModule::DumpWrapper(StoreEntry *e)
 571 {
 572     PconnModule::GetInstance()->dump(e);
 573 }
 574