src/pconn.cc

   1 /*
   2  * Copyright (C) 1996-2020 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 48    Persistent Connections */
  10
  11 #include "squid.h"
  12 #include "CachePeer.h"
  13 #include "comm.h"
  14 #include "comm/Connection.h"
  15 #include "comm/Read.h"
  16 #include "fd.h"
  17 #include "fde.h"
  18 #include "globals.h"
  19 #include "mgr/Registration.h"
  20 #include "neighbors.h"
  21 #include "pconn.h"
  22 #include "PeerPoolMgr.h"
  23 #include "SquidConfig.h"
  24 #include "Store.h"
  25
  26 #define PCONN_FDS_SZ    8   /* pconn set size, increase for better memcache hit rate */
  27
  28 //TODO: re-attach to MemPools. WAS: static MemAllocator *pconn_fds_pool = NULL;
  29 PconnModule * PconnModule::instance = NULL;
  30 CBDATA_CLASS_INIT(IdleConnList);
  31
  32 /* ========== IdleConnList ============================================ */
  33
  34 IdleConnList::IdleConnList(const char *aKey, PconnPool *thePool) :
  35     capacity_(PCONN_FDS_SZ),
  36     size_(0),
  37     parent_(thePool)
  38 {
  39     //Initialize hash_link members
  40     key = xstrdup(aKey);
  41     next = NULL;
  42
  43     theList_ = new Comm::ConnectionPointer[capacity_];
  44
  45     registerRunner();
  46
  47 // TODO: re-attach to MemPools. WAS: theList = (?? *)pconn_fds_pool->alloc();
  48 }
  49
  50 IdleConnList::~IdleConnList()
  51 {
  52     if (parent_)
  53         parent_->unlinkList(this);
  54
  55     if (size_) {
  56         parent_ = NULL; // prevent reentrant notifications and deletions
  57         closeN(size_);
  58     }
  59
  60     delete[] theList_;
  61
  62     xfree(key);
  63 }
  64
  65 /** Search the list. Matches by FD socket number.
  66  * Performed from the end of list where newest entries are.
  67  *
  68  * \retval <0   The connection is not listed
  69  * \retval >=0  The connection array index
  70  */
  71 int
  72 IdleConnList::findIndexOf(const Comm::ConnectionPointer &conn) const
  73 {
  74     for (int index = size_ - 1; index >= 0; --index) {
  75         if (conn->fd == theList_[index]->fd) {
  76             debugs(48, 3, HERE << "found " << conn << " at index " << index);
  77             return index;
  78         }
  79     }
  80
  81     debugs(48, 2, HERE << conn << " NOT FOUND!");
  82     return -1;
  83 }
  84
  85 /** Remove the entry at specified index.
  86  * May perform a shuffle of list entries to fill the gap.
  87  * \retval false The index is not an in-use entry.
  88  */
  89 bool
  90 IdleConnList::removeAt(int index)
  91 {
  92     if (index < 0 || index >= size_)
  93         return false;
  94
  95     // shuffle the remaining entries to fill the new gap.
  96     for (; index < size_ - 1; ++index)
  97         theList_[index] = theList_[index + 1];
  98     theList_[--size_] = NULL;
  99
 100     if (parent_) {
 101         parent_->noteConnectionRemoved();
 102         if (size_ == 0) {
 103             debugs(48, 3, "deleting " << hashKeyStr(this));
 104             delete this;
 105         }
 106     }
 107
 108     return true;
 109 }
 110
 111 // almost a duplicate of removeFD. But drops multiple entries.
 112 void
 113 IdleConnList::closeN(size_t n)
 114 {
 115     if (n < 1) {
 116         debugs(48, 2, HERE << "Nothing to do.");
 117         return;
 118     } else if (n >= (size_t)size_) {
 119         debugs(48, 2, HERE << "Closing all entries.");
 120         while (size_ > 0) {
 121             const Comm::ConnectionPointer conn = theList_[--size_];
 122             theList_[size_] = NULL;
 123             clearHandlers(conn);
 124             conn->close();
 125             if (parent_)
 126                 parent_->noteConnectionRemoved();
 127         }
 128     } else { //if (n < size_)
 129         debugs(48, 2, HERE << "Closing " << n << " of " << size_ << " entries.");
 130
 131         size_t index;
 132         // ensure the first N entries are closed
 133         for (index = 0; index < n; ++index) {
 134             const Comm::ConnectionPointer conn = theList_[index];
 135             theList_[index] = NULL;
 136             clearHandlers(conn);
 137             conn->close();
 138             if (parent_)
 139                 parent_->noteConnectionRemoved();
 140         }
 141         // shuffle the list N down.
 142         for (index = 0; index < (size_t)size_ - n; ++index) {
 143             theList_[index] = theList_[index + n];
 144         }
 145         // ensure the last N entries are unset
 146         while (index < ((size_t)size_)) {
 147             theList_[index] = NULL;
 148             ++index;
 149         }
 150         size_ -= n;
 151     }
 152
 153     if (parent_ && size_ == 0) {
 154         debugs(48, 3, "deleting " << hashKeyStr(this));
 155         delete this;
 156     }
 157 }
 158
 159 void
 160 IdleConnList::clearHandlers(const Comm::ConnectionPointer &conn)
 161 {
 162     debugs(48, 3, HERE << "removing close handler for " << conn);
 163     comm_read_cancel(conn->fd, IdleConnList::Read, this);
 164     commUnsetConnTimeout(conn);
 165 }
 166
 167 void
 168 IdleConnList::push(const Comm::ConnectionPointer &conn)
 169 {
 170     if (size_ == capacity_) {
 171         debugs(48, 3, HERE << "growing idle Connection array");
 172         capacity_ <<= 1;
 173         const Comm::ConnectionPointer *oldList = theList_;
 174         theList_ = new Comm::ConnectionPointer[capacity_];
 175         for (int index = 0; index < size_; ++index)
 176             theList_[index] = oldList[index];
 177
 178         delete[] oldList;
 179     }
 180
 181     if (parent_)
 182         parent_->noteConnectionAdded();
 183
 184     theList_[size_] = conn;
 185     ++size_;
 186     AsyncCall::Pointer readCall = commCbCall(5,4, "IdleConnList::Read",
 187                                   CommIoCbPtrFun(IdleConnList::Read, this));
 188     comm_read(conn, fakeReadBuf_, sizeof(fakeReadBuf_), readCall);
 189     AsyncCall::Pointer timeoutCall = commCbCall(5,4, "IdleConnList::Timeout",
 190                                      CommTimeoutCbPtrFun(IdleConnList::Timeout, this));
 191     commSetConnTimeout(conn, conn->timeLeft(Config.Timeout.serverIdlePconn), timeoutCall);
 192 }
 193
 194 /// Determine whether an entry in the idle list is available for use.
 195 /// Returns false if the entry is unset, closed or closing.
 196 bool
 197 IdleConnList::isAvailable(int i) const
 198 {
 199     const Comm::ConnectionPointer &conn = theList_[i];
 200
 201     // connection already closed. useless.
 202     if (!Comm::IsConnOpen(conn))
 203         return false;
 204
 205     // our connection early-read/close handler is scheduled to run already. unsafe
 206     if (!COMMIO_FD_READCB(conn->fd)->active())
 207         return false;
 208
 209     return true;
 210 }
 211
 212 Comm::ConnectionPointer
 213 IdleConnList::pop()
 214 {
 215     for (int i=size_-1; i>=0; --i) {
 216
 217         if (!isAvailable(i))
 218             continue;
 219
 220         // our connection timeout handler is scheduled to run already. unsafe for now.
 221         // TODO: cancel the pending timeout callback and allow re-use of the conn.
 222         if (fd_table[theList_[i]->fd].timeoutHandler == NULL)
 223             continue;
 224
 225         // finally, a match. pop and return it.
 226         Comm::ConnectionPointer result = theList_[i];
 227         clearHandlers(result);
 228         /* may delete this */
 229         removeAt(i);
 230         return result;
 231     }
 232
 233     return Comm::ConnectionPointer();
 234 }
 235
 236 /*
 237  * XXX this routine isn't terribly efficient - if there's a pending
 238  * read event (which signifies the fd will close in the next IO loop!)
 239  * we ignore the FD and move onto the next one. This means, as an example,
 240  * if we have a lot of FDs open to a very popular server and we get a bunch
 241  * of requests JUST as they timeout (say, it shuts down) we'll be wasting
 242  * quite a bit of CPU. Just keep it in mind.
 243  */
 244 Comm::ConnectionPointer
 245 IdleConnList::findUseable(const Comm::ConnectionPointer &aKey)
 246 {
 247     assert(size_);
 248
 249     // small optimization: do the constant bool tests only once.
 250     const bool keyCheckAddr = !aKey->local.isAnyAddr();
 251     const bool keyCheckPort = aKey->local.port() > 0;
 252
 253     for (int i=size_-1; i>=0; --i) {
 254
 255         if (!isAvailable(i))
 256             continue;
 257
 258         // local end port is required, but do not match.
 259         if (keyCheckPort && aKey->local.port() != theList_[i]->local.port())
 260             continue;
 261
 262         // local address is required, but does not match.
 263         if (keyCheckAddr && aKey->local.matchIPAddr(theList_[i]->local) != 0)
 264             continue;
 265
 266         // our connection timeout handler is scheduled to run already. unsafe for now.
 267         // TODO: cancel the pending timeout callback and allow re-use of the conn.
 268         if (fd_table[theList_[i]->fd].timeoutHandler == NULL)
 269             continue;
 270
 271         // finally, a match. pop and return it.
 272         Comm::ConnectionPointer result = theList_[i];
 273         clearHandlers(result);
 274         /* may delete this */
 275         removeAt(i);
 276         return result;
 277     }
 278
 279     return Comm::ConnectionPointer();
 280 }
 281
 282 /* might delete list */
 283 void
 284 IdleConnList::findAndClose(const Comm::ConnectionPointer &conn)
 285 {
 286     const int index = findIndexOf(conn);
 287     if (index >= 0) {
 288         if (parent_)
 289             parent_->notifyManager("idle conn closure");
 290         clearHandlers(conn);
 291         /* might delete this */
 292         removeAt(index);
 293         conn->close();
 294     }
 295 }
 296
 297 void
 298 IdleConnList::Read(const Comm::ConnectionPointer &conn, char *, size_t len, Comm::Flag flag, int, void *data)
 299 {
 300     debugs(48, 3, HERE << len << " bytes from " << conn);
 301
 302     if (flag == Comm::ERR_CLOSING) {
 303         debugs(48, 3, HERE << "Comm::ERR_CLOSING from " << conn);
 304         /* Bail out on Comm::ERR_CLOSING - may happen when shutdown aborts our idle FD */
 305         return;
 306     }
 307
 308     IdleConnList *list = (IdleConnList *) data;
 309     /* may delete list/data */
 310     list->findAndClose(conn);
 311 }
 312
 313 void
 314 IdleConnList::Timeout(const CommTimeoutCbParams &io)
 315 {
 316     debugs(48, 3, HERE << io.conn);
 317     IdleConnList *list = static_cast<IdleConnList *>(io.data);
 318     /* may delete list/data */
 319     list->findAndClose(io.conn);
 320 }
 321
 322 void
 323 IdleConnList::endingShutdown()
 324 {
 325     closeN(size_);
 326 }
 327
 328 /* ========== PconnPool PRIVATE FUNCTIONS ============================================ */
 329
 330 const char *
 331 PconnPool::key(const Comm::ConnectionPointer &destLink, const char *domain)
 332 {
 333     LOCAL_ARRAY(char, buf, SQUIDHOSTNAMELEN * 3 + 10);
 334
 335     destLink->remote.toUrl(buf, SQUIDHOSTNAMELEN * 3 + 10);
 336
 337     // when connecting through a cache_peer, ignore the final destination
 338     if (destLink->getPeer())
 339         domain = nullptr;
 340
 341     if (domain) {
 342         const int used = strlen(buf);
 343         snprintf(buf+used, SQUIDHOSTNAMELEN * 3 + 10-used, "/%s", domain);
 344     }
 345
 346     debugs(48,6,"PconnPool::key(" << destLink << ", " << (domain?domain:"[no domain]") << ") is {" << buf << "}" );
 347     return buf;
 348 }
 349
 350 void
 351 PconnPool::dumpHist(StoreEntry * e) const
 352 {
 353     storeAppendPrintf(e,
 354                       "%s persistent connection counts:\n"
 355                       "\n"
 356                       "\t Requests\t Connection Count\n"
 357                       "\t --------\t ----------------\n",
 358                       descr);
 359
 360     for (int i = 0; i < PCONN_HIST_SZ; ++i) {
 361         if (hist[i] == 0)
 362             continue;
 363
 364         storeAppendPrintf(e, "\t%d\t%d\n", i, hist[i]);
 365     }
 366 }
 367
 368 void
 369 PconnPool::dumpHash(StoreEntry *e) const
 370 {
 371     hash_table *hid = table;
 372     hash_first(hid);
 373
 374     int i = 0;
 375     for (hash_link *walker = hash_next(hid); walker; walker = hash_next(hid)) {
 376         storeAppendPrintf(e, "\t item %d:\t%s\n", i, (char *)(walker->key));
 377         ++i;
 378     }
 379 }
 380
 381 /* ========== PconnPool PUBLIC FUNCTIONS ============================================ */
 382
 383 PconnPool::PconnPool(const char *aDescr, const CbcPointer<PeerPoolMgr> &aMgr):
 384     table(NULL), descr(aDescr),
 385     mgr(aMgr),
 386     theCount(0)
 387 {
 388     int i;
 389     table = hash_create((HASHCMP *) strcmp, 229, hash_string);
 390
 391     for (i = 0; i < PCONN_HIST_SZ; ++i)
 392         hist[i] = 0;
 393
 394     PconnModule::GetInstance()->add(this);
 395 }
 396
 397 static void
 398 DeleteIdleConnList(void *hashItem)
 399 {
 400     delete static_cast<IdleConnList*>(hashItem);
 401 }
 402
 403 PconnPool::~PconnPool()
 404 {
 405     PconnModule::GetInstance()->remove(this);
 406     hashFreeItems(table, &DeleteIdleConnList);
 407     hashFreeMemory(table);
 408     descr = NULL;
 409 }
 410
 411 void
 412 PconnPool::push(const Comm::ConnectionPointer &conn, const char *domain)
 413 {
 414     if (fdUsageHigh()) {
 415         debugs(48, 3, HERE << "Not many unused FDs");
 416         conn->close();
 417         return;
 418     } else if (shutting_down) {
 419         conn->close();
 420         debugs(48, 3, HERE << "Squid is shutting down. Refusing to do anything");
 421         return;
 422     }
 423     // TODO: also close used pconns if we exceed peer max-conn limit
 424
 425     const char *aKey = key(conn, domain);
 426     IdleConnList *list = (IdleConnList *) hash_lookup(table, aKey);
 427
 428     if (list == NULL) {
 429         list = new IdleConnList(aKey, this);
 430         debugs(48, 3, "new IdleConnList for {" << hashKeyStr(list) << "}" );
 431         hash_join(table, list);
 432     } else {
 433         debugs(48, 3, "found IdleConnList for {" << hashKeyStr(list) << "}" );
 434     }
 435
 436     list->push(conn);
 437     assert(!comm_has_incomplete_write(conn->fd));
 438
 439     LOCAL_ARRAY(char, desc, FD_DESC_SZ);
 440     snprintf(desc, FD_DESC_SZ, "Idle server: %s", aKey);
 441     fd_note(conn->fd, desc);
 442     debugs(48, 3, HERE << "pushed " << conn << " for " << aKey);
 443
 444     // successful push notifications resume multi-connection opening sequence
 445     notifyManager("push");
 446 }
 447
 448 Comm::ConnectionPointer
 449 PconnPool::pop(const Comm::ConnectionPointer &dest, const char *domain, bool keepOpen)
 450 {
 451     // always call shared pool first because we need to close an idle
 452     // connection there if we have to use a standby connection.
 453     if (const auto direct = popStored(dest, domain, keepOpen))
 454         return direct;
 455
 456     // either there was no pconn to pop or this is not a retriable xaction
 457     if (const auto peer = dest->getPeer()) {
 458         if (peer->standby.pool)
 459             return peer->standby.pool->popStored(dest, domain, true);
 460     }
 461
 462     return nullptr;
 463 }
 464
 465 /// implements pop() API while disregarding peer standby pools
 466 /// \returns an open connection or nil
 467 Comm::ConnectionPointer
 468 PconnPool::popStored(const Comm::ConnectionPointer &dest, const char *domain, const bool keepOpen)
 469 {
 470     const char * aKey = key(dest, domain);
 471
 472     IdleConnList *list = (IdleConnList *)hash_lookup(table, aKey);
 473     if (list == NULL) {
 474         debugs(48, 3, HERE << "lookup for key {" << aKey << "} failed.");
 475         // failure notifications resume standby conn creation after fdUsageHigh
 476         notifyManager("pop lookup failure");
 477         return Comm::ConnectionPointer();
 478     } else {
 479         debugs(48, 3, "found " << hashKeyStr(list) <<
 480                (keepOpen ? " to use" : " to kill"));
 481     }
 482
 483     if (const auto popped = list->findUseable(dest)) { // may delete list
 484         // successful pop notifications replenish standby connections pool
 485         notifyManager("pop");
 486
 487         if (keepOpen)
 488             return popped;
 489
 490         popped->close();
 491         return Comm::ConnectionPointer();
 492     }
 493
 494     // failure notifications resume standby conn creation after fdUsageHigh
 495     notifyManager("pop usability failure");
 496     return Comm::ConnectionPointer();
 497 }
 498
 499 void
 500 PconnPool::notifyManager(const char *reason)
 501 {
 502     if (mgr.valid())
 503         PeerPoolMgr::Checkpoint(mgr, reason);
 504 }
 505
 506 void
 507 PconnPool::closeN(int n)
 508 {
 509     hash_table *hid = table;
 510     hash_first(hid);
 511
 512     // close N connections, one per list, to treat all lists "fairly"
 513     for (int i = 0; i < n && count(); ++i) {
 514
 515         hash_link *current = hash_next(hid);
 516         if (!current) {
 517             hash_first(hid);
 518             current = hash_next(hid);
 519             Must(current); // must have one because the count() was positive
 520         }
 521
 522         // may delete current
 523         static_cast<IdleConnList*>(current)->closeN(1);
 524     }
 525 }
 526
 527 void
 528 PconnPool::unlinkList(IdleConnList *list)
 529 {
 530     theCount -= list->count();
 531     assert(theCount >= 0);
 532     hash_remove_link(table, list);
 533 }
 534
 535 void
 536 PconnPool::noteUses(int uses)
 537 {
 538     if (uses >= PCONN_HIST_SZ)
 539         uses = PCONN_HIST_SZ - 1;
 540
 541     ++hist[uses];
 542 }
 543
 544 /* ========== PconnModule ============================================ */
 545
 546 /*
 547  * This simple class exists only for the cache manager
 548  */
 549
 550 PconnModule::PconnModule(): pools()
 551 {
 552     registerWithCacheManager();
 553 }
 554
 555 PconnModule *
 556 PconnModule::GetInstance()
 557 {
 558     if (instance == NULL)
 559         instance = new PconnModule;
 560
 561     return instance;
 562 }
 563
 564 void
 565 PconnModule::registerWithCacheManager(void)
 566 {
 567     Mgr::RegisterAction("pconn",
 568                         "Persistent Connection Utilization Histograms",
 569                         DumpWrapper, 0, 1);
 570 }
 571
 572 void
 573 PconnModule::add(PconnPool *aPool)
 574 {
 575     pools.insert(aPool);
 576 }
 577
 578 void
 579 PconnModule::remove(PconnPool *aPool)
 580 {
 581     pools.erase(aPool);
 582 }
 583
 584 void
 585 PconnModule::dump(StoreEntry *e)
 586 {
 587     typedef Pools::const_iterator PCI;
 588     int i = 0; // TODO: Why number pools if they all have names?
 589     for (PCI p = pools.begin(); p != pools.end(); ++p, ++i) {
 590         // TODO: Let each pool dump itself the way it wants to.
 591         storeAppendPrintf(e, "\n Pool %d Stats\n", i);
 592         (*p)->dumpHist(e);
 593         storeAppendPrintf(e, "\n Pool %d Hash Table\n",i);
 594         (*p)->dumpHash(e);
 595     }
 596 }
 597
 598 void
 599 PconnModule::DumpWrapper(StoreEntry *e)
 600 {
 601     PconnModule::GetInstance()->dump(e);
 602 }
 603