src/pconn.cc

   1 /*
   2  * DEBUG: section 48    Persistent Connections
   3  * AUTHOR: Duane Wessels
   4  *
   5  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   6  * ----------------------------------------------------------
   7  *
   8  *  Squid is the result of efforts by numerous individuals from
   9  *  the Internet community; see the CONTRIBUTORS file for full
  10  *  details.   Many organizations have provided support for Squid's
  11  *  development; see the SPONSORS file for full details.  Squid is
  12  *  Copyrighted (C) 2001 by the Regents of the University of
  13  *  California; see the COPYRIGHT file for full details.  Squid
  14  *  incorporates software developed and/or copyrighted by other
  15  *  sources; see the CREDITS file for full details.
  16  *
  17  *  This program is free software; you can redistribute it and/or modify
  18  *  it under the terms of the GNU General Public License as published by
  19  *  the Free Software Foundation; either version 2 of the License, or
  20  *  (at your option) any later version.
  21  *
  22  *  This program is distributed in the hope that it will be useful,
  23  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  24  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  25  *  GNU General Public License for more details.
  26  *
  27  *  You should have received a copy of the GNU General Public License
  28  *  along with this program; if not, write to the Free Software
  29  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  30  *
  31  */
  32
  33 #include "squid.h"
  34 #include "comm.h"
  35 #include "comm/Connection.h"
  36 #include "fd.h"
  37 #include "fde.h"
  38 #include "globals.h"
  39 #include "mgr/Registration.h"
  40 #include "pconn.h"
  41 #include "SquidConfig.h"
  42 #include "Store.h"
  43
  44 #define PCONN_FDS_SZ    8       /* pconn set size, increase for better memcache hit rate */
  45
  46 //TODO: re-attach to MemPools. WAS: static MemAllocator *pconn_fds_pool = NULL;
  47 PconnModule * PconnModule::instance = NULL;
  48 CBDATA_CLASS_INIT(IdleConnList);
  49
  50 /* ========== IdleConnList ============================================ */
  51
  52 IdleConnList::IdleConnList(const char *key, PconnPool *thePool) :
  53         capacity_(PCONN_FDS_SZ),
  54         size_(0),
  55         parent_(thePool)
  56 {
  57     hash.key = xstrdup(key);
  58     theList_ = new Comm::ConnectionPointer[capacity_];
  59 // TODO: re-attach to MemPools. WAS: theList = (?? *)pconn_fds_pool->alloc();
  60 }
  61
  62 IdleConnList::~IdleConnList()
  63 {
  64     if (parent_)
  65         parent_->unlinkList(this);
  66
  67     delete[] theList_;
  68
  69     xfree(hash.key);
  70 }
  71
  72 /** Search the list. Matches by FD socket number.
  73  * Performed from the end of list where newest entries are.
  74  *
  75  * \retval <0   The connection is not listed
  76  * \retval >=0  The connection array index
  77  */
  78 int
  79 IdleConnList::findIndexOf(const Comm::ConnectionPointer &conn) const
  80 {
  81     for (int index = size_ - 1; index >= 0; --index) {
  82         if (conn->fd == theList_[index]->fd) {
  83             debugs(48, 3, HERE << "found " << conn << " at index " << index);
  84             return index;
  85         }
  86     }
  87
  88     debugs(48, 2, HERE << conn << " NOT FOUND!");
  89     return -1;
  90 }
  91
  92 /** Remove the entry at specified index.
  93  * May perform a shuffle of list entries to fill the gap.
  94  * \retval false The index is not an in-use entry.
  95  */
  96 bool
  97 IdleConnList::removeAt(int index)
  98 {
  99     if (index < 0 || index >= size_)
 100         return false;
 101
 102     // shuffle the remaining entries to fill the new gap.
 103     for (; index < size_ - 1; ++index)
 104         theList_[index] = theList_[index + 1];
 105     theList_[--size_] = NULL;
 106
 107     if (parent_) {
 108         parent_->noteConnectionRemoved();
 109         if (size_ == 0) {
 110             debugs(48, 3, HERE << "deleting " << hashKeyStr(&hash));
 111             delete this;
 112         }
 113     }
 114
 115     return true;
 116 }
 117
 118 // almost a duplicate of removeFD. But drops multiple entries.
 119 void
 120 IdleConnList::closeN(size_t n)
 121 {
 122     if (n < 1) {
 123         debugs(48, 2, HERE << "Nothing to do.");
 124         return;
 125     } else if (n >= (size_t)size_) {
 126         debugs(48, 2, HERE << "Closing all entries.");
 127         while (size_ > 0) {
 128             const Comm::ConnectionPointer conn = theList_[--size_];
 129             theList_[size_] = NULL;
 130             clearHandlers(conn);
 131             conn->close();
 132             if (parent_)
 133                 parent_->noteConnectionRemoved();
 134         }
 135     } else { //if (n < size_)
 136         debugs(48, 2, HERE << "Closing " << n << " of " << size_ << " entries.");
 137
 138         size_t index;
 139         // ensure the first N entries are closed
 140         for (index = 0; index < n; ++index) {
 141             const Comm::ConnectionPointer conn = theList_[index];
 142             theList_[index] = NULL;
 143             clearHandlers(conn);
 144             conn->close();
 145             if (parent_)
 146                 parent_->noteConnectionRemoved();
 147         }
 148         // shuffle the list N down.
 149         for (index = 0; index < (size_t)size_ - n; ++index) {
 150             theList_[index] = theList_[index + n];
 151         }
 152         // ensure the last N entries are unset
 153         while (index < ((size_t)size_)) {
 154             theList_[index] = NULL;
 155             ++index;
 156         }
 157         size_ -= n;
 158     }
 159
 160     if (parent_ && size_ == 0) {
 161         debugs(48, 3, HERE << "deleting " << hashKeyStr(&hash));
 162         delete this;
 163     }
 164 }
 165
 166 void
 167 IdleConnList::clearHandlers(const Comm::ConnectionPointer &conn)
 168 {
 169     debugs(48, 3, HERE << "removing close handler for " << conn);
 170     comm_read_cancel(conn->fd, IdleConnList::Read, this);
 171     commUnsetConnTimeout(conn);
 172 }
 173
 174 void
 175 IdleConnList::push(const Comm::ConnectionPointer &conn)
 176 {
 177     if (size_ == capacity_) {
 178         debugs(48, 3, HERE << "growing idle Connection array");
 179         capacity_ <<= 1;
 180         const Comm::ConnectionPointer *oldList = theList_;
 181         theList_ = new Comm::ConnectionPointer[capacity_];
 182         for (int index = 0; index < size_; ++index)
 183             theList_[index] = oldList[index];
 184
 185         delete[] oldList;
 186     }
 187
 188     if (parent_)
 189         parent_->noteConnectionAdded();
 190
 191     theList_[size_] = conn;
 192     ++size_;
 193     AsyncCall::Pointer readCall = commCbCall(5,4, "IdleConnList::Read",
 194                                   CommIoCbPtrFun(IdleConnList::Read, this));
 195     comm_read(conn, fakeReadBuf_, sizeof(fakeReadBuf_), readCall);
 196     AsyncCall::Pointer timeoutCall = commCbCall(5,4, "IdleConnList::Timeout",
 197                                      CommTimeoutCbPtrFun(IdleConnList::Timeout, this));
 198     commSetConnTimeout(conn, Config.Timeout.serverIdlePconn, timeoutCall);
 199 }
 200
 201 /// Determine whether an entry in the idle list is available for use.
 202 /// Returns false if the entry is unset, closed or closing.
 203 bool
 204 IdleConnList::isAvailable(int i) const
 205 {
 206     const Comm::ConnectionPointer &conn = theList_[i];
 207
 208     // connection already closed. useless.
 209     if (!Comm::IsConnOpen(conn))
 210         return false;
 211
 212     // our connection early-read/close handler is scheduled to run already. unsafe
 213     if (!COMMIO_FD_READCB(conn->fd)->active())
 214         return false;
 215
 216     return true;
 217 }
 218
 219 Comm::ConnectionPointer
 220 IdleConnList::pop()
 221 {
 222     for (int i=size_-1; i>=0; --i) {
 223
 224         if (!isAvailable(i))
 225             continue;
 226
 227         // our connection timeout handler is scheduled to run already. unsafe for now.
 228         // TODO: cancel the pending timeout callback and allow re-use of the conn.
 229         if (fd_table[theList_[i]->fd].timeoutHandler == NULL)
 230             continue;
 231
 232         // finally, a match. pop and return it.
 233         Comm::ConnectionPointer result = theList_[i];
 234         /* may delete this */
 235         removeAt(i);
 236         clearHandlers(result);
 237         return result;
 238     }
 239
 240     return Comm::ConnectionPointer();
 241 }
 242
 243 /*
 244  * XXX this routine isn't terribly efficient - if there's a pending
 245  * read event (which signifies the fd will close in the next IO loop!)
 246  * we ignore the FD and move onto the next one. This means, as an example,
 247  * if we have a lot of FDs open to a very popular server and we get a bunch
 248  * of requests JUST as they timeout (say, it shuts down) we'll be wasting
 249  * quite a bit of CPU. Just keep it in mind.
 250  */
 251 Comm::ConnectionPointer
 252 IdleConnList::findUseable(const Comm::ConnectionPointer &key)
 253 {
 254     assert(size_);
 255
 256     // small optimization: do the constant bool tests only once.
 257     const bool keyCheckAddr = !key->local.IsAnyAddr();
 258     const bool keyCheckPort = key->local.GetPort() > 0;
 259
 260     for (int i=size_-1; i>=0; --i) {
 261
 262         if (!isAvailable(i))
 263             continue;
 264
 265         // local end port is required, but dont match.
 266         if (keyCheckPort && key->local.GetPort() != theList_[i]->local.GetPort())
 267             continue;
 268
 269         // local address is required, but does not match.
 270         if (keyCheckAddr && key->local.matchIPAddr(theList_[i]->local) != 0)
 271             continue;
 272
 273         // our connection timeout handler is scheduled to run already. unsafe for now.
 274         // TODO: cancel the pending timeout callback and allow re-use of the conn.
 275         if (fd_table[theList_[i]->fd].timeoutHandler == NULL)
 276             continue;
 277
 278         // finally, a match. pop and return it.
 279         Comm::ConnectionPointer result = theList_[i];
 280         /* may delete this */
 281         removeAt(i);
 282         clearHandlers(result);
 283         return result;
 284     }
 285
 286     return Comm::ConnectionPointer();
 287 }
 288
 289 /* might delete list */
 290 void
 291 IdleConnList::findAndClose(const Comm::ConnectionPointer &conn)
 292 {
 293     const int index = findIndexOf(conn);
 294     if (index >= 0) {
 295         /* might delete this */
 296         removeAt(index);
 297         clearHandlers(conn);
 298         conn->close();
 299     }
 300 }
 301
 302 void
 303 IdleConnList::Read(const Comm::ConnectionPointer &conn, char *buf, size_t len, comm_err_t flag, int xerrno, void *data)
 304 {
 305     debugs(48, 3, HERE << len << " bytes from " << conn);
 306
 307     if (flag == COMM_ERR_CLOSING) {
 308         debugs(48, 3, HERE << "COMM_ERR_CLOSING from " << conn);
 309         /* Bail out on COMM_ERR_CLOSING - may happen when shutdown aborts our idle FD */
 310         return;
 311     }
 312
 313     IdleConnList *list = (IdleConnList *) data;
 314     /* may delete list/data */
 315     list->findAndClose(conn);
 316 }
 317
 318 void
 319 IdleConnList::Timeout(const CommTimeoutCbParams &io)
 320 {
 321     debugs(48, 3, HERE << io.conn);
 322     IdleConnList *list = static_cast<IdleConnList *>(io.data);
 323     /* may delete list/data */
 324     list->findAndClose(io.conn);
 325 }
 326
 327 /* ========== PconnPool PRIVATE FUNCTIONS ============================================ */
 328
 329 const char *
 330 PconnPool::key(const Comm::ConnectionPointer &destLink, const char *domain)
 331 {
 332     LOCAL_ARRAY(char, buf, SQUIDHOSTNAMELEN * 3 + 10);
 333
 334     destLink->remote.ToURL(buf, SQUIDHOSTNAMELEN * 3 + 10);
 335     if (domain) {
 336         const int used = strlen(buf);
 337         snprintf(buf+used, SQUIDHOSTNAMELEN * 3 + 10-used, "/%s", domain);
 338     }
 339
 340     debugs(48,6,"PconnPool::key(" << destLink << ", " << (domain?domain:"[no domain]") << ") is {" << buf << "}" );
 341     return buf;
 342 }
 343
 344 void
 345 PconnPool::dumpHist(StoreEntry * e) const
 346 {
 347     storeAppendPrintf(e,
 348                       "%s persistent connection counts:\n"
 349                       "\n"
 350                       "\treq/\n"
 351                       "\tconn      count\n"
 352                       "\t----  ---------\n",
 353                       descr);
 354
 355     for (int i = 0; i < PCONN_HIST_SZ; ++i) {
 356         if (hist[i] == 0)
 357             continue;
 358
 359         storeAppendPrintf(e, "\t%4d  %9d\n", i, hist[i]);
 360     }
 361 }
 362
 363 void
 364 PconnPool::dumpHash(StoreEntry *e) const
 365 {
 366     hash_table *hid = table;
 367     hash_first(hid);
 368
 369     int i = 0;
 370     for (hash_link *walker = hid->next; walker; walker = hash_next(hid)) {
 371         storeAppendPrintf(e, "\t item %5d: %s\n", i, (char *)(walker->key));
 372         ++i;
 373     }
 374 }
 375
 376 /* ========== PconnPool PUBLIC FUNCTIONS ============================================ */
 377
 378 PconnPool::PconnPool(const char *aDescr) : table(NULL), descr(aDescr),
 379         theCount(0)
 380 {
 381     int i;
 382     table = hash_create((HASHCMP *) strcmp, 229, hash_string);
 383
 384     for (i = 0; i < PCONN_HIST_SZ; ++i)
 385         hist[i] = 0;
 386
 387     PconnModule::GetInstance()->add(this);
 388 }
 389
 390 PconnPool::~PconnPool()
 391 {
 392     descr = NULL;
 393     hashFreeMemory(table);
 394 }
 395
 396 void
 397 PconnPool::push(const Comm::ConnectionPointer &conn, const char *domain)
 398 {
 399     if (fdUsageHigh()) {
 400         debugs(48, 3, HERE << "Not many unused FDs");
 401         conn->close();
 402         return;
 403     } else if (shutting_down) {
 404         conn->close();
 405         debugs(48, 3, HERE << "Squid is shutting down. Refusing to do anything");
 406         return;
 407     }
 408
 409     const char *aKey = key(conn, domain);
 410     IdleConnList *list = (IdleConnList *) hash_lookup(table, aKey);
 411
 412     if (list == NULL) {
 413         list = new IdleConnList(aKey, this);
 414         debugs(48, 3, HERE << "new IdleConnList for {" << hashKeyStr(&list->hash) << "}" );
 415         hash_join(table, &list->hash);
 416     } else {
 417         debugs(48, 3, HERE << "found IdleConnList for {" << hashKeyStr(&list->hash) << "}" );
 418     }
 419
 420     list->push(conn);
 421     assert(!comm_has_incomplete_write(conn->fd));
 422
 423     LOCAL_ARRAY(char, desc, FD_DESC_SZ);
 424     snprintf(desc, FD_DESC_SZ, "Idle server: %s", aKey);
 425     fd_note(conn->fd, desc);
 426     debugs(48, 3, HERE << "pushed " << conn << " for " << aKey);
 427 }
 428
 429 Comm::ConnectionPointer
 430 PconnPool::pop(const Comm::ConnectionPointer &destLink, const char *domain, bool isRetriable)
 431 {
 432     const char * aKey = key(destLink, domain);
 433
 434     IdleConnList *list = (IdleConnList *)hash_lookup(table, aKey);
 435     if (list == NULL) {
 436         debugs(48, 3, HERE << "lookup for key {" << aKey << "} failed.");
 437         return Comm::ConnectionPointer();
 438     } else {
 439         debugs(48, 3, HERE << "found " << hashKeyStr(&list->hash) << (isRetriable?"(to use)":"(to kill)") );
 440     }
 441
 442     /* may delete list */
 443     Comm::ConnectionPointer temp = list->findUseable(destLink);
 444     if (!isRetriable && Comm::IsConnOpen(temp))
 445         temp->close();
 446
 447     return temp;
 448 }
 449
 450 void
 451 PconnPool::closeN(int n, const Comm::ConnectionPointer &destLink, const char *domain)
 452 {
 453     // TODO: optimize: we can probably do hash_lookup just once
 454     for (int i = 0; i < n; ++i)
 455         pop(destLink, domain, false); // may fail!
 456 }
 457
 458 void
 459 PconnPool::unlinkList(IdleConnList *list)
 460 {
 461     theCount -= list->count();
 462     assert(theCount >= 0);
 463     hash_remove_link(table, &list->hash);
 464 }
 465
 466 void
 467 PconnPool::noteUses(int uses)
 468 {
 469     if (uses >= PCONN_HIST_SZ)
 470         uses = PCONN_HIST_SZ - 1;
 471
 472     ++hist[uses];
 473 }
 474
 475 /* ========== PconnModule ============================================ */
 476
 477 /*
 478  * This simple class exists only for the cache manager
 479  */
 480
 481 PconnModule::PconnModule() : pools(NULL), poolCount(0)
 482 {
 483     pools = (PconnPool **) xcalloc(MAX_NUM_PCONN_POOLS, sizeof(*pools));
 484 //TODO: re-link to MemPools. WAS:    pconn_fds_pool = memPoolCreate("pconn_fds", PCONN_FDS_SZ * sizeof(int));
 485     debugs(48, DBG_CRITICAL, "persistent connection module initialized");
 486     registerWithCacheManager();
 487 }
 488
 489 PconnModule *
 490 PconnModule::GetInstance()
 491 {
 492     if (instance == NULL)
 493         instance = new PconnModule;
 494
 495     return instance;
 496 }
 497
 498 void
 499 PconnModule::registerWithCacheManager(void)
 500 {
 501     Mgr::RegisterAction("pconn",
 502                         "Persistent Connection Utilization Histograms",
 503                         DumpWrapper, 0, 1);
 504 }
 505
 506 void
 507 PconnModule::add(PconnPool *aPool)
 508 {
 509     assert(poolCount < MAX_NUM_PCONN_POOLS);
 510     *(pools+poolCount) = aPool;
 511     ++poolCount;
 512 }
 513
 514 void
 515 PconnModule::dump(StoreEntry *e)
 516 {
 517     int i;
 518
 519     for (i = 0; i < poolCount; ++i) {
 520         storeAppendPrintf(e, "\n Pool %d Stats\n", i);
 521         (*(pools+i))->dumpHist(e);
 522         storeAppendPrintf(e, "\n Pool %d Hash Table\n",i);
 523         (*(pools+i))->dumpHash(e);
 524     }
 525 }
 526
 527 void
 528 PconnModule::DumpWrapper(StoreEntry *e)
 529 {
 530     PconnModule::GetInstance()->dump(e);
 531 }