src/pconn.cc

   1 /*
   2  * $Id$
   3  *
   4  * DEBUG: section 48    Persistent Connections
   5  * AUTHOR: Duane Wessels
   6  *
   7  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   8  * ----------------------------------------------------------
   9  *
  10  *  Squid is the result of efforts by numerous individuals from
  11  *  the Internet community; see the CONTRIBUTORS file for full
  12  *  details.   Many organizations have provided support for Squid's
  13  *  development; see the SPONSORS file for full details.  Squid is
  14  *  Copyrighted (C) 2001 by the Regents of the University of
  15  *  California; see the COPYRIGHT file for full details.  Squid
  16  *  incorporates software developed and/or copyrighted by other
  17  *  sources; see the CREDITS file for full details.
  18  *
  19  *  This program is free software; you can redistribute it and/or modify
  20  *  it under the terms of the GNU General Public License as published by
  21  *  the Free Software Foundation; either version 2 of the License, or
  22  *  (at your option) any later version.
  23  *
  24  *  This program is distributed in the hope that it will be useful,
  25  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  26  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  27  *  GNU General Public License for more details.
  28  *
  29  *  You should have received a copy of the GNU General Public License
  30  *  along with this program; if not, write to the Free Software
  31  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  32  *
  33  */
  34
  35 #include "squid.h"
  36 #include "comm.h"
  37 #include "comm/Connection.h"
  38 #include "fde.h"
  39 #include "globals.h"
  40 #include "mgr/Registration.h"
  41 #include "pconn.h"
  42 #include "protos.h"
  43 #include "Store.h"
  44
  45 #define PCONN_FDS_SZ    8       /* pconn set size, increase for better memcache hit rate */
  46
  47 //TODO: re-attach to MemPools. WAS: static MemAllocator *pconn_fds_pool = NULL;
  48 PconnModule * PconnModule::instance = NULL;
  49 CBDATA_CLASS_INIT(IdleConnList);
  50
  51 /* ========== IdleConnList ============================================ */
  52
  53 IdleConnList::IdleConnList(const char *key, PconnPool *thePool) :
  54         capacity_(PCONN_FDS_SZ),
  55         size_(0),
  56         parent_(thePool)
  57 {
  58     hash.key = xstrdup(key);
  59     theList_ = new Comm::ConnectionPointer[capacity_];
  60 // TODO: re-attach to MemPools. WAS: theList = (?? *)pconn_fds_pool->alloc();
  61 }
  62
  63 IdleConnList::~IdleConnList()
  64 {
  65     if (parent_)
  66         parent_->unlinkList(this);
  67
  68     delete[] theList_;
  69
  70     xfree(hash.key);
  71 }
  72
  73 /** Search the list. Matches by FD socket number.
  74  * Performed from the end of list where newest entries are.
  75  *
  76  * \retval <0   The connection is not listed
  77  * \retval >=0  The connection array index
  78  */
  79 int
  80 IdleConnList::findIndexOf(const Comm::ConnectionPointer &conn) const
  81 {
  82     for (int index = size_ - 1; index >= 0; --index) {
  83         if (conn->fd == theList_[index]->fd) {
  84             debugs(48, 3, HERE << "found " << conn << " at index " << index);
  85             return index;
  86         }
  87     }
  88
  89     debugs(48, 2, HERE << conn << " NOT FOUND!");
  90     return -1;
  91 }
  92
  93 /** Remove the entry at specified index.
  94  * May perform a shuffle of list entries to fill the gap.
  95  * \retval false The index is not an in-use entry.
  96  */
  97 bool
  98 IdleConnList::removeAt(int index)
  99 {
 100     if (index < 0 || index >= size_)
 101         return false;
 102
 103     // shuffle the remaining entries to fill the new gap.
 104     for (; index < size_ - 1; ++index)
 105         theList_[index] = theList_[index + 1];
 106     theList_[--size_] = NULL;
 107
 108     if (parent_) {
 109         parent_->noteConnectionRemoved();
 110         if (size_ == 0) {
 111             debugs(48, 3, HERE << "deleting " << hashKeyStr(&hash));
 112             delete this;
 113         }
 114     }
 115
 116     return true;
 117 }
 118
 119 // almost a duplicate of removeFD. But drops multiple entries.
 120 void
 121 IdleConnList::closeN(size_t n)
 122 {
 123     if (n < 1) {
 124         debugs(48, 2, HERE << "Nothing to do.");
 125         return;
 126     } else if (n >= (size_t)size_) {
 127         debugs(48, 2, HERE << "Closing all entries.");
 128         while (size_ > 0) {
 129             const Comm::ConnectionPointer conn = theList_[--size_];
 130             theList_[size_] = NULL;
 131             clearHandlers(conn);
 132             conn->close();
 133             if (parent_)
 134                 parent_->noteConnectionRemoved();
 135         }
 136     } else { //if (n < size_)
 137         debugs(48, 2, HERE << "Closing " << n << " of " << size_ << " entries.");
 138
 139         size_t index;
 140         // ensure the first N entries are closed
 141         for (index = 0; index < n; ++index) {
 142             const Comm::ConnectionPointer conn = theList_[index];
 143             theList_[index] = NULL;
 144             clearHandlers(conn);
 145             conn->close();
 146             if (parent_)
 147                 parent_->noteConnectionRemoved();
 148         }
 149         // shuffle the list N down.
 150         for (index = 0; index < (size_t)size_ - n; ++index) {
 151             theList_[index] = theList_[index + n];
 152         }
 153         // ensure the last N entries are unset
 154         while (index < ((size_t)size_)) {
 155             theList_[index] = NULL;
 156             ++index;
 157         }
 158         size_ -= n;
 159     }
 160
 161     if (parent_ && size_ == 0) {
 162         debugs(48, 3, HERE << "deleting " << hashKeyStr(&hash));
 163         delete this;
 164     }
 165 }
 166
 167 void
 168 IdleConnList::clearHandlers(const Comm::ConnectionPointer &conn)
 169 {
 170     debugs(48, 3, HERE << "removing close handler for " << conn);
 171     comm_read_cancel(conn->fd, IdleConnList::Read, this);
 172     commUnsetConnTimeout(conn);
 173 }
 174
 175 void
 176 IdleConnList::push(const Comm::ConnectionPointer &conn)
 177 {
 178     if (size_ == capacity_) {
 179         debugs(48, 3, HERE << "growing idle Connection array");
 180         capacity_ <<= 1;
 181         const Comm::ConnectionPointer *oldList = theList_;
 182         theList_ = new Comm::ConnectionPointer[capacity_];
 183         for (int index = 0; index < size_; ++index)
 184             theList_[index] = oldList[index];
 185
 186         delete[] oldList;
 187     }
 188
 189     if (parent_)
 190         parent_->noteConnectionAdded();
 191
 192     theList_[size_] = conn;
 193     ++size_;
 194     AsyncCall::Pointer readCall = commCbCall(5,4, "IdleConnList::Read",
 195                                   CommIoCbPtrFun(IdleConnList::Read, this));
 196     comm_read(conn, fakeReadBuf_, sizeof(fakeReadBuf_), readCall);
 197     AsyncCall::Pointer timeoutCall = commCbCall(5,4, "IdleConnList::Timeout",
 198                                      CommTimeoutCbPtrFun(IdleConnList::Timeout, this));
 199     commSetConnTimeout(conn, Config.Timeout.serverIdlePconn, timeoutCall);
 200 }
 201
 202 /// Determine whether an entry in the idle list is available for use.
 203 /// Returns false if the entry is unset, closed or closing.
 204 bool
 205 IdleConnList::isAvailable(int i) const
 206 {
 207     const Comm::ConnectionPointer &conn = theList_[i];
 208
 209     // connection already closed. useless.
 210     if (!Comm::IsConnOpen(conn))
 211         return false;
 212
 213     // our connection early-read/close handler is scheduled to run already. unsafe
 214     if (!COMMIO_FD_READCB(conn->fd)->active())
 215         return false;
 216
 217     return true;
 218 }
 219
 220 Comm::ConnectionPointer
 221 IdleConnList::pop()
 222 {
 223     for (int i=size_-1; i>=0; --i) {
 224
 225         if (!isAvailable(i))
 226             continue;
 227
 228         // our connection timeout handler is scheduled to run already. unsafe for now.
 229         // TODO: cancel the pending timeout callback and allow re-use of the conn.
 230         if (fd_table[theList_[i]->fd].timeoutHandler == NULL)
 231             continue;
 232
 233         // finally, a match. pop and return it.
 234         Comm::ConnectionPointer result = theList_[i];
 235         /* may delete this */
 236         removeAt(i);
 237         clearHandlers(result);
 238         return result;
 239     }
 240
 241     return Comm::ConnectionPointer();
 242 }
 243
 244 /*
 245  * XXX this routine isn't terribly efficient - if there's a pending
 246  * read event (which signifies the fd will close in the next IO loop!)
 247  * we ignore the FD and move onto the next one. This means, as an example,
 248  * if we have a lot of FDs open to a very popular server and we get a bunch
 249  * of requests JUST as they timeout (say, it shuts down) we'll be wasting
 250  * quite a bit of CPU. Just keep it in mind.
 251  */
 252 Comm::ConnectionPointer
 253 IdleConnList::findUseable(const Comm::ConnectionPointer &key)
 254 {
 255     assert(size_);
 256
 257     // small optimization: do the constant bool tests only once.
 258     const bool keyCheckAddr = !key->local.IsAnyAddr();
 259     const bool keyCheckPort = key->local.GetPort() > 0;
 260
 261     for (int i=size_-1; i>=0; --i) {
 262
 263         if (!isAvailable(i))
 264             continue;
 265
 266         // local end port is required, but dont match.
 267         if (keyCheckPort && key->local.GetPort() != theList_[i]->local.GetPort())
 268             continue;
 269
 270         // local address is required, but does not match.
 271         if (keyCheckAddr && key->local.matchIPAddr(theList_[i]->local) != 0)
 272             continue;
 273
 274         // our connection timeout handler is scheduled to run already. unsafe for now.
 275         // TODO: cancel the pending timeout callback and allow re-use of the conn.
 276         if (fd_table[theList_[i]->fd].timeoutHandler == NULL)
 277             continue;
 278
 279         // finally, a match. pop and return it.
 280         Comm::ConnectionPointer result = theList_[i];
 281         /* may delete this */
 282         removeAt(i);
 283         clearHandlers(result);
 284         return result;
 285     }
 286
 287     return Comm::ConnectionPointer();
 288 }
 289
 290 /* might delete list */
 291 void
 292 IdleConnList::findAndClose(const Comm::ConnectionPointer &conn)
 293 {
 294     const int index = findIndexOf(conn);
 295     if (index >= 0) {
 296         /* might delete this */
 297         removeAt(index);
 298         clearHandlers(conn);
 299         conn->close();
 300     }
 301 }
 302
 303 void
 304 IdleConnList::Read(const Comm::ConnectionPointer &conn, char *buf, size_t len, comm_err_t flag, int xerrno, void *data)
 305 {
 306     debugs(48, 3, HERE << len << " bytes from " << conn);
 307
 308     if (flag == COMM_ERR_CLOSING) {
 309         debugs(48, 3, HERE << "COMM_ERR_CLOSING from " << conn);
 310         /* Bail out on COMM_ERR_CLOSING - may happen when shutdown aborts our idle FD */
 311         return;
 312     }
 313
 314     IdleConnList *list = (IdleConnList *) data;
 315     /* may delete list/data */
 316     list->findAndClose(conn);
 317 }
 318
 319 void
 320 IdleConnList::Timeout(const CommTimeoutCbParams &io)
 321 {
 322     debugs(48, 3, HERE << io.conn);
 323     IdleConnList *list = static_cast<IdleConnList *>(io.data);
 324     /* may delete list/data */
 325     list->findAndClose(io.conn);
 326 }
 327
 328 /* ========== PconnPool PRIVATE FUNCTIONS ============================================ */
 329
 330 const char *
 331 PconnPool::key(const Comm::ConnectionPointer &destLink, const char *domain)
 332 {
 333     LOCAL_ARRAY(char, buf, SQUIDHOSTNAMELEN * 3 + 10);
 334
 335     destLink->remote.ToURL(buf, SQUIDHOSTNAMELEN * 3 + 10);
 336     if (domain) {
 337         const int used = strlen(buf);
 338         snprintf(buf+used, SQUIDHOSTNAMELEN * 3 + 10-used, "/%s", domain);
 339     }
 340
 341     debugs(48,6,"PconnPool::key(" << destLink << ", " << (domain?domain:"[no domain]") << ") is {" << buf << "}" );
 342     return buf;
 343 }
 344
 345 void
 346 PconnPool::dumpHist(StoreEntry * e) const
 347 {
 348     storeAppendPrintf(e,
 349                       "%s persistent connection counts:\n"
 350                       "\n"
 351                       "\treq/\n"
 352                       "\tconn      count\n"
 353                       "\t----  ---------\n",
 354                       descr);
 355
 356     for (int i = 0; i < PCONN_HIST_SZ; ++i) {
 357         if (hist[i] == 0)
 358             continue;
 359
 360         storeAppendPrintf(e, "\t%4d  %9d\n", i, hist[i]);
 361     }
 362 }
 363
 364 void
 365 PconnPool::dumpHash(StoreEntry *e) const
 366 {
 367     hash_table *hid = table;
 368     hash_first(hid);
 369
 370     int i = 0;
 371     for (hash_link *walker = hid->next; walker; walker = hash_next(hid)) {
 372         storeAppendPrintf(e, "\t item %5d: %s\n", i, (char *)(walker->key));
 373         ++i;
 374     }
 375 }
 376
 377 /* ========== PconnPool PUBLIC FUNCTIONS ============================================ */
 378
 379 PconnPool::PconnPool(const char *aDescr) : table(NULL), descr(aDescr),
 380         theCount(0)
 381 {
 382     int i;
 383     table = hash_create((HASHCMP *) strcmp, 229, hash_string);
 384
 385     for (i = 0; i < PCONN_HIST_SZ; ++i)
 386         hist[i] = 0;
 387
 388     PconnModule::GetInstance()->add(this);
 389 }
 390
 391 PconnPool::~PconnPool()
 392 {
 393     descr = NULL;
 394     hashFreeMemory(table);
 395 }
 396
 397 void
 398 PconnPool::push(const Comm::ConnectionPointer &conn, const char *domain)
 399 {
 400     if (fdUsageHigh()) {
 401         debugs(48, 3, HERE << "Not many unused FDs");
 402         conn->close();
 403         return;
 404     } else if (shutting_down) {
 405         conn->close();
 406         debugs(48, 3, HERE << "Squid is shutting down. Refusing to do anything");
 407         return;
 408     }
 409
 410     const char *aKey = key(conn, domain);
 411     IdleConnList *list = (IdleConnList *) hash_lookup(table, aKey);
 412
 413     if (list == NULL) {
 414         list = new IdleConnList(aKey, this);
 415         debugs(48, 3, HERE << "new IdleConnList for {" << hashKeyStr(&list->hash) << "}" );
 416         hash_join(table, &list->hash);
 417     } else {
 418         debugs(48, 3, HERE << "found IdleConnList for {" << hashKeyStr(&list->hash) << "}" );
 419     }
 420
 421     list->push(conn);
 422     assert(!comm_has_incomplete_write(conn->fd));
 423
 424     LOCAL_ARRAY(char, desc, FD_DESC_SZ);
 425     snprintf(desc, FD_DESC_SZ, "Idle server: %s", aKey);
 426     fd_note(conn->fd, desc);
 427     debugs(48, 3, HERE << "pushed " << conn << " for " << aKey);
 428 }
 429
 430 Comm::ConnectionPointer
 431 PconnPool::pop(const Comm::ConnectionPointer &destLink, const char *domain, bool isRetriable)
 432 {
 433     const char * aKey = key(destLink, domain);
 434
 435     IdleConnList *list = (IdleConnList *)hash_lookup(table, aKey);
 436     if (list == NULL) {
 437         debugs(48, 3, HERE << "lookup for key {" << aKey << "} failed.");
 438         return Comm::ConnectionPointer();
 439     } else {
 440         debugs(48, 3, HERE << "found " << hashKeyStr(&list->hash) << (isRetriable?"(to use)":"(to kill)") );
 441     }
 442
 443     /* may delete list */
 444     Comm::ConnectionPointer temp = list->findUseable(destLink);
 445     if (!isRetriable && Comm::IsConnOpen(temp))
 446         temp->close();
 447
 448     return temp;
 449 }
 450
 451 void
 452 PconnPool::closeN(int n, const Comm::ConnectionPointer &destLink, const char *domain)
 453 {
 454     // TODO: optimize: we can probably do hash_lookup just once
 455     for (int i = 0; i < n; ++i)
 456         pop(destLink, domain, false); // may fail!
 457 }
 458
 459 void
 460 PconnPool::unlinkList(IdleConnList *list)
 461 {
 462     theCount -= list->count();
 463     assert(theCount >= 0);
 464     hash_remove_link(table, &list->hash);
 465 }
 466
 467 void
 468 PconnPool::noteUses(int uses)
 469 {
 470     if (uses >= PCONN_HIST_SZ)
 471         uses = PCONN_HIST_SZ - 1;
 472
 473     ++hist[uses];
 474 }
 475
 476 /* ========== PconnModule ============================================ */
 477
 478 /*
 479  * This simple class exists only for the cache manager
 480  */
 481
 482 PconnModule::PconnModule() : pools(NULL), poolCount(0)
 483 {
 484     pools = (PconnPool **) xcalloc(MAX_NUM_PCONN_POOLS, sizeof(*pools));
 485 //TODO: re-link to MemPools. WAS:    pconn_fds_pool = memPoolCreate("pconn_fds", PCONN_FDS_SZ * sizeof(int));
 486     debugs(48, DBG_CRITICAL, "persistent connection module initialized");
 487     registerWithCacheManager();
 488 }
 489
 490 PconnModule *
 491 PconnModule::GetInstance()
 492 {
 493     if (instance == NULL)
 494         instance = new PconnModule;
 495
 496     return instance;
 497 }
 498
 499 void
 500 PconnModule::registerWithCacheManager(void)
 501 {
 502     Mgr::RegisterAction("pconn",
 503                         "Persistent Connection Utilization Histograms",
 504                         DumpWrapper, 0, 1);
 505 }
 506
 507 void
 508 PconnModule::add(PconnPool *aPool)
 509 {
 510     assert(poolCount < MAX_NUM_PCONN_POOLS);
 511     *(pools+poolCount) = aPool;
 512     ++poolCount;
 513 }
 514
 515 void
 516 PconnModule::dump(StoreEntry *e)
 517 {
 518     int i;
 519
 520     for (i = 0; i < poolCount; ++i) {
 521         storeAppendPrintf(e, "\n Pool %d Stats\n", i);
 522         (*(pools+i))->dumpHist(e);
 523         storeAppendPrintf(e, "\n Pool %d Hash Table\n",i);
 524         (*(pools+i))->dumpHash(e);
 525     }
 526 }
 527
 528 void
 529 PconnModule::DumpWrapper(StoreEntry *e)
 530 {
 531     PconnModule::GetInstance()->dump(e);
 532 }