src/pconn.cc

   1 /*
   2  * $Id$
   3  *
   4  * DEBUG: section 48    Persistent Connections
   5  * AUTHOR: Duane Wessels
   6  *
   7  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   8  * ----------------------------------------------------------
   9  *
  10  *  Squid is the result of efforts by numerous individuals from
  11  *  the Internet community; see the CONTRIBUTORS file for full
  12  *  details.   Many organizations have provided support for Squid's
  13  *  development; see the SPONSORS file for full details.  Squid is
  14  *  Copyrighted (C) 2001 by the Regents of the University of
  15  *  California; see the COPYRIGHT file for full details.  Squid
  16  *  incorporates software developed and/or copyrighted by other
  17  *  sources; see the CREDITS file for full details.
  18  *
  19  *  This program is free software; you can redistribute it and/or modify
  20  *  it under the terms of the GNU General Public License as published by
  21  *  the Free Software Foundation; either version 2 of the License, or
  22  *  (at your option) any later version.
  23  *
  24  *  This program is distributed in the hope that it will be useful,
  25  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  26  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  27  *  GNU General Public License for more details.
  28  *
  29  *  You should have received a copy of the GNU General Public License
  30  *  along with this program; if not, write to the Free Software
  31  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  32  *
  33  */
  34
  35 #include "squid.h"
  36 #include "comm/Connection.h"
  37 #include "mgr/Registration.h"
  38 #include "Store.h"
  39 #include "pconn.h"
  40 #include "comm.h"
  41 #include "fde.h"
  42
  43 #define PCONN_FDS_SZ    8       /* pconn set size, increase for better memcache hit rate */
  44
  45 //TODO: re-attach to MemPools. WAS: static MemAllocator *pconn_fds_pool = NULL;
  46 PconnModule * PconnModule::instance = NULL;
  47 CBDATA_CLASS_INIT(IdleConnList);
  48
  49 /* ========== IdleConnList ============================================ */
  50
  51 IdleConnList::IdleConnList(const char *key, PconnPool *thePool) :
  52         capacity_(PCONN_FDS_SZ),
  53         size_(0),
  54         parent_(thePool)
  55 {
  56     hash.key = xstrdup(key);
  57     theList_ = new Comm::ConnectionPointer[capacity_];
  58 // TODO: re-attach to MemPools. WAS: theList = (?? *)pconn_fds_pool->alloc();
  59 }
  60
  61 IdleConnList::~IdleConnList()
  62 {
  63     if (parent_)
  64         parent_->unlinkList(this);
  65
  66     delete[] theList_;
  67
  68     xfree(hash.key);
  69 }
  70
  71 /** Search the list. Matches by FD socket number.
  72  * Performed from the end of list where newest entries are.
  73  *
  74  * \retval <0   The connection is not listed
  75  * \retval >=0  The connection array index
  76  */
  77 int
  78 IdleConnList::findIndexOf(const Comm::ConnectionPointer &conn) const
  79 {
  80     for (int index = size_ - 1; index >= 0; --index) {
  81         if (conn->fd == theList_[index]->fd) {
  82             debugs(48, 3, HERE << "found " << conn << " at index " << index);
  83             return index;
  84         }
  85     }
  86
  87     debugs(48, 2, HERE << conn << " NOT FOUND!");
  88     return -1;
  89 }
  90
  91 /** Remove the entry at specified index.
  92  * May perform a shuffle of list entries to fill the gap.
  93  * \retval false The index is not an in-use entry.
  94  */
  95 bool
  96 IdleConnList::removeAt(int index)
  97 {
  98     if (index < 0 || index >= size_)
  99         return false;
 100
 101     // shuffle the remaining entries to fill the new gap.
 102     for (; index < size_ - 1; index++)
 103         theList_[index] = theList_[index + 1];
 104     theList_[--size_] = NULL;
 105
 106     if (parent_) {
 107         parent_->noteConnectionRemoved();
 108         if (size_ == 0) {
 109             debugs(48, 3, HERE << "deleting " << hashKeyStr(&hash));
 110             delete this;
 111         }
 112     }
 113
 114     return true;
 115 }
 116
 117 // almost a duplicate of removeFD. But drops multiple entries.
 118 void
 119 IdleConnList::closeN(size_t n)
 120 {
 121     if (n < 1) {
 122         debugs(48, 2, HERE << "Nothing to do.");
 123         return;
 124     } else if (n >= (size_t)size_) {
 125         debugs(48, 2, HERE << "Closing all entries.");
 126         while (size_ > 0) {
 127             const Comm::ConnectionPointer conn = theList_[--size_];
 128             theList_[size_] = NULL;
 129             clearHandlers(conn);
 130             conn->close();
 131             if (parent_)
 132                 parent_->noteConnectionRemoved();
 133         }
 134     } else { //if (n < size_)
 135         debugs(48, 2, HERE << "Closing " << n << " of " << size_ << " entries.");
 136
 137         size_t index;
 138         // ensure the first N entries are closed
 139         for (index = 0; index < n; index++) {
 140             const Comm::ConnectionPointer conn = theList_[index];
 141             theList_[index] = NULL;
 142             clearHandlers(conn);
 143             conn->close();
 144             if (parent_)
 145                 parent_->noteConnectionRemoved();
 146         }
 147         // shuffle the list N down.
 148         for (index = 0; index < (size_t)size_ - n; index++) {
 149             theList_[index] = theList_[index + n];
 150         }
 151         // ensure the last N entries are unset
 152         while (index < ((size_t)size_)) {
 153             theList_[index++] = NULL;
 154         }
 155         size_ -= n;
 156     }
 157
 158     if (parent_ && size_ == 0) {
 159         debugs(48, 3, HERE << "deleting " << hashKeyStr(&hash));
 160         delete this;
 161     }
 162 }
 163
 164 void
 165 IdleConnList::clearHandlers(const Comm::ConnectionPointer &conn)
 166 {
 167     debugs(48, 3, HERE << "removing close handler for " << conn);
 168     comm_read_cancel(conn->fd, IdleConnList::Read, this);
 169     commUnsetConnTimeout(conn);
 170 }
 171
 172 void
 173 IdleConnList::push(const Comm::ConnectionPointer &conn)
 174 {
 175     if (size_ == capacity_) {
 176         debugs(48, 3, HERE << "growing idle Connection array");
 177         capacity_ <<= 1;
 178         const Comm::ConnectionPointer *oldList = theList_;
 179         theList_ = new Comm::ConnectionPointer[capacity_];
 180         for (int index = 0; index < size_; index++)
 181             theList_[index] = oldList[index];
 182
 183         delete[] oldList;
 184     }
 185
 186     if (parent_)
 187         parent_->noteConnectionAdded();
 188
 189     theList_[size_++] = conn;
 190     AsyncCall::Pointer readCall = commCbCall(5,4, "IdleConnList::Read",
 191                                   CommIoCbPtrFun(IdleConnList::Read, this));
 192     comm_read(conn, fakeReadBuf_, sizeof(fakeReadBuf_), readCall);
 193     AsyncCall::Pointer timeoutCall = commCbCall(5,4, "IdleConnList::Timeout",
 194                                      CommTimeoutCbPtrFun(IdleConnList::Timeout, this));
 195     commSetConnTimeout(conn, Config.Timeout.serverIdlePconn, timeoutCall);
 196 }
 197
 198 /// Determine whether an entry in the idle list is available for use.
 199 /// Returns false if the entry is unset, closed or closing.
 200 bool
 201 IdleConnList::isAvailable(int i) const
 202 {
 203     const Comm::ConnectionPointer &conn = theList_[i];
 204
 205     // connection already closed. useless.
 206     if (!Comm::IsConnOpen(conn))
 207         return false;
 208
 209     // our connection early-read/close handler is scheduled to run already. unsafe
 210     if (!COMMIO_FD_READCB(conn->fd)->active())
 211         return false;
 212
 213     return true;
 214 }
 215
 216 Comm::ConnectionPointer
 217 IdleConnList::pop()
 218 {
 219     for (int i=size_-1; i>=0; i--) {
 220
 221         if (!isAvailable(i))
 222             continue;
 223
 224         // our connection timeout handler is scheduled to run already. unsafe for now.
 225         // TODO: cancel the pending timeout callback and allow re-use of the conn.
 226         if (fd_table[theList_[i]->fd].timeoutHandler == NULL)
 227             continue;
 228
 229         // finally, a match. pop and return it.
 230         Comm::ConnectionPointer result = theList_[i];
 231         /* may delete this */
 232         removeAt(i);
 233         clearHandlers(result);
 234         return result;
 235     }
 236
 237     return Comm::ConnectionPointer();
 238 }
 239
 240 /*
 241  * XXX this routine isn't terribly efficient - if there's a pending
 242  * read event (which signifies the fd will close in the next IO loop!)
 243  * we ignore the FD and move onto the next one. This means, as an example,
 244  * if we have a lot of FDs open to a very popular server and we get a bunch
 245  * of requests JUST as they timeout (say, it shuts down) we'll be wasting
 246  * quite a bit of CPU. Just keep it in mind.
 247  */
 248 Comm::ConnectionPointer
 249 IdleConnList::findUseable(const Comm::ConnectionPointer &key)
 250 {
 251     assert(size_);
 252
 253     // small optimization: do the constant bool tests only once.
 254     const bool keyCheckAddr = !key->local.IsAnyAddr();
 255     const bool keyCheckPort = key->local.GetPort() > 0;
 256
 257     for (int i=size_-1; i>=0; i--) {
 258
 259         if (!isAvailable(i))
 260             continue;
 261
 262         // local end port is required, but dont match.
 263         if (keyCheckPort && key->local.GetPort() != theList_[i]->local.GetPort())
 264             continue;
 265
 266         // local address is required, but does not match.
 267         if (keyCheckAddr && key->local.matchIPAddr(theList_[i]->local) != 0)
 268             continue;
 269
 270         // our connection timeout handler is scheduled to run already. unsafe for now.
 271         // TODO: cancel the pending timeout callback and allow re-use of the conn.
 272         if (fd_table[theList_[i]->fd].timeoutHandler == NULL)
 273             continue;
 274
 275         // finally, a match. pop and return it.
 276         Comm::ConnectionPointer result = theList_[i];
 277         /* may delete this */
 278         removeAt(i);
 279         clearHandlers(result);
 280         return result;
 281     }
 282
 283     return Comm::ConnectionPointer();
 284 }
 285
 286 /* might delete list */
 287 void
 288 IdleConnList::findAndClose(const Comm::ConnectionPointer &conn)
 289 {
 290     const int index = findIndexOf(conn);
 291     if (index >= 0) {
 292         /* might delete this */
 293         removeAt(index);
 294         clearHandlers(conn);
 295         conn->close();
 296     }
 297 }
 298
 299 void
 300 IdleConnList::Read(const Comm::ConnectionPointer &conn, char *buf, size_t len, comm_err_t flag, int xerrno, void *data)
 301 {
 302     debugs(48, 3, HERE << len << " bytes from " << conn);
 303
 304     if (flag == COMM_ERR_CLOSING) {
 305         debugs(48, 3, HERE << "COMM_ERR_CLOSING from " << conn);
 306         /* Bail out on COMM_ERR_CLOSING - may happen when shutdown aborts our idle FD */
 307         return;
 308     }
 309
 310     IdleConnList *list = (IdleConnList *) data;
 311     /* may delete list/data */
 312     list->findAndClose(conn);
 313 }
 314
 315 void
 316 IdleConnList::Timeout(const CommTimeoutCbParams &io)
 317 {
 318     debugs(48, 3, HERE << io.conn);
 319     IdleConnList *list = static_cast<IdleConnList *>(io.data);
 320     /* may delete list/data */
 321     list->findAndClose(io.conn);
 322 }
 323
 324 /* ========== PconnPool PRIVATE FUNCTIONS ============================================ */
 325
 326 const char *
 327 PconnPool::key(const Comm::ConnectionPointer &destLink, const char *domain)
 328 {
 329     LOCAL_ARRAY(char, buf, SQUIDHOSTNAMELEN * 3 + 10);
 330
 331     destLink->remote.ToURL(buf, SQUIDHOSTNAMELEN * 3 + 10);
 332     if (domain) {
 333         const int used = strlen(buf);
 334         snprintf(buf+used, SQUIDHOSTNAMELEN * 3 + 10-used, "/%s", domain);
 335     }
 336
 337     debugs(48,6,"PconnPool::key(" << destLink << ", " << (domain?domain:"[no domain]") << ") is {" << buf << "}" );
 338     return buf;
 339 }
 340
 341 void
 342 PconnPool::dumpHist(StoreEntry * e) const
 343 {
 344     storeAppendPrintf(e,
 345                       "%s persistent connection counts:\n"
 346                       "\n"
 347                       "\treq/\n"
 348                       "\tconn      count\n"
 349                       "\t----  ---------\n",
 350                       descr);
 351
 352     for (int i = 0; i < PCONN_HIST_SZ; i++) {
 353         if (hist[i] == 0)
 354             continue;
 355
 356         storeAppendPrintf(e, "\t%4d  %9d\n", i, hist[i]);
 357     }
 358 }
 359
 360 void
 361 PconnPool::dumpHash(StoreEntry *e) const
 362 {
 363     hash_table *hid = table;
 364     hash_first(hid);
 365
 366     int i = 0;
 367     for (hash_link *walker = hid->next; walker; walker = hash_next(hid)) {
 368         storeAppendPrintf(e, "\t item %5d: %s\n", i++, (char *)(walker->key));
 369     }
 370 }
 371
 372 /* ========== PconnPool PUBLIC FUNCTIONS ============================================ */
 373
 374 PconnPool::PconnPool(const char *aDescr) : table(NULL), descr(aDescr),
 375         theCount(0)
 376 {
 377     int i;
 378     table = hash_create((HASHCMP *) strcmp, 229, hash_string);
 379
 380     for (i = 0; i < PCONN_HIST_SZ; i++)
 381         hist[i] = 0;
 382
 383     PconnModule::GetInstance()->add(this);
 384 }
 385
 386 PconnPool::~PconnPool()
 387 {
 388     descr = NULL;
 389     hashFreeMemory(table);
 390 }
 391
 392 void
 393 PconnPool::push(const Comm::ConnectionPointer &conn, const char *domain)
 394 {
 395     if (fdUsageHigh()) {
 396         debugs(48, 3, HERE << "Not many unused FDs");
 397         conn->close();
 398         return;
 399     } else if (shutting_down) {
 400         conn->close();
 401         debugs(48, 3, HERE << "Squid is shutting down. Refusing to do anything");
 402         return;
 403     }
 404
 405     const char *aKey = key(conn, domain);
 406     IdleConnList *list = (IdleConnList *) hash_lookup(table, aKey);
 407
 408     if (list == NULL) {
 409         list = new IdleConnList(aKey, this);
 410         debugs(48, 3, HERE << "new IdleConnList for {" << hashKeyStr(&list->hash) << "}" );
 411         hash_join(table, &list->hash);
 412     } else {
 413         debugs(48, 3, HERE << "found IdleConnList for {" << hashKeyStr(&list->hash) << "}" );
 414     }
 415
 416     list->push(conn);
 417     assert(!comm_has_incomplete_write(conn->fd));
 418
 419     LOCAL_ARRAY(char, desc, FD_DESC_SZ);
 420     snprintf(desc, FD_DESC_SZ, "Idle server: %s", aKey);
 421     fd_note(conn->fd, desc);
 422     debugs(48, 3, HERE << "pushed " << conn << " for " << aKey);
 423 }
 424
 425 Comm::ConnectionPointer
 426 PconnPool::pop(const Comm::ConnectionPointer &destLink, const char *domain, bool isRetriable)
 427 {
 428     const char * aKey = key(destLink, domain);
 429
 430     IdleConnList *list = (IdleConnList *)hash_lookup(table, aKey);
 431     if (list == NULL) {
 432         debugs(48, 3, HERE << "lookup for key {" << aKey << "} failed.");
 433         return Comm::ConnectionPointer();
 434     } else {
 435         debugs(48, 3, HERE << "found " << hashKeyStr(&list->hash) << (isRetriable?"(to use)":"(to kill)") );
 436     }
 437
 438     /* may delete list */
 439     Comm::ConnectionPointer temp = list->findUseable(destLink);
 440     if (!isRetriable && Comm::IsConnOpen(temp))
 441         temp->close();
 442
 443     return temp;
 444 }
 445
 446 void
 447 PconnPool::closeN(int n, const Comm::ConnectionPointer &destLink, const char *domain)
 448 {
 449     // TODO: optimize: we can probably do hash_lookup just once
 450     for (int i = 0; i < n; ++i)
 451         pop(destLink, domain, false); // may fail!
 452 }
 453
 454 void
 455 PconnPool::unlinkList(IdleConnList *list)
 456 {
 457     theCount -= list->count();
 458     assert(theCount >= 0);
 459     hash_remove_link(table, &list->hash);
 460 }
 461
 462 void
 463 PconnPool::noteUses(int uses)
 464 {
 465     if (uses >= PCONN_HIST_SZ)
 466         uses = PCONN_HIST_SZ - 1;
 467
 468     hist[uses]++;
 469 }
 470
 471 /* ========== PconnModule ============================================ */
 472
 473 /*
 474  * This simple class exists only for the cache manager
 475  */
 476
 477 PconnModule::PconnModule() : pools(NULL), poolCount(0)
 478 {
 479     pools = (PconnPool **) xcalloc(MAX_NUM_PCONN_POOLS, sizeof(*pools));
 480 //TODO: re-link to MemPools. WAS:    pconn_fds_pool = memPoolCreate("pconn_fds", PCONN_FDS_SZ * sizeof(int));
 481     debugs(48, 0, "persistent connection module initialized");
 482     registerWithCacheManager();
 483 }
 484
 485 PconnModule *
 486 PconnModule::GetInstance()
 487 {
 488     if (instance == NULL)
 489         instance = new PconnModule;
 490
 491     return instance;
 492 }
 493
 494 void
 495 PconnModule::registerWithCacheManager(void)
 496 {
 497     Mgr::RegisterAction("pconn",
 498                         "Persistent Connection Utilization Histograms",
 499                         DumpWrapper, 0, 1);
 500 }
 501
 502 void
 503 PconnModule::add(PconnPool *aPool)
 504 {
 505     assert(poolCount < MAX_NUM_PCONN_POOLS);
 506     *(pools+poolCount) = aPool;
 507     poolCount++;
 508 }
 509
 510 void
 511 PconnModule::dump(StoreEntry *e)
 512 {
 513     int i;
 514
 515     for (i = 0; i < poolCount; i++) {
 516         storeAppendPrintf(e, "\n Pool %d Stats\n", i);
 517         (*(pools+i))->dumpHist(e);
 518         storeAppendPrintf(e, "\n Pool %d Hash Table\n",i);
 519         (*(pools+i))->dumpHash(e);
 520     }
 521 }
 522
 523 void
 524 PconnModule::DumpWrapper(StoreEntry *e)
 525 {
 526     PconnModule::GetInstance()->dump(e);
 527 }