src/client_db.cc

   1 /*
   2  * DEBUG: section 00    Client Database
   3  * AUTHOR: Duane Wessels
   4  *
   5  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   6  * ----------------------------------------------------------
   7  *
   8  *  Squid is the result of efforts by numerous individuals from
   9  *  the Internet community; see the CONTRIBUTORS file for full
  10  *  details.   Many organizations have provided support for Squid's
  11  *  development; see the SPONSORS file for full details.  Squid is
  12  *  Copyrighted (C) 2001 by the Regents of the University of
  13  *  California; see the COPYRIGHT file for full details.  Squid
  14  *  incorporates software developed and/or copyrighted by other
  15  *  sources; see the CREDITS file for full details.
  16  *
  17  *  This program is free software; you can redistribute it and/or modify
  18  *  it under the terms of the GNU General Public License as published by
  19  *  the Free Software Foundation; either version 2 of the License, or
  20  *  (at your option) any later version.
  21  *
  22  *  This program is distributed in the hope that it will be useful,
  23  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  24  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  25  *  GNU General Public License for more details.
  26  *
  27  *  You should have received a copy of the GNU General Public License
  28  *  along with this program; if not, write to the Free Software
  29  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  30  *
  31  */
  32
  33 #include "squid.h"
  34 #include "client_db.h"
  35 #include "event.h"
  36 #include "format/Token.h"
  37 #include "ClientInfo.h"
  38 #include "fqdncache.h"
  39 #include "ip/Address.h"
  40 #include "log/access_log.h"
  41 #include "Mem.h"
  42 #include "mgr/Registration.h"
  43 #include "SquidConfig.h"
  44 #include "SquidMath.h"
  45 #include "SquidTime.h"
  46 #include "StatCounters.h"
  47 #include "Store.h"
  48 #include "tools.h"
  49
  50 #if SQUID_SNMP
  51 #include "snmp_core.h"
  52 #endif
  53
  54 static hash_table *client_table = NULL;
  55
  56 static ClientInfo *clientdbAdd(const Ip::Address &addr);
  57 static FREE clientdbFreeItem;
  58 static void clientdbStartGC(void);
  59 static void clientdbScheduledGC(void *);
  60
  61 #if USE_DELAY_POOLS
  62 static int max_clients = 32768;
  63 #else
  64 static int max_clients = 32;
  65 #endif
  66
  67 static int cleanup_running = 0;
  68 static int cleanup_scheduled = 0;
  69 static int cleanup_removed;
  70
  71 #if USE_DELAY_POOLS
  72 #define CLIENT_DB_HASH_SIZE 65357
  73 #else
  74 #define CLIENT_DB_HASH_SIZE 467
  75 #endif
  76
  77 static ClientInfo *
  78
  79 clientdbAdd(const Ip::Address &addr)
  80 {
  81     ClientInfo *c;
  82     char *buf = static_cast<char*>(xmalloc(MAX_IPSTRLEN)); // becomes hash.key
  83     c = (ClientInfo *)memAllocate(MEM_CLIENT_INFO);
  84     debugs(77, 9, "ClientInfo constructed, this=" << c);
  85     c->hash.key = addr.NtoA(buf,MAX_IPSTRLEN);
  86     c->addr = addr;
  87 #if USE_DELAY_POOLS
  88     /* setup default values for client write limiter */
  89     c->writeLimitingActive=false;
  90     c->writeSpeedLimit=0;
  91     c->bucketSize = 0;
  92     c->firstTimeConnection=true;
  93     c->quotaQueue = NULL;
  94     c->rationedQuota = 0;
  95     c->rationedCount = 0;
  96     c->selectWaiting = false;
  97     c->eventWaiting = false;
  98
  99     /* get current time */
 100     getCurrentTime();
 101     c->prevTime=current_dtime;/* put current time to have something sensible here */
 102 #endif
 103     hash_join(client_table, &c->hash);
 104     ++statCounter.client_http.clients;
 105
 106     if ((statCounter.client_http.clients > max_clients) && !cleanup_running && cleanup_scheduled < 2) {
 107         ++cleanup_scheduled;
 108         eventAdd("client_db garbage collector", clientdbScheduledGC, NULL, 90, 0);
 109     }
 110
 111     return c;
 112 }
 113
 114 static void
 115 clientdbRegisterWithCacheManager(void)
 116 {
 117     Mgr::RegisterAction("client_list", "Cache Client List", clientdbDump, 0, 1);
 118 }
 119
 120 void
 121 clientdbInit(void)
 122 {
 123     clientdbRegisterWithCacheManager();
 124
 125     if (client_table)
 126         return;
 127
 128     client_table = hash_create((HASHCMP *) strcmp, CLIENT_DB_HASH_SIZE, hash_string);
 129 }
 130
 131 #if USE_DELAY_POOLS
 132 /* returns ClientInfo for given IP addr
 133    Returns NULL if no such client (or clientdb turned off)
 134    (it is assumed that clientdbEstablished will be called before and create client record if needed)
 135 */
 136 ClientInfo * clientdbGetInfo(const Ip::Address &addr)
 137 {
 138     char key[MAX_IPSTRLEN];
 139     ClientInfo *c;
 140
 141     if (!Config.onoff.client_db)
 142         return NULL;
 143
 144     addr.NtoA(key,MAX_IPSTRLEN);
 145
 146     c = (ClientInfo *) hash_lookup(client_table, key);
 147     if (c==NULL) {
 148         debugs(77, DBG_IMPORTANT,"Client db does not contain information for given IP address "<<(const char*)key);
 149         return NULL;
 150     }
 151     return c;
 152 }
 153 #endif
 154 void
 155 clientdbUpdate(const Ip::Address &addr, LogTags ltype, AnyP::ProtocolType p, size_t size)
 156 {
 157     char key[MAX_IPSTRLEN];
 158     ClientInfo *c;
 159
 160     if (!Config.onoff.client_db)
 161         return;
 162
 163     addr.NtoA(key,MAX_IPSTRLEN);
 164
 165     c = (ClientInfo *) hash_lookup(client_table, key);
 166
 167     if (c == NULL)
 168         c = clientdbAdd(addr);
 169
 170     if (c == NULL)
 171         debug_trap("clientdbUpdate: Failed to add entry");
 172
 173     if (p == AnyP::PROTO_HTTP) {
 174         ++ c->Http.n_requests;
 175         ++ c->Http.result_hist[ltype];
 176         kb_incr(&c->Http.kbytes_out, size);
 177
 178         if (logTypeIsATcpHit(ltype))
 179             kb_incr(&c->Http.hit_kbytes_out, size);
 180     } else if (p == AnyP::PROTO_ICP) {
 181         ++ c->Icp.n_requests;
 182         ++ c->Icp.result_hist[ltype];
 183         kb_incr(&c->Icp.kbytes_out, size);
 184
 185         if (LOG_UDP_HIT == ltype)
 186             kb_incr(&c->Icp.hit_kbytes_out, size);
 187     }
 188
 189     c->last_seen = squid_curtime;
 190 }
 191
 192 /**
 193  * This function tracks the number of currently established connections
 194  * for a client IP address.  When a connection is accepted, call this
 195  * with delta = 1.  When the connection is closed, call with delta =
 196  * -1.  To get the current value, simply call with delta = 0.
 197  */
 198 int
 199 clientdbEstablished(const Ip::Address &addr, int delta)
 200 {
 201     char key[MAX_IPSTRLEN];
 202     ClientInfo *c;
 203
 204     if (!Config.onoff.client_db)
 205         return 0;
 206
 207     addr.NtoA(key,MAX_IPSTRLEN);
 208
 209     c = (ClientInfo *) hash_lookup(client_table, key);
 210
 211     if (c == NULL) {
 212         c = clientdbAdd(addr);
 213     }
 214
 215     if (c == NULL)
 216         debug_trap("clientdbUpdate: Failed to add entry");
 217
 218     c->n_established += delta;
 219
 220     return c->n_established;
 221 }
 222
 223 #define CUTOFF_SECONDS 3600
 224 int
 225
 226 clientdbCutoffDenied(const Ip::Address &addr)
 227 {
 228     char key[MAX_IPSTRLEN];
 229     int NR;
 230     int ND;
 231     double p;
 232     ClientInfo *c;
 233
 234     if (!Config.onoff.client_db)
 235         return 0;
 236
 237     addr.NtoA(key,MAX_IPSTRLEN);
 238
 239     c = (ClientInfo *) hash_lookup(client_table, key);
 240
 241     if (c == NULL)
 242         return 0;
 243
 244     /*
 245      * If we are in a cutoff window, we don't send a reply
 246      */
 247     if (squid_curtime - c->cutoff.time < CUTOFF_SECONDS)
 248         return 1;
 249
 250     /*
 251      * Calculate the percent of DENIED replies since the last
 252      * cutoff time.
 253      */
 254     NR = c->Icp.n_requests - c->cutoff.n_req;
 255
 256     if (NR < 150)
 257         NR = 150;
 258
 259     ND = c->Icp.result_hist[LOG_UDP_DENIED] - c->cutoff.n_denied;
 260
 261     p = 100.0 * ND / NR;
 262
 263     if (p < 95.0)
 264         return 0;
 265
 266     debugs(1, DBG_CRITICAL, "WARNING: Probable misconfigured neighbor at " << key);
 267
 268     debugs(1, DBG_CRITICAL, "WARNING: " << ND << " of the last " << NR <<
 269            " ICP replies are DENIED");
 270
 271     debugs(1, DBG_CRITICAL, "WARNING: No replies will be sent for the next " <<
 272            CUTOFF_SECONDS << " seconds");
 273
 274     c->cutoff.time = squid_curtime;
 275
 276     c->cutoff.n_req = c->Icp.n_requests;
 277
 278     c->cutoff.n_denied = c->Icp.result_hist[LOG_UDP_DENIED];
 279
 280     return 1;
 281 }
 282
 283 void
 284 clientdbDump(StoreEntry * sentry)
 285 {
 286     const char *name;
 287     ClientInfo *c;
 288     int icp_total = 0;
 289     int icp_hits = 0;
 290     int http_total = 0;
 291     int http_hits = 0;
 292     storeAppendPrintf(sentry, "Cache Clients:\n");
 293     hash_first(client_table);
 294
 295     while ((c = (ClientInfo *) hash_next(client_table))) {
 296         storeAppendPrintf(sentry, "Address: %s\n", hashKeyStr(&c->hash));
 297         if ( (name = fqdncache_gethostbyaddr(c->addr, 0)) ) {
 298             storeAppendPrintf(sentry, "Name:    %s\n", name);
 299         }
 300         storeAppendPrintf(sentry, "Currently established connections: %d\n",
 301                           c->n_established);
 302         storeAppendPrintf(sentry, "    ICP  Requests %d\n",
 303                           c->Icp.n_requests);
 304
 305         for (LogTags l = LOG_TAG_NONE; l < LOG_TYPE_MAX; ++l) {
 306             if (c->Icp.result_hist[l] == 0)
 307                 continue;
 308
 309             icp_total += c->Icp.result_hist[l];
 310
 311             if (LOG_UDP_HIT == l)
 312                 icp_hits += c->Icp.result_hist[l];
 313
 314             storeAppendPrintf(sentry, "        %-20.20s %7d %3d%%\n",LogTags_str[l], c->Icp.result_hist[l], Math::intPercent(c->Icp.result_hist[l], c->Icp.n_requests));
 315         }
 316
 317         storeAppendPrintf(sentry, "    HTTP Requests %d\n", c->Http.n_requests);
 318
 319         for (LogTags l = LOG_TAG_NONE; l < LOG_TYPE_MAX; ++l) {
 320             if (c->Http.result_hist[l] == 0)
 321                 continue;
 322
 323             http_total += c->Http.result_hist[l];
 324
 325             if (logTypeIsATcpHit(l))
 326                 http_hits += c->Http.result_hist[l];
 327
 328             storeAppendPrintf(sentry,
 329                               "        %-20.20s %7d %3d%%\n",
 330                               LogTags_str[l],
 331                               c->Http.result_hist[l],
 332                               Math::intPercent(c->Http.result_hist[l], c->Http.n_requests));
 333         }
 334
 335         storeAppendPrintf(sentry, "\n");
 336     }
 337
 338     storeAppendPrintf(sentry, "TOTALS\n");
 339     storeAppendPrintf(sentry, "ICP : %d Queries, %d Hits (%3d%%)\n",
 340                       icp_total, icp_hits, Math::intPercent(icp_hits, icp_total));
 341     storeAppendPrintf(sentry, "HTTP: %d Requests, %d Hits (%3d%%)\n",
 342                       http_total, http_hits, Math::intPercent(http_hits, http_total));
 343 }
 344
 345 static void
 346 clientdbFreeItem(void *data)
 347 {
 348     ClientInfo *c = (ClientInfo *)data;
 349     safe_free(c->hash.key);
 350
 351 #if USE_DELAY_POOLS
 352     if (CommQuotaQueue *q = c->quotaQueue) {
 353         q->clientInfo = NULL;
 354         delete q; // invalidates cbdata, cancelling any pending kicks
 355     }
 356 #endif
 357
 358     debugs(77, 9, "ClientInfo destructed, this=" << c);
 359     memFree(c, MEM_CLIENT_INFO);
 360 }
 361
 362 void
 363 clientdbFreeMemory(void)
 364 {
 365     hashFreeItems(client_table, clientdbFreeItem);
 366     hashFreeMemory(client_table);
 367     client_table = NULL;
 368 }
 369
 370 static void
 371 clientdbScheduledGC(void *unused)
 372 {
 373     cleanup_scheduled = 0;
 374     clientdbStartGC();
 375 }
 376
 377 static void
 378 clientdbGC(void *unused)
 379 {
 380     static int bucket = 0;
 381     hash_link *link_next;
 382
 383     link_next = hash_get_bucket(client_table, bucket++);
 384
 385     while (link_next != NULL) {
 386         ClientInfo *c = (ClientInfo *)link_next;
 387         int age = squid_curtime - c->last_seen;
 388         link_next = link_next->next;
 389
 390         if (c->n_established)
 391             continue;
 392
 393         if (age < 24 * 3600 && c->Http.n_requests > 100)
 394             continue;
 395
 396         if (age < 4 * 3600 && (c->Http.n_requests > 10 || c->Icp.n_requests > 10))
 397             continue;
 398
 399         if (age < 5 * 60 && (c->Http.n_requests > 1 || c->Icp.n_requests > 1))
 400             continue;
 401
 402         if (age < 60)
 403             continue;
 404
 405         hash_remove_link(client_table, &c->hash);
 406
 407         clientdbFreeItem(c);
 408
 409         --statCounter.client_http.clients;
 410
 411         ++cleanup_removed;
 412     }
 413
 414     if (bucket < CLIENT_DB_HASH_SIZE)
 415         eventAdd("client_db garbage collector", clientdbGC, NULL, 0.15, 0);
 416     else {
 417         bucket = 0;
 418         cleanup_running = 0;
 419         max_clients = statCounter.client_http.clients * 3 / 2;
 420
 421         if (!cleanup_scheduled) {
 422             cleanup_scheduled = 1;
 423             eventAdd("client_db garbage collector", clientdbScheduledGC, NULL, 6 * 3600, 0);
 424         }
 425
 426         debugs(49, 2, "clientdbGC: Removed " << cleanup_removed << " entries");
 427     }
 428 }
 429
 430 static void
 431 clientdbStartGC(void)
 432 {
 433     max_clients = statCounter.client_http.clients;
 434     cleanup_running = 1;
 435     cleanup_removed = 0;
 436     clientdbGC(NULL);
 437 }
 438
 439 #if SQUID_SNMP
 440
 441 Ip::Address *
 442 client_entry(Ip::Address *current)
 443 {
 444     ClientInfo *c = NULL;
 445     char key[MAX_IPSTRLEN];
 446
 447     if (current) {
 448         current->NtoA(key,MAX_IPSTRLEN);
 449         hash_first(client_table);
 450         while ((c = (ClientInfo *) hash_next(client_table))) {
 451             if (!strcmp(key, hashKeyStr(&c->hash)))
 452                 break;
 453         }
 454
 455         c = (ClientInfo *) hash_next(client_table);
 456     } else {
 457         hash_first(client_table);
 458         c = (ClientInfo *) hash_next(client_table);
 459     }
 460
 461     hash_last(client_table);
 462
 463     if (c)
 464         return (&c->addr);
 465     else
 466         return (NULL);
 467
 468 }
 469
 470 variable_list *
 471 snmp_meshCtblFn(variable_list * Var, snint * ErrP)
 472 {
 473     char key[MAX_IPSTRLEN];
 474     ClientInfo *c = NULL;
 475     Ip::Address keyIp;
 476
 477     *ErrP = SNMP_ERR_NOERROR;
 478     MemBuf tmp;
 479     debugs(49, 6, HERE << "Current : length=" << Var->name_length << ": " << snmpDebugOid(Var->name, Var->name_length, tmp));
 480     if (Var->name_length == 16) {
 481         oid2addr(&(Var->name[12]), keyIp, 4);
 482     } else if (Var->name_length == 28) {
 483         oid2addr(&(Var->name[12]), keyIp, 16);
 484     } else {
 485         *ErrP = SNMP_ERR_NOSUCHNAME;
 486         return NULL;
 487     }
 488
 489     keyIp.NtoA(key, sizeof(key));
 490     debugs(49, 5, HERE << "[" << key << "] requested!");
 491     c = (ClientInfo *) hash_lookup(client_table, key);
 492
 493     if (c == NULL) {
 494         debugs(49, 5, HERE << "not found.");
 495         *ErrP = SNMP_ERR_NOSUCHNAME;
 496         return NULL;
 497     }
 498
 499     variable_list *Answer = NULL;
 500     int aggr = 0;
 501
 502     switch (Var->name[LEN_SQ_NET + 2]) {
 503
 504     case MESH_CTBL_ADDR_TYPE: {
 505         int ival;
 506         ival = c->addr.IsIPv4() ? INETADDRESSTYPE_IPV4 : INETADDRESSTYPE_IPV6 ;
 507         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 508                                       ival, SMI_INTEGER);
 509     }
 510     break;
 511
 512     case MESH_CTBL_ADDR: {
 513         Answer = snmp_var_new(Var->name, Var->name_length);
 514         // InetAddress doesn't have its own ASN.1 type,
 515         // like IpAddr does (SMI_IPADDRESS)
 516         // See: rfc4001.txt
 517         Answer->type = ASN_OCTET_STR;
 518         char client[MAX_IPSTRLEN];
 519         c->addr.NtoA(client,MAX_IPSTRLEN);
 520         Answer->val_len = strlen(client);
 521         Answer->val.string =  (u_char *) xstrdup(client);
 522     }
 523     break;
 524     case MESH_CTBL_HTBYTES:
 525         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 526                                       (snint) c->Http.kbytes_out.kb,
 527                                       SMI_COUNTER32);
 528         break;
 529
 530     case MESH_CTBL_HTREQ:
 531         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 532                                       (snint) c->Http.n_requests,
 533                                       SMI_COUNTER32);
 534         break;
 535
 536     case MESH_CTBL_HTHITS:
 537         aggr = 0;
 538
 539         for (LogTags l = LOG_TAG_NONE; l < LOG_TYPE_MAX; ++l) {
 540             if (logTypeIsATcpHit(l))
 541                 aggr += c->Http.result_hist[l];
 542         }
 543
 544         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 545                                       (snint) aggr,
 546                                       SMI_COUNTER32);
 547         break;
 548
 549     case MESH_CTBL_HTHITBYTES:
 550         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 551                                       (snint) c->Http.hit_kbytes_out.kb,
 552                                       SMI_COUNTER32);
 553         break;
 554
 555     case MESH_CTBL_ICPBYTES:
 556         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 557                                       (snint) c->Icp.kbytes_out.kb,
 558                                       SMI_COUNTER32);
 559         break;
 560
 561     case MESH_CTBL_ICPREQ:
 562         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 563                                       (snint) c->Icp.n_requests,
 564                                       SMI_COUNTER32);
 565         break;
 566
 567     case MESH_CTBL_ICPHITS:
 568         aggr = c->Icp.result_hist[LOG_UDP_HIT];
 569         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 570                                       (snint) aggr,
 571                                       SMI_COUNTER32);
 572         break;
 573
 574     case MESH_CTBL_ICPHITBYTES:
 575         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 576                                       (snint) c->Icp.hit_kbytes_out.kb,
 577                                       SMI_COUNTER32);
 578         break;
 579
 580     default:
 581         *ErrP = SNMP_ERR_NOSUCHNAME;
 582         debugs(49, 5, "snmp_meshCtblFn: illegal column.");
 583         break;
 584     }
 585
 586     return Answer;
 587 }
 588
 589 #endif /*SQUID_SNMP */