src/client_db.cc

   1 /*
   2  * DEBUG: section 00    Client Database
   3  * AUTHOR: Duane Wessels
   4  *
   5  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   6  * ----------------------------------------------------------
   7  *
   8  *  Squid is the result of efforts by numerous individuals from
   9  *  the Internet community; see the CONTRIBUTORS file for full
  10  *  details.   Many organizations have provided support for Squid's
  11  *  development; see the SPONSORS file for full details.  Squid is
  12  *  Copyrighted (C) 2001 by the Regents of the University of
  13  *  California; see the COPYRIGHT file for full details.  Squid
  14  *  incorporates software developed and/or copyrighted by other
  15  *  sources; see the CREDITS file for full details.
  16  *
  17  *  This program is free software; you can redistribute it and/or modify
  18  *  it under the terms of the GNU General Public License as published by
  19  *  the Free Software Foundation; either version 2 of the License, or
  20  *  (at your option) any later version.
  21  *
  22  *  This program is distributed in the hope that it will be useful,
  23  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  24  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  25  *  GNU General Public License for more details.
  26  *
  27  *  You should have received a copy of the GNU General Public License
  28  *  along with this program; if not, write to the Free Software
  29  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  30  *
  31  */
  32
  33 #include "squid.h"
  34 #include "client_db.h"
  35 #include "event.h"
  36 #include "format/Token.h"
  37 #include "ClientInfo.h"
  38 #include "fqdncache.h"
  39 #include "ip/Address.h"
  40 #include "log/access_log.h"
  41 #include "Mem.h"
  42 #include "mgr/Registration.h"
  43 #include "SquidMath.h"
  44 #include "SquidTime.h"
  45 #include "StatCounters.h"
  46 #include "Store.h"
  47 #include "tools.h"
  48
  49 #if SQUID_SNMP
  50 #include "snmp_core.h"
  51 #endif
  52
  53 static hash_table *client_table = NULL;
  54
  55 static ClientInfo *clientdbAdd(const Ip::Address &addr);
  56 static FREE clientdbFreeItem;
  57 static void clientdbStartGC(void);
  58 static void clientdbScheduledGC(void *);
  59
  60 #if USE_DELAY_POOLS
  61 static int max_clients = 32768;
  62 #else
  63 static int max_clients = 32;
  64 #endif
  65
  66 static int cleanup_running = 0;
  67 static int cleanup_scheduled = 0;
  68 static int cleanup_removed;
  69
  70 #if USE_DELAY_POOLS
  71 #define CLIENT_DB_HASH_SIZE 65357
  72 #else
  73 #define CLIENT_DB_HASH_SIZE 467
  74 #endif
  75
  76 static ClientInfo *
  77
  78 clientdbAdd(const Ip::Address &addr)
  79 {
  80     ClientInfo *c;
  81     char *buf = new char[MAX_IPSTRLEN];
  82     c = (ClientInfo *)memAllocate(MEM_CLIENT_INFO);
  83     c->hash.key = addr.NtoA(buf,MAX_IPSTRLEN);
  84     c->addr = addr;
  85 #if USE_DELAY_POOLS
  86     /* setup default values for client write limiter */
  87     c->writeLimitingActive=false;
  88     c->writeSpeedLimit=0;
  89     c->bucketSize = 0;
  90     c->firstTimeConnection=true;
  91     c->quotaQueue = NULL;
  92     c->rationedQuota = 0;
  93     c->rationedCount = 0;
  94     c->selectWaiting = false;
  95     c->eventWaiting = false;
  96
  97     /* get current time */
  98     getCurrentTime();
  99     c->prevTime=current_dtime;/* put current time to have something sensible here */
 100 #endif
 101     hash_join(client_table, &c->hash);
 102     ++statCounter.client_http.clients;
 103
 104     if ((statCounter.client_http.clients > max_clients) && !cleanup_running && cleanup_scheduled < 2) {
 105         ++cleanup_scheduled;
 106         eventAdd("client_db garbage collector", clientdbScheduledGC, NULL, 90, 0);
 107     }
 108
 109     return c;
 110 }
 111
 112 static void
 113 clientdbRegisterWithCacheManager(void)
 114 {
 115     Mgr::RegisterAction("client_list", "Cache Client List", clientdbDump, 0, 1);
 116 }
 117
 118 void
 119 clientdbInit(void)
 120 {
 121     clientdbRegisterWithCacheManager();
 122
 123     if (client_table)
 124         return;
 125
 126     client_table = hash_create((HASHCMP *) strcmp, CLIENT_DB_HASH_SIZE, hash_string);
 127 }
 128
 129 #if USE_DELAY_POOLS
 130 /* returns ClientInfo for given IP addr
 131    Returns NULL if no such client (or clientdb turned off)
 132    (it is assumed that clientdbEstablished will be called before and create client record if needed)
 133 */
 134 ClientInfo * clientdbGetInfo(const Ip::Address &addr)
 135 {
 136     char key[MAX_IPSTRLEN];
 137     ClientInfo *c;
 138
 139     if (!Config.onoff.client_db)
 140         return NULL;
 141
 142     addr.NtoA(key,MAX_IPSTRLEN);
 143
 144     c = (ClientInfo *) hash_lookup(client_table, key);
 145     if (c==NULL) {
 146         debugs(77, DBG_IMPORTANT,"Client db does not contain information for given IP address "<<(const char*)key);
 147         return NULL;
 148     }
 149     return c;
 150 }
 151 #endif
 152 void
 153 clientdbUpdate(const Ip::Address &addr, log_type ltype, AnyP::ProtocolType p, size_t size)
 154 {
 155     char key[MAX_IPSTRLEN];
 156     ClientInfo *c;
 157
 158     if (!Config.onoff.client_db)
 159         return;
 160
 161     addr.NtoA(key,MAX_IPSTRLEN);
 162
 163     c = (ClientInfo *) hash_lookup(client_table, key);
 164
 165     if (c == NULL)
 166         c = clientdbAdd(addr);
 167
 168     if (c == NULL)
 169         debug_trap("clientdbUpdate: Failed to add entry");
 170
 171     if (p == AnyP::PROTO_HTTP) {
 172         ++ c->Http.n_requests;
 173         ++ c->Http.result_hist[ltype];
 174         kb_incr(&c->Http.kbytes_out, size);
 175
 176         if (logTypeIsATcpHit(ltype))
 177             kb_incr(&c->Http.hit_kbytes_out, size);
 178     } else if (p == AnyP::PROTO_ICP) {
 179         ++ c->Icp.n_requests;
 180         ++ c->Icp.result_hist[ltype];
 181         kb_incr(&c->Icp.kbytes_out, size);
 182
 183         if (LOG_UDP_HIT == ltype)
 184             kb_incr(&c->Icp.hit_kbytes_out, size);
 185     }
 186
 187     c->last_seen = squid_curtime;
 188 }
 189
 190 /**
 191  * This function tracks the number of currently established connections
 192  * for a client IP address.  When a connection is accepted, call this
 193  * with delta = 1.  When the connection is closed, call with delta =
 194  * -1.  To get the current value, simply call with delta = 0.
 195  */
 196 int
 197 clientdbEstablished(const Ip::Address &addr, int delta)
 198 {
 199     char key[MAX_IPSTRLEN];
 200     ClientInfo *c;
 201
 202     if (!Config.onoff.client_db)
 203         return 0;
 204
 205     addr.NtoA(key,MAX_IPSTRLEN);
 206
 207     c = (ClientInfo *) hash_lookup(client_table, key);
 208
 209     if (c == NULL) {
 210         c = clientdbAdd(addr);
 211     }
 212
 213     if (c == NULL)
 214         debug_trap("clientdbUpdate: Failed to add entry");
 215
 216     c->n_established += delta;
 217
 218     return c->n_established;
 219 }
 220
 221 #define CUTOFF_SECONDS 3600
 222 int
 223
 224 clientdbCutoffDenied(const Ip::Address &addr)
 225 {
 226     char key[MAX_IPSTRLEN];
 227     int NR;
 228     int ND;
 229     double p;
 230     ClientInfo *c;
 231
 232     if (!Config.onoff.client_db)
 233         return 0;
 234
 235     addr.NtoA(key,MAX_IPSTRLEN);
 236
 237     c = (ClientInfo *) hash_lookup(client_table, key);
 238
 239     if (c == NULL)
 240         return 0;
 241
 242     /*
 243      * If we are in a cutoff window, we don't send a reply
 244      */
 245     if (squid_curtime - c->cutoff.time < CUTOFF_SECONDS)
 246         return 1;
 247
 248     /*
 249      * Calculate the percent of DENIED replies since the last
 250      * cutoff time.
 251      */
 252     NR = c->Icp.n_requests - c->cutoff.n_req;
 253
 254     if (NR < 150)
 255         NR = 150;
 256
 257     ND = c->Icp.result_hist[LOG_UDP_DENIED] - c->cutoff.n_denied;
 258
 259     p = 100.0 * ND / NR;
 260
 261     if (p < 95.0)
 262         return 0;
 263
 264     debugs(1, DBG_CRITICAL, "WARNING: Probable misconfigured neighbor at " << key);
 265
 266     debugs(1, DBG_CRITICAL, "WARNING: " << ND << " of the last " << NR <<
 267            " ICP replies are DENIED");
 268
 269     debugs(1, DBG_CRITICAL, "WARNING: No replies will be sent for the next " <<
 270            CUTOFF_SECONDS << " seconds");
 271
 272     c->cutoff.time = squid_curtime;
 273
 274     c->cutoff.n_req = c->Icp.n_requests;
 275
 276     c->cutoff.n_denied = c->Icp.result_hist[LOG_UDP_DENIED];
 277
 278     return 1;
 279 }
 280
 281 log_type &operator++ (log_type &aLogType)
 282 {
 283     int tmp = (int)aLogType;
 284     aLogType = (log_type)(++tmp);
 285     return aLogType;
 286 }
 287
 288 void
 289 clientdbDump(StoreEntry * sentry)
 290 {
 291     const char *name;
 292     ClientInfo *c;
 293     log_type l;
 294     int icp_total = 0;
 295     int icp_hits = 0;
 296     int http_total = 0;
 297     int http_hits = 0;
 298     storeAppendPrintf(sentry, "Cache Clients:\n");
 299     hash_first(client_table);
 300
 301     while ((c = (ClientInfo *) hash_next(client_table))) {
 302         storeAppendPrintf(sentry, "Address: %s\n", hashKeyStr(&c->hash));
 303         if ( (name = fqdncache_gethostbyaddr(c->addr, 0)) ) {
 304             storeAppendPrintf(sentry, "Name:    %s\n", name);
 305         }
 306         storeAppendPrintf(sentry, "Currently established connections: %d\n",
 307                           c->n_established);
 308         storeAppendPrintf(sentry, "    ICP  Requests %d\n",
 309                           c->Icp.n_requests);
 310
 311         for (l = LOG_TAG_NONE; l < LOG_TYPE_MAX; ++l) {
 312             if (c->Icp.result_hist[l] == 0)
 313                 continue;
 314
 315             icp_total += c->Icp.result_hist[l];
 316
 317             if (LOG_UDP_HIT == l)
 318                 icp_hits += c->Icp.result_hist[l];
 319
 320             storeAppendPrintf(sentry, "        %-20.20s %7d %3d%%\n",Format::log_tags[l], c->Icp.result_hist[l], Math::intPercent(c->Icp.result_hist[l], c->Icp.n_requests));
 321         }
 322
 323         storeAppendPrintf(sentry, "    HTTP Requests %d\n", c->Http.n_requests);
 324
 325         for (l = LOG_TAG_NONE; l < LOG_TYPE_MAX; ++l) {
 326             if (c->Http.result_hist[l] == 0)
 327                 continue;
 328
 329             http_total += c->Http.result_hist[l];
 330
 331             if (logTypeIsATcpHit(l))
 332                 http_hits += c->Http.result_hist[l];
 333
 334             storeAppendPrintf(sentry,
 335                               "        %-20.20s %7d %3d%%\n",
 336                               Format::log_tags[l],
 337                               c->Http.result_hist[l],
 338                               Math::intPercent(c->Http.result_hist[l], c->Http.n_requests));
 339         }
 340
 341         storeAppendPrintf(sentry, "\n");
 342     }
 343
 344     storeAppendPrintf(sentry, "TOTALS\n");
 345     storeAppendPrintf(sentry, "ICP : %d Queries, %d Hits (%3d%%)\n",
 346                       icp_total, icp_hits, Math::intPercent(icp_hits, icp_total));
 347     storeAppendPrintf(sentry, "HTTP: %d Requests, %d Hits (%3d%%)\n",
 348                       http_total, http_hits, Math::intPercent(http_hits, http_total));
 349 }
 350
 351 static void
 352 clientdbFreeItem(void *data)
 353 {
 354     ClientInfo *c = (ClientInfo *)data;
 355     safe_free(c->hash.key);
 356
 357 #if USE_DELAY_POOLS
 358     if (CommQuotaQueue *q = c->quotaQueue) {
 359         q->clientInfo = NULL;
 360         delete q; // invalidates cbdata, cancelling any pending kicks
 361     }
 362 #endif
 363
 364     memFree(c, MEM_CLIENT_INFO);
 365 }
 366
 367 void
 368 clientdbFreeMemory(void)
 369 {
 370     hashFreeItems(client_table, clientdbFreeItem);
 371     hashFreeMemory(client_table);
 372     client_table = NULL;
 373 }
 374
 375 static void
 376 clientdbScheduledGC(void *unused)
 377 {
 378     cleanup_scheduled = 0;
 379     clientdbStartGC();
 380 }
 381
 382 static void
 383 clientdbGC(void *unused)
 384 {
 385     static int bucket = 0;
 386     hash_link *link_next;
 387
 388     link_next = hash_get_bucket(client_table, bucket++);
 389
 390     while (link_next != NULL) {
 391         ClientInfo *c = (ClientInfo *)link_next;
 392         int age = squid_curtime - c->last_seen;
 393         link_next = link_next->next;
 394
 395         if (c->n_established)
 396             continue;
 397
 398         if (age < 24 * 3600 && c->Http.n_requests > 100)
 399             continue;
 400
 401         if (age < 4 * 3600 && (c->Http.n_requests > 10 || c->Icp.n_requests > 10))
 402             continue;
 403
 404         if (age < 5 * 60 && (c->Http.n_requests > 1 || c->Icp.n_requests > 1))
 405             continue;
 406
 407         if (age < 60)
 408             continue;
 409
 410         hash_remove_link(client_table, &c->hash);
 411
 412         clientdbFreeItem(c);
 413
 414         --statCounter.client_http.clients;
 415
 416         ++cleanup_removed;
 417     }
 418
 419     if (bucket < CLIENT_DB_HASH_SIZE)
 420         eventAdd("client_db garbage collector", clientdbGC, NULL, 0.15, 0);
 421     else {
 422         bucket = 0;
 423         cleanup_running = 0;
 424         max_clients = statCounter.client_http.clients * 3 / 2;
 425
 426         if (!cleanup_scheduled) {
 427             cleanup_scheduled = 1;
 428             eventAdd("client_db garbage collector", clientdbScheduledGC, NULL, 6 * 3600, 0);
 429         }
 430
 431         debugs(49, 2, "clientdbGC: Removed " << cleanup_removed << " entries");
 432     }
 433 }
 434
 435 static void
 436 clientdbStartGC(void)
 437 {
 438     max_clients = statCounter.client_http.clients;
 439     cleanup_running = 1;
 440     cleanup_removed = 0;
 441     clientdbGC(NULL);
 442 }
 443
 444 #if SQUID_SNMP
 445
 446 Ip::Address *
 447 client_entry(Ip::Address *current)
 448 {
 449     ClientInfo *c = NULL;
 450     char key[MAX_IPSTRLEN];
 451
 452     if (current) {
 453         current->NtoA(key,MAX_IPSTRLEN);
 454         hash_first(client_table);
 455         while ((c = (ClientInfo *) hash_next(client_table))) {
 456             if (!strcmp(key, hashKeyStr(&c->hash)))
 457                 break;
 458         }
 459
 460         c = (ClientInfo *) hash_next(client_table);
 461     } else {
 462         hash_first(client_table);
 463         c = (ClientInfo *) hash_next(client_table);
 464     }
 465
 466     hash_last(client_table);
 467
 468     if (c)
 469         return (&c->addr);
 470     else
 471         return (NULL);
 472
 473 }
 474
 475 variable_list *
 476 snmp_meshCtblFn(variable_list * Var, snint * ErrP)
 477 {
 478     char key[MAX_IPSTRLEN];
 479     ClientInfo *c = NULL;
 480     Ip::Address keyIp;
 481
 482     *ErrP = SNMP_ERR_NOERROR;
 483     MemBuf tmp;
 484     debugs(49, 6, HERE << "Current : length=" << Var->name_length << ": " << snmpDebugOid(Var->name, Var->name_length, tmp));
 485     if (Var->name_length == 16) {
 486         oid2addr(&(Var->name[12]), keyIp, 4);
 487     } else if (Var->name_length == 28) {
 488         oid2addr(&(Var->name[12]), keyIp, 16);
 489     } else {
 490         *ErrP = SNMP_ERR_NOSUCHNAME;
 491         return NULL;
 492     }
 493
 494     keyIp.NtoA(key, sizeof(key));
 495     debugs(49, 5, HERE << "[" << key << "] requested!");
 496     c = (ClientInfo *) hash_lookup(client_table, key);
 497
 498     if (c == NULL) {
 499         debugs(49, 5, HERE << "not found.");
 500         *ErrP = SNMP_ERR_NOSUCHNAME;
 501         return NULL;
 502     }
 503
 504     variable_list *Answer = NULL;
 505     int aggr = 0;
 506     log_type l;
 507
 508     switch (Var->name[LEN_SQ_NET + 2]) {
 509
 510     case MESH_CTBL_ADDR_TYPE: {
 511         int ival;
 512         ival = c->addr.IsIPv4() ? INETADDRESSTYPE_IPV4 : INETADDRESSTYPE_IPV6 ;
 513         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 514                                       ival, SMI_INTEGER);
 515     }
 516     break;
 517
 518     case MESH_CTBL_ADDR: {
 519         Answer = snmp_var_new(Var->name, Var->name_length);
 520         // InetAddress doesn't have its own ASN.1 type,
 521         // like IpAddr does (SMI_IPADDRESS)
 522         // See: rfc4001.txt
 523         Answer->type = ASN_OCTET_STR;
 524         char client[MAX_IPSTRLEN];
 525         c->addr.NtoA(client,MAX_IPSTRLEN);
 526         Answer->val_len = strlen(client);
 527         Answer->val.string =  (u_char *) xstrdup(client);
 528     }
 529     break;
 530     case MESH_CTBL_HTBYTES:
 531         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 532                                       (snint) c->Http.kbytes_out.kb,
 533                                       SMI_COUNTER32);
 534         break;
 535
 536     case MESH_CTBL_HTREQ:
 537         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 538                                       (snint) c->Http.n_requests,
 539                                       SMI_COUNTER32);
 540         break;
 541
 542     case MESH_CTBL_HTHITS:
 543         aggr = 0;
 544
 545         for (l = LOG_TAG_NONE; l < LOG_TYPE_MAX; ++l) {
 546             if (logTypeIsATcpHit(l))
 547                 aggr += c->Http.result_hist[l];
 548         }
 549
 550         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 551                                       (snint) aggr,
 552                                       SMI_COUNTER32);
 553         break;
 554
 555     case MESH_CTBL_HTHITBYTES:
 556         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 557                                       (snint) c->Http.hit_kbytes_out.kb,
 558                                       SMI_COUNTER32);
 559         break;
 560
 561     case MESH_CTBL_ICPBYTES:
 562         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 563                                       (snint) c->Icp.kbytes_out.kb,
 564                                       SMI_COUNTER32);
 565         break;
 566
 567     case MESH_CTBL_ICPREQ:
 568         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 569                                       (snint) c->Icp.n_requests,
 570                                       SMI_COUNTER32);
 571         break;
 572
 573     case MESH_CTBL_ICPHITS:
 574         aggr = c->Icp.result_hist[LOG_UDP_HIT];
 575         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 576                                       (snint) aggr,
 577                                       SMI_COUNTER32);
 578         break;
 579
 580     case MESH_CTBL_ICPHITBYTES:
 581         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 582                                       (snint) c->Icp.hit_kbytes_out.kb,
 583                                       SMI_COUNTER32);
 584         break;
 585
 586     default:
 587         *ErrP = SNMP_ERR_NOSUCHNAME;
 588         debugs(49, 5, "snmp_meshCtblFn: illegal column.");
 589         break;
 590     }
 591
 592     return Answer;
 593 }
 594
 595 #endif /*SQUID_SNMP */