src/client_db.cc

   1 /*
   2  * DEBUG: section 00    Client Database
   3  * AUTHOR: Duane Wessels
   4  *
   5  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   6  * ----------------------------------------------------------
   7  *
   8  *  Squid is the result of efforts by numerous individuals from
   9  *  the Internet community; see the CONTRIBUTORS file for full
  10  *  details.   Many organizations have provided support for Squid's
  11  *  development; see the SPONSORS file for full details.  Squid is
  12  *  Copyrighted (C) 2001 by the Regents of the University of
  13  *  California; see the COPYRIGHT file for full details.  Squid
  14  *  incorporates software developed and/or copyrighted by other
  15  *  sources; see the CREDITS file for full details.
  16  *
  17  *  This program is free software; you can redistribute it and/or modify
  18  *  it under the terms of the GNU General Public License as published by
  19  *  the Free Software Foundation; either version 2 of the License, or
  20  *  (at your option) any later version.
  21  *
  22  *  This program is distributed in the hope that it will be useful,
  23  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  24  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  25  *  GNU General Public License for more details.
  26  *
  27  *  You should have received a copy of the GNU General Public License
  28  *  along with this program; if not, write to the Free Software
  29  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  30  *
  31  */
  32
  33 #include "squid.h"
  34 #include "client_db.h"
  35 #include "event.h"
  36 #include "format/Token.h"
  37 #include "ClientInfo.h"
  38 #include "fqdncache.h"
  39 #include "ip/Address.h"
  40 #include "log/access_log.h"
  41 #include "Mem.h"
  42 #include "mgr/Registration.h"
  43 #include "SquidConfig.h"
  44 #include "SquidMath.h"
  45 #include "SquidTime.h"
  46 #include "StatCounters.h"
  47 #include "Store.h"
  48 #include "tools.h"
  49
  50 #if SQUID_SNMP
  51 #include "snmp_core.h"
  52 #endif
  53
  54 static hash_table *client_table = NULL;
  55
  56 static ClientInfo *clientdbAdd(const Ip::Address &addr);
  57 static FREE clientdbFreeItem;
  58 static void clientdbStartGC(void);
  59 static void clientdbScheduledGC(void *);
  60
  61 #if USE_DELAY_POOLS
  62 static int max_clients = 32768;
  63 #else
  64 static int max_clients = 32;
  65 #endif
  66
  67 static int cleanup_running = 0;
  68 static int cleanup_scheduled = 0;
  69 static int cleanup_removed;
  70
  71 #if USE_DELAY_POOLS
  72 #define CLIENT_DB_HASH_SIZE 65357
  73 #else
  74 #define CLIENT_DB_HASH_SIZE 467
  75 #endif
  76
  77 static ClientInfo *
  78
  79 clientdbAdd(const Ip::Address &addr)
  80 {
  81     ClientInfo *c;
  82     char *buf = new char[MAX_IPSTRLEN];
  83     c = (ClientInfo *)memAllocate(MEM_CLIENT_INFO);
  84     c->hash.key = addr.NtoA(buf,MAX_IPSTRLEN);
  85     c->addr = addr;
  86 #if USE_DELAY_POOLS
  87     /* setup default values for client write limiter */
  88     c->writeLimitingActive=false;
  89     c->writeSpeedLimit=0;
  90     c->bucketSize = 0;
  91     c->firstTimeConnection=true;
  92     c->quotaQueue = NULL;
  93     c->rationedQuota = 0;
  94     c->rationedCount = 0;
  95     c->selectWaiting = false;
  96     c->eventWaiting = false;
  97
  98     /* get current time */
  99     getCurrentTime();
 100     c->prevTime=current_dtime;/* put current time to have something sensible here */
 101 #endif
 102     hash_join(client_table, &c->hash);
 103     ++statCounter.client_http.clients;
 104
 105     if ((statCounter.client_http.clients > max_clients) && !cleanup_running && cleanup_scheduled < 2) {
 106         ++cleanup_scheduled;
 107         eventAdd("client_db garbage collector", clientdbScheduledGC, NULL, 90, 0);
 108     }
 109
 110     return c;
 111 }
 112
 113 static void
 114 clientdbRegisterWithCacheManager(void)
 115 {
 116     Mgr::RegisterAction("client_list", "Cache Client List", clientdbDump, 0, 1);
 117 }
 118
 119 void
 120 clientdbInit(void)
 121 {
 122     clientdbRegisterWithCacheManager();
 123
 124     if (client_table)
 125         return;
 126
 127     client_table = hash_create((HASHCMP *) strcmp, CLIENT_DB_HASH_SIZE, hash_string);
 128 }
 129
 130 #if USE_DELAY_POOLS
 131 /* returns ClientInfo for given IP addr
 132    Returns NULL if no such client (or clientdb turned off)
 133    (it is assumed that clientdbEstablished will be called before and create client record if needed)
 134 */
 135 ClientInfo * clientdbGetInfo(const Ip::Address &addr)
 136 {
 137     char key[MAX_IPSTRLEN];
 138     ClientInfo *c;
 139
 140     if (!Config.onoff.client_db)
 141         return NULL;
 142
 143     addr.NtoA(key,MAX_IPSTRLEN);
 144
 145     c = (ClientInfo *) hash_lookup(client_table, key);
 146     if (c==NULL) {
 147         debugs(77, DBG_IMPORTANT,"Client db does not contain information for given IP address "<<(const char*)key);
 148         return NULL;
 149     }
 150     return c;
 151 }
 152 #endif
 153 void
 154 clientdbUpdate(const Ip::Address &addr, log_type ltype, AnyP::ProtocolType p, size_t size)
 155 {
 156     char key[MAX_IPSTRLEN];
 157     ClientInfo *c;
 158
 159     if (!Config.onoff.client_db)
 160         return;
 161
 162     addr.NtoA(key,MAX_IPSTRLEN);
 163
 164     c = (ClientInfo *) hash_lookup(client_table, key);
 165
 166     if (c == NULL)
 167         c = clientdbAdd(addr);
 168
 169     if (c == NULL)
 170         debug_trap("clientdbUpdate: Failed to add entry");
 171
 172     if (p == AnyP::PROTO_HTTP) {
 173         ++ c->Http.n_requests;
 174         ++ c->Http.result_hist[ltype];
 175         kb_incr(&c->Http.kbytes_out, size);
 176
 177         if (logTypeIsATcpHit(ltype))
 178             kb_incr(&c->Http.hit_kbytes_out, size);
 179     } else if (p == AnyP::PROTO_ICP) {
 180         ++ c->Icp.n_requests;
 181         ++ c->Icp.result_hist[ltype];
 182         kb_incr(&c->Icp.kbytes_out, size);
 183
 184         if (LOG_UDP_HIT == ltype)
 185             kb_incr(&c->Icp.hit_kbytes_out, size);
 186     }
 187
 188     c->last_seen = squid_curtime;
 189 }
 190
 191 /**
 192  * This function tracks the number of currently established connections
 193  * for a client IP address.  When a connection is accepted, call this
 194  * with delta = 1.  When the connection is closed, call with delta =
 195  * -1.  To get the current value, simply call with delta = 0.
 196  */
 197 int
 198 clientdbEstablished(const Ip::Address &addr, int delta)
 199 {
 200     char key[MAX_IPSTRLEN];
 201     ClientInfo *c;
 202
 203     if (!Config.onoff.client_db)
 204         return 0;
 205
 206     addr.NtoA(key,MAX_IPSTRLEN);
 207
 208     c = (ClientInfo *) hash_lookup(client_table, key);
 209
 210     if (c == NULL) {
 211         c = clientdbAdd(addr);
 212     }
 213
 214     if (c == NULL)
 215         debug_trap("clientdbUpdate: Failed to add entry");
 216
 217     c->n_established += delta;
 218
 219     return c->n_established;
 220 }
 221
 222 #define CUTOFF_SECONDS 3600
 223 int
 224
 225 clientdbCutoffDenied(const Ip::Address &addr)
 226 {
 227     char key[MAX_IPSTRLEN];
 228     int NR;
 229     int ND;
 230     double p;
 231     ClientInfo *c;
 232
 233     if (!Config.onoff.client_db)
 234         return 0;
 235
 236     addr.NtoA(key,MAX_IPSTRLEN);
 237
 238     c = (ClientInfo *) hash_lookup(client_table, key);
 239
 240     if (c == NULL)
 241         return 0;
 242
 243     /*
 244      * If we are in a cutoff window, we don't send a reply
 245      */
 246     if (squid_curtime - c->cutoff.time < CUTOFF_SECONDS)
 247         return 1;
 248
 249     /*
 250      * Calculate the percent of DENIED replies since the last
 251      * cutoff time.
 252      */
 253     NR = c->Icp.n_requests - c->cutoff.n_req;
 254
 255     if (NR < 150)
 256         NR = 150;
 257
 258     ND = c->Icp.result_hist[LOG_UDP_DENIED] - c->cutoff.n_denied;
 259
 260     p = 100.0 * ND / NR;
 261
 262     if (p < 95.0)
 263         return 0;
 264
 265     debugs(1, DBG_CRITICAL, "WARNING: Probable misconfigured neighbor at " << key);
 266
 267     debugs(1, DBG_CRITICAL, "WARNING: " << ND << " of the last " << NR <<
 268            " ICP replies are DENIED");
 269
 270     debugs(1, DBG_CRITICAL, "WARNING: No replies will be sent for the next " <<
 271            CUTOFF_SECONDS << " seconds");
 272
 273     c->cutoff.time = squid_curtime;
 274
 275     c->cutoff.n_req = c->Icp.n_requests;
 276
 277     c->cutoff.n_denied = c->Icp.result_hist[LOG_UDP_DENIED];
 278
 279     return 1;
 280 }
 281
 282 log_type &operator++ (log_type &aLogType)
 283 {
 284     int tmp = (int)aLogType;
 285     aLogType = (log_type)(++tmp);
 286     return aLogType;
 287 }
 288
 289 void
 290 clientdbDump(StoreEntry * sentry)
 291 {
 292     const char *name;
 293     ClientInfo *c;
 294     log_type l;
 295     int icp_total = 0;
 296     int icp_hits = 0;
 297     int http_total = 0;
 298     int http_hits = 0;
 299     storeAppendPrintf(sentry, "Cache Clients:\n");
 300     hash_first(client_table);
 301
 302     while ((c = (ClientInfo *) hash_next(client_table))) {
 303         storeAppendPrintf(sentry, "Address: %s\n", hashKeyStr(&c->hash));
 304         if ( (name = fqdncache_gethostbyaddr(c->addr, 0)) ) {
 305             storeAppendPrintf(sentry, "Name:    %s\n", name);
 306         }
 307         storeAppendPrintf(sentry, "Currently established connections: %d\n",
 308                           c->n_established);
 309         storeAppendPrintf(sentry, "    ICP  Requests %d\n",
 310                           c->Icp.n_requests);
 311
 312         for (l = LOG_TAG_NONE; l < LOG_TYPE_MAX; ++l) {
 313             if (c->Icp.result_hist[l] == 0)
 314                 continue;
 315
 316             icp_total += c->Icp.result_hist[l];
 317
 318             if (LOG_UDP_HIT == l)
 319                 icp_hits += c->Icp.result_hist[l];
 320
 321             storeAppendPrintf(sentry, "        %-20.20s %7d %3d%%\n",Format::log_tags[l], c->Icp.result_hist[l], Math::intPercent(c->Icp.result_hist[l], c->Icp.n_requests));
 322         }
 323
 324         storeAppendPrintf(sentry, "    HTTP Requests %d\n", c->Http.n_requests);
 325
 326         for (l = LOG_TAG_NONE; l < LOG_TYPE_MAX; ++l) {
 327             if (c->Http.result_hist[l] == 0)
 328                 continue;
 329
 330             http_total += c->Http.result_hist[l];
 331
 332             if (logTypeIsATcpHit(l))
 333                 http_hits += c->Http.result_hist[l];
 334
 335             storeAppendPrintf(sentry,
 336                               "        %-20.20s %7d %3d%%\n",
 337                               Format::log_tags[l],
 338                               c->Http.result_hist[l],
 339                               Math::intPercent(c->Http.result_hist[l], c->Http.n_requests));
 340         }
 341
 342         storeAppendPrintf(sentry, "\n");
 343     }
 344
 345     storeAppendPrintf(sentry, "TOTALS\n");
 346     storeAppendPrintf(sentry, "ICP : %d Queries, %d Hits (%3d%%)\n",
 347                       icp_total, icp_hits, Math::intPercent(icp_hits, icp_total));
 348     storeAppendPrintf(sentry, "HTTP: %d Requests, %d Hits (%3d%%)\n",
 349                       http_total, http_hits, Math::intPercent(http_hits, http_total));
 350 }
 351
 352 static void
 353 clientdbFreeItem(void *data)
 354 {
 355     ClientInfo *c = (ClientInfo *)data;
 356     safe_free(c->hash.key);
 357
 358 #if USE_DELAY_POOLS
 359     if (CommQuotaQueue *q = c->quotaQueue) {
 360         q->clientInfo = NULL;
 361         delete q; // invalidates cbdata, cancelling any pending kicks
 362     }
 363 #endif
 364
 365     memFree(c, MEM_CLIENT_INFO);
 366 }
 367
 368 void
 369 clientdbFreeMemory(void)
 370 {
 371     hashFreeItems(client_table, clientdbFreeItem);
 372     hashFreeMemory(client_table);
 373     client_table = NULL;
 374 }
 375
 376 static void
 377 clientdbScheduledGC(void *unused)
 378 {
 379     cleanup_scheduled = 0;
 380     clientdbStartGC();
 381 }
 382
 383 static void
 384 clientdbGC(void *unused)
 385 {
 386     static int bucket = 0;
 387     hash_link *link_next;
 388
 389     link_next = hash_get_bucket(client_table, bucket++);
 390
 391     while (link_next != NULL) {
 392         ClientInfo *c = (ClientInfo *)link_next;
 393         int age = squid_curtime - c->last_seen;
 394         link_next = link_next->next;
 395
 396         if (c->n_established)
 397             continue;
 398
 399         if (age < 24 * 3600 && c->Http.n_requests > 100)
 400             continue;
 401
 402         if (age < 4 * 3600 && (c->Http.n_requests > 10 || c->Icp.n_requests > 10))
 403             continue;
 404
 405         if (age < 5 * 60 && (c->Http.n_requests > 1 || c->Icp.n_requests > 1))
 406             continue;
 407
 408         if (age < 60)
 409             continue;
 410
 411         hash_remove_link(client_table, &c->hash);
 412
 413         clientdbFreeItem(c);
 414
 415         --statCounter.client_http.clients;
 416
 417         ++cleanup_removed;
 418     }
 419
 420     if (bucket < CLIENT_DB_HASH_SIZE)
 421         eventAdd("client_db garbage collector", clientdbGC, NULL, 0.15, 0);
 422     else {
 423         bucket = 0;
 424         cleanup_running = 0;
 425         max_clients = statCounter.client_http.clients * 3 / 2;
 426
 427         if (!cleanup_scheduled) {
 428             cleanup_scheduled = 1;
 429             eventAdd("client_db garbage collector", clientdbScheduledGC, NULL, 6 * 3600, 0);
 430         }
 431
 432         debugs(49, 2, "clientdbGC: Removed " << cleanup_removed << " entries");
 433     }
 434 }
 435
 436 static void
 437 clientdbStartGC(void)
 438 {
 439     max_clients = statCounter.client_http.clients;
 440     cleanup_running = 1;
 441     cleanup_removed = 0;
 442     clientdbGC(NULL);
 443 }
 444
 445 #if SQUID_SNMP
 446
 447 Ip::Address *
 448 client_entry(Ip::Address *current)
 449 {
 450     ClientInfo *c = NULL;
 451     char key[MAX_IPSTRLEN];
 452
 453     if (current) {
 454         current->NtoA(key,MAX_IPSTRLEN);
 455         hash_first(client_table);
 456         while ((c = (ClientInfo *) hash_next(client_table))) {
 457             if (!strcmp(key, hashKeyStr(&c->hash)))
 458                 break;
 459         }
 460
 461         c = (ClientInfo *) hash_next(client_table);
 462     } else {
 463         hash_first(client_table);
 464         c = (ClientInfo *) hash_next(client_table);
 465     }
 466
 467     hash_last(client_table);
 468
 469     if (c)
 470         return (&c->addr);
 471     else
 472         return (NULL);
 473
 474 }
 475
 476 variable_list *
 477 snmp_meshCtblFn(variable_list * Var, snint * ErrP)
 478 {
 479     char key[MAX_IPSTRLEN];
 480     ClientInfo *c = NULL;
 481     Ip::Address keyIp;
 482
 483     *ErrP = SNMP_ERR_NOERROR;
 484     MemBuf tmp;
 485     debugs(49, 6, HERE << "Current : length=" << Var->name_length << ": " << snmpDebugOid(Var->name, Var->name_length, tmp));
 486     if (Var->name_length == 16) {
 487         oid2addr(&(Var->name[12]), keyIp, 4);
 488     } else if (Var->name_length == 28) {
 489         oid2addr(&(Var->name[12]), keyIp, 16);
 490     } else {
 491         *ErrP = SNMP_ERR_NOSUCHNAME;
 492         return NULL;
 493     }
 494
 495     keyIp.NtoA(key, sizeof(key));
 496     debugs(49, 5, HERE << "[" << key << "] requested!");
 497     c = (ClientInfo *) hash_lookup(client_table, key);
 498
 499     if (c == NULL) {
 500         debugs(49, 5, HERE << "not found.");
 501         *ErrP = SNMP_ERR_NOSUCHNAME;
 502         return NULL;
 503     }
 504
 505     variable_list *Answer = NULL;
 506     int aggr = 0;
 507     log_type l;
 508
 509     switch (Var->name[LEN_SQ_NET + 2]) {
 510
 511     case MESH_CTBL_ADDR_TYPE: {
 512         int ival;
 513         ival = c->addr.IsIPv4() ? INETADDRESSTYPE_IPV4 : INETADDRESSTYPE_IPV6 ;
 514         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 515                                       ival, SMI_INTEGER);
 516     }
 517     break;
 518
 519     case MESH_CTBL_ADDR: {
 520         Answer = snmp_var_new(Var->name, Var->name_length);
 521         // InetAddress doesn't have its own ASN.1 type,
 522         // like IpAddr does (SMI_IPADDRESS)
 523         // See: rfc4001.txt
 524         Answer->type = ASN_OCTET_STR;
 525         char client[MAX_IPSTRLEN];
 526         c->addr.NtoA(client,MAX_IPSTRLEN);
 527         Answer->val_len = strlen(client);
 528         Answer->val.string =  (u_char *) xstrdup(client);
 529     }
 530     break;
 531     case MESH_CTBL_HTBYTES:
 532         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 533                                       (snint) c->Http.kbytes_out.kb,
 534                                       SMI_COUNTER32);
 535         break;
 536
 537     case MESH_CTBL_HTREQ:
 538         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 539                                       (snint) c->Http.n_requests,
 540                                       SMI_COUNTER32);
 541         break;
 542
 543     case MESH_CTBL_HTHITS:
 544         aggr = 0;
 545
 546         for (l = LOG_TAG_NONE; l < LOG_TYPE_MAX; ++l) {
 547             if (logTypeIsATcpHit(l))
 548                 aggr += c->Http.result_hist[l];
 549         }
 550
 551         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 552                                       (snint) aggr,
 553                                       SMI_COUNTER32);
 554         break;
 555
 556     case MESH_CTBL_HTHITBYTES:
 557         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 558                                       (snint) c->Http.hit_kbytes_out.kb,
 559                                       SMI_COUNTER32);
 560         break;
 561
 562     case MESH_CTBL_ICPBYTES:
 563         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 564                                       (snint) c->Icp.kbytes_out.kb,
 565                                       SMI_COUNTER32);
 566         break;
 567
 568     case MESH_CTBL_ICPREQ:
 569         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 570                                       (snint) c->Icp.n_requests,
 571                                       SMI_COUNTER32);
 572         break;
 573
 574     case MESH_CTBL_ICPHITS:
 575         aggr = c->Icp.result_hist[LOG_UDP_HIT];
 576         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 577                                       (snint) aggr,
 578                                       SMI_COUNTER32);
 579         break;
 580
 581     case MESH_CTBL_ICPHITBYTES:
 582         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 583                                       (snint) c->Icp.hit_kbytes_out.kb,
 584                                       SMI_COUNTER32);
 585         break;
 586
 587     default:
 588         *ErrP = SNMP_ERR_NOSUCHNAME;
 589         debugs(49, 5, "snmp_meshCtblFn: illegal column.");
 590         break;
 591     }
 592
 593     return Answer;
 594 }
 595
 596 #endif /*SQUID_SNMP */