src/client_db.cc

   1 /*
   2  * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 00    Client Database */
  10
  11 #include "squid.h"
  12 #include "base/RunnersRegistry.h"
  13 #include "client_db.h"
  14 #include "ClientInfo.h"
  15 #include "event.h"
  16 #include "format/Token.h"
  17 #include "fqdncache.h"
  18 #include "ip/Address.h"
  19 #include "log/access_log.h"
  20 #include "mgr/Registration.h"
  21 #include "SquidConfig.h"
  22 #include "SquidMath.h"
  23 #include "SquidTime.h"
  24 #include "StatCounters.h"
  25 #include "Store.h"
  26 #include "tools.h"
  27
  28 #if SQUID_SNMP
  29 #include "snmp_core.h"
  30 #endif
  31
  32 static hash_table *client_table = NULL;
  33
  34 static ClientInfo *clientdbAdd(const Ip::Address &addr);
  35 static FREE clientdbFreeItem;
  36 static void clientdbStartGC(void);
  37 static void clientdbScheduledGC(void *);
  38
  39 #if USE_DELAY_POOLS
  40 static int max_clients = 32768;
  41 #else
  42 static int max_clients = 32;
  43 #endif
  44
  45 static int cleanup_running = 0;
  46 static int cleanup_scheduled = 0;
  47 static int cleanup_removed;
  48
  49 #if USE_DELAY_POOLS
  50 #define CLIENT_DB_HASH_SIZE 65357
  51 #else
  52 #define CLIENT_DB_HASH_SIZE 467
  53 #endif
  54
  55 static ClientInfo *
  56
  57 clientdbAdd(const Ip::Address &addr)
  58 {
  59     ClientInfo *c;
  60     char *buf = static_cast<char*>(xmalloc(MAX_IPSTRLEN)); // becomes hash.key
  61     c = (ClientInfo *)memAllocate(MEM_CLIENT_INFO);
  62     debugs(77, 9, "ClientInfo constructed, this=" << c);
  63     c->hash.key = addr.toStr(buf,MAX_IPSTRLEN);
  64     c->addr = addr;
  65 #if USE_DELAY_POOLS
  66     /* setup default values for client write limiter */
  67     c->writeLimitingActive=false;
  68     c->writeSpeedLimit=0;
  69     c->bucketSize = 0;
  70     c->firstTimeConnection=true;
  71     c->quotaQueue = NULL;
  72     c->rationedQuota = 0;
  73     c->rationedCount = 0;
  74     c->selectWaiting = false;
  75     c->eventWaiting = false;
  76
  77     /* get current time */
  78     getCurrentTime();
  79     c->prevTime=current_dtime;/* put current time to have something sensible here */
  80 #endif
  81     hash_join(client_table, &c->hash);
  82     ++statCounter.client_http.clients;
  83
  84     if ((statCounter.client_http.clients > max_clients) && !cleanup_running && cleanup_scheduled < 2) {
  85         ++cleanup_scheduled;
  86         eventAdd("client_db garbage collector", clientdbScheduledGC, NULL, 90, 0);
  87     }
  88
  89     return c;
  90 }
  91
  92 static void
  93 clientdbInit(void)
  94 {
  95     if (client_table)
  96         return;
  97
  98     client_table = hash_create((HASHCMP *) strcmp, CLIENT_DB_HASH_SIZE, hash_string);
  99 }
 100
 101 class ClientDbRr: public RegisteredRunner
 102 {
 103 public:
 104     /* RegisteredRunner API */
 105     virtual void useConfig();
 106 };
 107 RunnerRegistrationEntry(ClientDbRr);
 108
 109 void
 110 ClientDbRr::useConfig()
 111 {
 112     clientdbInit();
 113     Mgr::RegisterAction("client_list", "Cache Client List", clientdbDump, 0, 1);
 114 }
 115
 116 #if USE_DELAY_POOLS
 117 /* returns ClientInfo for given IP addr
 118    Returns NULL if no such client (or clientdb turned off)
 119    (it is assumed that clientdbEstablished will be called before and create client record if needed)
 120 */
 121 ClientInfo * clientdbGetInfo(const Ip::Address &addr)
 122 {
 123     char key[MAX_IPSTRLEN];
 124     ClientInfo *c;
 125
 126     if (!Config.onoff.client_db)
 127         return NULL;
 128
 129     addr.toStr(key,MAX_IPSTRLEN);
 130
 131     c = (ClientInfo *) hash_lookup(client_table, key);
 132     if (c==NULL) {
 133         debugs(77, DBG_IMPORTANT,"Client db does not contain information for given IP address "<<(const char*)key);
 134         return NULL;
 135     }
 136     return c;
 137 }
 138 #endif
 139 void
 140 clientdbUpdate(const Ip::Address &addr, LogTags ltype, AnyP::ProtocolType p, size_t size)
 141 {
 142     char key[MAX_IPSTRLEN];
 143     ClientInfo *c;
 144
 145     if (!Config.onoff.client_db)
 146         return;
 147
 148     addr.toStr(key,MAX_IPSTRLEN);
 149
 150     c = (ClientInfo *) hash_lookup(client_table, key);
 151
 152     if (c == NULL)
 153         c = clientdbAdd(addr);
 154
 155     if (c == NULL)
 156         debug_trap("clientdbUpdate: Failed to add entry");
 157
 158     if (p == AnyP::PROTO_HTTP) {
 159         ++ c->Http.n_requests;
 160         ++ c->Http.result_hist[ltype];
 161         kb_incr(&c->Http.kbytes_out, size);
 162
 163         if (logTypeIsATcpHit(ltype))
 164             kb_incr(&c->Http.hit_kbytes_out, size);
 165     } else if (p == AnyP::PROTO_ICP) {
 166         ++ c->Icp.n_requests;
 167         ++ c->Icp.result_hist[ltype];
 168         kb_incr(&c->Icp.kbytes_out, size);
 169
 170         if (LOG_UDP_HIT == ltype)
 171             kb_incr(&c->Icp.hit_kbytes_out, size);
 172     }
 173
 174     c->last_seen = squid_curtime;
 175 }
 176
 177 /**
 178  * This function tracks the number of currently established connections
 179  * for a client IP address.  When a connection is accepted, call this
 180  * with delta = 1.  When the connection is closed, call with delta =
 181  * -1.  To get the current value, simply call with delta = 0.
 182  */
 183 int
 184 clientdbEstablished(const Ip::Address &addr, int delta)
 185 {
 186     char key[MAX_IPSTRLEN];
 187     ClientInfo *c;
 188
 189     if (!Config.onoff.client_db)
 190         return 0;
 191
 192     addr.toStr(key,MAX_IPSTRLEN);
 193
 194     c = (ClientInfo *) hash_lookup(client_table, key);
 195
 196     if (c == NULL) {
 197         c = clientdbAdd(addr);
 198     }
 199
 200     if (c == NULL)
 201         debug_trap("clientdbUpdate: Failed to add entry");
 202
 203     c->n_established += delta;
 204
 205     return c->n_established;
 206 }
 207
 208 #define CUTOFF_SECONDS 3600
 209 int
 210
 211 clientdbCutoffDenied(const Ip::Address &addr)
 212 {
 213     char key[MAX_IPSTRLEN];
 214     int NR;
 215     int ND;
 216     double p;
 217     ClientInfo *c;
 218
 219     if (!Config.onoff.client_db)
 220         return 0;
 221
 222     addr.toStr(key,MAX_IPSTRLEN);
 223
 224     c = (ClientInfo *) hash_lookup(client_table, key);
 225
 226     if (c == NULL)
 227         return 0;
 228
 229     /*
 230      * If we are in a cutoff window, we don't send a reply
 231      */
 232     if (squid_curtime - c->cutoff.time < CUTOFF_SECONDS)
 233         return 1;
 234
 235     /*
 236      * Calculate the percent of DENIED replies since the last
 237      * cutoff time.
 238      */
 239     NR = c->Icp.n_requests - c->cutoff.n_req;
 240
 241     if (NR < 150)
 242         NR = 150;
 243
 244     ND = c->Icp.result_hist[LOG_UDP_DENIED] - c->cutoff.n_denied;
 245
 246     p = 100.0 * ND / NR;
 247
 248     if (p < 95.0)
 249         return 0;
 250
 251     debugs(1, DBG_CRITICAL, "WARNING: Probable misconfigured neighbor at " << key);
 252
 253     debugs(1, DBG_CRITICAL, "WARNING: " << ND << " of the last " << NR <<
 254            " ICP replies are DENIED");
 255
 256     debugs(1, DBG_CRITICAL, "WARNING: No replies will be sent for the next " <<
 257            CUTOFF_SECONDS << " seconds");
 258
 259     c->cutoff.time = squid_curtime;
 260
 261     c->cutoff.n_req = c->Icp.n_requests;
 262
 263     c->cutoff.n_denied = c->Icp.result_hist[LOG_UDP_DENIED];
 264
 265     return 1;
 266 }
 267
 268 void
 269 clientdbDump(StoreEntry * sentry)
 270 {
 271     const char *name;
 272     ClientInfo *c;
 273     int icp_total = 0;
 274     int icp_hits = 0;
 275     int http_total = 0;
 276     int http_hits = 0;
 277     storeAppendPrintf(sentry, "Cache Clients:\n");
 278     hash_first(client_table);
 279
 280     while ((c = (ClientInfo *) hash_next(client_table))) {
 281         storeAppendPrintf(sentry, "Address: %s\n", hashKeyStr(&c->hash));
 282         if ( (name = fqdncache_gethostbyaddr(c->addr, 0)) ) {
 283             storeAppendPrintf(sentry, "Name:    %s\n", name);
 284         }
 285         storeAppendPrintf(sentry, "Currently established connections: %d\n",
 286                           c->n_established);
 287         storeAppendPrintf(sentry, "    ICP  Requests %d\n",
 288                           c->Icp.n_requests);
 289
 290         for (LogTags l = LOG_TAG_NONE; l < LOG_TYPE_MAX; ++l) {
 291             if (c->Icp.result_hist[l] == 0)
 292                 continue;
 293
 294             icp_total += c->Icp.result_hist[l];
 295
 296             if (LOG_UDP_HIT == l)
 297                 icp_hits += c->Icp.result_hist[l];
 298
 299             storeAppendPrintf(sentry, "        %-20.20s %7d %3d%%\n",LogTags_str[l], c->Icp.result_hist[l], Math::intPercent(c->Icp.result_hist[l], c->Icp.n_requests));
 300         }
 301
 302         storeAppendPrintf(sentry, "    HTTP Requests %d\n", c->Http.n_requests);
 303
 304         for (LogTags l = LOG_TAG_NONE; l < LOG_TYPE_MAX; ++l) {
 305             if (c->Http.result_hist[l] == 0)
 306                 continue;
 307
 308             http_total += c->Http.result_hist[l];
 309
 310             if (logTypeIsATcpHit(l))
 311                 http_hits += c->Http.result_hist[l];
 312
 313             storeAppendPrintf(sentry,
 314                               "        %-20.20s %7d %3d%%\n",
 315                               LogTags_str[l],
 316                               c->Http.result_hist[l],
 317                               Math::intPercent(c->Http.result_hist[l], c->Http.n_requests));
 318         }
 319
 320         storeAppendPrintf(sentry, "\n");
 321     }
 322
 323     storeAppendPrintf(sentry, "TOTALS\n");
 324     storeAppendPrintf(sentry, "ICP : %d Queries, %d Hits (%3d%%)\n",
 325                       icp_total, icp_hits, Math::intPercent(icp_hits, icp_total));
 326     storeAppendPrintf(sentry, "HTTP: %d Requests, %d Hits (%3d%%)\n",
 327                       http_total, http_hits, Math::intPercent(http_hits, http_total));
 328 }
 329
 330 static void
 331 clientdbFreeItem(void *data)
 332 {
 333     ClientInfo *c = (ClientInfo *)data;
 334     safe_free(c->hash.key);
 335
 336 #if USE_DELAY_POOLS
 337     if (CommQuotaQueue *q = c->quotaQueue) {
 338         q->clientInfo = NULL;
 339         delete q; // invalidates cbdata, cancelling any pending kicks
 340     }
 341 #endif
 342
 343     debugs(77, 9, "ClientInfo destructed, this=" << c);
 344     memFree(c, MEM_CLIENT_INFO);
 345 }
 346
 347 void
 348 clientdbFreeMemory(void)
 349 {
 350     hashFreeItems(client_table, clientdbFreeItem);
 351     hashFreeMemory(client_table);
 352     client_table = NULL;
 353 }
 354
 355 static void
 356 clientdbScheduledGC(void *)
 357 {
 358     cleanup_scheduled = 0;
 359     clientdbStartGC();
 360 }
 361
 362 static void
 363 clientdbGC(void *)
 364 {
 365     static int bucket = 0;
 366     hash_link *link_next;
 367
 368     link_next = hash_get_bucket(client_table, bucket++);
 369
 370     while (link_next != NULL) {
 371         ClientInfo *c = (ClientInfo *)link_next;
 372         int age = squid_curtime - c->last_seen;
 373         link_next = link_next->next;
 374
 375         if (c->n_established)
 376             continue;
 377
 378         if (age < 24 * 3600 && c->Http.n_requests > 100)
 379             continue;
 380
 381         if (age < 4 * 3600 && (c->Http.n_requests > 10 || c->Icp.n_requests > 10))
 382             continue;
 383
 384         if (age < 5 * 60 && (c->Http.n_requests > 1 || c->Icp.n_requests > 1))
 385             continue;
 386
 387         if (age < 60)
 388             continue;
 389
 390         hash_remove_link(client_table, &c->hash);
 391
 392         clientdbFreeItem(c);
 393
 394         --statCounter.client_http.clients;
 395
 396         ++cleanup_removed;
 397     }
 398
 399     if (bucket < CLIENT_DB_HASH_SIZE)
 400         eventAdd("client_db garbage collector", clientdbGC, NULL, 0.15, 0);
 401     else {
 402         bucket = 0;
 403         cleanup_running = 0;
 404         max_clients = statCounter.client_http.clients * 3 / 2;
 405
 406         if (!cleanup_scheduled) {
 407             cleanup_scheduled = 1;
 408             eventAdd("client_db garbage collector", clientdbScheduledGC, NULL, 6 * 3600, 0);
 409         }
 410
 411         debugs(49, 2, "clientdbGC: Removed " << cleanup_removed << " entries");
 412     }
 413 }
 414
 415 static void
 416 clientdbStartGC(void)
 417 {
 418     max_clients = statCounter.client_http.clients;
 419     cleanup_running = 1;
 420     cleanup_removed = 0;
 421     clientdbGC(NULL);
 422 }
 423
 424 #if SQUID_SNMP
 425
 426 Ip::Address *
 427 client_entry(Ip::Address *current)
 428 {
 429     ClientInfo *c = NULL;
 430     char key[MAX_IPSTRLEN];
 431
 432     if (current) {
 433         current->toStr(key,MAX_IPSTRLEN);
 434         hash_first(client_table);
 435         while ((c = (ClientInfo *) hash_next(client_table))) {
 436             if (!strcmp(key, hashKeyStr(&c->hash)))
 437                 break;
 438         }
 439
 440         c = (ClientInfo *) hash_next(client_table);
 441     } else {
 442         hash_first(client_table);
 443         c = (ClientInfo *) hash_next(client_table);
 444     }
 445
 446     hash_last(client_table);
 447
 448     if (c)
 449         return (&c->addr);
 450     else
 451         return (NULL);
 452
 453 }
 454
 455 variable_list *
 456 snmp_meshCtblFn(variable_list * Var, snint * ErrP)
 457 {
 458     char key[MAX_IPSTRLEN];
 459     ClientInfo *c = NULL;
 460     Ip::Address keyIp;
 461
 462     *ErrP = SNMP_ERR_NOERROR;
 463     MemBuf tmp;
 464     debugs(49, 6, HERE << "Current : length=" << Var->name_length << ": " << snmpDebugOid(Var->name, Var->name_length, tmp));
 465     if (Var->name_length == 16) {
 466         oid2addr(&(Var->name[12]), keyIp, 4);
 467     } else if (Var->name_length == 28) {
 468         oid2addr(&(Var->name[12]), keyIp, 16);
 469     } else {
 470         *ErrP = SNMP_ERR_NOSUCHNAME;
 471         return NULL;
 472     }
 473
 474     keyIp.toStr(key, sizeof(key));
 475     debugs(49, 5, HERE << "[" << key << "] requested!");
 476     c = (ClientInfo *) hash_lookup(client_table, key);
 477
 478     if (c == NULL) {
 479         debugs(49, 5, HERE << "not found.");
 480         *ErrP = SNMP_ERR_NOSUCHNAME;
 481         return NULL;
 482     }
 483
 484     variable_list *Answer = NULL;
 485     int aggr = 0;
 486
 487     switch (Var->name[LEN_SQ_NET + 2]) {
 488
 489     case MESH_CTBL_ADDR_TYPE: {
 490         int ival;
 491         ival = c->addr.isIPv4() ? INETADDRESSTYPE_IPV4 : INETADDRESSTYPE_IPV6 ;
 492         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 493                                       ival, SMI_INTEGER);
 494     }
 495     break;
 496
 497     case MESH_CTBL_ADDR: {
 498         Answer = snmp_var_new(Var->name, Var->name_length);
 499         // InetAddress doesn't have its own ASN.1 type,
 500         // like IpAddr does (SMI_IPADDRESS)
 501         // See: rfc4001.txt
 502         Answer->type = ASN_OCTET_STR;
 503         char client[MAX_IPSTRLEN];
 504         c->addr.toStr(client,MAX_IPSTRLEN);
 505         Answer->val_len = strlen(client);
 506         Answer->val.string =  (u_char *) xstrdup(client);
 507     }
 508     break;
 509     case MESH_CTBL_HTBYTES:
 510         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 511                                       (snint) c->Http.kbytes_out.kb,
 512                                       SMI_COUNTER32);
 513         break;
 514
 515     case MESH_CTBL_HTREQ:
 516         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 517                                       (snint) c->Http.n_requests,
 518                                       SMI_COUNTER32);
 519         break;
 520
 521     case MESH_CTBL_HTHITS:
 522         aggr = 0;
 523
 524         for (LogTags l = LOG_TAG_NONE; l < LOG_TYPE_MAX; ++l) {
 525             if (logTypeIsATcpHit(l))
 526                 aggr += c->Http.result_hist[l];
 527         }
 528
 529         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 530                                       (snint) aggr,
 531                                       SMI_COUNTER32);
 532         break;
 533
 534     case MESH_CTBL_HTHITBYTES:
 535         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 536                                       (snint) c->Http.hit_kbytes_out.kb,
 537                                       SMI_COUNTER32);
 538         break;
 539
 540     case MESH_CTBL_ICPBYTES:
 541         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 542                                       (snint) c->Icp.kbytes_out.kb,
 543                                       SMI_COUNTER32);
 544         break;
 545
 546     case MESH_CTBL_ICPREQ:
 547         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 548                                       (snint) c->Icp.n_requests,
 549                                       SMI_COUNTER32);
 550         break;
 551
 552     case MESH_CTBL_ICPHITS:
 553         aggr = c->Icp.result_hist[LOG_UDP_HIT];
 554         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 555                                       (snint) aggr,
 556                                       SMI_COUNTER32);
 557         break;
 558
 559     case MESH_CTBL_ICPHITBYTES:
 560         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 561                                       (snint) c->Icp.hit_kbytes_out.kb,
 562                                       SMI_COUNTER32);
 563         break;
 564
 565     default:
 566         *ErrP = SNMP_ERR_NOSUCHNAME;
 567         debugs(49, 5, "snmp_meshCtblFn: illegal column.");
 568         break;
 569     }
 570
 571     return Answer;
 572 }
 573
 574 #endif /*SQUID_SNMP */
 575