src/client_db.cc

   1 /*
   2  * Copyright (C) 1996-2017 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 00    Client Database */
  10
  11 #include "squid.h"
  12 #include "base/RunnersRegistry.h"
  13 #include "client_db.h"
  14 #include "ClientInfo.h"
  15 #include "event.h"
  16 #include "format/Token.h"
  17 #include "fqdncache.h"
  18 #include "ip/Address.h"
  19 #include "log/access_log.h"
  20 #include "mgr/Registration.h"
  21 #include "SquidConfig.h"
  22 #include "SquidMath.h"
  23 #include "SquidTime.h"
  24 #include "StatCounters.h"
  25 #include "Store.h"
  26 #include "tools.h"
  27
  28 #if SQUID_SNMP
  29 #include "snmp_core.h"
  30 #endif
  31
  32 static hash_table *client_table = NULL;
  33
  34 static ClientInfo *clientdbAdd(const Ip::Address &addr);
  35 static FREE clientdbFreeItem;
  36 static void clientdbStartGC(void);
  37 static void clientdbScheduledGC(void *);
  38
  39 #if USE_DELAY_POOLS
  40 static int max_clients = 32768;
  41 #else
  42 static int max_clients = 32;
  43 #endif
  44
  45 static int cleanup_running = 0;
  46 static int cleanup_scheduled = 0;
  47 static int cleanup_removed;
  48
  49 #if USE_DELAY_POOLS
  50 #define CLIENT_DB_HASH_SIZE 65357
  51 #else
  52 #define CLIENT_DB_HASH_SIZE 467
  53 #endif
  54
  55 ClientInfo::ClientInfo(const Ip::Address &ip) :
  56     addr(ip),
  57     n_established(0),
  58     last_seen(0)
  59 #if USE_DELAY_POOLS
  60     , writeSpeedLimit(0),
  61     prevTime(0),
  62     bucketSize(0),
  63     bucketSizeLimit(0),
  64     writeLimitingActive(false),
  65     firstTimeConnection(true),
  66     quotaQueue(nullptr),
  67     rationedQuota(0),
  68     rationedCount(0),
  69     selectWaiting(false),
  70     eventWaiting(false)
  71 #endif
  72 {
  73     debugs(77, 9, "ClientInfo constructed, this=" << static_cast<void*>(this));
  74
  75 #if USE_DELAY_POOLS
  76     getCurrentTime();
  77     /* put current time to have something sensible here */
  78     prevTime = current_dtime;
  79 #endif
  80
  81     char *buf = static_cast<char*>(xmalloc(MAX_IPSTRLEN)); // becomes hash.key
  82     hash.key = addr.toStr(buf,MAX_IPSTRLEN);
  83 }
  84
  85 static ClientInfo *
  86 clientdbAdd(const Ip::Address &addr)
  87 {
  88     ClientInfo *c = new ClientInfo(addr);
  89     hash_join(client_table, &c->hash);
  90     ++statCounter.client_http.clients;
  91
  92     if ((statCounter.client_http.clients > max_clients) && !cleanup_running && cleanup_scheduled < 2) {
  93         ++cleanup_scheduled;
  94         eventAdd("client_db garbage collector", clientdbScheduledGC, NULL, 90, 0);
  95     }
  96
  97     return c;
  98 }
  99
 100 static void
 101 clientdbInit(void)
 102 {
 103     if (client_table)
 104         return;
 105
 106     client_table = hash_create((HASHCMP *) strcmp, CLIENT_DB_HASH_SIZE, hash_string);
 107 }
 108
 109 class ClientDbRr: public RegisteredRunner
 110 {
 111 public:
 112     /* RegisteredRunner API */
 113     virtual void useConfig();
 114 };
 115 RunnerRegistrationEntry(ClientDbRr);
 116
 117 void
 118 ClientDbRr::useConfig()
 119 {
 120     clientdbInit();
 121     Mgr::RegisterAction("client_list", "Cache Client List", clientdbDump, 0, 1);
 122 }
 123
 124 #if USE_DELAY_POOLS
 125 /* returns ClientInfo for given IP addr
 126    Returns NULL if no such client (or clientdb turned off)
 127    (it is assumed that clientdbEstablished will be called before and create client record if needed)
 128 */
 129 ClientInfo * clientdbGetInfo(const Ip::Address &addr)
 130 {
 131     char key[MAX_IPSTRLEN];
 132     ClientInfo *c;
 133
 134     if (!Config.onoff.client_db)
 135         return NULL;
 136
 137     addr.toStr(key,MAX_IPSTRLEN);
 138
 139     c = (ClientInfo *) hash_lookup(client_table, key);
 140     if (c==NULL) {
 141         debugs(77, DBG_IMPORTANT,"Client db does not contain information for given IP address "<<(const char*)key);
 142         return NULL;
 143     }
 144     return c;
 145 }
 146 #endif
 147 void
 148 clientdbUpdate(const Ip::Address &addr, const LogTags &ltype, AnyP::ProtocolType p, size_t size)
 149 {
 150     char key[MAX_IPSTRLEN];
 151     ClientInfo *c;
 152
 153     if (!Config.onoff.client_db)
 154         return;
 155
 156     addr.toStr(key,MAX_IPSTRLEN);
 157
 158     c = (ClientInfo *) hash_lookup(client_table, key);
 159
 160     if (c == NULL)
 161         c = clientdbAdd(addr);
 162
 163     if (c == NULL)
 164         debug_trap("clientdbUpdate: Failed to add entry");
 165
 166     if (p == AnyP::PROTO_HTTP) {
 167         ++ c->Http.n_requests;
 168         ++ c->Http.result_hist[ltype.oldType];
 169         c->Http.kbytes_out += size;
 170
 171         if (ltype.isTcpHit())
 172             c->Http.hit_kbytes_out += size;
 173     } else if (p == AnyP::PROTO_ICP) {
 174         ++ c->Icp.n_requests;
 175         ++ c->Icp.result_hist[ltype.oldType];
 176         c->Icp.kbytes_out += size;
 177
 178         if (LOG_UDP_HIT == ltype.oldType)
 179             c->Icp.hit_kbytes_out += size;
 180     }
 181
 182     c->last_seen = squid_curtime;
 183 }
 184
 185 /**
 186  * This function tracks the number of currently established connections
 187  * for a client IP address.  When a connection is accepted, call this
 188  * with delta = 1.  When the connection is closed, call with delta =
 189  * -1.  To get the current value, simply call with delta = 0.
 190  */
 191 int
 192 clientdbEstablished(const Ip::Address &addr, int delta)
 193 {
 194     char key[MAX_IPSTRLEN];
 195     ClientInfo *c;
 196
 197     if (!Config.onoff.client_db)
 198         return 0;
 199
 200     addr.toStr(key,MAX_IPSTRLEN);
 201
 202     c = (ClientInfo *) hash_lookup(client_table, key);
 203
 204     if (c == NULL) {
 205         c = clientdbAdd(addr);
 206     }
 207
 208     if (c == NULL)
 209         debug_trap("clientdbUpdate: Failed to add entry");
 210
 211     c->n_established += delta;
 212
 213     return c->n_established;
 214 }
 215
 216 #define CUTOFF_SECONDS 3600
 217 int
 218
 219 clientdbCutoffDenied(const Ip::Address &addr)
 220 {
 221     char key[MAX_IPSTRLEN];
 222     int NR;
 223     int ND;
 224     double p;
 225     ClientInfo *c;
 226
 227     if (!Config.onoff.client_db)
 228         return 0;
 229
 230     addr.toStr(key,MAX_IPSTRLEN);
 231
 232     c = (ClientInfo *) hash_lookup(client_table, key);
 233
 234     if (c == NULL)
 235         return 0;
 236
 237     /*
 238      * If we are in a cutoff window, we don't send a reply
 239      */
 240     if (squid_curtime - c->cutoff.time < CUTOFF_SECONDS)
 241         return 1;
 242
 243     /*
 244      * Calculate the percent of DENIED replies since the last
 245      * cutoff time.
 246      */
 247     NR = c->Icp.n_requests - c->cutoff.n_req;
 248
 249     if (NR < 150)
 250         NR = 150;
 251
 252     ND = c->Icp.result_hist[LOG_UDP_DENIED] - c->cutoff.n_denied;
 253
 254     p = 100.0 * ND / NR;
 255
 256     if (p < 95.0)
 257         return 0;
 258
 259     debugs(1, DBG_CRITICAL, "WARNING: Probable misconfigured neighbor at " << key);
 260
 261     debugs(1, DBG_CRITICAL, "WARNING: " << ND << " of the last " << NR <<
 262            " ICP replies are DENIED");
 263
 264     debugs(1, DBG_CRITICAL, "WARNING: No replies will be sent for the next " <<
 265            CUTOFF_SECONDS << " seconds");
 266
 267     c->cutoff.time = squid_curtime;
 268
 269     c->cutoff.n_req = c->Icp.n_requests;
 270
 271     c->cutoff.n_denied = c->Icp.result_hist[LOG_UDP_DENIED];
 272
 273     return 1;
 274 }
 275
 276 void
 277 clientdbDump(StoreEntry * sentry)
 278 {
 279     const char *name;
 280     ClientInfo *c;
 281     int icp_total = 0;
 282     int icp_hits = 0;
 283     int http_total = 0;
 284     int http_hits = 0;
 285     storeAppendPrintf(sentry, "Cache Clients:\n");
 286     hash_first(client_table);
 287
 288     while ((c = (ClientInfo *) hash_next(client_table))) {
 289         storeAppendPrintf(sentry, "Address: %s\n", hashKeyStr(&c->hash));
 290         if ( (name = fqdncache_gethostbyaddr(c->addr, 0)) ) {
 291             storeAppendPrintf(sentry, "Name:    %s\n", name);
 292         }
 293         storeAppendPrintf(sentry, "Currently established connections: %d\n",
 294                           c->n_established);
 295         storeAppendPrintf(sentry, "    ICP  Requests %d\n",
 296                           c->Icp.n_requests);
 297
 298         for (LogTags_ot l = LOG_TAG_NONE; l < LOG_TYPE_MAX; ++l) {
 299             if (c->Icp.result_hist[l] == 0)
 300                 continue;
 301
 302             icp_total += c->Icp.result_hist[l];
 303
 304             if (LOG_UDP_HIT == l)
 305                 icp_hits += c->Icp.result_hist[l];
 306
 307             storeAppendPrintf(sentry, "        %-20.20s %7d %3d%%\n", LogTags(l).c_str(), c->Icp.result_hist[l], Math::intPercent(c->Icp.result_hist[l], c->Icp.n_requests));
 308         }
 309
 310         storeAppendPrintf(sentry, "    HTTP Requests %d\n", c->Http.n_requests);
 311
 312         for (LogTags_ot l = LOG_TAG_NONE; l < LOG_TYPE_MAX; ++l) {
 313             if (c->Http.result_hist[l] == 0)
 314                 continue;
 315
 316             http_total += c->Http.result_hist[l];
 317
 318             if (LogTags(l).isTcpHit())
 319                 http_hits += c->Http.result_hist[l];
 320
 321             storeAppendPrintf(sentry,
 322                               "        %-20.20s %7d %3d%%\n",
 323                               LogTags(l).c_str(),
 324                               c->Http.result_hist[l],
 325                               Math::intPercent(c->Http.result_hist[l], c->Http.n_requests));
 326         }
 327
 328         storeAppendPrintf(sentry, "\n");
 329     }
 330
 331     storeAppendPrintf(sentry, "TOTALS\n");
 332     storeAppendPrintf(sentry, "ICP : %d Queries, %d Hits (%3d%%)\n",
 333                       icp_total, icp_hits, Math::intPercent(icp_hits, icp_total));
 334     storeAppendPrintf(sentry, "HTTP: %d Requests, %d Hits (%3d%%)\n",
 335                       http_total, http_hits, Math::intPercent(http_hits, http_total));
 336 }
 337
 338 static void
 339 clientdbFreeItem(void *data)
 340 {
 341     ClientInfo *c = (ClientInfo *)data;
 342     delete c;
 343 }
 344
 345 ClientInfo::~ClientInfo()
 346 {
 347     safe_free(hash.key);
 348
 349 #if USE_DELAY_POOLS
 350     if (CommQuotaQueue *q = quotaQueue) {
 351         q->clientInfo = NULL;
 352         delete q; // invalidates cbdata, cancelling any pending kicks
 353     }
 354 #endif
 355
 356     debugs(77, 9, "ClientInfo destructed, this=" << static_cast<void*>(this));
 357 }
 358
 359 void
 360 clientdbFreeMemory(void)
 361 {
 362     hashFreeItems(client_table, clientdbFreeItem);
 363     hashFreeMemory(client_table);
 364     client_table = NULL;
 365 }
 366
 367 static void
 368 clientdbScheduledGC(void *)
 369 {
 370     cleanup_scheduled = 0;
 371     clientdbStartGC();
 372 }
 373
 374 static void
 375 clientdbGC(void *)
 376 {
 377     static int bucket = 0;
 378     hash_link *link_next;
 379
 380     link_next = hash_get_bucket(client_table, bucket++);
 381
 382     while (link_next != NULL) {
 383         ClientInfo *c = (ClientInfo *)link_next;
 384         int age = squid_curtime - c->last_seen;
 385         link_next = link_next->next;
 386
 387         if (c->n_established)
 388             continue;
 389
 390         if (age < 24 * 3600 && c->Http.n_requests > 100)
 391             continue;
 392
 393         if (age < 4 * 3600 && (c->Http.n_requests > 10 || c->Icp.n_requests > 10))
 394             continue;
 395
 396         if (age < 5 * 60 && (c->Http.n_requests > 1 || c->Icp.n_requests > 1))
 397             continue;
 398
 399         if (age < 60)
 400             continue;
 401
 402         hash_remove_link(client_table, &c->hash);
 403
 404         clientdbFreeItem(c);
 405
 406         --statCounter.client_http.clients;
 407
 408         ++cleanup_removed;
 409     }
 410
 411     if (bucket < CLIENT_DB_HASH_SIZE)
 412         eventAdd("client_db garbage collector", clientdbGC, NULL, 0.15, 0);
 413     else {
 414         bucket = 0;
 415         cleanup_running = 0;
 416         max_clients = statCounter.client_http.clients * 3 / 2;
 417
 418         if (!cleanup_scheduled) {
 419             cleanup_scheduled = 1;
 420             eventAdd("client_db garbage collector", clientdbScheduledGC, NULL, 6 * 3600, 0);
 421         }
 422
 423         debugs(49, 2, "clientdbGC: Removed " << cleanup_removed << " entries");
 424     }
 425 }
 426
 427 static void
 428 clientdbStartGC(void)
 429 {
 430     max_clients = statCounter.client_http.clients;
 431     cleanup_running = 1;
 432     cleanup_removed = 0;
 433     clientdbGC(NULL);
 434 }
 435
 436 #if SQUID_SNMP
 437
 438 Ip::Address *
 439 client_entry(Ip::Address *current)
 440 {
 441     ClientInfo *c = NULL;
 442     char key[MAX_IPSTRLEN];
 443
 444     if (current) {
 445         current->toStr(key,MAX_IPSTRLEN);
 446         hash_first(client_table);
 447         while ((c = (ClientInfo *) hash_next(client_table))) {
 448             if (!strcmp(key, hashKeyStr(&c->hash)))
 449                 break;
 450         }
 451
 452         c = (ClientInfo *) hash_next(client_table);
 453     } else {
 454         hash_first(client_table);
 455         c = (ClientInfo *) hash_next(client_table);
 456     }
 457
 458     hash_last(client_table);
 459
 460     if (c)
 461         return (&c->addr);
 462     else
 463         return (NULL);
 464
 465 }
 466
 467 variable_list *
 468 snmp_meshCtblFn(variable_list * Var, snint * ErrP)
 469 {
 470     char key[MAX_IPSTRLEN];
 471     ClientInfo *c = NULL;
 472     Ip::Address keyIp;
 473
 474     *ErrP = SNMP_ERR_NOERROR;
 475     MemBuf tmp;
 476     debugs(49, 6, HERE << "Current : length=" << Var->name_length << ": " << snmpDebugOid(Var->name, Var->name_length, tmp));
 477     if (Var->name_length == 16) {
 478         oid2addr(&(Var->name[12]), keyIp, 4);
 479     } else if (Var->name_length == 28) {
 480         oid2addr(&(Var->name[12]), keyIp, 16);
 481     } else {
 482         *ErrP = SNMP_ERR_NOSUCHNAME;
 483         return NULL;
 484     }
 485
 486     keyIp.toStr(key, sizeof(key));
 487     debugs(49, 5, HERE << "[" << key << "] requested!");
 488     c = (ClientInfo *) hash_lookup(client_table, key);
 489
 490     if (c == NULL) {
 491         debugs(49, 5, HERE << "not found.");
 492         *ErrP = SNMP_ERR_NOSUCHNAME;
 493         return NULL;
 494     }
 495
 496     variable_list *Answer = NULL;
 497     int aggr = 0;
 498
 499     switch (Var->name[LEN_SQ_NET + 2]) {
 500
 501     case MESH_CTBL_ADDR_TYPE: {
 502         int ival;
 503         ival = c->addr.isIPv4() ? INETADDRESSTYPE_IPV4 : INETADDRESSTYPE_IPV6 ;
 504         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 505                                       ival, SMI_INTEGER);
 506     }
 507     break;
 508
 509     case MESH_CTBL_ADDR: {
 510         Answer = snmp_var_new(Var->name, Var->name_length);
 511         // InetAddress doesn't have its own ASN.1 type,
 512         // like IpAddr does (SMI_IPADDRESS)
 513         // See: rfc4001.txt
 514         Answer->type = ASN_OCTET_STR;
 515         char client[MAX_IPSTRLEN];
 516         c->addr.toStr(client,MAX_IPSTRLEN);
 517         Answer->val_len = strlen(client);
 518         Answer->val.string =  (u_char *) xstrdup(client);
 519     }
 520     break;
 521     case MESH_CTBL_HTBYTES:
 522         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 523                                       (snint) c->Http.kbytes_out.kb,
 524                                       SMI_COUNTER32);
 525         break;
 526
 527     case MESH_CTBL_HTREQ:
 528         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 529                                       (snint) c->Http.n_requests,
 530                                       SMI_COUNTER32);
 531         break;
 532
 533     case MESH_CTBL_HTHITS:
 534         aggr = 0;
 535
 536         for (LogTags_ot l = LOG_TAG_NONE; l < LOG_TYPE_MAX; ++l) {
 537             if (LogTags(l).isTcpHit())
 538                 aggr += c->Http.result_hist[l];
 539         }
 540
 541         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 542                                       (snint) aggr,
 543                                       SMI_COUNTER32);
 544         break;
 545
 546     case MESH_CTBL_HTHITBYTES:
 547         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 548                                       (snint) c->Http.hit_kbytes_out.kb,
 549                                       SMI_COUNTER32);
 550         break;
 551
 552     case MESH_CTBL_ICPBYTES:
 553         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 554                                       (snint) c->Icp.kbytes_out.kb,
 555                                       SMI_COUNTER32);
 556         break;
 557
 558     case MESH_CTBL_ICPREQ:
 559         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 560                                       (snint) c->Icp.n_requests,
 561                                       SMI_COUNTER32);
 562         break;
 563
 564     case MESH_CTBL_ICPHITS:
 565         aggr = c->Icp.result_hist[LOG_UDP_HIT];
 566         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 567                                       (snint) aggr,
 568                                       SMI_COUNTER32);
 569         break;
 570
 571     case MESH_CTBL_ICPHITBYTES:
 572         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 573                                       (snint) c->Icp.hit_kbytes_out.kb,
 574                                       SMI_COUNTER32);
 575         break;
 576
 577     default:
 578         *ErrP = SNMP_ERR_NOSUCHNAME;
 579         debugs(49, 5, "snmp_meshCtblFn: illegal column.");
 580         break;
 581     }
 582
 583     return Answer;
 584 }
 585
 586 #endif /*SQUID_SNMP */
 587