src/client_db.cc

   1 /*
   2  * Copyright (C) 1996-2020 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 00    Client Database */
  10
  11 #include "squid.h"
  12 #include "base/RunnersRegistry.h"
  13 #include "client_db.h"
  14 #include "ClientInfo.h"
  15 #include "event.h"
  16 #include "format/Token.h"
  17 #include "fqdncache.h"
  18 #include "ip/Address.h"
  19 #include "log/access_log.h"
  20 #include "mgr/Registration.h"
  21 #include "SquidConfig.h"
  22 #include "SquidMath.h"
  23 #include "SquidTime.h"
  24 #include "StatCounters.h"
  25 #include "Store.h"
  26 #include "tools.h"
  27
  28 #if SQUID_SNMP
  29 #include "snmp_core.h"
  30 #endif
  31
  32 static hash_table *client_table = NULL;
  33
  34 static ClientInfo *clientdbAdd(const Ip::Address &addr);
  35 static FREE clientdbFreeItem;
  36 static void clientdbStartGC(void);
  37 static void clientdbScheduledGC(void *);
  38
  39 #if USE_DELAY_POOLS
  40 static int max_clients = 32768;
  41 #else
  42 static int max_clients = 32;
  43 #endif
  44
  45 static int cleanup_running = 0;
  46 static int cleanup_scheduled = 0;
  47 static int cleanup_removed;
  48
  49 #if USE_DELAY_POOLS
  50 #define CLIENT_DB_HASH_SIZE 65357
  51 #else
  52 #define CLIENT_DB_HASH_SIZE 467
  53 #endif
  54
  55 ClientInfo::ClientInfo(const Ip::Address &ip) :
  56 #if USE_DELAY_POOLS
  57     BandwidthBucket(0, 0, 0),
  58 #endif
  59     addr(ip),
  60     n_established(0),
  61     last_seen(0)
  62 #if USE_DELAY_POOLS
  63     , writeLimitingActive(false),
  64     firstTimeConnection(true),
  65     quotaQueue(nullptr),
  66     rationedQuota(0),
  67     rationedCount(0),
  68     eventWaiting(false)
  69 #endif
  70 {
  71     debugs(77, 9, "ClientInfo constructed, this=" << static_cast<void*>(this));
  72     char *buf = static_cast<char*>(xmalloc(MAX_IPSTRLEN)); // becomes hash.key
  73     key = addr.toStr(buf,MAX_IPSTRLEN);
  74 }
  75
  76 static ClientInfo *
  77 clientdbAdd(const Ip::Address &addr)
  78 {
  79     ClientInfo *c = new ClientInfo(addr);
  80     hash_join(client_table, static_cast<hash_link*>(c));
  81     ++statCounter.client_http.clients;
  82
  83     if ((statCounter.client_http.clients > max_clients) && !cleanup_running && cleanup_scheduled < 2) {
  84         ++cleanup_scheduled;
  85         eventAdd("client_db garbage collector", clientdbScheduledGC, NULL, 90, 0);
  86     }
  87
  88     return c;
  89 }
  90
  91 static void
  92 clientdbInit(void)
  93 {
  94     if (client_table)
  95         return;
  96
  97     client_table = hash_create((HASHCMP *) strcmp, CLIENT_DB_HASH_SIZE, hash_string);
  98 }
  99
 100 class ClientDbRr: public RegisteredRunner
 101 {
 102 public:
 103     /* RegisteredRunner API */
 104     virtual void useConfig();
 105 };
 106 RunnerRegistrationEntry(ClientDbRr);
 107
 108 void
 109 ClientDbRr::useConfig()
 110 {
 111     clientdbInit();
 112     Mgr::RegisterAction("client_list", "Cache Client List", clientdbDump, 0, 1);
 113 }
 114
 115 #if USE_DELAY_POOLS
 116 /* returns ClientInfo for given IP addr
 117    Returns NULL if no such client (or clientdb turned off)
 118    (it is assumed that clientdbEstablished will be called before and create client record if needed)
 119 */
 120 ClientInfo * clientdbGetInfo(const Ip::Address &addr)
 121 {
 122     char key[MAX_IPSTRLEN];
 123     ClientInfo *c;
 124
 125     if (!Config.onoff.client_db)
 126         return NULL;
 127
 128     addr.toStr(key,MAX_IPSTRLEN);
 129
 130     c = (ClientInfo *) hash_lookup(client_table, key);
 131     if (c==NULL) {
 132         debugs(77, DBG_IMPORTANT,"Client db does not contain information for given IP address "<<(const char*)key);
 133         return NULL;
 134     }
 135     return c;
 136 }
 137 #endif
 138 void
 139 clientdbUpdate(const Ip::Address &addr, const LogTags &ltype, AnyP::ProtocolType p, size_t size)
 140 {
 141     char key[MAX_IPSTRLEN];
 142     ClientInfo *c;
 143
 144     if (!Config.onoff.client_db)
 145         return;
 146
 147     addr.toStr(key,MAX_IPSTRLEN);
 148
 149     c = (ClientInfo *) hash_lookup(client_table, key);
 150
 151     if (c == NULL)
 152         c = clientdbAdd(addr);
 153
 154     if (c == NULL)
 155         debug_trap("clientdbUpdate: Failed to add entry");
 156
 157     if (p == AnyP::PROTO_HTTP) {
 158         ++ c->Http.n_requests;
 159         ++ c->Http.result_hist[ltype.oldType];
 160         c->Http.kbytes_out += size;
 161
 162         if (ltype.isTcpHit())
 163             c->Http.hit_kbytes_out += size;
 164     } else if (p == AnyP::PROTO_ICP) {
 165         ++ c->Icp.n_requests;
 166         ++ c->Icp.result_hist[ltype.oldType];
 167         c->Icp.kbytes_out += size;
 168
 169         if (LOG_UDP_HIT == ltype.oldType)
 170             c->Icp.hit_kbytes_out += size;
 171     }
 172
 173     c->last_seen = squid_curtime;
 174 }
 175
 176 /**
 177  * This function tracks the number of currently established connections
 178  * for a client IP address.  When a connection is accepted, call this
 179  * with delta = 1.  When the connection is closed, call with delta =
 180  * -1.  To get the current value, simply call with delta = 0.
 181  */
 182 int
 183 clientdbEstablished(const Ip::Address &addr, int delta)
 184 {
 185     char key[MAX_IPSTRLEN];
 186     ClientInfo *c;
 187
 188     if (!Config.onoff.client_db)
 189         return 0;
 190
 191     addr.toStr(key,MAX_IPSTRLEN);
 192
 193     c = (ClientInfo *) hash_lookup(client_table, key);
 194
 195     if (c == NULL) {
 196         c = clientdbAdd(addr);
 197     }
 198
 199     if (c == NULL)
 200         debug_trap("clientdbUpdate: Failed to add entry");
 201
 202     c->n_established += delta;
 203
 204     return c->n_established;
 205 }
 206
 207 #define CUTOFF_SECONDS 3600
 208 int
 209
 210 clientdbCutoffDenied(const Ip::Address &addr)
 211 {
 212     char key[MAX_IPSTRLEN];
 213     int NR;
 214     int ND;
 215     double p;
 216     ClientInfo *c;
 217
 218     if (!Config.onoff.client_db)
 219         return 0;
 220
 221     addr.toStr(key,MAX_IPSTRLEN);
 222
 223     c = (ClientInfo *) hash_lookup(client_table, key);
 224
 225     if (c == NULL)
 226         return 0;
 227
 228     /*
 229      * If we are in a cutoff window, we don't send a reply
 230      */
 231     if (squid_curtime - c->cutoff.time < CUTOFF_SECONDS)
 232         return 1;
 233
 234     /*
 235      * Calculate the percent of DENIED replies since the last
 236      * cutoff time.
 237      */
 238     NR = c->Icp.n_requests - c->cutoff.n_req;
 239
 240     if (NR < 150)
 241         NR = 150;
 242
 243     ND = c->Icp.result_hist[LOG_UDP_DENIED] - c->cutoff.n_denied;
 244
 245     p = 100.0 * ND / NR;
 246
 247     if (p < 95.0)
 248         return 0;
 249
 250     debugs(1, DBG_CRITICAL, "WARNING: Probable misconfigured neighbor at " << key);
 251
 252     debugs(1, DBG_CRITICAL, "WARNING: " << ND << " of the last " << NR <<
 253            " ICP replies are DENIED");
 254
 255     debugs(1, DBG_CRITICAL, "WARNING: No replies will be sent for the next " <<
 256            CUTOFF_SECONDS << " seconds");
 257
 258     c->cutoff.time = squid_curtime;
 259
 260     c->cutoff.n_req = c->Icp.n_requests;
 261
 262     c->cutoff.n_denied = c->Icp.result_hist[LOG_UDP_DENIED];
 263
 264     return 1;
 265 }
 266
 267 void
 268 clientdbDump(StoreEntry * sentry)
 269 {
 270     const char *name;
 271     int icp_total = 0;
 272     int icp_hits = 0;
 273     int http_total = 0;
 274     int http_hits = 0;
 275     storeAppendPrintf(sentry, "Cache Clients:\n");
 276     hash_first(client_table);
 277
 278     while (hash_link *hash = hash_next(client_table)) {
 279         const ClientInfo *c = static_cast<const ClientInfo *>(hash);
 280         storeAppendPrintf(sentry, "Address: %s\n", hashKeyStr(hash));
 281         if ( (name = fqdncache_gethostbyaddr(c->addr, 0)) ) {
 282             storeAppendPrintf(sentry, "Name:    %s\n", name);
 283         }
 284         storeAppendPrintf(sentry, "Currently established connections: %d\n",
 285                           c->n_established);
 286         storeAppendPrintf(sentry, "    ICP  Requests %d\n",
 287                           c->Icp.n_requests);
 288
 289         for (LogTags_ot l = LOG_TAG_NONE; l < LOG_TYPE_MAX; ++l) {
 290             if (c->Icp.result_hist[l] == 0)
 291                 continue;
 292
 293             icp_total += c->Icp.result_hist[l];
 294
 295             if (LOG_UDP_HIT == l)
 296                 icp_hits += c->Icp.result_hist[l];
 297
 298             storeAppendPrintf(sentry, "        %-20.20s %7d %3d%%\n", LogTags(l).c_str(), c->Icp.result_hist[l], Math::intPercent(c->Icp.result_hist[l], c->Icp.n_requests));
 299         }
 300
 301         storeAppendPrintf(sentry, "    HTTP Requests %d\n", c->Http.n_requests);
 302
 303         for (LogTags_ot l = LOG_TAG_NONE; l < LOG_TYPE_MAX; ++l) {
 304             if (c->Http.result_hist[l] == 0)
 305                 continue;
 306
 307             http_total += c->Http.result_hist[l];
 308
 309             if (LogTags(l).isTcpHit())
 310                 http_hits += c->Http.result_hist[l];
 311
 312             storeAppendPrintf(sentry,
 313                               "        %-20.20s %7d %3d%%\n",
 314                               LogTags(l).c_str(),
 315                               c->Http.result_hist[l],
 316                               Math::intPercent(c->Http.result_hist[l], c->Http.n_requests));
 317         }
 318
 319         storeAppendPrintf(sentry, "\n");
 320     }
 321
 322     storeAppendPrintf(sentry, "TOTALS\n");
 323     storeAppendPrintf(sentry, "ICP : %d Queries, %d Hits (%3d%%)\n",
 324                       icp_total, icp_hits, Math::intPercent(icp_hits, icp_total));
 325     storeAppendPrintf(sentry, "HTTP: %d Requests, %d Hits (%3d%%)\n",
 326                       http_total, http_hits, Math::intPercent(http_hits, http_total));
 327 }
 328
 329 static void
 330 clientdbFreeItem(void *data)
 331 {
 332     ClientInfo *c = (ClientInfo *)data;
 333     delete c;
 334 }
 335
 336 ClientInfo::~ClientInfo()
 337 {
 338     safe_free(key);
 339
 340 #if USE_DELAY_POOLS
 341     if (CommQuotaQueue *q = quotaQueue) {
 342         q->clientInfo = NULL;
 343         delete q; // invalidates cbdata, cancelling any pending kicks
 344     }
 345 #endif
 346
 347     debugs(77, 9, "ClientInfo destructed, this=" << static_cast<void*>(this));
 348 }
 349
 350 void
 351 clientdbFreeMemory(void)
 352 {
 353     hashFreeItems(client_table, clientdbFreeItem);
 354     hashFreeMemory(client_table);
 355     client_table = NULL;
 356 }
 357
 358 static void
 359 clientdbScheduledGC(void *)
 360 {
 361     cleanup_scheduled = 0;
 362     clientdbStartGC();
 363 }
 364
 365 static void
 366 clientdbGC(void *)
 367 {
 368     static int bucket = 0;
 369     hash_link *link_next;
 370
 371     link_next = hash_get_bucket(client_table, bucket++);
 372
 373     while (link_next != NULL) {
 374         ClientInfo *c = (ClientInfo *)link_next;
 375         int age = squid_curtime - c->last_seen;
 376         link_next = link_next->next;
 377
 378         if (c->n_established)
 379             continue;
 380
 381         if (age < 24 * 3600 && c->Http.n_requests > 100)
 382             continue;
 383
 384         if (age < 4 * 3600 && (c->Http.n_requests > 10 || c->Icp.n_requests > 10))
 385             continue;
 386
 387         if (age < 5 * 60 && (c->Http.n_requests > 1 || c->Icp.n_requests > 1))
 388             continue;
 389
 390         if (age < 60)
 391             continue;
 392
 393         hash_remove_link(client_table, static_cast<hash_link*>(c));
 394
 395         clientdbFreeItem(c);
 396
 397         --statCounter.client_http.clients;
 398
 399         ++cleanup_removed;
 400     }
 401
 402     if (bucket < CLIENT_DB_HASH_SIZE)
 403         eventAdd("client_db garbage collector", clientdbGC, NULL, 0.15, 0);
 404     else {
 405         bucket = 0;
 406         cleanup_running = 0;
 407         max_clients = statCounter.client_http.clients * 3 / 2;
 408
 409         if (!cleanup_scheduled) {
 410             cleanup_scheduled = 1;
 411             eventAdd("client_db garbage collector", clientdbScheduledGC, NULL, 6 * 3600, 0);
 412         }
 413
 414         debugs(49, 2, "clientdbGC: Removed " << cleanup_removed << " entries");
 415     }
 416 }
 417
 418 static void
 419 clientdbStartGC(void)
 420 {
 421     max_clients = statCounter.client_http.clients;
 422     cleanup_running = 1;
 423     cleanup_removed = 0;
 424     clientdbGC(NULL);
 425 }
 426
 427 #if SQUID_SNMP
 428
 429 Ip::Address *
 430 client_entry(Ip::Address *current)
 431 {
 432     char key[MAX_IPSTRLEN];
 433     hash_first(client_table);
 434
 435     if (current) {
 436         current->toStr(key,MAX_IPSTRLEN);
 437         while (hash_link *hash = hash_next(client_table)) {
 438             if (!strcmp(key, hashKeyStr(hash)))
 439                 break;
 440         }
 441     }
 442
 443     ClientInfo *c = static_cast<ClientInfo *>(hash_next(client_table));
 444
 445     hash_last(client_table);
 446
 447     return c ? &c->addr : nullptr;
 448 }
 449
 450 variable_list *
 451 snmp_meshCtblFn(variable_list * Var, snint * ErrP)
 452 {
 453     char key[MAX_IPSTRLEN];
 454     ClientInfo *c = NULL;
 455     Ip::Address keyIp;
 456
 457     *ErrP = SNMP_ERR_NOERROR;
 458     MemBuf tmp;
 459     debugs(49, 6, HERE << "Current : length=" << Var->name_length << ": " << snmpDebugOid(Var->name, Var->name_length, tmp));
 460     if (Var->name_length == 16) {
 461         oid2addr(&(Var->name[12]), keyIp, 4);
 462     } else if (Var->name_length == 28) {
 463         oid2addr(&(Var->name[12]), keyIp, 16);
 464     } else {
 465         *ErrP = SNMP_ERR_NOSUCHNAME;
 466         return NULL;
 467     }
 468
 469     keyIp.toStr(key, sizeof(key));
 470     debugs(49, 5, HERE << "[" << key << "] requested!");
 471     c = (ClientInfo *) hash_lookup(client_table, key);
 472
 473     if (c == NULL) {
 474         debugs(49, 5, HERE << "not found.");
 475         *ErrP = SNMP_ERR_NOSUCHNAME;
 476         return NULL;
 477     }
 478
 479     variable_list *Answer = NULL;
 480     int aggr = 0;
 481
 482     switch (Var->name[LEN_SQ_NET + 2]) {
 483
 484     case MESH_CTBL_ADDR_TYPE: {
 485         int ival;
 486         ival = c->addr.isIPv4() ? INETADDRESSTYPE_IPV4 : INETADDRESSTYPE_IPV6 ;
 487         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 488                                       ival, SMI_INTEGER);
 489     }
 490     break;
 491
 492     case MESH_CTBL_ADDR: {
 493         Answer = snmp_var_new(Var->name, Var->name_length);
 494         // InetAddress doesn't have its own ASN.1 type,
 495         // like IpAddr does (SMI_IPADDRESS)
 496         // See: rfc4001.txt
 497         Answer->type = ASN_OCTET_STR;
 498         char client[MAX_IPSTRLEN];
 499         c->addr.toStr(client,MAX_IPSTRLEN);
 500         Answer->val_len = strlen(client);
 501         Answer->val.string =  (u_char *) xstrdup(client);
 502     }
 503     break;
 504     case MESH_CTBL_HTBYTES:
 505         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 506                                       (snint) c->Http.kbytes_out.kb,
 507                                       SMI_COUNTER32);
 508         break;
 509
 510     case MESH_CTBL_HTREQ:
 511         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 512                                       (snint) c->Http.n_requests,
 513                                       SMI_COUNTER32);
 514         break;
 515
 516     case MESH_CTBL_HTHITS:
 517         aggr = 0;
 518
 519         for (LogTags_ot l = LOG_TAG_NONE; l < LOG_TYPE_MAX; ++l) {
 520             if (LogTags(l).isTcpHit())
 521                 aggr += c->Http.result_hist[l];
 522         }
 523
 524         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 525                                       (snint) aggr,
 526                                       SMI_COUNTER32);
 527         break;
 528
 529     case MESH_CTBL_HTHITBYTES:
 530         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 531                                       (snint) c->Http.hit_kbytes_out.kb,
 532                                       SMI_COUNTER32);
 533         break;
 534
 535     case MESH_CTBL_ICPBYTES:
 536         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 537                                       (snint) c->Icp.kbytes_out.kb,
 538                                       SMI_COUNTER32);
 539         break;
 540
 541     case MESH_CTBL_ICPREQ:
 542         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 543                                       (snint) c->Icp.n_requests,
 544                                       SMI_COUNTER32);
 545         break;
 546
 547     case MESH_CTBL_ICPHITS:
 548         aggr = c->Icp.result_hist[LOG_UDP_HIT];
 549         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 550                                       (snint) aggr,
 551                                       SMI_COUNTER32);
 552         break;
 553
 554     case MESH_CTBL_ICPHITBYTES:
 555         Answer = snmp_var_new_integer(Var->name, Var->name_length,
 556                                       (snint) c->Icp.hit_kbytes_out.kb,
 557                                       SMI_COUNTER32);
 558         break;
 559
 560     default:
 561         *ErrP = SNMP_ERR_NOSUCHNAME;
 562         debugs(49, 5, "snmp_meshCtblFn: illegal column.");
 563         break;
 564     }
 565
 566     return Answer;
 567 }
 568
 569 #endif /*SQUID_SNMP */
 570