src/external_acl.cc

   1 /*
   2  * Copyright (C) 1996-2023 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 82    External ACL */
  10
  11 #include "squid.h"
  12 #include "acl/Acl.h"
  13 #include "acl/FilledChecklist.h"
  14 #include "cache_cf.h"
  15 #include "client_side.h"
  16 #include "client_side_request.h"
  17 #include "comm/Connection.h"
  18 #include "ConfigParser.h"
  19 #include "ExternalACL.h"
  20 #include "ExternalACLEntry.h"
  21 #include "fde.h"
  22 #include "format/Token.h"
  23 #include "helper.h"
  24 #include "helper/Reply.h"
  25 #include "http/Stream.h"
  26 #include "HttpHeaderTools.h"
  27 #include "HttpReply.h"
  28 #include "HttpRequest.h"
  29 #include "ip/tools.h"
  30 #include "MemBuf.h"
  31 #include "mgr/Registration.h"
  32 #include "rfc1738.h"
  33 #include "SquidConfig.h"
  34 #include "SquidString.h"
  35 #include "Store.h"
  36 #include "tools.h"
  37 #include "wordlist.h"
  38 #if USE_OPENSSL
  39 #include "ssl/ServerBump.h"
  40 #include "ssl/support.h"
  41 #endif
  42 #if USE_AUTH
  43 #include "auth/Acl.h"
  44 #include "auth/Gadgets.h"
  45 #include "auth/UserRequest.h"
  46 #endif
  47
  48 #ifndef DEFAULT_EXTERNAL_ACL_TTL
  49 #define DEFAULT_EXTERNAL_ACL_TTL 1 * 60 * 60
  50 #endif
  51 #ifndef DEFAULT_EXTERNAL_ACL_CHILDREN
  52 #define DEFAULT_EXTERNAL_ACL_CHILDREN 5
  53 #endif
  54
  55 static void external_acl_cache_delete(external_acl * def, const ExternalACLEntryPointer &entry);
  56 static int external_acl_entry_expired(external_acl * def, const ExternalACLEntryPointer &entry);
  57 static int external_acl_grace_expired(external_acl * def, const ExternalACLEntryPointer &entry);
  58 static void external_acl_cache_touch(external_acl * def, const ExternalACLEntryPointer &entry);
  59 static ExternalACLEntryPointer external_acl_cache_add(external_acl * def, const char *key, ExternalACLEntryData const &data);
  60
  61 /******************************************************************
  62  * external_acl directive
  63  */
  64
  65 class external_acl
  66 {
  67     /* XXX: These are not really cbdata, but it is an easy way
  68      * to get them pooled, refcounted, accounted and freed properly...
  69      * Use RefCountable MEMPROXY_CLASS instead
  70      */
  71     CBDATA_CLASS(external_acl);
  72
  73 public:
  74     external_acl();
  75     ~external_acl();
  76
  77     external_acl *next;
  78
  79     void add(const ExternalACLEntryPointer &);
  80
  81     void trimCache();
  82
  83     bool maybeCacheable(const Acl::Answer &) const;
  84
  85     int ttl;
  86
  87     int negative_ttl;
  88
  89     int grace;
  90
  91     char *name;
  92
  93     Format::Format format;
  94
  95     wordlist *cmdline;
  96
  97     Helper::ChildConfig children;
  98
  99     Helper::Client::Pointer theHelper;
 100
 101     hash_table *cache;
 102
 103     dlink_list lru_list;
 104
 105     int cache_size;
 106
 107     int cache_entries;
 108
 109     dlink_list queue;
 110
 111 #if USE_AUTH
 112     /**
 113      * Configuration flag. May only be altered by the configuration parser.
 114      *
 115      * Indicates that all uses of this external_acl_type helper require authentication
 116      * details to be processed. If none are available its a fail match.
 117      */
 118     bool require_auth;
 119 #endif
 120
 121     Format::Quoting quote; // default quoting to use, set by protocol= parameter
 122
 123     Ip::Address local_addr;
 124 };
 125
 126 CBDATA_CLASS_INIT(external_acl);
 127
 128 external_acl::external_acl() :
 129     next(nullptr),
 130     ttl(DEFAULT_EXTERNAL_ACL_TTL),
 131     negative_ttl(-1),
 132     grace(1),
 133     name(nullptr),
 134     format("external_acl_type"),
 135     cmdline(nullptr),
 136     children(DEFAULT_EXTERNAL_ACL_CHILDREN),
 137     theHelper(nullptr),
 138     cache(nullptr),
 139     cache_size(256*1024),
 140     cache_entries(0),
 141 #if USE_AUTH
 142     require_auth(0),
 143 #endif
 144     quote(Format::LOG_QUOTE_URL)
 145 {
 146     local_addr.setLocalhost();
 147 }
 148
 149 external_acl::~external_acl()
 150 {
 151     xfree(name);
 152     wordlistDestroy(&cmdline);
 153
 154     if (theHelper) {
 155         helperShutdown(theHelper);
 156         theHelper = nullptr;
 157     }
 158
 159     while (lru_list.tail) {
 160         ExternalACLEntryPointer e(static_cast<ExternalACLEntry *>(lru_list.tail->data));
 161         external_acl_cache_delete(this, e);
 162     }
 163     if (cache)
 164         hashFreeMemory(cache);
 165
 166     while (next) {
 167         external_acl *node = next;
 168         next = node->next;
 169         node->next = nullptr; // prevent recursion
 170         delete node;
 171     }
 172 }
 173
 174 void
 175 parse_externalAclHelper(external_acl ** list)
 176 {
 177     char *token = ConfigParser::NextToken();
 178
 179     if (!token) {
 180         self_destruct();
 181         return;
 182     }
 183
 184     external_acl *a = new external_acl;
 185     a->name = xstrdup(token);
 186
 187     // Allow supported %macros inside quoted tokens
 188     ConfigParser::EnableMacros();
 189     token = ConfigParser::NextToken();
 190
 191     /* Parse options */
 192     while (token) {
 193         if (strncmp(token, "ttl=", 4) == 0) {
 194             a->ttl = atoi(token + 4);
 195         } else if (strncmp(token, "negative_ttl=", 13) == 0) {
 196             a->negative_ttl = atoi(token + 13);
 197         } else if (strncmp(token, "children=", 9) == 0) {
 198             a->children.n_max = atoi(token + 9);
 199             debugs(0, DBG_CRITICAL, "WARNING: external_acl_type option children=N has been deprecated in favor of children-max=N and children-startup=N");
 200         } else if (strncmp(token, "children-max=", 13) == 0) {
 201             a->children.n_max = atoi(token + 13);
 202         } else if (strncmp(token, "children-startup=", 17) == 0) {
 203             a->children.n_startup = atoi(token + 17);
 204         } else if (strncmp(token, "children-idle=", 14) == 0) {
 205             a->children.n_idle = atoi(token + 14);
 206         } else if (strncmp(token, "concurrency=", 12) == 0) {
 207             a->children.concurrency = atoi(token + 12);
 208         } else if (strncmp(token, "queue-size=", 11) == 0) {
 209             a->children.queue_size = atoi(token + 11);
 210             a->children.defaultQueueSize = false;
 211         } else if (strncmp(token, "cache=", 6) == 0) {
 212             a->cache_size = atoi(token + 6);
 213         } else if (strncmp(token, "grace=", 6) == 0) {
 214             a->grace = atoi(token + 6);
 215         } else if (strcmp(token, "protocol=2.5") == 0) {
 216             a->quote = Format::LOG_QUOTE_SHELL;
 217         } else if (strcmp(token, "protocol=3.0") == 0) {
 218             debugs(3, DBG_PARSE_NOTE(2), "WARNING: external_acl_type option protocol=3.0 is deprecated. Remove this from your config.");
 219             a->quote = Format::LOG_QUOTE_URL;
 220         } else if (strcmp(token, "quote=url") == 0) {
 221             debugs(3, DBG_PARSE_NOTE(2), "WARNING: external_acl_type option quote=url is deprecated. Remove this from your config.");
 222             a->quote = Format::LOG_QUOTE_URL;
 223         } else if (strcmp(token, "quote=shell") == 0) {
 224             debugs(3, DBG_PARSE_NOTE(2), "WARNING: external_acl_type option quote=shell is deprecated. Use protocol=2.5 if still needed.");
 225             a->quote = Format::LOG_QUOTE_SHELL;
 226
 227             /* INET6: allow admin to configure some helpers explicitly to
 228                       bind to IPv4/v6 localhost port. */
 229         } else if (strcmp(token, "ipv4") == 0) {
 230             if ( !a->local_addr.setIPv4() ) {
 231                 debugs(3, DBG_CRITICAL, "WARNING: Error converting " << a->local_addr << " to IPv4 in " << a->name );
 232             }
 233         } else if (strcmp(token, "ipv6") == 0) {
 234             if (!Ip::EnableIpv6)
 235                 debugs(3, DBG_CRITICAL, "WARNING: --enable-ipv6 required for external ACL helpers to use IPv6: " << a->name );
 236             // else nothing to do.
 237         } else {
 238             break;
 239         }
 240
 241         token = ConfigParser::NextToken();
 242     }
 243     ConfigParser::DisableMacros();
 244
 245     /* check that child startup value is sane. */
 246     if (a->children.n_startup > a->children.n_max)
 247         a->children.n_startup = a->children.n_max;
 248
 249     /* check that child idle value is sane. */
 250     if (a->children.n_idle > a->children.n_max)
 251         a->children.n_idle = a->children.n_max;
 252     if (a->children.n_idle < 1)
 253         a->children.n_idle = 1;
 254
 255     if (a->negative_ttl == -1)
 256         a->negative_ttl = a->ttl;
 257
 258     if (a->children.defaultQueueSize)
 259         a->children.queue_size = 2 * a->children.n_max;
 260
 261     /* Legacy external_acl_type format parser.
 262      * Handles a series of %... tokens where any non-% means
 263      * the start of another parameter field (ie the path to binary).
 264      */
 265     enum Format::Quoting quote = Format::LOG_QUOTE_NONE;
 266     Format::Token **fmt = &a->format.format;
 267     bool data_used = false;
 268     while (token) {
 269         /* stop on first non-% token found */
 270         if (*token != '%')
 271             break;
 272
 273         *fmt = new Format::Token;
 274         // these tokens are whitespace delimited
 275         (*fmt)->space = true;
 276
 277         // set the default encoding to match the protocol= config
 278         // this will be overridden by explicit %macro attributes
 279         (*fmt)->quote = a->quote;
 280
 281         // compatibility for old tokens incompatible with Format::Token syntax
 282 #if USE_OPENSSL // do not bother unless we have to.
 283         if (strncmp(token, "%USER_CERT_", 11) == 0) {
 284             (*fmt)->type = Format::LFT_EXT_ACL_USER_CERT;
 285             (*fmt)->data.string = xstrdup(token + 11);
 286             (*fmt)->data.header.header = (*fmt)->data.string;
 287         } else if (strncmp(token, "%USER_CA_CERT_", 14) == 0) {
 288             (*fmt)->type = Format::LFT_EXT_ACL_USER_CA_CERT;
 289             (*fmt)->data.string = xstrdup(token + 14);
 290             (*fmt)->data.header.header = (*fmt)->data.string;
 291         } else if (strncmp(token, "%CA_CERT_", 9) == 0) {
 292             debugs(82, DBG_PARSE_NOTE(DBG_IMPORTANT), "WARNING: external_acl_type %CA_CERT_* code is obsolete. Use %USER_CA_CERT_* instead");
 293             (*fmt)->type = Format::LFT_EXT_ACL_USER_CA_CERT;
 294             (*fmt)->data.string = xstrdup(token + 9);
 295             (*fmt)->data.header.header = (*fmt)->data.string;
 296         } else
 297 #endif
 298             if (strncmp(token,"%<{", 3) == 0) {
 299                 SBuf tmp("%<h");
 300                 tmp.append(token+2);
 301                 debugs(82, DBG_PARSE_NOTE(DBG_IMPORTANT), "WARNING: external_acl_type format %<{...} is deprecated. Use " << tmp);
 302                 const size_t parsedLen = (*fmt)->parse(tmp.c_str(), &quote);
 303                 assert(parsedLen == tmp.length());
 304                 assert((*fmt)->type == Format::LFT_REPLY_HEADER ||
 305                        (*fmt)->type == Format::LFT_REPLY_HEADER_ELEM);
 306
 307             } else if (strncmp(token,"%>{", 3) == 0) {
 308                 SBuf tmp("%>ha");
 309                 tmp.append(token+2);
 310                 debugs(82, DBG_PARSE_NOTE(DBG_IMPORTANT), "WARNING: external_acl_type format %>{...} is deprecated. Use " << tmp);
 311                 const size_t parsedLen = (*fmt)->parse(tmp.c_str(), &quote);
 312                 assert(parsedLen == tmp.length());
 313                 assert((*fmt)->type == Format::LFT_ADAPTED_REQUEST_HEADER ||
 314                        (*fmt)->type == Format::LFT_ADAPTED_REQUEST_HEADER_ELEM);
 315
 316             } else {
 317                 // we can use the Format::Token::parse() method since it
 318                 // only pulls off one token. Since we already checked
 319                 // for '%' prefix above this is guaranteed to be a token.
 320                 const size_t len = (*fmt)->parse(token, &quote);
 321                 assert(len == strlen(token));
 322             }
 323
 324         // process special token-specific actions (only if necessary)
 325 #if USE_AUTH
 326         if ((*fmt)->type == Format::LFT_USER_LOGIN)
 327             a->require_auth = true;
 328 #endif
 329
 330         if ((*fmt)->type == Format::LFT_EXT_ACL_DATA)
 331             data_used = true;
 332
 333         fmt = &((*fmt)->next);
 334         token = ConfigParser::NextToken();
 335     }
 336
 337     /* There must be at least one format token */
 338     if (!a->format.format) {
 339         delete a;
 340         self_destruct();
 341         return;
 342     }
 343
 344     // format has implicit %DATA on the end if not used explicitly
 345     if (!data_used) {
 346         *fmt = new Format::Token;
 347         (*fmt)->type = Format::LFT_EXT_ACL_DATA;
 348         (*fmt)->quote = Format::LOG_QUOTE_NONE;
 349     }
 350
 351     /* helper */
 352     if (!token) {
 353         delete a;
 354         self_destruct();
 355         return;
 356     }
 357
 358     wordlistAdd(&a->cmdline, token);
 359
 360     /* arguments */
 361     parse_wordlist(&a->cmdline);
 362
 363     while (*list)
 364         list = &(*list)->next;
 365
 366     *list = a;
 367 }
 368
 369 void
 370 dump_externalAclHelper(StoreEntry * sentry, const char *name, const external_acl * list)
 371 {
 372     const external_acl *node;
 373     const wordlist *word;
 374
 375     for (node = list; node; node = node->next) {
 376         storeAppendPrintf(sentry, "%s %s", name, node->name);
 377
 378         if (!node->local_addr.isIPv6())
 379             storeAppendPrintf(sentry, " ipv4");
 380         else
 381             storeAppendPrintf(sentry, " ipv6");
 382
 383         if (node->ttl != DEFAULT_EXTERNAL_ACL_TTL)
 384             storeAppendPrintf(sentry, " ttl=%d", node->ttl);
 385
 386         if (node->negative_ttl != node->ttl)
 387             storeAppendPrintf(sentry, " negative_ttl=%d", node->negative_ttl);
 388
 389         if (node->grace)
 390             storeAppendPrintf(sentry, " grace=%d", node->grace);
 391
 392         if (node->children.n_max != DEFAULT_EXTERNAL_ACL_CHILDREN)
 393             storeAppendPrintf(sentry, " children-max=%d", node->children.n_max);
 394
 395         if (node->children.n_startup != 0) // sync with helper/ChildConfig.cc default
 396             storeAppendPrintf(sentry, " children-startup=%d", node->children.n_startup);
 397
 398         if (node->children.n_idle != 1) // sync with helper/ChildConfig.cc default
 399             storeAppendPrintf(sentry, " children-idle=%d", node->children.n_idle);
 400
 401         if (node->children.concurrency != 0)
 402             storeAppendPrintf(sentry, " concurrency=%d", node->children.concurrency);
 403
 404         if (node->cache)
 405             storeAppendPrintf(sentry, " cache=%d", node->cache_size);
 406
 407         if (node->quote == Format::LOG_QUOTE_SHELL)
 408             storeAppendPrintf(sentry, " protocol=2.5");
 409
 410         node->format.dump(sentry, nullptr, false);
 411
 412         for (word = node->cmdline; word; word = word->next)
 413             storeAppendPrintf(sentry, " %s", word->key);
 414
 415         storeAppendPrintf(sentry, "\n");
 416     }
 417 }
 418
 419 void
 420 free_externalAclHelper(external_acl ** list)
 421 {
 422     delete *list;
 423     *list = nullptr;
 424 }
 425
 426 static external_acl *
 427 find_externalAclHelper(const char *name)
 428 {
 429     external_acl *node;
 430
 431     for (node = Config.externalAclHelperList; node; node = node->next) {
 432         if (strcmp(node->name, name) == 0)
 433             return node;
 434     }
 435
 436     return nullptr;
 437 }
 438
 439 void
 440 external_acl::add(const ExternalACLEntryPointer &anEntry)
 441 {
 442     trimCache();
 443     assert(anEntry != nullptr);
 444     assert (anEntry->def == nullptr);
 445     anEntry->def = this;
 446     ExternalACLEntry *e = const_cast<ExternalACLEntry *>(anEntry.getRaw()); // XXX: make hash a std::map of Pointer.
 447     hash_join(cache, e);
 448     dlinkAdd(e, &e->lru, &lru_list);
 449     e->lock(); //cbdataReference(e); // lock it on behalf of the hash
 450     ++cache_entries;
 451 }
 452
 453 void
 454 external_acl::trimCache()
 455 {
 456     if (cache_size && cache_entries >= cache_size) {
 457         ExternalACLEntryPointer e(static_cast<ExternalACLEntry *>(lru_list.tail->data));
 458         external_acl_cache_delete(this, e);
 459     }
 460 }
 461
 462 bool
 463 external_acl::maybeCacheable(const Acl::Answer &result) const
 464 {
 465     if (cache_size <= 0)
 466         return false; // cache is disabled
 467
 468     if (result == ACCESS_DUNNO)
 469         return false; // non-cacheable response
 470
 471     if ((result.allowed() ? ttl : negative_ttl) <= 0)
 472         return false; // not caching this type of response
 473
 474     return true;
 475 }
 476
 477 /******************************************************************
 478  * external acl type
 479  */
 480
 481 class external_acl_data
 482 {
 483     CBDATA_CLASS(external_acl_data);
 484
 485 public:
 486     explicit external_acl_data(external_acl * const aDef): def(cbdataReference(aDef)), arguments(nullptr) {}
 487     ~external_acl_data();
 488
 489     external_acl *def;
 490     SBuf name;
 491     wordlist *arguments;
 492 };
 493
 494 CBDATA_CLASS_INIT(external_acl_data);
 495
 496 external_acl_data::~external_acl_data()
 497 {
 498     wordlistDestroy(&arguments);
 499     cbdataReferenceDone(def);
 500 }
 501
 502 void
 503 ACLExternal::parse()
 504 {
 505     if (data) {
 506         self_destruct();
 507         return;
 508     }
 509
 510     char *token = ConfigParser::strtokFile();
 511
 512     if (!token) {
 513         self_destruct();
 514         return;
 515     }
 516
 517     data = new external_acl_data(find_externalAclHelper(token));
 518
 519     if (!data->def) {
 520         delete data;
 521         self_destruct();
 522         return;
 523     }
 524
 525     // def->name is the name of the external_acl_type.
 526     // this is the name of the 'acl' directive being tested
 527     data->name = name;
 528
 529     while ((token = ConfigParser::strtokFile())) {
 530         wordlistAdd(&data->arguments, token);
 531     }
 532 }
 533
 534 bool
 535 ACLExternal::valid () const
 536 {
 537 #if USE_AUTH
 538     if (data->def->require_auth) {
 539         if (authenticateSchemeCount() == 0) {
 540             debugs(28, DBG_CRITICAL, "ERROR: Cannot use proxy auth because no authentication schemes were compiled.");
 541             return false;
 542         }
 543
 544         if (authenticateActiveSchemeCount() == 0) {
 545             debugs(28, DBG_CRITICAL, "ERROR: Cannot use proxy auth because no authentication schemes are fully configured.");
 546             return false;
 547         }
 548     }
 549 #endif
 550
 551     return true;
 552 }
 553
 554 bool
 555 ACLExternal::empty () const
 556 {
 557     return false;
 558 }
 559
 560 ACLExternal::~ACLExternal()
 561 {
 562     delete data;
 563     xfree(class_);
 564 }
 565
 566 static void
 567 copyResultsFromEntry(const HttpRequest::Pointer &req, const ExternalACLEntryPointer &entry)
 568 {
 569     if (req) {
 570 #if USE_AUTH
 571         if (entry->user.size())
 572             req->extacl_user = entry->user;
 573
 574         if (entry->password.size())
 575             req->extacl_passwd = entry->password;
 576 #endif
 577         if (!req->tag.size())
 578             req->tag = entry->tag;
 579
 580         if (entry->log.size())
 581             req->extacl_log = entry->log;
 582
 583         if (entry->message.size())
 584             req->extacl_message = entry->message;
 585
 586         // attach the helper kv-pair to the transaction
 587         UpdateRequestNotes(req->clientConnectionManager.get(), *req, entry->notes);
 588     }
 589 }
 590
 591 // TODO: Diff reduction. Rename this helper method to match_() or similar.
 592 Acl::Answer
 593 ACLExternal::aclMatchExternal(external_acl_data *acl, ACLFilledChecklist *ch) const
 594 {
 595     debugs(82, 9, "acl=\"" << acl->def->name << "\"");
 596     ExternalACLEntryPointer entry = ch->extacl_entry;
 597
 598     external_acl_message = "MISSING REQUIRED INFORMATION";
 599
 600     if (entry != nullptr) {
 601         if (entry->def == acl->def) {
 602             /* Ours, use it.. if the key matches */
 603             const char *key = makeExternalAclKey(ch, acl);
 604             if (!key)
 605                 return ACCESS_DUNNO; // insufficient data to continue
 606             if (strcmp(key, (char*)entry->key) != 0) {
 607                 debugs(82, 9, "entry key='" << (char *)entry->key << "', our key='" << key << "' do not match. Discarded.");
 608                 // too bad. need a new lookup.
 609                 entry = ch->extacl_entry = nullptr;
 610             }
 611         } else {
 612             /* Not ours.. get rid of it */
 613             debugs(82, 9, "entry " << entry << " not valid or not ours. Discarded.");
 614             if (entry != nullptr) {
 615                 debugs(82, 9, "entry def=" << entry->def << ", our def=" << acl->def);
 616                 const char *key = makeExternalAclKey(ch, acl); // may be nil
 617                 debugs(82, 9, "entry key='" << (char *)entry->key << "', our key='" << key << "'");
 618             }
 619             entry = ch->extacl_entry = nullptr;
 620         }
 621     }
 622
 623     if (!entry) {
 624         debugs(82, 9, "No helper entry available");
 625 #if USE_AUTH
 626         if (acl->def->require_auth) {
 627             /* Make sure the user is authenticated */
 628             debugs(82, 3, acl->def->name << " check user authenticated.");
 629             const auto ti = AuthenticateAcl(ch, *this);
 630             if (!ti.allowed()) {
 631                 debugs(82, 2, acl->def->name << " user not authenticated (" << ti << ")");
 632                 return ti;
 633             }
 634             debugs(82, 3, acl->def->name << " user is authenticated.");
 635         }
 636 #endif
 637         const char *key = makeExternalAclKey(ch, acl);
 638
 639         if (!key) {
 640             /* Not sufficient data to process */
 641             return ACCESS_DUNNO;
 642         }
 643
 644         entry = static_cast<ExternalACLEntry *>(hash_lookup(acl->def->cache, key));
 645
 646         const ExternalACLEntryPointer staleEntry = entry;
 647         if (entry != nullptr && external_acl_entry_expired(acl->def, entry))
 648             entry = nullptr;
 649
 650         if (entry != nullptr && external_acl_grace_expired(acl->def, entry)) {
 651             // refresh in the background
 652             startLookup(ch, acl, true);
 653             debugs(82, 4, "no need to wait for the refresh of '" <<
 654                    key << "' in '" << acl->def->name << "' (ch=" << ch << ").");
 655         }
 656
 657         if (!entry) {
 658             debugs(82, 2, acl->def->name << "(\"" << key << "\") = lookup needed");
 659
 660             // TODO: All other helpers allow temporary overload. Should not we?
 661             if (!acl->def->theHelper->willOverload()) {
 662                 debugs(82, 2, "\"" << key << "\": queueing a call.");
 663                 if (!ch->goAsync(StartLookup, *this))
 664                     debugs(82, 2, "\"" << key << "\": no async support!");
 665                 debugs(82, 2, "\"" << key << "\": return -1.");
 666                 return ACCESS_DUNNO; // expired cached or simply absent entry
 667             } else {
 668                 if (!staleEntry) {
 669                     debugs(82, DBG_IMPORTANT, "WARNING: external ACL '" << acl->def->name <<
 670                            "' queue full. Request rejected '" << key << "'.");
 671                     external_acl_message = "SYSTEM TOO BUSY, TRY AGAIN LATER";
 672                     return ACCESS_DUNNO;
 673                 } else {
 674                     debugs(82, DBG_IMPORTANT, "WARNING: external ACL '" << acl->def->name <<
 675                            "' queue full. Using stale result. '" << key << "'.");
 676                     entry = staleEntry;
 677                     /* Fall thru to processing below */
 678                 }
 679             }
 680         }
 681     }
 682
 683     debugs(82, 4, "entry = { date=" <<
 684            (long unsigned int) entry->date <<
 685            ", result=" << entry->result <<
 686            " tag=" << entry->tag <<
 687            " log=" << entry->log << " }");
 688 #if USE_AUTH
 689     debugs(82, 4, "entry user=" << entry->user);
 690 #endif
 691
 692     external_acl_cache_touch(acl->def, entry);
 693     external_acl_message = entry->message.termedBuf();
 694
 695     debugs(82, 2, acl->def->name << " = " << entry->result);
 696     copyResultsFromEntry(ch->request, entry);
 697     return entry->result;
 698 }
 699
 700 int
 701 ACLExternal::match(ACLChecklist *checklist)
 702 {
 703     auto answer = aclMatchExternal(data, Filled(checklist));
 704
 705     // convert to tri-state ACL match 1,0,-1
 706     switch (answer) {
 707     case ACCESS_ALLOWED:
 708         return 1; // match
 709
 710     case ACCESS_DENIED:
 711         return 0; // non-match
 712
 713     case ACCESS_DUNNO:
 714     case ACCESS_AUTH_REQUIRED:
 715     default:
 716         // If the answer is not allowed or denied (matches/not matches) and
 717         // async authentication is not in progress, then we are done.
 718         if (checklist->keepMatching())
 719             checklist->markFinished(answer, "aclMatchExternal exception");
 720         return -1; // other
 721     }
 722 }
 723
 724 SBufList
 725 ACLExternal::dump() const
 726 {
 727     external_acl_data const *acl = data;
 728     SBufList rv;
 729     rv.push_back(SBuf(acl->def->name));
 730
 731     for (wordlist *arg = acl->arguments; arg; arg = arg->next) {
 732         SBuf s;
 733         s.Printf(" %s", arg->key);
 734         rv.push_back(s);
 735     }
 736
 737     return rv;
 738 }
 739
 740 /******************************************************************
 741  * external_acl cache
 742  */
 743
 744 static void
 745 external_acl_cache_touch(external_acl * def, const ExternalACLEntryPointer &entry)
 746 {
 747     // this must not be done when nothing is being cached.
 748     if (!def->maybeCacheable(entry->result))
 749         return;
 750
 751     dlinkDelete(&entry->lru, &def->lru_list);
 752     ExternalACLEntry *e = const_cast<ExternalACLEntry *>(entry.getRaw()); // XXX: make hash a std::map of Pointer.
 753     dlinkAdd(e, &entry->lru, &def->lru_list);
 754 }
 755
 756 char *
 757 ACLExternal::makeExternalAclKey(ACLFilledChecklist * ch, external_acl_data * acl_data) const
 758 {
 759     static MemBuf mb;
 760     mb.reset();
 761
 762     // check for special case tokens in the format
 763     for (Format::Token *t = acl_data->def->format.format; t ; t = t->next) {
 764
 765         if (t->type == Format::LFT_EXT_ACL_NAME) {
 766             // setup for %ACL
 767             ch->al->lastAclName = acl_data->name;
 768         }
 769
 770         if (t->type == Format::LFT_EXT_ACL_DATA) {
 771             // setup string for %DATA
 772             SBuf sb;
 773             for (auto arg = acl_data->arguments; arg; arg = arg->next) {
 774                 if (sb.length())
 775                     sb.append(" ", 1);
 776
 777                 if (acl_data->def->quote == Format::LOG_QUOTE_URL) {
 778                     const char *quoted = rfc1738_escape(arg->key);
 779                     sb.append(quoted, strlen(quoted));
 780                 } else {
 781                     static MemBuf mb2;
 782                     mb2.init();
 783                     strwordquote(&mb2, arg->key);
 784                     sb.append(mb2.buf, mb2.size);
 785                     mb2.clean();
 786                 }
 787             }
 788
 789             ch->al->lastAclData = sb;
 790         }
 791     }
 792
 793     // assemble the full helper lookup string
 794     acl_data->def->format.assemble(mb, ch->al, 0);
 795
 796     return mb.buf;
 797 }
 798
 799 static int
 800 external_acl_entry_expired(external_acl * def, const ExternalACLEntryPointer &entry)
 801 {
 802     if (def->cache_size <= 0 || entry->result == ACCESS_DUNNO)
 803         return 1;
 804
 805     if (entry->date + (entry->result.allowed() ? def->ttl : def->negative_ttl) < squid_curtime)
 806         return 1;
 807     else
 808         return 0;
 809 }
 810
 811 static int
 812 external_acl_grace_expired(external_acl * def, const ExternalACLEntryPointer &entry)
 813 {
 814     if (def->cache_size <= 0 || entry->result == ACCESS_DUNNO)
 815         return 1;
 816
 817     int ttl;
 818     ttl = entry->result.allowed() ? def->ttl : def->negative_ttl;
 819     ttl = (ttl * (100 - def->grace)) / 100;
 820
 821     if (entry->date + ttl <= squid_curtime)
 822         return 1;
 823     else
 824         return 0;
 825 }
 826
 827 static ExternalACLEntryPointer
 828 external_acl_cache_add(external_acl * def, const char *key, ExternalACLEntryData const & data)
 829 {
 830     ExternalACLEntryPointer entry;
 831
 832     if (!def->maybeCacheable(data.result)) {
 833         debugs(82,6, MYNAME);
 834
 835         if (data.result == ACCESS_DUNNO) {
 836             if (const ExternalACLEntryPointer oldentry = static_cast<ExternalACLEntry *>(hash_lookup(def->cache, key)))
 837                 external_acl_cache_delete(def, oldentry);
 838         }
 839         entry = new ExternalACLEntry;
 840         entry->key = xstrdup(key);
 841         entry->update(data);
 842         entry->def = def;
 843         return entry;
 844     }
 845
 846     entry = static_cast<ExternalACLEntry *>(hash_lookup(def->cache, key));
 847     debugs(82, 2, "external_acl_cache_add: Adding '" << key << "' = " << data.result);
 848
 849     if (entry != nullptr) {
 850         debugs(82, 3, "updating existing entry");
 851         entry->update(data);
 852         external_acl_cache_touch(def, entry);
 853         return entry;
 854     }
 855
 856     entry = new ExternalACLEntry;
 857     entry->key = xstrdup(key);
 858     entry->update(data);
 859
 860     def->add(entry);
 861
 862     return entry;
 863 }
 864
 865 static void
 866 external_acl_cache_delete(external_acl * def, const ExternalACLEntryPointer &entry)
 867 {
 868     assert(entry != nullptr);
 869     assert(def->cache_size > 0 && entry->def == def);
 870     ExternalACLEntry *e = const_cast<ExternalACLEntry *>(entry.getRaw()); // XXX: make hash a std::map of Pointer.
 871     hash_remove_link(def->cache, e);
 872     dlinkDelete(&e->lru, &def->lru_list);
 873     e->unlock(); // unlock on behalf of the hash
 874     def->cache_entries -= 1;
 875 }
 876
 877 /******************************************************************
 878  * external_acl helpers
 879  */
 880
 881 class externalAclState
 882 {
 883     CBDATA_CLASS(externalAclState);
 884
 885 public:
 886     externalAclState(external_acl* aDef, const char *aKey) :
 887         callback(nullptr),
 888         callback_data(nullptr),
 889         key(xstrdup(aKey)),
 890         def(cbdataReference(aDef)),
 891         queue(nullptr)
 892     {}
 893     ~externalAclState();
 894
 895     EAH *callback;
 896     void *callback_data;
 897     char *key;
 898     external_acl *def;
 899     dlink_node list;
 900     externalAclState *queue;
 901 };
 902
 903 CBDATA_CLASS_INIT(externalAclState);
 904
 905 externalAclState::~externalAclState()
 906 {
 907     xfree(key);
 908     cbdataReferenceDone(callback_data);
 909     cbdataReferenceDone(def);
 910 }
 911
 912 /*
 913  * The helper program receives queries on stdin, one
 914  * per line, and must return the result on on stdout
 915  *
 916  * General result syntax:
 917  *
 918  *   OK/ERR keyword=value ...
 919  *
 920  * Keywords:
 921  *
 922  *   user=      The users name (login)
 923  *   message=   Message describing the reason
 924  *   tag=   A string tag to be applied to the request that triggered the acl match.
 925  *          applies to both OK and ERR responses.
 926  *          Won't override existing request tags.
 927  *   log=   A string to be used in access logging
 928  *
 929  * Other keywords may be added to the protocol later
 930  *
 931  * value needs to be URL-encoded or enclosed in double quotes (")
 932  * with \-escaping on any whitespace, quotes, or slashes (\).
 933  */
 934 static void
 935 externalAclHandleReply(void *data, const Helper::Reply &reply)
 936 {
 937     externalAclState *state = static_cast<externalAclState *>(data);
 938     externalAclState *next;
 939     ExternalACLEntryData entryData;
 940
 941     debugs(82, 2, "reply=" << reply);
 942
 943     if (reply.result == Helper::Okay)
 944         entryData.result = ACCESS_ALLOWED;
 945     else if (reply.result == Helper::Error)
 946         entryData.result = ACCESS_DENIED;
 947     else //BrokenHelper,TimedOut or Unknown. Should not cached.
 948         entryData.result = ACCESS_DUNNO;
 949
 950     // XXX: make entryData store a proper Helper::Reply object instead of copying.
 951
 952     entryData.notes.append(&reply.notes);
 953
 954     const char *label = reply.notes.findFirst("tag");
 955     if (label != nullptr && *label != '\0')
 956         entryData.tag = label;
 957
 958     label = reply.notes.findFirst("message");
 959     if (label != nullptr && *label != '\0')
 960         entryData.message = label;
 961
 962     label = reply.notes.findFirst("log");
 963     if (label != nullptr && *label != '\0')
 964         entryData.log = label;
 965
 966 #if USE_AUTH
 967     label = reply.notes.findFirst("user");
 968     if (label != nullptr && *label != '\0')
 969         entryData.user = label;
 970
 971     label = reply.notes.findFirst("password");
 972     if (label != nullptr && *label != '\0')
 973         entryData.password = label;
 974 #endif
 975
 976     // XXX: This state->def access conflicts with the cbdata validity check
 977     // below.
 978     dlinkDelete(&state->list, &state->def->queue);
 979
 980     ExternalACLEntryPointer entry;
 981     if (cbdataReferenceValid(state->def))
 982         entry = external_acl_cache_add(state->def, state->key, entryData);
 983
 984     do {
 985         void *cbdata;
 986         if (state->callback && cbdataReferenceValidDone(state->callback_data, &cbdata))
 987             state->callback(cbdata, entry);
 988
 989         next = state->queue;
 990         state->queue = nullptr;
 991
 992         delete state;
 993
 994         state = next;
 995     } while (state);
 996 }
 997
 998 /// Asks the helper (if needed) or returns the [cached] result (otherwise).
 999 /// Does not support "background" lookups. See also: ACLExternal::Start().
1000 void
1001 ACLExternal::StartLookup(ACLFilledChecklist &checklist, const Acl::Node &acl)
1002 {
1003     const auto &me = dynamic_cast<const ACLExternal&>(acl);
1004     me.startLookup(&checklist, me.data, false);
1005 }
1006
1007 // If possible, starts an asynchronous lookup of an external ACL.
1008 // Otherwise, asserts (or bails if background refresh is requested).
1009 void
1010 ACLExternal::startLookup(ACLFilledChecklist *ch, external_acl_data *acl, bool inBackground) const
1011 {
1012     external_acl *def = acl->def;
1013
1014     const char *key = makeExternalAclKey(ch, acl);
1015     assert(key);
1016
1017     debugs(82, 2, (inBackground ? "bg" : "fg") << " lookup in '" <<
1018            def->name << "' for '" << key << "'");
1019
1020     /* Check for a pending lookup to hook into */
1021     // only possible if we are caching results.
1022     externalAclState *oldstate = nullptr;
1023     if (def->cache_size > 0) {
1024         for (dlink_node *node = def->queue.head; node; node = node->next) {
1025             externalAclState *oldstatetmp = static_cast<externalAclState *>(node->data);
1026
1027             if (strcmp(key, oldstatetmp->key) == 0) {
1028                 oldstate = oldstatetmp;
1029                 break;
1030             }
1031         }
1032     }
1033
1034     // A background refresh has no need to piggiback on a pending request:
1035     // When the pending request completes, the cache will be refreshed anyway.
1036     if (oldstate && inBackground) {
1037         debugs(82, 7, "'" << def->name << "' queue is already being refreshed (ch=" << ch << ")");
1038         return;
1039     }
1040
1041     externalAclState *state = new externalAclState(def, key);
1042
1043     if (!inBackground) {
1044         state->callback = &LookupDone;
1045         state->callback_data = cbdataReference(ch);
1046     }
1047
1048     if (oldstate) {
1049         /* Hook into pending lookup */
1050         state->queue = oldstate->queue;
1051         oldstate->queue = state;
1052     } else {
1053         /* No pending lookup found. Sumbit to helper */
1054
1055         MemBuf buf;
1056         buf.init();
1057         buf.appendf("%s\n", key);
1058         debugs(82, 4, "externalAclLookup: looking up for '" << key << "' in '" << def->name << "'.");
1059
1060         if (!def->theHelper->trySubmit(buf.buf, externalAclHandleReply, state)) {
1061             debugs(82, 7, "'" << def->name << "' submit to helper failed");
1062             assert(inBackground); // or the caller should have checked
1063             delete state;
1064             return;
1065         }
1066
1067         dlinkAdd(state, &state->list, &def->queue);
1068
1069         buf.clean();
1070     }
1071
1072     debugs(82, 4, "externalAclLookup: will wait for the result of '" << key <<
1073            "' in '" << def->name << "' (ch=" << ch << ").");
1074 }
1075
1076 static void
1077 externalAclStats(StoreEntry * sentry)
1078 {
1079     for (external_acl *p = Config.externalAclHelperList; p; p = p->next) {
1080         storeAppendPrintf(sentry, "External ACL Statistics: %s\n", p->name);
1081         storeAppendPrintf(sentry, "Cache size: %d\n", p->cache->count);
1082         assert(p->theHelper);
1083         p->theHelper->packStatsInto(sentry);
1084         storeAppendPrintf(sentry, "\n");
1085     }
1086 }
1087
1088 static void
1089 externalAclRegisterWithCacheManager(void)
1090 {
1091     Mgr::RegisterAction("external_acl",
1092                         "External ACL stats",
1093                         externalAclStats, 0, 1);
1094 }
1095
1096 void
1097 externalAclInit(void)
1098 {
1099     for (external_acl *p = Config.externalAclHelperList; p; p = p->next) {
1100         if (!p->cache)
1101             p->cache = hash_create((HASHCMP *) strcmp, hashPrime(1024), hash4);
1102
1103         if (!p->theHelper)
1104             p->theHelper = Helper::Client::Make("external_acl_type");
1105
1106         p->theHelper->cmdline = p->cmdline;
1107
1108         p->theHelper->childs.updateLimits(p->children);
1109
1110         p->theHelper->ipc_type = IPC_TCP_SOCKET;
1111
1112         p->theHelper->addr = p->local_addr;
1113
1114         p->theHelper->openSessions();
1115     }
1116
1117     externalAclRegisterWithCacheManager();
1118 }
1119
1120 void
1121 externalAclShutdown(void)
1122 {
1123     external_acl *p;
1124
1125     for (p = Config.externalAclHelperList; p; p = p->next) {
1126         helperShutdown(p->theHelper);
1127     }
1128 }
1129
1130 /// Called when an async lookup returns
1131 void
1132 ACLExternal::LookupDone(void *data, const ExternalACLEntryPointer &result)
1133 {
1134     ACLFilledChecklist *checklist = Filled(static_cast<ACLChecklist*>(data));
1135     checklist->extacl_entry = result;
1136     checklist->resumeNonBlockingCheck();
1137 }
1138
1139 ACLExternal::ACLExternal(char const *theClass) : data(nullptr), class_(xstrdup(theClass))
1140 {}
1141
1142 char const *
1143 ACLExternal::typeString() const
1144 {
1145     return class_;
1146 }
1147
1148 bool
1149 ACLExternal::isProxyAuth() const
1150 {
1151 #if USE_AUTH
1152     return data->def->require_auth;
1153 #else
1154     return false;
1155 #endif
1156 }
1157