src/external_acl.cc

   1 /*
   2  * Copyright (C) 1996-2025 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 82    External ACL */
  10
  11 #include "squid.h"
  12 #include "acl/Acl.h"
  13 #include "acl/FilledChecklist.h"
  14 #include "cache_cf.h"
  15 #include "client_side.h"
  16 #include "client_side_request.h"
  17 #include "comm/Connection.h"
  18 #include "ConfigParser.h"
  19 #include "ExternalACL.h"
  20 #include "ExternalACLEntry.h"
  21 #include "fde.h"
  22 #include "format/Token.h"
  23 #include "helper.h"
  24 #include "helper/Reply.h"
  25 #include "http/Stream.h"
  26 #include "HttpHeaderTools.h"
  27 #include "HttpReply.h"
  28 #include "HttpRequest.h"
  29 #include "ip/tools.h"
  30 #include "MemBuf.h"
  31 #include "mgr/Registration.h"
  32 #include "rfc1738.h"
  33 #include "SquidConfig.h"
  34 #include "SquidString.h"
  35 #include "Store.h"
  36 #include "tools.h"
  37 #include "wordlist.h"
  38 #if USE_OPENSSL
  39 #include "ssl/ServerBump.h"
  40 #include "ssl/support.h"
  41 #endif
  42 #if USE_AUTH
  43 #include "auth/Acl.h"
  44 #include "auth/Gadgets.h"
  45 #include "auth/UserRequest.h"
  46 #endif
  47
  48 #ifndef DEFAULT_EXTERNAL_ACL_TTL
  49 #define DEFAULT_EXTERNAL_ACL_TTL 1 * 60 * 60
  50 #endif
  51 #ifndef DEFAULT_EXTERNAL_ACL_CHILDREN
  52 #define DEFAULT_EXTERNAL_ACL_CHILDREN 5
  53 #endif
  54
  55 static void external_acl_cache_delete(external_acl * def, const ExternalACLEntryPointer &entry);
  56 static int external_acl_entry_expired(external_acl * def, const ExternalACLEntryPointer &entry);
  57 static int external_acl_grace_expired(external_acl * def, const ExternalACLEntryPointer &entry);
  58 static void external_acl_cache_touch(external_acl * def, const ExternalACLEntryPointer &entry);
  59 static ExternalACLEntryPointer external_acl_cache_add(external_acl * def, const char *key, ExternalACLEntryData const &data);
  60
  61 /******************************************************************
  62  * external_acl directive
  63  */
  64
  65 class external_acl
  66 {
  67     /* XXX: These are not really cbdata, but it is an easy way
  68      * to get them pooled, refcounted, accounted and freed properly...
  69      * Use RefCountable MEMPROXY_CLASS instead
  70      */
  71     CBDATA_CLASS(external_acl);
  72
  73 public:
  74     external_acl();
  75     ~external_acl();
  76
  77     external_acl *next;
  78
  79     void add(const ExternalACLEntryPointer &);
  80
  81     void trimCache();
  82
  83     bool maybeCacheable(const Acl::Answer &) const;
  84
  85     int ttl;
  86
  87     int negative_ttl;
  88
  89     int grace;
  90
  91     char *name;
  92
  93     Format::Format format;
  94
  95     wordlist *cmdline;
  96
  97     Helper::ChildConfig children;
  98
  99     Helper::Client::Pointer theHelper;
 100
 101     hash_table *cache;
 102
 103     dlink_list lru_list;
 104
 105     int cache_size;
 106
 107     int cache_entries;
 108
 109     dlink_list queue;
 110
 111 #if USE_AUTH
 112     /**
 113      * Configuration flag. May only be altered by the configuration parser.
 114      *
 115      * Indicates that all uses of this external_acl_type helper require authentication
 116      * details to be processed. If none are available its a fail match.
 117      */
 118     bool require_auth;
 119 #endif
 120
 121     Format::Quoting quote; // default quoting to use, set by protocol= parameter
 122
 123     Ip::Address local_addr;
 124 };
 125
 126 CBDATA_CLASS_INIT(external_acl);
 127
 128 external_acl::external_acl() :
 129     next(nullptr),
 130     ttl(DEFAULT_EXTERNAL_ACL_TTL),
 131     negative_ttl(-1),
 132     grace(1),
 133     name(nullptr),
 134     format("external_acl_type"),
 135     cmdline(nullptr),
 136     children(DEFAULT_EXTERNAL_ACL_CHILDREN),
 137     theHelper(nullptr),
 138     cache(nullptr),
 139     cache_size(256*1024),
 140     cache_entries(0),
 141 #if USE_AUTH
 142     require_auth(0),
 143 #endif
 144     quote(Format::LOG_QUOTE_URL)
 145 {
 146     local_addr.setLocalhost();
 147 }
 148
 149 external_acl::~external_acl()
 150 {
 151     xfree(name);
 152     wordlistDestroy(&cmdline);
 153
 154     if (theHelper) {
 155         helperShutdown(theHelper);
 156         theHelper = nullptr;
 157     }
 158
 159     while (lru_list.tail) {
 160         ExternalACLEntryPointer e(static_cast<ExternalACLEntry *>(lru_list.tail->data));
 161         external_acl_cache_delete(this, e);
 162     }
 163     if (cache)
 164         hashFreeMemory(cache);
 165
 166     while (next) {
 167         external_acl *node = next;
 168         next = node->next;
 169         node->next = nullptr; // prevent recursion
 170         delete node;
 171     }
 172 }
 173
 174 void
 175 parse_externalAclHelper(external_acl ** list)
 176 {
 177     char *token = ConfigParser::NextToken();
 178
 179     if (!token) {
 180         self_destruct();
 181         return;
 182     }
 183
 184     external_acl *a = new external_acl;
 185     a->name = xstrdup(token);
 186
 187     // Allow supported %macros inside quoted tokens
 188     ConfigParser::EnableMacros();
 189     token = ConfigParser::NextToken();
 190
 191     /* Parse options */
 192     while (token) {
 193         if (strncmp(token, "ttl=", 4) == 0) {
 194             a->ttl = atoi(token + 4);
 195         } else if (strncmp(token, "negative_ttl=", 13) == 0) {
 196             a->negative_ttl = atoi(token + 13);
 197         } else if (strncmp(token, "children=", 9) == 0) {
 198             a->children.n_max = atoi(token + 9);
 199             debugs(0, DBG_CRITICAL, "WARNING: external_acl_type option children=N has been deprecated in favor of children-max=N and children-startup=N");
 200         } else if (strncmp(token, "children-max=", 13) == 0) {
 201             a->children.n_max = atoi(token + 13);
 202         } else if (strncmp(token, "children-startup=", 17) == 0) {
 203             a->children.n_startup = atoi(token + 17);
 204         } else if (strncmp(token, "children-idle=", 14) == 0) {
 205             a->children.n_idle = atoi(token + 14);
 206         } else if (strncmp(token, "concurrency=", 12) == 0) {
 207             a->children.concurrency = atoi(token + 12);
 208         } else if (strncmp(token, "queue-size=", 11) == 0) {
 209             a->children.queue_size = atoi(token + 11);
 210             a->children.defaultQueueSize = false;
 211         } else if (strncmp(token, "cache=", 6) == 0) {
 212             a->cache_size = atoi(token + 6);
 213         } else if (strncmp(token, "grace=", 6) == 0) {
 214             a->grace = atoi(token + 6);
 215         } else if (strcmp(token, "protocol=2.5") == 0) {
 216             a->quote = Format::LOG_QUOTE_SHELL;
 217         } else if (strcmp(token, "protocol=3.0") == 0) {
 218             debugs(3, DBG_PARSE_NOTE(2), "WARNING: external_acl_type option protocol=3.0 is deprecated. Remove this from your config.");
 219             a->quote = Format::LOG_QUOTE_URL;
 220         } else if (strcmp(token, "quote=url") == 0) {
 221             debugs(3, DBG_PARSE_NOTE(2), "WARNING: external_acl_type option quote=url is deprecated. Remove this from your config.");
 222             a->quote = Format::LOG_QUOTE_URL;
 223         } else if (strcmp(token, "quote=shell") == 0) {
 224             debugs(3, DBG_PARSE_NOTE(2), "WARNING: external_acl_type option quote=shell is deprecated. Use protocol=2.5 if still needed.");
 225             a->quote = Format::LOG_QUOTE_SHELL;
 226
 227             /* INET6: allow admin to configure some helpers explicitly to
 228                       bind to IPv4/v6 localhost port. */
 229         } else if (strcmp(token, "ipv4") == 0) {
 230             if ( !a->local_addr.setIPv4() ) {
 231                 debugs(3, DBG_CRITICAL, "WARNING: Error converting " << a->local_addr << " to IPv4 in " << a->name );
 232             }
 233         } else if (strcmp(token, "ipv6") == 0) {
 234             if (!Ip::EnableIpv6)
 235                 debugs(3, DBG_CRITICAL, "WARNING: --enable-ipv6 required for external ACL helpers to use IPv6: " << a->name );
 236             // else nothing to do.
 237         } else {
 238             break;
 239         }
 240
 241         token = ConfigParser::NextToken();
 242     }
 243     ConfigParser::DisableMacros();
 244
 245     /* check that child startup value is sane. */
 246     if (a->children.n_startup > a->children.n_max)
 247         a->children.n_startup = a->children.n_max;
 248
 249     /* check that child idle value is sane. */
 250     if (a->children.n_idle > a->children.n_max)
 251         a->children.n_idle = a->children.n_max;
 252     if (a->children.n_idle < 1)
 253         a->children.n_idle = 1;
 254
 255     if (a->negative_ttl == -1)
 256         a->negative_ttl = a->ttl;
 257
 258     if (a->children.defaultQueueSize)
 259         a->children.queue_size = 2 * a->children.n_max;
 260
 261     /* Legacy external_acl_type format parser.
 262      * Handles a series of %... tokens where any non-% means
 263      * the start of another parameter field (ie the path to binary).
 264      */
 265     enum Format::Quoting quote = Format::LOG_QUOTE_NONE;
 266     Format::Token **fmt = &a->format.format;
 267     bool data_used = false;
 268     while (token) {
 269         /* stop on first non-% token found */
 270         if (*token != '%')
 271             break;
 272
 273         *fmt = new Format::Token;
 274         // these tokens are whitespace delimited
 275         (*fmt)->space = true;
 276
 277         // set the default encoding to match the protocol= config
 278         // this will be overridden by explicit %macro attributes
 279         (*fmt)->quote = a->quote;
 280
 281         // compatibility for old tokens incompatible with Format::Token syntax
 282 #if USE_OPENSSL // do not bother unless we have to.
 283         if (strncmp(token, "%USER_CERT_", 11) == 0) {
 284             (*fmt)->type = Format::LFT_EXT_ACL_USER_CERT;
 285             (*fmt)->data.string = xstrdup(token + 11);
 286             (*fmt)->data.header.header = (*fmt)->data.string;
 287         } else if (strncmp(token, "%USER_CA_CERT_", 14) == 0) {
 288             (*fmt)->type = Format::LFT_EXT_ACL_USER_CA_CERT;
 289             (*fmt)->data.string = xstrdup(token + 14);
 290             (*fmt)->data.header.header = (*fmt)->data.string;
 291         } else if (strncmp(token, "%CA_CERT_", 9) == 0) {
 292             debugs(82, DBG_PARSE_NOTE(DBG_IMPORTANT), "WARNING: external_acl_type %CA_CERT_* code is obsolete. Use %USER_CA_CERT_* instead");
 293             (*fmt)->type = Format::LFT_EXT_ACL_USER_CA_CERT;
 294             (*fmt)->data.string = xstrdup(token + 9);
 295             (*fmt)->data.header.header = (*fmt)->data.string;
 296         } else
 297 #endif
 298             if (strncmp(token,"%<{", 3) == 0) {
 299                 SBuf tmp("%<h");
 300                 tmp.append(token+2);
 301                 debugs(82, DBG_PARSE_NOTE(DBG_IMPORTANT), "WARNING: external_acl_type format %<{...} is deprecated. Use " << tmp);
 302                 const size_t parsedLen = (*fmt)->parse(tmp.c_str(), &quote);
 303                 assert(parsedLen == tmp.length());
 304                 assert((*fmt)->type == Format::LFT_REPLY_HEADER ||
 305                        (*fmt)->type == Format::LFT_REPLY_HEADER_ELEM);
 306
 307             } else if (strncmp(token,"%>{", 3) == 0) {
 308                 SBuf tmp("%>ha");
 309                 tmp.append(token+2);
 310                 debugs(82, DBG_PARSE_NOTE(DBG_IMPORTANT), "WARNING: external_acl_type format %>{...} is deprecated. Use " << tmp);
 311                 const size_t parsedLen = (*fmt)->parse(tmp.c_str(), &quote);
 312                 assert(parsedLen == tmp.length());
 313                 assert((*fmt)->type == Format::LFT_ADAPTED_REQUEST_HEADER ||
 314                        (*fmt)->type == Format::LFT_ADAPTED_REQUEST_HEADER_ELEM);
 315
 316             } else {
 317                 // we can use the Format::Token::parse() method since it
 318                 // only pulls off one token. Since we already checked
 319                 // for '%' prefix above this is guaranteed to be a token.
 320                 const size_t len = (*fmt)->parse(token, &quote);
 321                 assert(len == strlen(token));
 322             }
 323
 324         // process special token-specific actions (only if necessary)
 325 #if USE_AUTH
 326         if ((*fmt)->type == Format::LFT_USER_LOGIN)
 327             a->require_auth = true;
 328 #endif
 329
 330         if ((*fmt)->type == Format::LFT_EXT_ACL_DATA)
 331             data_used = true;
 332
 333         fmt = &((*fmt)->next);
 334         token = ConfigParser::NextToken();
 335     }
 336
 337     /* There must be at least one format token */
 338     if (!a->format.format) {
 339         delete a;
 340         self_destruct();
 341         return;
 342     }
 343
 344     // format has implicit %DATA on the end if not used explicitly
 345     if (!data_used) {
 346         *fmt = new Format::Token;
 347         (*fmt)->type = Format::LFT_EXT_ACL_DATA;
 348         (*fmt)->quote = Format::LOG_QUOTE_NONE;
 349     }
 350
 351     /* helper */
 352     if (!token) {
 353         delete a;
 354         self_destruct();
 355         return;
 356     }
 357
 358     wordlistAdd(&a->cmdline, token);
 359
 360     /* arguments */
 361     parse_wordlist(&a->cmdline);
 362
 363     while (*list)
 364         list = &(*list)->next;
 365
 366     *list = a;
 367 }
 368
 369 void
 370 dump_externalAclHelper(StoreEntry * sentry, const char *name, const external_acl * list)
 371 {
 372     const external_acl *node;
 373     const wordlist *word;
 374
 375     for (node = list; node; node = node->next) {
 376         storeAppendPrintf(sentry, "%s %s", name, node->name);
 377
 378         if (!node->local_addr.isIPv6())
 379             storeAppendPrintf(sentry, " ipv4");
 380         else
 381             storeAppendPrintf(sentry, " ipv6");
 382
 383         if (node->ttl != DEFAULT_EXTERNAL_ACL_TTL)
 384             storeAppendPrintf(sentry, " ttl=%d", node->ttl);
 385
 386         if (node->negative_ttl != node->ttl)
 387             storeAppendPrintf(sentry, " negative_ttl=%d", node->negative_ttl);
 388
 389         if (node->grace)
 390             storeAppendPrintf(sentry, " grace=%d", node->grace);
 391
 392         if (node->children.n_max != DEFAULT_EXTERNAL_ACL_CHILDREN)
 393             storeAppendPrintf(sentry, " children-max=%d", node->children.n_max);
 394
 395         if (node->children.n_startup != 0) // sync with helper/ChildConfig.cc default
 396             storeAppendPrintf(sentry, " children-startup=%d", node->children.n_startup);
 397
 398         if (node->children.n_idle != 1) // sync with helper/ChildConfig.cc default
 399             storeAppendPrintf(sentry, " children-idle=%d", node->children.n_idle);
 400
 401         if (node->children.concurrency != 0)
 402             storeAppendPrintf(sentry, " concurrency=%d", node->children.concurrency);
 403
 404         if (node->cache)
 405             storeAppendPrintf(sentry, " cache=%d", node->cache_size);
 406
 407         if (node->quote == Format::LOG_QUOTE_SHELL)
 408             storeAppendPrintf(sentry, " protocol=2.5");
 409
 410         node->format.dump(sentry, nullptr, false);
 411
 412         for (word = node->cmdline; word; word = word->next)
 413             storeAppendPrintf(sentry, " %s", word->key);
 414
 415         storeAppendPrintf(sentry, "\n");
 416     }
 417 }
 418
 419 void
 420 free_externalAclHelper(external_acl ** list)
 421 {
 422     delete *list;
 423     *list = nullptr;
 424 }
 425
 426 static external_acl *
 427 find_externalAclHelper(const char *name)
 428 {
 429     external_acl *node;
 430
 431     for (node = Config.externalAclHelperList; node; node = node->next) {
 432         if (strcmp(node->name, name) == 0)
 433             return node;
 434     }
 435
 436     return nullptr;
 437 }
 438
 439 void
 440 external_acl::add(const ExternalACLEntryPointer &anEntry)
 441 {
 442     trimCache();
 443     assert(anEntry != nullptr);
 444     assert (anEntry->def == nullptr);
 445     anEntry->def = this;
 446     ExternalACLEntry *e = const_cast<ExternalACLEntry *>(anEntry.getRaw()); // XXX: make hash a std::map of Pointer.
 447     hash_join(cache, e);
 448     dlinkAdd(e, &e->lru, &lru_list);
 449     e->lock(); //cbdataReference(e); // lock it on behalf of the hash
 450     ++cache_entries;
 451 }
 452
 453 void
 454 external_acl::trimCache()
 455 {
 456     if (cache_size && cache_entries >= cache_size) {
 457         ExternalACLEntryPointer e(static_cast<ExternalACLEntry *>(lru_list.tail->data));
 458         external_acl_cache_delete(this, e);
 459     }
 460 }
 461
 462 bool
 463 external_acl::maybeCacheable(const Acl::Answer &result) const
 464 {
 465     if (cache_size <= 0)
 466         return false; // cache is disabled
 467
 468     if (result == ACCESS_DUNNO)
 469         return false; // non-cacheable response
 470
 471     if ((result.allowed() ? ttl : negative_ttl) <= 0)
 472         return false; // not caching this type of response
 473
 474     return true;
 475 }
 476
 477 /******************************************************************
 478  * external acl type
 479  */
 480
 481 class external_acl_data
 482 {
 483     CBDATA_CLASS(external_acl_data);
 484
 485 public:
 486     explicit external_acl_data(external_acl * const aDef): def(cbdataReference(aDef)), arguments(nullptr) {}
 487     ~external_acl_data();
 488
 489     external_acl *def;
 490     SBuf name;
 491     wordlist *arguments;
 492 };
 493
 494 CBDATA_CLASS_INIT(external_acl_data);
 495
 496 external_acl_data::~external_acl_data()
 497 {
 498     wordlistDestroy(&arguments);
 499     cbdataReferenceDone(def);
 500 }
 501
 502 void
 503 ACLExternal::parse()
 504 {
 505     if (data) {
 506         self_destruct();
 507         return;
 508     }
 509
 510     char *token = ConfigParser::strtokFile();
 511
 512     if (!token) {
 513         self_destruct();
 514         return;
 515     }
 516
 517     data = new external_acl_data(find_externalAclHelper(token));
 518
 519     if (!data->def) {
 520         delete data;
 521         self_destruct();
 522         return;
 523     }
 524
 525     // def->name is the name of the external_acl_type.
 526     // this is the name of the 'acl' directive being tested
 527     data->name = name;
 528
 529     while ((token = ConfigParser::strtokFile())) {
 530         wordlistAdd(&data->arguments, token);
 531     }
 532 }
 533
 534 bool
 535 ACLExternal::valid () const
 536 {
 537 #if USE_AUTH
 538     if (data->def->require_auth) {
 539         if (authenticateSchemeCount() == 0) {
 540             debugs(28, DBG_CRITICAL, "ERROR: Cannot use proxy auth because no authentication schemes were compiled.");
 541             return false;
 542         }
 543
 544         if (authenticateActiveSchemeCount() == 0) {
 545             debugs(28, DBG_CRITICAL, "ERROR: Cannot use proxy auth because no authentication schemes are fully configured.");
 546             return false;
 547         }
 548     }
 549 #endif
 550
 551     return true;
 552 }
 553
 554 bool
 555 ACLExternal::empty () const
 556 {
 557     return false;
 558 }
 559
 560 ACLExternal::~ACLExternal()
 561 {
 562     delete data;
 563     xfree(class_);
 564 }
 565
 566 static void
 567 copyResultsFromEntry(const HttpRequest::Pointer &req, const ExternalACLEntryPointer &entry)
 568 {
 569     if (req) {
 570 #if USE_AUTH
 571         if (entry->user.size())
 572             req->extacl_user = entry->user;
 573
 574         if (entry->password.size())
 575             req->extacl_passwd = entry->password;
 576 #endif
 577         if (!req->tag.size())
 578             req->tag = entry->tag;
 579
 580         if (entry->log.size())
 581             req->extacl_log = entry->log;
 582
 583         if (entry->message.size())
 584             req->extacl_message = entry->message;
 585
 586         // attach the helper kv-pair to the transaction
 587         UpdateRequestNotes(req->clientConnectionManager.get(), *req, entry->notes);
 588     }
 589 }
 590
 591 // TODO: Diff reduction. Rename this helper method to match_() or similar.
 592 Acl::Answer
 593 ACLExternal::aclMatchExternal(external_acl_data *acl, ACLFilledChecklist *ch) const
 594 {
 595     // Despite its external_acl C++ type name, acl->def is not an ACL (i.e. not
 596     // a reference-counted Acl::Node) and gets invalidated by reconfiguration.
 597     // TODO: RefCount external_acl, so that we do not have to bail here.
 598     if (!cbdataReferenceValid(acl->def)) {
 599         debugs(82, 3, "cannot resume matching; external_acl gone");
 600         return ACCESS_DUNNO;
 601     }
 602
 603     debugs(82, 9, "acl=\"" << acl->def->name << "\"");
 604     ExternalACLEntryPointer entry = ch->extacl_entry;
 605
 606     external_acl_message = "MISSING REQUIRED INFORMATION";
 607
 608     if (entry != nullptr) {
 609         if (entry->def == acl->def) {
 610             /* Ours, use it.. if the key matches */
 611             const char *key = makeExternalAclKey(ch, acl);
 612             if (!key)
 613                 return ACCESS_DUNNO; // insufficient data to continue
 614             if (strcmp(key, (char*)entry->key) != 0) {
 615                 debugs(82, 9, "entry key='" << (char *)entry->key << "', our key='" << key << "' do not match. Discarded.");
 616                 // too bad. need a new lookup.
 617                 entry = ch->extacl_entry = nullptr;
 618             }
 619         } else {
 620             /* Not ours.. get rid of it */
 621             debugs(82, 9, "entry " << entry << " not valid or not ours. Discarded.");
 622             if (entry != nullptr) {
 623                 debugs(82, 9, "entry def=" << entry->def << ", our def=" << acl->def);
 624                 const char *key = makeExternalAclKey(ch, acl); // may be nil
 625                 debugs(82, 9, "entry key='" << (char *)entry->key << "', our key='" << key << "'");
 626             }
 627             entry = ch->extacl_entry = nullptr;
 628         }
 629     }
 630
 631     if (!entry) {
 632         debugs(82, 9, "No helper entry available");
 633 #if USE_AUTH
 634         if (acl->def->require_auth) {
 635             /* Make sure the user is authenticated */
 636             debugs(82, 3, acl->def->name << " check user authenticated.");
 637             const auto ti = AuthenticateAcl(ch, *this);
 638             if (!ti.allowed()) {
 639                 debugs(82, 2, acl->def->name << " user not authenticated (" << ti << ")");
 640                 return ti;
 641             }
 642             debugs(82, 3, acl->def->name << " user is authenticated.");
 643         }
 644 #endif
 645         const char *key = makeExternalAclKey(ch, acl);
 646
 647         if (!key) {
 648             /* Not sufficient data to process */
 649             return ACCESS_DUNNO;
 650         }
 651
 652         entry = static_cast<ExternalACLEntry *>(hash_lookup(acl->def->cache, key));
 653
 654         const ExternalACLEntryPointer staleEntry = entry;
 655         if (entry != nullptr && external_acl_entry_expired(acl->def, entry))
 656             entry = nullptr;
 657
 658         if (entry != nullptr && external_acl_grace_expired(acl->def, entry)) {
 659             // refresh in the background
 660             startLookup(ch, acl, true);
 661             debugs(82, 4, "no need to wait for the refresh of '" <<
 662                    key << "' in '" << acl->def->name << "' (ch=" << ch << ").");
 663         }
 664
 665         if (!entry) {
 666             debugs(82, 2, acl->def->name << "(\"" << key << "\") = lookup needed");
 667
 668             // TODO: All other helpers allow temporary overload. Should not we?
 669             if (!acl->def->theHelper->willOverload()) {
 670                 debugs(82, 2, "\"" << key << "\": queueing a call.");
 671                 if (!ch->goAsync(StartLookup, *this))
 672                     debugs(82, 2, "\"" << key << "\": no async support!");
 673                 debugs(82, 2, "\"" << key << "\": return -1.");
 674                 return ACCESS_DUNNO; // expired cached or simply absent entry
 675             } else {
 676                 if (!staleEntry) {
 677                     debugs(82, DBG_IMPORTANT, "WARNING: external ACL '" << acl->def->name <<
 678                            "' queue full. Request rejected '" << key << "'.");
 679                     external_acl_message = "SYSTEM TOO BUSY, TRY AGAIN LATER";
 680                     return ACCESS_DUNNO;
 681                 } else {
 682                     debugs(82, DBG_IMPORTANT, "WARNING: external ACL '" << acl->def->name <<
 683                            "' queue full. Using stale result. '" << key << "'.");
 684                     entry = staleEntry;
 685                     /* Fall thru to processing below */
 686                 }
 687             }
 688         }
 689     }
 690
 691     debugs(82, 4, "entry = { date=" <<
 692            (long unsigned int) entry->date <<
 693            ", result=" << entry->result <<
 694            " tag=" << entry->tag <<
 695            " log=" << entry->log << " }");
 696 #if USE_AUTH
 697     debugs(82, 4, "entry user=" << entry->user);
 698 #endif
 699
 700     external_acl_cache_touch(acl->def, entry);
 701     external_acl_message = entry->message.termedBuf();
 702
 703     debugs(82, 2, acl->def->name << " = " << entry->result);
 704     copyResultsFromEntry(ch->request, entry);
 705     return entry->result;
 706 }
 707
 708 int
 709 ACLExternal::match(ACLChecklist *checklist)
 710 {
 711     auto answer = aclMatchExternal(data, Filled(checklist));
 712
 713     // convert to tri-state ACL match 1,0,-1
 714     switch (answer) {
 715     case ACCESS_ALLOWED:
 716         return 1; // match
 717
 718     case ACCESS_DENIED:
 719         return 0; // non-match
 720
 721     case ACCESS_DUNNO:
 722     case ACCESS_AUTH_REQUIRED:
 723     default:
 724         // If the answer is not allowed or denied (matches/not matches) and
 725         // async authentication is not in progress, then we are done.
 726         if (checklist->keepMatching())
 727             checklist->markFinished(answer, "aclMatchExternal exception");
 728         return -1; // other
 729     }
 730 }
 731
 732 SBufList
 733 ACLExternal::dump() const
 734 {
 735     external_acl_data const *acl = data;
 736     SBufList rv;
 737     rv.push_back(SBuf(acl->def->name));
 738
 739     for (wordlist *arg = acl->arguments; arg; arg = arg->next) {
 740         SBuf s;
 741         s.Printf(" %s", arg->key);
 742         rv.push_back(s);
 743     }
 744
 745     return rv;
 746 }
 747
 748 /******************************************************************
 749  * external_acl cache
 750  */
 751
 752 static void
 753 external_acl_cache_touch(external_acl * def, const ExternalACLEntryPointer &entry)
 754 {
 755     // this must not be done when nothing is being cached.
 756     if (!def->maybeCacheable(entry->result))
 757         return;
 758
 759     dlinkDelete(&entry->lru, &def->lru_list);
 760     ExternalACLEntry *e = const_cast<ExternalACLEntry *>(entry.getRaw()); // XXX: make hash a std::map of Pointer.
 761     dlinkAdd(e, &entry->lru, &def->lru_list);
 762 }
 763
 764 char *
 765 ACLExternal::makeExternalAclKey(ACLFilledChecklist * ch, external_acl_data * acl_data) const
 766 {
 767     static MemBuf mb;
 768     mb.reset();
 769
 770     // check for special case tokens in the format
 771     for (Format::Token *t = acl_data->def->format.format; t ; t = t->next) {
 772
 773         if (t->type == Format::LFT_EXT_ACL_NAME) {
 774             // setup for %ACL
 775             ch->al->lastAclName = acl_data->name;
 776         }
 777
 778         if (t->type == Format::LFT_EXT_ACL_DATA) {
 779             // setup string for %DATA
 780             SBuf sb;
 781             for (auto arg = acl_data->arguments; arg; arg = arg->next) {
 782                 if (sb.length())
 783                     sb.append(" ", 1);
 784
 785                 if (acl_data->def->quote == Format::LOG_QUOTE_URL) {
 786                     const char *quoted = rfc1738_escape(arg->key);
 787                     sb.append(quoted, strlen(quoted));
 788                 } else {
 789                     static MemBuf mb2;
 790                     mb2.init();
 791                     strwordquote(&mb2, arg->key);
 792                     sb.append(mb2.buf, mb2.size);
 793                     mb2.clean();
 794                 }
 795             }
 796
 797             ch->al->lastAclData = sb;
 798         }
 799     }
 800
 801     // assemble the full helper lookup string
 802     acl_data->def->format.assemble(mb, ch->al, 0);
 803
 804     return mb.buf;
 805 }
 806
 807 static int
 808 external_acl_entry_expired(external_acl * def, const ExternalACLEntryPointer &entry)
 809 {
 810     if (def->cache_size <= 0 || entry->result == ACCESS_DUNNO)
 811         return 1;
 812
 813     if (entry->date + (entry->result.allowed() ? def->ttl : def->negative_ttl) < squid_curtime)
 814         return 1;
 815     else
 816         return 0;
 817 }
 818
 819 static int
 820 external_acl_grace_expired(external_acl * def, const ExternalACLEntryPointer &entry)
 821 {
 822     if (def->cache_size <= 0 || entry->result == ACCESS_DUNNO)
 823         return 1;
 824
 825     int ttl;
 826     ttl = entry->result.allowed() ? def->ttl : def->negative_ttl;
 827     ttl = (ttl * (100 - def->grace)) / 100;
 828
 829     if (entry->date + ttl <= squid_curtime)
 830         return 1;
 831     else
 832         return 0;
 833 }
 834
 835 static ExternalACLEntryPointer
 836 external_acl_cache_add(external_acl * def, const char *key, ExternalACLEntryData const & data)
 837 {
 838     ExternalACLEntryPointer entry;
 839
 840     if (!def->maybeCacheable(data.result)) {
 841         debugs(82,6, MYNAME);
 842
 843         if (data.result == ACCESS_DUNNO) {
 844             if (const ExternalACLEntryPointer oldentry = static_cast<ExternalACLEntry *>(hash_lookup(def->cache, key)))
 845                 external_acl_cache_delete(def, oldentry);
 846         }
 847         entry = new ExternalACLEntry;
 848         entry->key = xstrdup(key);
 849         entry->update(data);
 850         entry->def = def;
 851         return entry;
 852     }
 853
 854     entry = static_cast<ExternalACLEntry *>(hash_lookup(def->cache, key));
 855     debugs(82, 2, "external_acl_cache_add: Adding '" << key << "' = " << data.result);
 856
 857     if (entry != nullptr) {
 858         debugs(82, 3, "updating existing entry");
 859         entry->update(data);
 860         external_acl_cache_touch(def, entry);
 861         return entry;
 862     }
 863
 864     entry = new ExternalACLEntry;
 865     entry->key = xstrdup(key);
 866     entry->update(data);
 867
 868     def->add(entry);
 869
 870     return entry;
 871 }
 872
 873 static void
 874 external_acl_cache_delete(external_acl * def, const ExternalACLEntryPointer &entry)
 875 {
 876     assert(entry != nullptr);
 877     assert(def->cache_size > 0 && entry->def == def);
 878     ExternalACLEntry *e = const_cast<ExternalACLEntry *>(entry.getRaw()); // XXX: make hash a std::map of Pointer.
 879     hash_remove_link(def->cache, e);
 880     dlinkDelete(&e->lru, &def->lru_list);
 881     e->unlock(); // unlock on behalf of the hash
 882     def->cache_entries -= 1;
 883 }
 884
 885 /******************************************************************
 886  * external_acl helpers
 887  */
 888
 889 class externalAclState
 890 {
 891     CBDATA_CLASS(externalAclState);
 892
 893 public:
 894     externalAclState(external_acl* aDef, const char *aKey) :
 895         callback(nullptr),
 896         callback_data(nullptr),
 897         key(xstrdup(aKey)),
 898         def(cbdataReference(aDef)),
 899         queue(nullptr)
 900     {}
 901     ~externalAclState();
 902
 903     EAH *callback;
 904     void *callback_data;
 905     char *key;
 906     external_acl *def;
 907     dlink_node list;
 908     externalAclState *queue;
 909 };
 910
 911 CBDATA_CLASS_INIT(externalAclState);
 912
 913 externalAclState::~externalAclState()
 914 {
 915     xfree(key);
 916     cbdataReferenceDone(callback_data);
 917     cbdataReferenceDone(def);
 918 }
 919
 920 /*
 921  * The helper program receives queries on stdin, one
 922  * per line, and must return the result on on stdout
 923  *
 924  * General result syntax:
 925  *
 926  *   OK/ERR keyword=value ...
 927  *
 928  * Keywords:
 929  *
 930  *   user=      The users name (login)
 931  *   message=   Message describing the reason
 932  *   tag=   A string tag to be applied to the request that triggered the acl match.
 933  *          applies to both OK and ERR responses.
 934  *          Won't override existing request tags.
 935  *   log=   A string to be used in access logging
 936  *
 937  * Other keywords may be added to the protocol later
 938  *
 939  * value needs to be URL-encoded or enclosed in double quotes (")
 940  * with \-escaping on any whitespace, quotes, or slashes (\).
 941  */
 942 static void
 943 externalAclHandleReply(void *data, const Helper::Reply &reply)
 944 {
 945     externalAclState *state = static_cast<externalAclState *>(data);
 946     externalAclState *next;
 947     ExternalACLEntryData entryData;
 948
 949     debugs(82, 2, "reply=" << reply);
 950
 951     if (reply.result == Helper::Okay)
 952         entryData.result = ACCESS_ALLOWED;
 953     else if (reply.result == Helper::Error)
 954         entryData.result = ACCESS_DENIED;
 955     else //BrokenHelper,TimedOut or Unknown. Should not cached.
 956         entryData.result = ACCESS_DUNNO;
 957
 958     // XXX: make entryData store a proper Helper::Reply object instead of copying.
 959
 960     entryData.notes.append(&reply.notes);
 961
 962     const char *label = reply.notes.findFirst("tag");
 963     if (label != nullptr && *label != '\0')
 964         entryData.tag = label;
 965
 966     label = reply.notes.findFirst("message");
 967     if (label != nullptr && *label != '\0')
 968         entryData.message = label;
 969
 970     label = reply.notes.findFirst("log");
 971     if (label != nullptr && *label != '\0')
 972         entryData.log = label;
 973
 974 #if USE_AUTH
 975     label = reply.notes.findFirst("user");
 976     if (label != nullptr && *label != '\0')
 977         entryData.user = label;
 978
 979     label = reply.notes.findFirst("password");
 980     if (label != nullptr && *label != '\0')
 981         entryData.password = label;
 982 #endif
 983
 984     // XXX: This state->def access conflicts with the cbdata validity check
 985     // below.
 986     dlinkDelete(&state->list, &state->def->queue);
 987
 988     ExternalACLEntryPointer entry;
 989     if (cbdataReferenceValid(state->def))
 990         entry = external_acl_cache_add(state->def, state->key, entryData);
 991
 992     do {
 993         void *cbdata;
 994         if (state->callback && cbdataReferenceValidDone(state->callback_data, &cbdata))
 995             state->callback(cbdata, entry);
 996
 997         next = state->queue;
 998         state->queue = nullptr;
 999
1000         delete state;
1001
1002         state = next;
1003     } while (state);
1004 }
1005
1006 /// Asks the helper (if needed) or returns the [cached] result (otherwise).
1007 /// Does not support "background" lookups. See also: ACLExternal::Start().
1008 void
1009 ACLExternal::StartLookup(ACLFilledChecklist &checklist, const Acl::Node &acl)
1010 {
1011     const auto &me = dynamic_cast<const ACLExternal&>(acl);
1012     me.startLookup(&checklist, me.data, false);
1013 }
1014
1015 // If possible, starts an asynchronous lookup of an external ACL.
1016 // Otherwise, asserts (or bails if background refresh is requested).
1017 void
1018 ACLExternal::startLookup(ACLFilledChecklist *ch, external_acl_data *acl, bool inBackground) const
1019 {
1020     external_acl *def = acl->def;
1021
1022     const char *key = makeExternalAclKey(ch, acl);
1023     assert(key);
1024
1025     debugs(82, 2, (inBackground ? "bg" : "fg") << " lookup in '" <<
1026            def->name << "' for '" << key << "'");
1027
1028     /* Check for a pending lookup to hook into */
1029     // only possible if we are caching results.
1030     externalAclState *oldstate = nullptr;
1031     if (def->cache_size > 0) {
1032         for (dlink_node *node = def->queue.head; node; node = node->next) {
1033             externalAclState *oldstatetmp = static_cast<externalAclState *>(node->data);
1034
1035             if (strcmp(key, oldstatetmp->key) == 0) {
1036                 oldstate = oldstatetmp;
1037                 break;
1038             }
1039         }
1040     }
1041
1042     // A background refresh has no need to piggiback on a pending request:
1043     // When the pending request completes, the cache will be refreshed anyway.
1044     if (oldstate && inBackground) {
1045         debugs(82, 7, "'" << def->name << "' queue is already being refreshed (ch=" << ch << ")");
1046         return;
1047     }
1048
1049     externalAclState *state = new externalAclState(def, key);
1050
1051     if (!inBackground) {
1052         state->callback = &LookupDone;
1053         state->callback_data = cbdataReference(ch);
1054     }
1055
1056     if (oldstate) {
1057         /* Hook into pending lookup */
1058         state->queue = oldstate->queue;
1059         oldstate->queue = state;
1060     } else {
1061         /* No pending lookup found. Sumbit to helper */
1062
1063         MemBuf buf;
1064         buf.init();
1065         buf.appendf("%s\n", key);
1066         debugs(82, 4, "externalAclLookup: looking up for '" << key << "' in '" << def->name << "'.");
1067
1068         if (!def->theHelper->trySubmit(buf.buf, externalAclHandleReply, state)) {
1069             debugs(82, 7, "'" << def->name << "' submit to helper failed");
1070             assert(inBackground); // or the caller should have checked
1071             delete state;
1072             return;
1073         }
1074
1075         dlinkAdd(state, &state->list, &def->queue);
1076
1077         buf.clean();
1078     }
1079
1080     debugs(82, 4, "externalAclLookup: will wait for the result of '" << key <<
1081            "' in '" << def->name << "' (ch=" << ch << ").");
1082 }
1083
1084 static void
1085 externalAclStats(StoreEntry * sentry)
1086 {
1087     for (external_acl *p = Config.externalAclHelperList; p; p = p->next) {
1088         storeAppendPrintf(sentry, "External ACL Statistics: %s\n", p->name);
1089         storeAppendPrintf(sentry, "Cache size: %d\n", p->cache->count);
1090         assert(p->theHelper);
1091         p->theHelper->packStatsInto(sentry);
1092         storeAppendPrintf(sentry, "\n");
1093     }
1094 }
1095
1096 static void
1097 externalAclRegisterWithCacheManager(void)
1098 {
1099     Mgr::RegisterAction("external_acl",
1100                         "External ACL stats",
1101                         externalAclStats, 0, 1);
1102 }
1103
1104 void
1105 externalAclInit(void)
1106 {
1107     for (external_acl *p = Config.externalAclHelperList; p; p = p->next) {
1108         if (!p->cache)
1109             p->cache = hash_create((HASHCMP *) strcmp, hashPrime(1024), hash4);
1110
1111         if (!p->theHelper)
1112             p->theHelper = Helper::Client::Make("external_acl_type");
1113
1114         p->theHelper->cmdline = p->cmdline;
1115
1116         p->theHelper->childs.updateLimits(p->children);
1117
1118         p->theHelper->ipc_type = IPC_TCP_SOCKET;
1119
1120         p->theHelper->addr = p->local_addr;
1121
1122         p->theHelper->openSessions();
1123     }
1124
1125     externalAclRegisterWithCacheManager();
1126 }
1127
1128 void
1129 externalAclShutdown(void)
1130 {
1131     external_acl *p;
1132
1133     for (p = Config.externalAclHelperList; p; p = p->next) {
1134         helperShutdown(p->theHelper);
1135     }
1136 }
1137
1138 /// Called when an async lookup returns
1139 void
1140 ACLExternal::LookupDone(void *data, const ExternalACLEntryPointer &result)
1141 {
1142     ACLFilledChecklist *checklist = Filled(static_cast<ACLChecklist*>(data));
1143     checklist->extacl_entry = result;
1144     checklist->resumeNonBlockingCheck();
1145 }
1146
1147 ACLExternal::ACLExternal(char const *theClass) : data(nullptr), class_(xstrdup(theClass))
1148 {}
1149
1150 char const *
1151 ACLExternal::typeString() const
1152 {
1153     return class_;
1154 }
1155
1156 bool
1157 ACLExternal::isProxyAuth() const
1158 {
1159 #if USE_AUTH
1160     return data->def->require_auth;
1161 #else
1162     return false;
1163 #endif
1164 }
1165