src/external_acl.cc

   1 /*
   2  * Copyright (C) 1996-2020 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 82    External ACL */
  10
  11 #include "squid.h"
  12 #include "acl/Acl.h"
  13 #include "acl/FilledChecklist.h"
  14 #include "cache_cf.h"
  15 #include "client_side.h"
  16 #include "client_side_request.h"
  17 #include "comm/Connection.h"
  18 #include "ConfigParser.h"
  19 #include "ExternalACL.h"
  20 #include "ExternalACLEntry.h"
  21 #include "fde.h"
  22 #include "format/Token.h"
  23 #include "helper.h"
  24 #include "helper/Reply.h"
  25 #include "http/Stream.h"
  26 #include "HttpHeaderTools.h"
  27 #include "HttpReply.h"
  28 #include "HttpRequest.h"
  29 #include "ip/tools.h"
  30 #include "MemBuf.h"
  31 #include "mgr/Registration.h"
  32 #include "rfc1738.h"
  33 #include "SquidConfig.h"
  34 #include "SquidString.h"
  35 #include "SquidTime.h"
  36 #include "Store.h"
  37 #include "tools.h"
  38 #include "wordlist.h"
  39 #if USE_OPENSSL
  40 #include "ssl/ServerBump.h"
  41 #include "ssl/support.h"
  42 #endif
  43 #if USE_AUTH
  44 #include "auth/Acl.h"
  45 #include "auth/Gadgets.h"
  46 #include "auth/UserRequest.h"
  47 #endif
  48 #if USE_IDENT
  49 #include "ident/AclIdent.h"
  50 #endif
  51
  52 #ifndef DEFAULT_EXTERNAL_ACL_TTL
  53 #define DEFAULT_EXTERNAL_ACL_TTL 1 * 60 * 60
  54 #endif
  55 #ifndef DEFAULT_EXTERNAL_ACL_CHILDREN
  56 #define DEFAULT_EXTERNAL_ACL_CHILDREN 5
  57 #endif
  58
  59 static char *makeExternalAclKey(ACLFilledChecklist * ch, external_acl_data * acl_data);
  60 static void external_acl_cache_delete(external_acl * def, const ExternalACLEntryPointer &entry);
  61 static int external_acl_entry_expired(external_acl * def, const ExternalACLEntryPointer &entry);
  62 static int external_acl_grace_expired(external_acl * def, const ExternalACLEntryPointer &entry);
  63 static void external_acl_cache_touch(external_acl * def, const ExternalACLEntryPointer &entry);
  64 static ExternalACLEntryPointer external_acl_cache_add(external_acl * def, const char *key, ExternalACLEntryData const &data);
  65
  66 /******************************************************************
  67  * external_acl directive
  68  */
  69
  70 class external_acl
  71 {
  72     /* FIXME: These are not really cbdata, but it is an easy way
  73      * to get them pooled, refcounted, accounted and freed properly...
  74      */
  75     CBDATA_CLASS(external_acl);
  76
  77 public:
  78     external_acl();
  79     ~external_acl();
  80
  81     external_acl *next;
  82
  83     void add(const ExternalACLEntryPointer &);
  84
  85     void trimCache();
  86
  87     bool maybeCacheable(const Acl::Answer &) const;
  88
  89     int ttl;
  90
  91     int negative_ttl;
  92
  93     int grace;
  94
  95     char *name;
  96
  97     Format::Format format;
  98
  99     wordlist *cmdline;
 100
 101     Helper::ChildConfig children;
 102
 103     helper *theHelper;
 104
 105     hash_table *cache;
 106
 107     dlink_list lru_list;
 108
 109     int cache_size;
 110
 111     int cache_entries;
 112
 113     dlink_list queue;
 114
 115 #if USE_AUTH
 116     /**
 117      * Configuration flag. May only be altered by the configuration parser.
 118      *
 119      * Indicates that all uses of this external_acl_type helper require authentication
 120      * details to be processed. If none are available its a fail match.
 121      */
 122     bool require_auth;
 123 #endif
 124
 125     Format::Quoting quote; // default quoting to use, set by protocol= parameter
 126
 127     Ip::Address local_addr;
 128 };
 129
 130 CBDATA_CLASS_INIT(external_acl);
 131
 132 external_acl::external_acl() :
 133     next(NULL),
 134     ttl(DEFAULT_EXTERNAL_ACL_TTL),
 135     negative_ttl(-1),
 136     grace(1),
 137     name(NULL),
 138     format("external_acl_type"),
 139     cmdline(NULL),
 140     children(DEFAULT_EXTERNAL_ACL_CHILDREN),
 141     theHelper(NULL),
 142     cache(NULL),
 143     cache_size(256*1024),
 144     cache_entries(0),
 145 #if USE_AUTH
 146     require_auth(0),
 147 #endif
 148     quote(Format::LOG_QUOTE_URL)
 149 {
 150     local_addr.setLocalhost();
 151 }
 152
 153 external_acl::~external_acl()
 154 {
 155     xfree(name);
 156     wordlistDestroy(&cmdline);
 157
 158     if (theHelper) {
 159         helperShutdown(theHelper);
 160         delete theHelper;
 161         theHelper = NULL;
 162     }
 163
 164     while (lru_list.tail) {
 165         ExternalACLEntryPointer e(static_cast<ExternalACLEntry *>(lru_list.tail->data));
 166         external_acl_cache_delete(this, e);
 167     }
 168     if (cache)
 169         hashFreeMemory(cache);
 170
 171     while (next) {
 172         external_acl *node = next;
 173         next = node->next;
 174         node->next = NULL; // prevent recursion
 175         delete node;
 176     }
 177 }
 178
 179 void
 180 parse_externalAclHelper(external_acl ** list)
 181 {
 182     char *token = ConfigParser::NextToken();
 183
 184     if (!token) {
 185         self_destruct();
 186         return;
 187     }
 188
 189     external_acl *a = new external_acl;
 190     a->name = xstrdup(token);
 191
 192     // Allow supported %macros inside quoted tokens
 193     ConfigParser::EnableMacros();
 194     token = ConfigParser::NextToken();
 195
 196     /* Parse options */
 197     while (token) {
 198         if (strncmp(token, "ttl=", 4) == 0) {
 199             a->ttl = atoi(token + 4);
 200         } else if (strncmp(token, "negative_ttl=", 13) == 0) {
 201             a->negative_ttl = atoi(token + 13);
 202         } else if (strncmp(token, "children=", 9) == 0) {
 203             a->children.n_max = atoi(token + 9);
 204             debugs(0, DBG_CRITICAL, "WARNING: external_acl_type option children=N has been deprecated in favor of children-max=N and children-startup=N");
 205         } else if (strncmp(token, "children-max=", 13) == 0) {
 206             a->children.n_max = atoi(token + 13);
 207         } else if (strncmp(token, "children-startup=", 17) == 0) {
 208             a->children.n_startup = atoi(token + 17);
 209         } else if (strncmp(token, "children-idle=", 14) == 0) {
 210             a->children.n_idle = atoi(token + 14);
 211         } else if (strncmp(token, "concurrency=", 12) == 0) {
 212             a->children.concurrency = atoi(token + 12);
 213         } else if (strncmp(token, "queue-size=", 11) == 0) {
 214             a->children.queue_size = atoi(token + 11);
 215             a->children.defaultQueueSize = false;
 216         } else if (strncmp(token, "cache=", 6) == 0) {
 217             a->cache_size = atoi(token + 6);
 218         } else if (strncmp(token, "grace=", 6) == 0) {
 219             a->grace = atoi(token + 6);
 220         } else if (strcmp(token, "protocol=2.5") == 0) {
 221             a->quote = Format::LOG_QUOTE_SHELL;
 222         } else if (strcmp(token, "protocol=3.0") == 0) {
 223             debugs(3, DBG_PARSE_NOTE(2), "WARNING: external_acl_type option protocol=3.0 is deprecated. Remove this from your config.");
 224             a->quote = Format::LOG_QUOTE_URL;
 225         } else if (strcmp(token, "quote=url") == 0) {
 226             debugs(3, DBG_PARSE_NOTE(2), "WARNING: external_acl_type option quote=url is deprecated. Remove this from your config.");
 227             a->quote = Format::LOG_QUOTE_URL;
 228         } else if (strcmp(token, "quote=shell") == 0) {
 229             debugs(3, DBG_PARSE_NOTE(2), "WARNING: external_acl_type option quote=shell is deprecated. Use protocol=2.5 if still needed.");
 230             a->quote = Format::LOG_QUOTE_SHELL;
 231
 232             /* INET6: allow admin to configure some helpers explicitly to
 233                       bind to IPv4/v6 localhost port. */
 234         } else if (strcmp(token, "ipv4") == 0) {
 235             if ( !a->local_addr.setIPv4() ) {
 236                 debugs(3, DBG_CRITICAL, "WARNING: Error converting " << a->local_addr << " to IPv4 in " << a->name );
 237             }
 238         } else if (strcmp(token, "ipv6") == 0) {
 239             if (!Ip::EnableIpv6)
 240                 debugs(3, DBG_CRITICAL, "WARNING: --enable-ipv6 required for external ACL helpers to use IPv6: " << a->name );
 241             // else nothing to do.
 242         } else {
 243             break;
 244         }
 245
 246         token = ConfigParser::NextToken();
 247     }
 248     ConfigParser::DisableMacros();
 249
 250     /* check that child startup value is sane. */
 251     if (a->children.n_startup > a->children.n_max)
 252         a->children.n_startup = a->children.n_max;
 253
 254     /* check that child idle value is sane. */
 255     if (a->children.n_idle > a->children.n_max)
 256         a->children.n_idle = a->children.n_max;
 257     if (a->children.n_idle < 1)
 258         a->children.n_idle = 1;
 259
 260     if (a->negative_ttl == -1)
 261         a->negative_ttl = a->ttl;
 262
 263     if (a->children.defaultQueueSize)
 264         a->children.queue_size = 2 * a->children.n_max;
 265
 266     /* Legacy external_acl_type format parser.
 267      * Handles a series of %... tokens where any non-% means
 268      * the start of another parameter field (ie the path to binary).
 269      */
 270     enum Format::Quoting quote = Format::LOG_QUOTE_NONE;
 271     Format::Token **fmt = &a->format.format;
 272     bool data_used = false;
 273     while (token) {
 274         /* stop on first non-% token found */
 275         if (*token != '%')
 276             break;
 277
 278         *fmt = new Format::Token;
 279         // these tokens are whitespace delimited
 280         (*fmt)->space = true;
 281
 282         // set the default encoding to match the protocol= config
 283         // this will be overridden by explicit %macro attributes
 284         (*fmt)->quote = a->quote;
 285
 286         // compatibility for old tokens incompatible with Format::Token syntax
 287 #if USE_OPENSSL // do not bother unless we have to.
 288         if (strncmp(token, "%USER_CERT_", 11) == 0) {
 289             (*fmt)->type = Format::LFT_EXT_ACL_USER_CERT;
 290             (*fmt)->data.string = xstrdup(token + 11);
 291             (*fmt)->data.header.header = (*fmt)->data.string;
 292         } else if (strncmp(token, "%USER_CA_CERT_", 14) == 0) {
 293             (*fmt)->type = Format::LFT_EXT_ACL_USER_CA_CERT;
 294             (*fmt)->data.string = xstrdup(token + 14);
 295             (*fmt)->data.header.header = (*fmt)->data.string;
 296         } else if (strncmp(token, "%CA_CERT_", 9) == 0) {
 297             debugs(82, DBG_PARSE_NOTE(DBG_IMPORTANT), "WARNING: external_acl_type %CA_CERT_* code is obsolete. Use %USER_CA_CERT_* instead");
 298             (*fmt)->type = Format::LFT_EXT_ACL_USER_CA_CERT;
 299             (*fmt)->data.string = xstrdup(token + 9);
 300             (*fmt)->data.header.header = (*fmt)->data.string;
 301         } else
 302 #endif
 303             if (strncmp(token,"%<{", 3) == 0) {
 304                 SBuf tmp("%<h");
 305                 tmp.append(token+2);
 306                 debugs(82, DBG_PARSE_NOTE(DBG_IMPORTANT), "WARNING: external_acl_type format %<{...} is deprecated. Use " << tmp);
 307                 const size_t parsedLen = (*fmt)->parse(tmp.c_str(), &quote);
 308                 assert(parsedLen == tmp.length());
 309                 assert((*fmt)->type == Format::LFT_REPLY_HEADER ||
 310                        (*fmt)->type == Format::LFT_REPLY_HEADER_ELEM);
 311
 312             } else if (strncmp(token,"%>{", 3) == 0) {
 313                 SBuf tmp("%>ha");
 314                 tmp.append(token+2);
 315                 debugs(82, DBG_PARSE_NOTE(DBG_IMPORTANT), "WARNING: external_acl_type format %>{...} is deprecated. Use " << tmp);
 316                 const size_t parsedLen = (*fmt)->parse(tmp.c_str(), &quote);
 317                 assert(parsedLen == tmp.length());
 318                 assert((*fmt)->type == Format::LFT_ADAPTED_REQUEST_HEADER ||
 319                        (*fmt)->type == Format::LFT_ADAPTED_REQUEST_HEADER_ELEM);
 320
 321             } else {
 322                 // we can use the Format::Token::parse() method since it
 323                 // only pulls off one token. Since we already checked
 324                 // for '%' prefix above this is guaranteed to be a token.
 325                 const size_t len = (*fmt)->parse(token, &quote);
 326                 assert(len == strlen(token));
 327             }
 328
 329         // process special token-specific actions (only if necessary)
 330 #if USE_AUTH
 331         if ((*fmt)->type == Format::LFT_USER_LOGIN)
 332             a->require_auth = true;
 333 #endif
 334
 335         if ((*fmt)->type == Format::LFT_EXT_ACL_DATA)
 336             data_used = true;
 337
 338         fmt = &((*fmt)->next);
 339         token = ConfigParser::NextToken();
 340     }
 341
 342     /* There must be at least one format token */
 343     if (!a->format.format) {
 344         delete a;
 345         self_destruct();
 346         return;
 347     }
 348
 349     // format has implicit %DATA on the end if not used explicitly
 350     if (!data_used) {
 351         *fmt = new Format::Token;
 352         (*fmt)->type = Format::LFT_EXT_ACL_DATA;
 353         (*fmt)->quote = Format::LOG_QUOTE_NONE;
 354     }
 355
 356     /* helper */
 357     if (!token) {
 358         delete a;
 359         self_destruct();
 360         return;
 361     }
 362
 363     wordlistAdd(&a->cmdline, token);
 364
 365     /* arguments */
 366     parse_wordlist(&a->cmdline);
 367
 368     while (*list)
 369         list = &(*list)->next;
 370
 371     *list = a;
 372 }
 373
 374 void
 375 dump_externalAclHelper(StoreEntry * sentry, const char *name, const external_acl * list)
 376 {
 377     const external_acl *node;
 378     const wordlist *word;
 379
 380     for (node = list; node; node = node->next) {
 381         storeAppendPrintf(sentry, "%s %s", name, node->name);
 382
 383         if (!node->local_addr.isIPv6())
 384             storeAppendPrintf(sentry, " ipv4");
 385         else
 386             storeAppendPrintf(sentry, " ipv6");
 387
 388         if (node->ttl != DEFAULT_EXTERNAL_ACL_TTL)
 389             storeAppendPrintf(sentry, " ttl=%d", node->ttl);
 390
 391         if (node->negative_ttl != node->ttl)
 392             storeAppendPrintf(sentry, " negative_ttl=%d", node->negative_ttl);
 393
 394         if (node->grace)
 395             storeAppendPrintf(sentry, " grace=%d", node->grace);
 396
 397         if (node->children.n_max != DEFAULT_EXTERNAL_ACL_CHILDREN)
 398             storeAppendPrintf(sentry, " children-max=%d", node->children.n_max);
 399
 400         if (node->children.n_startup != 0) // sync with helper/ChildConfig.cc default
 401             storeAppendPrintf(sentry, " children-startup=%d", node->children.n_startup);
 402
 403         if (node->children.n_idle != 1) // sync with helper/ChildConfig.cc default
 404             storeAppendPrintf(sentry, " children-idle=%d", node->children.n_idle);
 405
 406         if (node->children.concurrency != 0)
 407             storeAppendPrintf(sentry, " concurrency=%d", node->children.concurrency);
 408
 409         if (node->cache)
 410             storeAppendPrintf(sentry, " cache=%d", node->cache_size);
 411
 412         if (node->quote == Format::LOG_QUOTE_SHELL)
 413             storeAppendPrintf(sentry, " protocol=2.5");
 414
 415         node->format.dump(sentry, NULL, false);
 416
 417         for (word = node->cmdline; word; word = word->next)
 418             storeAppendPrintf(sentry, " %s", word->key);
 419
 420         storeAppendPrintf(sentry, "\n");
 421     }
 422 }
 423
 424 void
 425 free_externalAclHelper(external_acl ** list)
 426 {
 427     delete *list;
 428     *list = NULL;
 429 }
 430
 431 static external_acl *
 432 find_externalAclHelper(const char *name)
 433 {
 434     external_acl *node;
 435
 436     for (node = Config.externalAclHelperList; node; node = node->next) {
 437         if (strcmp(node->name, name) == 0)
 438             return node;
 439     }
 440
 441     return NULL;
 442 }
 443
 444 void
 445 external_acl::add(const ExternalACLEntryPointer &anEntry)
 446 {
 447     trimCache();
 448     assert(anEntry != NULL);
 449     assert (anEntry->def == NULL);
 450     anEntry->def = this;
 451     ExternalACLEntry *e = const_cast<ExternalACLEntry *>(anEntry.getRaw()); // XXX: make hash a std::map of Pointer.
 452     hash_join(cache, e);
 453     dlinkAdd(e, &e->lru, &lru_list);
 454     e->lock(); //cbdataReference(e); // lock it on behalf of the hash
 455     ++cache_entries;
 456 }
 457
 458 void
 459 external_acl::trimCache()
 460 {
 461     if (cache_size && cache_entries >= cache_size) {
 462         ExternalACLEntryPointer e(static_cast<ExternalACLEntry *>(lru_list.tail->data));
 463         external_acl_cache_delete(this, e);
 464     }
 465 }
 466
 467 bool
 468 external_acl::maybeCacheable(const Acl::Answer &result) const
 469 {
 470     if (cache_size <= 0)
 471         return false; // cache is disabled
 472
 473     if (result == ACCESS_DUNNO)
 474         return false; // non-cacheable response
 475
 476     if ((result.allowed() ? ttl : negative_ttl) <= 0)
 477         return false; // not caching this type of response
 478
 479     return true;
 480 }
 481
 482 /******************************************************************
 483  * external acl type
 484  */
 485
 486 class external_acl_data
 487 {
 488     CBDATA_CLASS(external_acl_data);
 489
 490 public:
 491     explicit external_acl_data(external_acl *aDef) : def(cbdataReference(aDef)), name(NULL), arguments(NULL) {}
 492     ~external_acl_data();
 493
 494     external_acl *def;
 495     const char *name;
 496     wordlist *arguments;
 497 };
 498
 499 CBDATA_CLASS_INIT(external_acl_data);
 500
 501 external_acl_data::~external_acl_data()
 502 {
 503     xfree(name);
 504     wordlistDestroy(&arguments);
 505     cbdataReferenceDone(def);
 506 }
 507
 508 void
 509 ACLExternal::parse()
 510 {
 511     if (data) {
 512         self_destruct();
 513         return;
 514     }
 515
 516     char *token = ConfigParser::strtokFile();
 517
 518     if (!token) {
 519         self_destruct();
 520         return;
 521     }
 522
 523     data = new external_acl_data(find_externalAclHelper(token));
 524
 525     if (!data->def) {
 526         delete data;
 527         self_destruct();
 528         return;
 529     }
 530
 531     // def->name is the name of the external_acl_type.
 532     // this is the name of the 'acl' directive being tested
 533     data->name = xstrdup(AclMatchedName);
 534
 535     while ((token = ConfigParser::strtokFile())) {
 536         wordlistAdd(&data->arguments, token);
 537     }
 538 }
 539
 540 bool
 541 ACLExternal::valid () const
 542 {
 543 #if USE_AUTH
 544     if (data->def->require_auth) {
 545         if (authenticateSchemeCount() == 0) {
 546             debugs(28, DBG_CRITICAL, "Can't use proxy auth because no authentication schemes were compiled.");
 547             return false;
 548         }
 549
 550         if (authenticateActiveSchemeCount() == 0) {
 551             debugs(28, DBG_CRITICAL, "Can't use proxy auth because no authentication schemes are fully configured.");
 552             return false;
 553         }
 554     }
 555 #endif
 556
 557     return true;
 558 }
 559
 560 bool
 561 ACLExternal::empty () const
 562 {
 563     return false;
 564 }
 565
 566 ACLExternal::~ACLExternal()
 567 {
 568     delete data;
 569     xfree(class_);
 570 }
 571
 572 static void
 573 copyResultsFromEntry(HttpRequest *req, const ExternalACLEntryPointer &entry)
 574 {
 575     if (req) {
 576 #if USE_AUTH
 577         if (entry->user.size())
 578             req->extacl_user = entry->user;
 579
 580         if (entry->password.size())
 581             req->extacl_passwd = entry->password;
 582 #endif
 583         if (!req->tag.size())
 584             req->tag = entry->tag;
 585
 586         if (entry->log.size())
 587             req->extacl_log = entry->log;
 588
 589         if (entry->message.size())
 590             req->extacl_message = entry->message;
 591
 592         // attach the helper kv-pair to the transaction
 593         UpdateRequestNotes(req->clientConnectionManager.get(), *req, entry->notes);
 594     }
 595 }
 596
 597 static Acl::Answer
 598 aclMatchExternal(external_acl_data *acl, ACLFilledChecklist *ch)
 599 {
 600     debugs(82, 9, HERE << "acl=\"" << acl->def->name << "\"");
 601     ExternalACLEntryPointer entry = ch->extacl_entry;
 602
 603     external_acl_message = "MISSING REQUIRED INFORMATION";
 604
 605     if (entry != NULL) {
 606         if (entry->def == acl->def) {
 607             /* Ours, use it.. if the key matches */
 608             const char *key = makeExternalAclKey(ch, acl);
 609             if (!key)
 610                 return ACCESS_DUNNO; // insufficent data to continue
 611             if (strcmp(key, (char*)entry->key) != 0) {
 612                 debugs(82, 9, "entry key='" << (char *)entry->key << "', our key='" << key << "' do not match. Discarded.");
 613                 // too bad. need a new lookup.
 614                 entry = ch->extacl_entry = NULL;
 615             }
 616         } else {
 617             /* Not ours.. get rid of it */
 618             debugs(82, 9, "entry " << entry << " not valid or not ours. Discarded.");
 619             if (entry != NULL) {
 620                 debugs(82, 9, "entry def=" << entry->def << ", our def=" << acl->def);
 621                 const char *key = makeExternalAclKey(ch, acl); // may be nil
 622                 debugs(82, 9, "entry key='" << (char *)entry->key << "', our key='" << key << "'");
 623             }
 624             entry = ch->extacl_entry = NULL;
 625         }
 626     }
 627
 628     if (!entry) {
 629         debugs(82, 9, HERE << "No helper entry available");
 630 #if USE_AUTH
 631         if (acl->def->require_auth) {
 632             /* Make sure the user is authenticated */
 633             debugs(82, 3, HERE << acl->def->name << " check user authenticated.");
 634             const auto ti = AuthenticateAcl(ch);
 635             if (!ti.allowed()) {
 636                 debugs(82, 2, HERE << acl->def->name << " user not authenticated (" << ti << ")");
 637                 return ti;
 638             }
 639             debugs(82, 3, HERE << acl->def->name << " user is authenticated.");
 640         }
 641 #endif
 642         const char *key = makeExternalAclKey(ch, acl);
 643
 644         if (!key) {
 645             /* Not sufficient data to process */
 646             return ACCESS_DUNNO;
 647         }
 648
 649         entry = static_cast<ExternalACLEntry *>(hash_lookup(acl->def->cache, key));
 650
 651         const ExternalACLEntryPointer staleEntry = entry;
 652         if (entry != NULL && external_acl_entry_expired(acl->def, entry))
 653             entry = NULL;
 654
 655         if (entry != NULL && external_acl_grace_expired(acl->def, entry)) {
 656             // refresh in the background
 657             ExternalACLLookup::Start(ch, acl, true);
 658             debugs(82, 4, HERE << "no need to wait for the refresh of '" <<
 659                    key << "' in '" << acl->def->name << "' (ch=" << ch << ").");
 660         }
 661
 662         if (!entry) {
 663             debugs(82, 2, HERE << acl->def->name << "(\"" << key << "\") = lookup needed");
 664
 665             // TODO: All other helpers allow temporary overload. Should not we?
 666             if (!acl->def->theHelper->willOverload()) {
 667                 debugs(82, 2, HERE << "\"" << key << "\": queueing a call.");
 668                 if (!ch->goAsync(ExternalACLLookup::Instance()))
 669                     debugs(82, 2, "\"" << key << "\": no async support!");
 670                 debugs(82, 2, HERE << "\"" << key << "\": return -1.");
 671                 return ACCESS_DUNNO; // expired cached or simply absent entry
 672             } else {
 673                 if (!staleEntry) {
 674                     debugs(82, DBG_IMPORTANT, "WARNING: external ACL '" << acl->def->name <<
 675                            "' queue full. Request rejected '" << key << "'.");
 676                     external_acl_message = "SYSTEM TOO BUSY, TRY AGAIN LATER";
 677                     return ACCESS_DUNNO;
 678                 } else {
 679                     debugs(82, DBG_IMPORTANT, "WARNING: external ACL '" << acl->def->name <<
 680                            "' queue full. Using stale result. '" << key << "'.");
 681                     entry = staleEntry;
 682                     /* Fall thru to processing below */
 683                 }
 684             }
 685         }
 686     }
 687
 688     debugs(82, 4, HERE << "entry = { date=" <<
 689            (long unsigned int) entry->date <<
 690            ", result=" << entry->result <<
 691            " tag=" << entry->tag <<
 692            " log=" << entry->log << " }");
 693 #if USE_AUTH
 694     debugs(82, 4, HERE << "entry user=" << entry->user);
 695 #endif
 696
 697     external_acl_cache_touch(acl->def, entry);
 698     external_acl_message = entry->message.termedBuf();
 699
 700     debugs(82, 2, HERE << acl->def->name << " = " << entry->result);
 701     copyResultsFromEntry(ch->request, entry);
 702     return entry->result;
 703 }
 704
 705 int
 706 ACLExternal::match(ACLChecklist *checklist)
 707 {
 708     auto answer = aclMatchExternal(data, Filled(checklist));
 709
 710     // convert to tri-state ACL match 1,0,-1
 711     switch (answer) {
 712     case ACCESS_ALLOWED:
 713         return 1; // match
 714
 715     case ACCESS_DENIED:
 716         return 0; // non-match
 717
 718     case ACCESS_DUNNO:
 719     case ACCESS_AUTH_REQUIRED:
 720     default:
 721         // If the answer is not allowed or denied (matches/not matches) and
 722         // async authentication is not in progress, then we are done.
 723         if (checklist->keepMatching())
 724             checklist->markFinished(answer, "aclMatchExternal exception");
 725         return -1; // other
 726     }
 727 }
 728
 729 SBufList
 730 ACLExternal::dump() const
 731 {
 732     external_acl_data const *acl = data;
 733     SBufList rv;
 734     rv.push_back(SBuf(acl->def->name));
 735
 736     for (wordlist *arg = acl->arguments; arg; arg = arg->next) {
 737         SBuf s;
 738         s.Printf(" %s", arg->key);
 739         rv.push_back(s);
 740     }
 741
 742     return rv;
 743 }
 744
 745 /******************************************************************
 746  * external_acl cache
 747  */
 748
 749 static void
 750 external_acl_cache_touch(external_acl * def, const ExternalACLEntryPointer &entry)
 751 {
 752     // this must not be done when nothing is being cached.
 753     if (!def->maybeCacheable(entry->result))
 754         return;
 755
 756     dlinkDelete(&entry->lru, &def->lru_list);
 757     ExternalACLEntry *e = const_cast<ExternalACLEntry *>(entry.getRaw()); // XXX: make hash a std::map of Pointer.
 758     dlinkAdd(e, &entry->lru, &def->lru_list);
 759 }
 760
 761 static char *
 762 makeExternalAclKey(ACLFilledChecklist * ch, external_acl_data * acl_data)
 763 {
 764     static MemBuf mb;
 765     mb.reset();
 766
 767     // check for special case tokens in the format
 768     for (Format::Token *t = acl_data->def->format.format; t ; t = t->next) {
 769
 770         if (t->type == Format::LFT_EXT_ACL_NAME) {
 771             // setup for %ACL
 772             safe_free(ch->al->lastAclName);
 773             ch->al->lastAclName = xstrdup(acl_data->name);
 774         }
 775
 776         if (t->type == Format::LFT_EXT_ACL_DATA) {
 777             // setup string for %DATA
 778             SBuf sb;
 779             for (auto arg = acl_data->arguments; arg; arg = arg->next) {
 780                 if (sb.length())
 781                     sb.append(" ", 1);
 782
 783                 if (acl_data->def->quote == Format::LOG_QUOTE_URL) {
 784                     const char *quoted = rfc1738_escape(arg->key);
 785                     sb.append(quoted, strlen(quoted));
 786                 } else {
 787                     static MemBuf mb2;
 788                     mb2.init();
 789                     strwordquote(&mb2, arg->key);
 790                     sb.append(mb2.buf, mb2.size);
 791                     mb2.clean();
 792                 }
 793             }
 794
 795             ch->al->lastAclData = sb;
 796         }
 797
 798 #if USE_IDENT
 799         if (t->type == Format::LFT_USER_IDENT) {
 800             if (!*ch->rfc931) {
 801                 // if we fail to go async, we still return NULL and the caller
 802                 // will detect the failure in ACLExternal::match().
 803                 (void)ch->goAsync(IdentLookup::Instance());
 804                 return NULL;
 805             }
 806         }
 807 #endif
 808     }
 809
 810     // assemble the full helper lookup string
 811     acl_data->def->format.assemble(mb, ch->al, 0);
 812
 813     return mb.buf;
 814 }
 815
 816 static int
 817 external_acl_entry_expired(external_acl * def, const ExternalACLEntryPointer &entry)
 818 {
 819     if (def->cache_size <= 0 || entry->result == ACCESS_DUNNO)
 820         return 1;
 821
 822     if (entry->date + (entry->result.allowed() ? def->ttl : def->negative_ttl) < squid_curtime)
 823         return 1;
 824     else
 825         return 0;
 826 }
 827
 828 static int
 829 external_acl_grace_expired(external_acl * def, const ExternalACLEntryPointer &entry)
 830 {
 831     if (def->cache_size <= 0 || entry->result == ACCESS_DUNNO)
 832         return 1;
 833
 834     int ttl;
 835     ttl = entry->result.allowed() ? def->ttl : def->negative_ttl;
 836     ttl = (ttl * (100 - def->grace)) / 100;
 837
 838     if (entry->date + ttl <= squid_curtime)
 839         return 1;
 840     else
 841         return 0;
 842 }
 843
 844 static ExternalACLEntryPointer
 845 external_acl_cache_add(external_acl * def, const char *key, ExternalACLEntryData const & data)
 846 {
 847     ExternalACLEntryPointer entry;
 848
 849     if (!def->maybeCacheable(data.result)) {
 850         debugs(82,6, HERE);
 851
 852         if (data.result == ACCESS_DUNNO) {
 853             if (const ExternalACLEntryPointer oldentry = static_cast<ExternalACLEntry *>(hash_lookup(def->cache, key)))
 854                 external_acl_cache_delete(def, oldentry);
 855         }
 856         entry = new ExternalACLEntry;
 857         entry->key = xstrdup(key);
 858         entry->update(data);
 859         entry->def = def;
 860         return entry;
 861     }
 862
 863     entry = static_cast<ExternalACLEntry *>(hash_lookup(def->cache, key));
 864     debugs(82, 2, "external_acl_cache_add: Adding '" << key << "' = " << data.result);
 865
 866     if (entry != NULL) {
 867         debugs(82, 3, "updating existing entry");
 868         entry->update(data);
 869         external_acl_cache_touch(def, entry);
 870         return entry;
 871     }
 872
 873     entry = new ExternalACLEntry;
 874     entry->key = xstrdup(key);
 875     entry->update(data);
 876
 877     def->add(entry);
 878
 879     return entry;
 880 }
 881
 882 static void
 883 external_acl_cache_delete(external_acl * def, const ExternalACLEntryPointer &entry)
 884 {
 885     assert(entry != NULL);
 886     assert(def->cache_size > 0 && entry->def == def);
 887     ExternalACLEntry *e = const_cast<ExternalACLEntry *>(entry.getRaw()); // XXX: make hash a std::map of Pointer.
 888     hash_remove_link(def->cache, e);
 889     dlinkDelete(&e->lru, &def->lru_list);
 890     e->unlock(); // unlock on behalf of the hash
 891     def->cache_entries -= 1;
 892 }
 893
 894 /******************************************************************
 895  * external_acl helpers
 896  */
 897
 898 class externalAclState
 899 {
 900     CBDATA_CLASS(externalAclState);
 901
 902 public:
 903     externalAclState(external_acl* aDef, const char *aKey) :
 904         callback(NULL),
 905         callback_data(NULL),
 906         key(xstrdup(aKey)),
 907         def(cbdataReference(aDef)),
 908         queue(NULL)
 909     {}
 910     ~externalAclState();
 911
 912     EAH *callback;
 913     void *callback_data;
 914     char *key;
 915     external_acl *def;
 916     dlink_node list;
 917     externalAclState *queue;
 918 };
 919
 920 CBDATA_CLASS_INIT(externalAclState);
 921
 922 externalAclState::~externalAclState()
 923 {
 924     xfree(key);
 925     cbdataReferenceDone(callback_data);
 926     cbdataReferenceDone(def);
 927 }
 928
 929 /*
 930  * The helper program receives queries on stdin, one
 931  * per line, and must return the result on on stdout
 932  *
 933  * General result syntax:
 934  *
 935  *   OK/ERR keyword=value ...
 936  *
 937  * Keywords:
 938  *
 939  *   user=      The users name (login)
 940  *   message=   Message describing the reason
 941  *   tag=   A string tag to be applied to the request that triggered the acl match.
 942  *          applies to both OK and ERR responses.
 943  *          Won't override existing request tags.
 944  *   log=   A string to be used in access logging
 945  *
 946  * Other keywords may be added to the protocol later
 947  *
 948  * value needs to be URL-encoded or enclosed in double quotes (")
 949  * with \-escaping on any whitespace, quotes, or slashes (\).
 950  */
 951 static void
 952 externalAclHandleReply(void *data, const Helper::Reply &reply)
 953 {
 954     externalAclState *state = static_cast<externalAclState *>(data);
 955     externalAclState *next;
 956     ExternalACLEntryData entryData;
 957
 958     debugs(82, 2, HERE << "reply=" << reply);
 959
 960     if (reply.result == Helper::Okay)
 961         entryData.result = ACCESS_ALLOWED;
 962     else if (reply.result == Helper::Error)
 963         entryData.result = ACCESS_DENIED;
 964     else //BrokenHelper,TimedOut or Unknown. Should not cached.
 965         entryData.result = ACCESS_DUNNO;
 966
 967     // XXX: make entryData store a proper Helper::Reply object instead of copying.
 968
 969     entryData.notes.append(&reply.notes);
 970
 971     const char *label = reply.notes.findFirst("tag");
 972     if (label != NULL && *label != '\0')
 973         entryData.tag = label;
 974
 975     label = reply.notes.findFirst("message");
 976     if (label != NULL && *label != '\0')
 977         entryData.message = label;
 978
 979     label = reply.notes.findFirst("log");
 980     if (label != NULL && *label != '\0')
 981         entryData.log = label;
 982
 983 #if USE_AUTH
 984     label = reply.notes.findFirst("user");
 985     if (label != NULL && *label != '\0')
 986         entryData.user = label;
 987
 988     label = reply.notes.findFirst("password");
 989     if (label != NULL && *label != '\0')
 990         entryData.password = label;
 991 #endif
 992
 993     // XXX: This state->def access conflicts with the cbdata validity check
 994     // below.
 995     dlinkDelete(&state->list, &state->def->queue);
 996
 997     ExternalACLEntryPointer entry;
 998     if (cbdataReferenceValid(state->def))
 999         entry = external_acl_cache_add(state->def, state->key, entryData);
1000
1001     do {
1002         void *cbdata;
1003         if (state->callback && cbdataReferenceValidDone(state->callback_data, &cbdata))
1004             state->callback(cbdata, entry);
1005
1006         next = state->queue;
1007         state->queue = NULL;
1008
1009         delete state;
1010
1011         state = next;
1012     } while (state);
1013 }
1014
1015 void
1016 ACLExternal::ExternalAclLookup(ACLChecklist *checklist, ACLExternal * me)
1017 {
1018     ExternalACLLookup::Start(checklist, me->data, false);
1019 }
1020
1021 void
1022 ExternalACLLookup::Start(ACLChecklist *checklist, external_acl_data *acl, bool inBackground)
1023 {
1024     external_acl *def = acl->def;
1025
1026     ACLFilledChecklist *ch = Filled(checklist);
1027     const char *key = makeExternalAclKey(ch, acl);
1028     assert(key); // XXX: will fail if EXT_ACL_IDENT case needs an async lookup
1029
1030     debugs(82, 2, HERE << (inBackground ? "bg" : "fg") << " lookup in '" <<
1031            def->name << "' for '" << key << "'");
1032
1033     /* Check for a pending lookup to hook into */
1034     // only possible if we are caching results.
1035     externalAclState *oldstate = NULL;
1036     if (def->cache_size > 0) {
1037         for (dlink_node *node = def->queue.head; node; node = node->next) {
1038             externalAclState *oldstatetmp = static_cast<externalAclState *>(node->data);
1039
1040             if (strcmp(key, oldstatetmp->key) == 0) {
1041                 oldstate = oldstatetmp;
1042                 break;
1043             }
1044         }
1045     }
1046
1047     // A background refresh has no need to piggiback on a pending request:
1048     // When the pending request completes, the cache will be refreshed anyway.
1049     if (oldstate && inBackground) {
1050         debugs(82, 7, HERE << "'" << def->name << "' queue is already being refreshed (ch=" << ch << ")");
1051         return;
1052     }
1053
1054     externalAclState *state = new externalAclState(def, key);
1055
1056     if (!inBackground) {
1057         state->callback = &ExternalACLLookup::LookupDone;
1058         state->callback_data = cbdataReference(checklist);
1059     }
1060
1061     if (oldstate) {
1062         /* Hook into pending lookup */
1063         state->queue = oldstate->queue;
1064         oldstate->queue = state;
1065     } else {
1066         /* No pending lookup found. Sumbit to helper */
1067
1068         MemBuf buf;
1069         buf.init();
1070         buf.appendf("%s\n", key);
1071         debugs(82, 4, "externalAclLookup: looking up for '" << key << "' in '" << def->name << "'.");
1072
1073         if (!def->theHelper->trySubmit(buf.buf, externalAclHandleReply, state)) {
1074             debugs(82, 7, HERE << "'" << def->name << "' submit to helper failed");
1075             assert(inBackground); // or the caller should have checked
1076             delete state;
1077             return;
1078         }
1079
1080         dlinkAdd(state, &state->list, &def->queue);
1081
1082         buf.clean();
1083     }
1084
1085     debugs(82, 4, "externalAclLookup: will wait for the result of '" << key <<
1086            "' in '" << def->name << "' (ch=" << ch << ").");
1087 }
1088
1089 static void
1090 externalAclStats(StoreEntry * sentry)
1091 {
1092     for (external_acl *p = Config.externalAclHelperList; p; p = p->next) {
1093         storeAppendPrintf(sentry, "External ACL Statistics: %s\n", p->name);
1094         storeAppendPrintf(sentry, "Cache size: %d\n", p->cache->count);
1095         assert(p->theHelper);
1096         p->theHelper->packStatsInto(sentry);
1097         storeAppendPrintf(sentry, "\n");
1098     }
1099 }
1100
1101 static void
1102 externalAclRegisterWithCacheManager(void)
1103 {
1104     Mgr::RegisterAction("external_acl",
1105                         "External ACL stats",
1106                         externalAclStats, 0, 1);
1107 }
1108
1109 void
1110 externalAclInit(void)
1111 {
1112     for (external_acl *p = Config.externalAclHelperList; p; p = p->next) {
1113         if (!p->cache)
1114             p->cache = hash_create((HASHCMP *) strcmp, hashPrime(1024), hash4);
1115
1116         if (!p->theHelper)
1117             p->theHelper = new helper(p->name);
1118
1119         p->theHelper->cmdline = p->cmdline;
1120
1121         p->theHelper->childs.updateLimits(p->children);
1122
1123         p->theHelper->ipc_type = IPC_TCP_SOCKET;
1124
1125         p->theHelper->addr = p->local_addr;
1126
1127         helperOpenServers(p->theHelper);
1128     }
1129
1130     externalAclRegisterWithCacheManager();
1131 }
1132
1133 void
1134 externalAclShutdown(void)
1135 {
1136     external_acl *p;
1137
1138     for (p = Config.externalAclHelperList; p; p = p->next) {
1139         helperShutdown(p->theHelper);
1140     }
1141 }
1142
1143 ExternalACLLookup ExternalACLLookup::instance_;
1144 ExternalACLLookup *
1145 ExternalACLLookup::Instance()
1146 {
1147     return &instance_;
1148 }
1149
1150 void
1151 ExternalACLLookup::checkForAsync(ACLChecklist *checklist)const
1152 {
1153     /* TODO: optimise this - we probably have a pointer to this
1154      * around somewhere */
1155     ACL *acl = ACL::FindByName(AclMatchedName);
1156     assert(acl);
1157     ACLExternal *me = dynamic_cast<ACLExternal *> (acl);
1158     assert (me);
1159     ACLExternal::ExternalAclLookup(checklist, me);
1160 }
1161
1162 /// Called when an async lookup returns
1163 void
1164 ExternalACLLookup::LookupDone(void *data, const ExternalACLEntryPointer &result)
1165 {
1166     ACLFilledChecklist *checklist = Filled(static_cast<ACLChecklist*>(data));
1167     checklist->extacl_entry = result;
1168     checklist->resumeNonBlockingCheck(ExternalACLLookup::Instance());
1169 }
1170
1171 ACL *
1172 ACLExternal::clone() const
1173 {
1174     return new ACLExternal(*this);
1175 }
1176
1177 ACLExternal::ACLExternal(char const *theClass) : data(NULL), class_(xstrdup(theClass))
1178 {}
1179
1180 ACLExternal::ACLExternal(ACLExternal const & old) : data(NULL), class_(old.class_ ? xstrdup(old.class_) : NULL)
1181 {
1182     /* we don't have copy constructors for the data yet */
1183     assert(!old.data);
1184 }
1185
1186 char const *
1187 ACLExternal::typeString() const
1188 {
1189     return class_;
1190 }
1191
1192 bool
1193 ACLExternal::isProxyAuth() const
1194 {
1195 #if USE_AUTH
1196     return data->def->require_auth;
1197 #else
1198     return false;
1199 #endif
1200 }
1201