src/external_acl.cc

   1 /*
   2  * Copyright (C) 1996-2017 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 82    External ACL */
  10
  11 #include "squid.h"
  12 #include "acl/Acl.h"
  13 #include "acl/FilledChecklist.h"
  14 #include "cache_cf.h"
  15 #include "client_side.h"
  16 #include "client_side_request.h"
  17 #include "comm/Connection.h"
  18 #include "ConfigParser.h"
  19 #include "ExternalACL.h"
  20 #include "ExternalACLEntry.h"
  21 #include "fde.h"
  22 #include "format/Token.h"
  23 #include "helper.h"
  24 #include "helper/Reply.h"
  25 #include "http/Stream.h"
  26 #include "HttpHeaderTools.h"
  27 #include "HttpReply.h"
  28 #include "HttpRequest.h"
  29 #include "ip/tools.h"
  30 #include "MemBuf.h"
  31 #include "mgr/Registration.h"
  32 #include "rfc1738.h"
  33 #include "SquidConfig.h"
  34 #include "SquidString.h"
  35 #include "SquidTime.h"
  36 #include "Store.h"
  37 #include "tools.h"
  38 #include "URL.h"
  39 #include "wordlist.h"
  40 #if USE_OPENSSL
  41 #include "ssl/ServerBump.h"
  42 #include "ssl/support.h"
  43 #endif
  44 #if USE_AUTH
  45 #include "auth/Acl.h"
  46 #include "auth/Gadgets.h"
  47 #include "auth/UserRequest.h"
  48 #endif
  49 #if USE_IDENT
  50 #include "ident/AclIdent.h"
  51 #endif
  52
  53 #ifndef DEFAULT_EXTERNAL_ACL_TTL
  54 #define DEFAULT_EXTERNAL_ACL_TTL 1 * 60 * 60
  55 #endif
  56 #ifndef DEFAULT_EXTERNAL_ACL_CHILDREN
  57 #define DEFAULT_EXTERNAL_ACL_CHILDREN 5
  58 #endif
  59
  60 static char *makeExternalAclKey(ACLFilledChecklist * ch, external_acl_data * acl_data);
  61 static void external_acl_cache_delete(external_acl * def, const ExternalACLEntryPointer &entry);
  62 static int external_acl_entry_expired(external_acl * def, const ExternalACLEntryPointer &entry);
  63 static int external_acl_grace_expired(external_acl * def, const ExternalACLEntryPointer &entry);
  64 static void external_acl_cache_touch(external_acl * def, const ExternalACLEntryPointer &entry);
  65 static ExternalACLEntryPointer external_acl_cache_add(external_acl * def, const char *key, ExternalACLEntryData const &data);
  66
  67 /******************************************************************
  68  * external_acl directive
  69  */
  70
  71 class external_acl
  72 {
  73     /* FIXME: These are not really cbdata, but it is an easy way
  74      * to get them pooled, refcounted, accounted and freed properly...
  75      */
  76     CBDATA_CLASS(external_acl);
  77
  78 public:
  79     external_acl();
  80     ~external_acl();
  81
  82     external_acl *next;
  83
  84     void add(const ExternalACLEntryPointer &);
  85
  86     void trimCache();
  87
  88     bool maybeCacheable(const allow_t &) const;
  89
  90     int ttl;
  91
  92     int negative_ttl;
  93
  94     int grace;
  95
  96     char *name;
  97
  98     Format::Format format;
  99
 100     wordlist *cmdline;
 101
 102     Helper::ChildConfig children;
 103
 104     helper *theHelper;
 105
 106     hash_table *cache;
 107
 108     dlink_list lru_list;
 109
 110     int cache_size;
 111
 112     int cache_entries;
 113
 114     dlink_list queue;
 115
 116 #if USE_AUTH
 117     /**
 118      * Configuration flag. May only be altered by the configuration parser.
 119      *
 120      * Indicates that all uses of this external_acl_type helper require authentication
 121      * details to be processed. If none are available its a fail match.
 122      */
 123     bool require_auth;
 124 #endif
 125
 126     Format::Quoting quote; // default quoting to use, set by protocol= parameter
 127
 128     Ip::Address local_addr;
 129 };
 130
 131 CBDATA_CLASS_INIT(external_acl);
 132
 133 external_acl::external_acl() :
 134     next(NULL),
 135     ttl(DEFAULT_EXTERNAL_ACL_TTL),
 136     negative_ttl(-1),
 137     grace(1),
 138     name(NULL),
 139     format("external_acl_type"),
 140     cmdline(NULL),
 141     children(DEFAULT_EXTERNAL_ACL_CHILDREN),
 142     theHelper(NULL),
 143     cache(NULL),
 144     cache_size(256*1024),
 145     cache_entries(0),
 146 #if USE_AUTH
 147     require_auth(0),
 148 #endif
 149     quote(Format::LOG_QUOTE_URL)
 150 {
 151     local_addr.setLocalhost();
 152 }
 153
 154 external_acl::~external_acl()
 155 {
 156     xfree(name);
 157     wordlistDestroy(&cmdline);
 158
 159     if (theHelper) {
 160         helperShutdown(theHelper);
 161         delete theHelper;
 162         theHelper = NULL;
 163     }
 164
 165     while (lru_list.tail) {
 166         ExternalACLEntryPointer e(static_cast<ExternalACLEntry *>(lru_list.tail->data));
 167         external_acl_cache_delete(this, e);
 168     }
 169     if (cache)
 170         hashFreeMemory(cache);
 171
 172     while (next) {
 173         external_acl *node = next;
 174         next = node->next;
 175         node->next = NULL; // prevent recursion
 176         delete node;
 177     }
 178 }
 179
 180 void
 181 parse_externalAclHelper(external_acl ** list)
 182 {
 183     char *token = ConfigParser::NextToken();
 184
 185     if (!token) {
 186         self_destruct();
 187         return;
 188     }
 189
 190     external_acl *a = new external_acl;
 191     a->name = xstrdup(token);
 192
 193     // Allow supported %macros inside quoted tokens
 194     ConfigParser::EnableMacros();
 195     token = ConfigParser::NextToken();
 196
 197     /* Parse options */
 198     while (token) {
 199         if (strncmp(token, "ttl=", 4) == 0) {
 200             a->ttl = atoi(token + 4);
 201         } else if (strncmp(token, "negative_ttl=", 13) == 0) {
 202             a->negative_ttl = atoi(token + 13);
 203         } else if (strncmp(token, "children=", 9) == 0) {
 204             a->children.n_max = atoi(token + 9);
 205             debugs(0, DBG_CRITICAL, "WARNING: external_acl_type option children=N has been deprecated in favor of children-max=N and children-startup=N");
 206         } else if (strncmp(token, "children-max=", 13) == 0) {
 207             a->children.n_max = atoi(token + 13);
 208         } else if (strncmp(token, "children-startup=", 17) == 0) {
 209             a->children.n_startup = atoi(token + 17);
 210         } else if (strncmp(token, "children-idle=", 14) == 0) {
 211             a->children.n_idle = atoi(token + 14);
 212         } else if (strncmp(token, "concurrency=", 12) == 0) {
 213             a->children.concurrency = atoi(token + 12);
 214         } else if (strncmp(token, "queue-size=", 11) == 0) {
 215             a->children.queue_size = atoi(token + 11);
 216             a->children.defaultQueueSize = false;
 217         } else if (strncmp(token, "cache=", 6) == 0) {
 218             a->cache_size = atoi(token + 6);
 219         } else if (strncmp(token, "grace=", 6) == 0) {
 220             a->grace = atoi(token + 6);
 221         } else if (strcmp(token, "protocol=2.5") == 0) {
 222             a->quote = Format::LOG_QUOTE_SHELL;
 223         } else if (strcmp(token, "protocol=3.0") == 0) {
 224             debugs(3, DBG_PARSE_NOTE(2), "WARNING: external_acl_type option protocol=3.0 is deprecated. Remove this from your config.");
 225             a->quote = Format::LOG_QUOTE_URL;
 226         } else if (strcmp(token, "quote=url") == 0) {
 227             debugs(3, DBG_PARSE_NOTE(2), "WARNING: external_acl_type option quote=url is deprecated. Remove this from your config.");
 228             a->quote = Format::LOG_QUOTE_URL;
 229         } else if (strcmp(token, "quote=shell") == 0) {
 230             debugs(3, DBG_PARSE_NOTE(2), "WARNING: external_acl_type option quote=shell is deprecated. Use protocol=2.5 if still needed.");
 231             a->quote = Format::LOG_QUOTE_SHELL;
 232
 233             /* INET6: allow admin to configure some helpers explicitly to
 234                       bind to IPv4/v6 localhost port. */
 235         } else if (strcmp(token, "ipv4") == 0) {
 236             if ( !a->local_addr.setIPv4() ) {
 237                 debugs(3, DBG_CRITICAL, "WARNING: Error converting " << a->local_addr << " to IPv4 in " << a->name );
 238             }
 239         } else if (strcmp(token, "ipv6") == 0) {
 240             if (!Ip::EnableIpv6)
 241                 debugs(3, DBG_CRITICAL, "WARNING: --enable-ipv6 required for external ACL helpers to use IPv6: " << a->name );
 242             // else nothing to do.
 243         } else {
 244             break;
 245         }
 246
 247         token = ConfigParser::NextToken();
 248     }
 249     ConfigParser::DisableMacros();
 250
 251     /* check that child startup value is sane. */
 252     if (a->children.n_startup > a->children.n_max)
 253         a->children.n_startup = a->children.n_max;
 254
 255     /* check that child idle value is sane. */
 256     if (a->children.n_idle > a->children.n_max)
 257         a->children.n_idle = a->children.n_max;
 258     if (a->children.n_idle < 1)
 259         a->children.n_idle = 1;
 260
 261     if (a->negative_ttl == -1)
 262         a->negative_ttl = a->ttl;
 263
 264     if (a->children.defaultQueueSize)
 265         a->children.queue_size = 2 * a->children.n_max;
 266
 267     /* Legacy external_acl_type format parser.
 268      * Handles a series of %... tokens where any non-% means
 269      * the start of another parameter field (ie the path to binary).
 270      */
 271     enum Format::Quoting quote = Format::LOG_QUOTE_NONE;
 272     Format::Token **fmt = &a->format.format;
 273     bool data_used = false;
 274     while (token) {
 275         /* stop on first non-% token found */
 276         if (*token != '%')
 277             break;
 278
 279         *fmt = new Format::Token;
 280         // these tokens are whitespace delimited
 281         (*fmt)->space = true;
 282
 283         // set the default encoding to match the protocol= config
 284         // this will be overridden by explicit %macro attributes
 285         (*fmt)->quote = a->quote;
 286
 287         // compatibility for old tokens incompatible with Format::Token syntax
 288 #if USE_OPENSSL // dont bother if we dont have to.
 289         if (strncmp(token, "%USER_CERT_", 11) == 0) {
 290             (*fmt)->type = Format::LFT_EXT_ACL_USER_CERT;
 291             (*fmt)->data.string = xstrdup(token + 11);
 292             (*fmt)->data.header.header = (*fmt)->data.string;
 293         } else if (strncmp(token, "%USER_CA_CERT_", 14) == 0) {
 294             (*fmt)->type = Format::LFT_EXT_ACL_USER_CA_CERT;
 295             (*fmt)->data.string = xstrdup(token + 14);
 296             (*fmt)->data.header.header = (*fmt)->data.string;
 297         } else if (strncmp(token, "%CA_CERT_", 9) == 0) {
 298             debugs(82, DBG_PARSE_NOTE(DBG_IMPORTANT), "WARNING: external_acl_type %CA_CERT_* code is obsolete. Use %USER_CA_CERT_* instead");
 299             (*fmt)->type = Format::LFT_EXT_ACL_USER_CA_CERT;
 300             (*fmt)->data.string = xstrdup(token + 9);
 301             (*fmt)->data.header.header = (*fmt)->data.string;
 302         } else
 303 #endif
 304             if (strncmp(token,"%<{", 3) == 0) {
 305                 SBuf tmp("%<h");
 306                 tmp.append(token+2);
 307                 debugs(82, DBG_PARSE_NOTE(DBG_IMPORTANT), "WARNING: external_acl_type format %<{...} is deprecated. Use " << tmp);
 308                 const size_t parsedLen = (*fmt)->parse(tmp.c_str(), &quote);
 309                 assert(parsedLen == tmp.length());
 310                 assert((*fmt)->type == Format::LFT_REPLY_HEADER ||
 311                        (*fmt)->type == Format::LFT_REPLY_HEADER_ELEM);
 312
 313             } else if (strncmp(token,"%>{", 3) == 0) {
 314                 SBuf tmp("%>ha");
 315                 tmp.append(token+2);
 316                 debugs(82, DBG_PARSE_NOTE(DBG_IMPORTANT), "WARNING: external_acl_type format %>{...} is deprecated. Use " << tmp);
 317                 const size_t parsedLen = (*fmt)->parse(tmp.c_str(), &quote);
 318                 assert(parsedLen == tmp.length());
 319                 assert((*fmt)->type == Format::LFT_ADAPTED_REQUEST_HEADER ||
 320                        (*fmt)->type == Format::LFT_ADAPTED_REQUEST_HEADER_ELEM);
 321
 322             } else {
 323                 // we can use the Format::Token::parse() method since it
 324                 // only pulls off one token. Since we already checked
 325                 // for '%' prefix above this is guaranteed to be a token.
 326                 const size_t len = (*fmt)->parse(token, &quote);
 327                 assert(len == strlen(token));
 328             }
 329
 330         // process special token-specific actions (only if necessary)
 331 #if USE_AUTH
 332         if ((*fmt)->type == Format::LFT_USER_LOGIN)
 333             a->require_auth = true;
 334 #endif
 335
 336         if ((*fmt)->type == Format::LFT_EXT_ACL_DATA)
 337             data_used = true;
 338
 339         fmt = &((*fmt)->next);
 340         token = ConfigParser::NextToken();
 341     }
 342
 343     /* There must be at least one format token */
 344     if (!a->format.format) {
 345         delete a;
 346         self_destruct();
 347         return;
 348     }
 349
 350     // format has implicit %DATA on the end if not used explicitly
 351     if (!data_used) {
 352         *fmt = new Format::Token;
 353         (*fmt)->type = Format::LFT_EXT_ACL_DATA;
 354         (*fmt)->quote = Format::LOG_QUOTE_NONE;
 355     }
 356
 357     /* helper */
 358     if (!token) {
 359         delete a;
 360         self_destruct();
 361         return;
 362     }
 363
 364     wordlistAdd(&a->cmdline, token);
 365
 366     /* arguments */
 367     parse_wordlist(&a->cmdline);
 368
 369     while (*list)
 370         list = &(*list)->next;
 371
 372     *list = a;
 373 }
 374
 375 void
 376 dump_externalAclHelper(StoreEntry * sentry, const char *name, const external_acl * list)
 377 {
 378     const external_acl *node;
 379     const wordlist *word;
 380
 381     for (node = list; node; node = node->next) {
 382         storeAppendPrintf(sentry, "%s %s", name, node->name);
 383
 384         if (!node->local_addr.isIPv6())
 385             storeAppendPrintf(sentry, " ipv4");
 386         else
 387             storeAppendPrintf(sentry, " ipv6");
 388
 389         if (node->ttl != DEFAULT_EXTERNAL_ACL_TTL)
 390             storeAppendPrintf(sentry, " ttl=%d", node->ttl);
 391
 392         if (node->negative_ttl != node->ttl)
 393             storeAppendPrintf(sentry, " negative_ttl=%d", node->negative_ttl);
 394
 395         if (node->grace)
 396             storeAppendPrintf(sentry, " grace=%d", node->grace);
 397
 398         if (node->children.n_max != DEFAULT_EXTERNAL_ACL_CHILDREN)
 399             storeAppendPrintf(sentry, " children-max=%d", node->children.n_max);
 400
 401         if (node->children.n_startup != 0) // sync with helper/ChildConfig.cc default
 402             storeAppendPrintf(sentry, " children-startup=%d", node->children.n_startup);
 403
 404         if (node->children.n_idle != 1) // sync with helper/ChildConfig.cc default
 405             storeAppendPrintf(sentry, " children-idle=%d", node->children.n_idle);
 406
 407         if (node->children.concurrency != 0)
 408             storeAppendPrintf(sentry, " concurrency=%d", node->children.concurrency);
 409
 410         if (node->cache)
 411             storeAppendPrintf(sentry, " cache=%d", node->cache_size);
 412
 413         if (node->quote == Format::LOG_QUOTE_SHELL)
 414             storeAppendPrintf(sentry, " protocol=2.5");
 415
 416         node->format.dump(sentry, NULL, false);
 417
 418         for (word = node->cmdline; word; word = word->next)
 419             storeAppendPrintf(sentry, " %s", word->key);
 420
 421         storeAppendPrintf(sentry, "\n");
 422     }
 423 }
 424
 425 void
 426 free_externalAclHelper(external_acl ** list)
 427 {
 428     delete *list;
 429     *list = NULL;
 430 }
 431
 432 static external_acl *
 433 find_externalAclHelper(const char *name)
 434 {
 435     external_acl *node;
 436
 437     for (node = Config.externalAclHelperList; node; node = node->next) {
 438         if (strcmp(node->name, name) == 0)
 439             return node;
 440     }
 441
 442     return NULL;
 443 }
 444
 445 void
 446 external_acl::add(const ExternalACLEntryPointer &anEntry)
 447 {
 448     trimCache();
 449     assert(anEntry != NULL);
 450     assert (anEntry->def == NULL);
 451     anEntry->def = this;
 452     ExternalACLEntry *e = const_cast<ExternalACLEntry *>(anEntry.getRaw()); // XXX: make hash a std::map of Pointer.
 453     hash_join(cache, e);
 454     dlinkAdd(e, &e->lru, &lru_list);
 455     e->lock(); //cbdataReference(e); // lock it on behalf of the hash
 456     ++cache_entries;
 457 }
 458
 459 void
 460 external_acl::trimCache()
 461 {
 462     if (cache_size && cache_entries >= cache_size) {
 463         ExternalACLEntryPointer e(static_cast<ExternalACLEntry *>(lru_list.tail->data));
 464         external_acl_cache_delete(this, e);
 465     }
 466 }
 467
 468 bool
 469 external_acl::maybeCacheable(const allow_t &result) const
 470 {
 471     if (cache_size <= 0)
 472         return false; // cache is disabled
 473
 474     if (result == ACCESS_DUNNO)
 475         return false; // non-cacheable response
 476
 477     if ((result.allowed() ? ttl : negative_ttl) <= 0)
 478         return false; // not caching this type of response
 479
 480     return true;
 481 }
 482
 483 /******************************************************************
 484  * external acl type
 485  */
 486
 487 class external_acl_data
 488 {
 489     CBDATA_CLASS(external_acl_data);
 490
 491 public:
 492     explicit external_acl_data(external_acl *aDef) : def(cbdataReference(aDef)), name(NULL), arguments(NULL) {}
 493     ~external_acl_data();
 494
 495     external_acl *def;
 496     const char *name;
 497     wordlist *arguments;
 498 };
 499
 500 CBDATA_CLASS_INIT(external_acl_data);
 501
 502 external_acl_data::~external_acl_data()
 503 {
 504     xfree(name);
 505     wordlistDestroy(&arguments);
 506     cbdataReferenceDone(def);
 507 }
 508
 509 void
 510 ACLExternal::parse()
 511 {
 512     if (data) {
 513         self_destruct();
 514         return;
 515     }
 516
 517     char *token = ConfigParser::strtokFile();
 518
 519     if (!token) {
 520         self_destruct();
 521         return;
 522     }
 523
 524     data = new external_acl_data(find_externalAclHelper(token));
 525
 526     if (!data->def) {
 527         delete data;
 528         self_destruct();
 529         return;
 530     }
 531
 532     // def->name is the name of the external_acl_type.
 533     // this is the name of the 'acl' directive being tested
 534     data->name = xstrdup(AclMatchedName);
 535
 536     while ((token = ConfigParser::strtokFile())) {
 537         wordlistAdd(&data->arguments, token);
 538     }
 539 }
 540
 541 bool
 542 ACLExternal::valid () const
 543 {
 544 #if USE_AUTH
 545     if (data->def->require_auth) {
 546         if (authenticateSchemeCount() == 0) {
 547             debugs(28, DBG_CRITICAL, "Can't use proxy auth because no authentication schemes were compiled.");
 548             return false;
 549         }
 550
 551         if (authenticateActiveSchemeCount() == 0) {
 552             debugs(28, DBG_CRITICAL, "Can't use proxy auth because no authentication schemes are fully configured.");
 553             return false;
 554         }
 555     }
 556 #endif
 557
 558     return true;
 559 }
 560
 561 bool
 562 ACLExternal::empty () const
 563 {
 564     return false;
 565 }
 566
 567 ACLExternal::~ACLExternal()
 568 {
 569     delete data;
 570     xfree(class_);
 571 }
 572
 573 static void
 574 copyResultsFromEntry(HttpRequest *req, const ExternalACLEntryPointer &entry)
 575 {
 576     if (req) {
 577 #if USE_AUTH
 578         if (entry->user.size())
 579             req->extacl_user = entry->user;
 580
 581         if (entry->password.size())
 582             req->extacl_passwd = entry->password;
 583 #endif
 584         if (!req->tag.size())
 585             req->tag = entry->tag;
 586
 587         if (entry->log.size())
 588             req->extacl_log = entry->log;
 589
 590         if (entry->message.size())
 591             req->extacl_message = entry->message;
 592
 593         // attach the helper kv-pair to the transaction
 594         UpdateRequestNotes(req->clientConnectionManager.get(), *req, entry->notes);
 595     }
 596 }
 597
 598 static allow_t
 599 aclMatchExternal(external_acl_data *acl, ACLFilledChecklist *ch)
 600 {
 601     debugs(82, 9, HERE << "acl=\"" << acl->def->name << "\"");
 602     ExternalACLEntryPointer entry = ch->extacl_entry;
 603
 604     external_acl_message = "MISSING REQUIRED INFORMATION";
 605
 606     if (entry != NULL) {
 607         if (entry->def == acl->def) {
 608             /* Ours, use it.. if the key matches */
 609             const char *key = makeExternalAclKey(ch, acl);
 610             if (!key)
 611                 return ACCESS_DUNNO; // insufficent data to continue
 612             if (strcmp(key, (char*)entry->key) != 0) {
 613                 debugs(82, 9, "entry key='" << (char *)entry->key << "', our key='" << key << "' dont match. Discarded.");
 614                 // too bad. need a new lookup.
 615                 entry = ch->extacl_entry = NULL;
 616             }
 617         } else {
 618             /* Not ours.. get rid of it */
 619             debugs(82, 9, "entry " << entry << " not valid or not ours. Discarded.");
 620             if (entry != NULL) {
 621                 debugs(82, 9, "entry def=" << entry->def << ", our def=" << acl->def);
 622                 const char *key = makeExternalAclKey(ch, acl); // may be nil
 623                 debugs(82, 9, "entry key='" << (char *)entry->key << "', our key='" << key << "'");
 624             }
 625             entry = ch->extacl_entry = NULL;
 626         }
 627     }
 628
 629     if (!entry) {
 630         debugs(82, 9, HERE << "No helper entry available");
 631 #if USE_AUTH
 632         if (acl->def->require_auth) {
 633             /* Make sure the user is authenticated */
 634             debugs(82, 3, HERE << acl->def->name << " check user authenticated.");
 635             const allow_t ti = AuthenticateAcl(ch);
 636             if (!ti.allowed()) {
 637                 debugs(82, 2, HERE << acl->def->name << " user not authenticated (" << ti << ")");
 638                 return ti;
 639             }
 640             debugs(82, 3, HERE << acl->def->name << " user is authenticated.");
 641         }
 642 #endif
 643         const char *key = makeExternalAclKey(ch, acl);
 644
 645         if (!key) {
 646             /* Not sufficient data to process */
 647             return ACCESS_DUNNO;
 648         }
 649
 650         entry = static_cast<ExternalACLEntry *>(hash_lookup(acl->def->cache, key));
 651
 652         const ExternalACLEntryPointer staleEntry = entry;
 653         if (entry != NULL && external_acl_entry_expired(acl->def, entry))
 654             entry = NULL;
 655
 656         if (entry != NULL && external_acl_grace_expired(acl->def, entry)) {
 657             // refresh in the background
 658             ExternalACLLookup::Start(ch, acl, true);
 659             debugs(82, 4, HERE << "no need to wait for the refresh of '" <<
 660                    key << "' in '" << acl->def->name << "' (ch=" << ch << ").");
 661         }
 662
 663         if (!entry) {
 664             debugs(82, 2, HERE << acl->def->name << "(\"" << key << "\") = lookup needed");
 665
 666             // TODO: All other helpers allow temporary overload. Should not we?
 667             if (!acl->def->theHelper->willOverload()) {
 668                 debugs(82, 2, HERE << "\"" << key << "\": queueing a call.");
 669                 if (!ch->goAsync(ExternalACLLookup::Instance()))
 670                     debugs(82, 2, "\"" << key << "\": no async support!");
 671                 debugs(82, 2, HERE << "\"" << key << "\": return -1.");
 672                 return ACCESS_DUNNO; // expired cached or simply absent entry
 673             } else {
 674                 if (!staleEntry) {
 675                     debugs(82, DBG_IMPORTANT, "WARNING: external ACL '" << acl->def->name <<
 676                            "' queue full. Request rejected '" << key << "'.");
 677                     external_acl_message = "SYSTEM TOO BUSY, TRY AGAIN LATER";
 678                     return ACCESS_DUNNO;
 679                 } else {
 680                     debugs(82, DBG_IMPORTANT, "WARNING: external ACL '" << acl->def->name <<
 681                            "' queue full. Using stale result. '" << key << "'.");
 682                     entry = staleEntry;
 683                     /* Fall thru to processing below */
 684                 }
 685             }
 686         }
 687     }
 688
 689     debugs(82, 4, HERE << "entry = { date=" <<
 690            (long unsigned int) entry->date <<
 691            ", result=" << entry->result <<
 692            " tag=" << entry->tag <<
 693            " log=" << entry->log << " }");
 694 #if USE_AUTH
 695     debugs(82, 4, HERE << "entry user=" << entry->user);
 696 #endif
 697
 698     external_acl_cache_touch(acl->def, entry);
 699     external_acl_message = entry->message.termedBuf();
 700
 701     debugs(82, 2, HERE << acl->def->name << " = " << entry->result);
 702     copyResultsFromEntry(ch->request, entry);
 703     return entry->result;
 704 }
 705
 706 int
 707 ACLExternal::match(ACLChecklist *checklist)
 708 {
 709     allow_t answer = aclMatchExternal(data, Filled(checklist));
 710
 711     // convert to tri-state ACL match 1,0,-1
 712     switch (answer) {
 713     case ACCESS_ALLOWED:
 714         return 1; // match
 715
 716     case ACCESS_DENIED:
 717         return 0; // non-match
 718
 719     case ACCESS_DUNNO:
 720     case ACCESS_AUTH_REQUIRED:
 721     default:
 722         // If the answer is not allowed or denied (matches/not matches) and
 723         // async authentication is not in progress, then we are done.
 724         if (checklist->keepMatching())
 725             checklist->markFinished(answer, "aclMatchExternal exception");
 726         return -1; // other
 727     }
 728 }
 729
 730 SBufList
 731 ACLExternal::dump() const
 732 {
 733     external_acl_data const *acl = data;
 734     SBufList rv;
 735     rv.push_back(SBuf(acl->def->name));
 736
 737     for (wordlist *arg = acl->arguments; arg; arg = arg->next) {
 738         SBuf s;
 739         s.Printf(" %s", arg->key);
 740         rv.push_back(s);
 741     }
 742
 743     return rv;
 744 }
 745
 746 /******************************************************************
 747  * external_acl cache
 748  */
 749
 750 static void
 751 external_acl_cache_touch(external_acl * def, const ExternalACLEntryPointer &entry)
 752 {
 753     // this must not be done when nothing is being cached.
 754     if (!def->maybeCacheable(entry->result))
 755         return;
 756
 757     dlinkDelete(&entry->lru, &def->lru_list);
 758     ExternalACLEntry *e = const_cast<ExternalACLEntry *>(entry.getRaw()); // XXX: make hash a std::map of Pointer.
 759     dlinkAdd(e, &entry->lru, &def->lru_list);
 760 }
 761
 762 static char *
 763 makeExternalAclKey(ACLFilledChecklist * ch, external_acl_data * acl_data)
 764 {
 765     static MemBuf mb;
 766     mb.reset();
 767
 768     // check for special case tokens in the format
 769     for (Format::Token *t = acl_data->def->format.format; t ; t = t->next) {
 770
 771         if (t->type == Format::LFT_EXT_ACL_NAME) {
 772             // setup for %ACL
 773             safe_free(ch->al->lastAclName);
 774             ch->al->lastAclName = xstrdup(acl_data->name);
 775         }
 776
 777         if (t->type == Format::LFT_EXT_ACL_DATA) {
 778             // setup string for %DATA
 779             SBuf sb;
 780             for (auto arg = acl_data->arguments; arg; arg = arg->next) {
 781                 if (sb.length())
 782                     sb.append(" ", 1);
 783
 784                 if (acl_data->def->quote == Format::LOG_QUOTE_URL) {
 785                     const char *quoted = rfc1738_escape(arg->key);
 786                     sb.append(quoted, strlen(quoted));
 787                 } else {
 788                     static MemBuf mb2;
 789                     mb2.init();
 790                     strwordquote(&mb2, arg->key);
 791                     sb.append(mb2.buf, mb2.size);
 792                     mb2.clean();
 793                 }
 794             }
 795
 796             ch->al->lastAclData = sb;
 797         }
 798
 799 #if USE_IDENT
 800         if (t->type == Format::LFT_USER_IDENT) {
 801             if (!*ch->rfc931) {
 802                 // if we fail to go async, we still return NULL and the caller
 803                 // will detect the failure in ACLExternal::match().
 804                 (void)ch->goAsync(IdentLookup::Instance());
 805                 return NULL;
 806             }
 807         }
 808 #endif
 809     }
 810
 811     // assemble the full helper lookup string
 812     acl_data->def->format.assemble(mb, ch->al, 0);
 813
 814     return mb.buf;
 815 }
 816
 817 static int
 818 external_acl_entry_expired(external_acl * def, const ExternalACLEntryPointer &entry)
 819 {
 820     if (def->cache_size <= 0 || entry->result == ACCESS_DUNNO)
 821         return 1;
 822
 823     if (entry->date + (entry->result.allowed() ? def->ttl : def->negative_ttl) < squid_curtime)
 824         return 1;
 825     else
 826         return 0;
 827 }
 828
 829 static int
 830 external_acl_grace_expired(external_acl * def, const ExternalACLEntryPointer &entry)
 831 {
 832     if (def->cache_size <= 0 || entry->result == ACCESS_DUNNO)
 833         return 1;
 834
 835     int ttl;
 836     ttl = entry->result.allowed() ? def->ttl : def->negative_ttl;
 837     ttl = (ttl * (100 - def->grace)) / 100;
 838
 839     if (entry->date + ttl <= squid_curtime)
 840         return 1;
 841     else
 842         return 0;
 843 }
 844
 845 static ExternalACLEntryPointer
 846 external_acl_cache_add(external_acl * def, const char *key, ExternalACLEntryData const & data)
 847 {
 848     ExternalACLEntryPointer entry;
 849
 850     if (!def->maybeCacheable(data.result)) {
 851         debugs(82,6, HERE);
 852
 853         if (data.result == ACCESS_DUNNO) {
 854             if (const ExternalACLEntryPointer oldentry = static_cast<ExternalACLEntry *>(hash_lookup(def->cache, key)))
 855                 external_acl_cache_delete(def, oldentry);
 856         }
 857         entry = new ExternalACLEntry;
 858         entry->key = xstrdup(key);
 859         entry->update(data);
 860         entry->def = def;
 861         return entry;
 862     }
 863
 864     entry = static_cast<ExternalACLEntry *>(hash_lookup(def->cache, key));
 865     debugs(82, 2, "external_acl_cache_add: Adding '" << key << "' = " << data.result);
 866
 867     if (entry != NULL) {
 868         debugs(82, 3, "updating existing entry");
 869         entry->update(data);
 870         external_acl_cache_touch(def, entry);
 871         return entry;
 872     }
 873
 874     entry = new ExternalACLEntry;
 875     entry->key = xstrdup(key);
 876     entry->update(data);
 877
 878     def->add(entry);
 879
 880     return entry;
 881 }
 882
 883 static void
 884 external_acl_cache_delete(external_acl * def, const ExternalACLEntryPointer &entry)
 885 {
 886     assert(entry != NULL);
 887     assert(def->cache_size > 0 && entry->def == def);
 888     ExternalACLEntry *e = const_cast<ExternalACLEntry *>(entry.getRaw()); // XXX: make hash a std::map of Pointer.
 889     hash_remove_link(def->cache, e);
 890     dlinkDelete(&e->lru, &def->lru_list);
 891     e->unlock(); // unlock on behalf of the hash
 892     def->cache_entries -= 1;
 893 }
 894
 895 /******************************************************************
 896  * external_acl helpers
 897  */
 898
 899 class externalAclState
 900 {
 901     CBDATA_CLASS(externalAclState);
 902
 903 public:
 904     externalAclState(external_acl* aDef, const char *aKey) :
 905         callback(NULL),
 906         callback_data(NULL),
 907         key(xstrdup(aKey)),
 908         def(cbdataReference(aDef)),
 909         queue(NULL)
 910     {}
 911     ~externalAclState();
 912
 913     EAH *callback;
 914     void *callback_data;
 915     char *key;
 916     external_acl *def;
 917     dlink_node list;
 918     externalAclState *queue;
 919 };
 920
 921 CBDATA_CLASS_INIT(externalAclState);
 922
 923 externalAclState::~externalAclState()
 924 {
 925     xfree(key);
 926     cbdataReferenceDone(callback_data);
 927     cbdataReferenceDone(def);
 928 }
 929
 930 /*
 931  * The helper program receives queries on stdin, one
 932  * per line, and must return the result on on stdout
 933  *
 934  * General result syntax:
 935  *
 936  *   OK/ERR keyword=value ...
 937  *
 938  * Keywords:
 939  *
 940  *   user=      The users name (login)
 941  *   message=   Message describing the reason
 942  *   tag=   A string tag to be applied to the request that triggered the acl match.
 943  *          applies to both OK and ERR responses.
 944  *          Won't override existing request tags.
 945  *   log=   A string to be used in access logging
 946  *
 947  * Other keywords may be added to the protocol later
 948  *
 949  * value needs to be URL-encoded or enclosed in double quotes (")
 950  * with \-escaping on any whitespace, quotes, or slashes (\).
 951  */
 952 static void
 953 externalAclHandleReply(void *data, const Helper::Reply &reply)
 954 {
 955     externalAclState *state = static_cast<externalAclState *>(data);
 956     externalAclState *next;
 957     ExternalACLEntryData entryData;
 958
 959     debugs(82, 2, HERE << "reply=" << reply);
 960
 961     if (reply.result == Helper::Okay)
 962         entryData.result = ACCESS_ALLOWED;
 963     else if (reply.result == Helper::Error)
 964         entryData.result = ACCESS_DENIED;
 965     else //BrokenHelper,TimedOut or Unknown. Should not cached.
 966         entryData.result = ACCESS_DUNNO;
 967
 968     // XXX: make entryData store a proper Helper::Reply object instead of copying.
 969
 970     entryData.notes.append(&reply.notes);
 971
 972     const char *label = reply.notes.findFirst("tag");
 973     if (label != NULL && *label != '\0')
 974         entryData.tag = label;
 975
 976     label = reply.notes.findFirst("message");
 977     if (label != NULL && *label != '\0')
 978         entryData.message = label;
 979
 980     label = reply.notes.findFirst("log");
 981     if (label != NULL && *label != '\0')
 982         entryData.log = label;
 983
 984 #if USE_AUTH
 985     label = reply.notes.findFirst("user");
 986     if (label != NULL && *label != '\0')
 987         entryData.user = label;
 988
 989     label = reply.notes.findFirst("password");
 990     if (label != NULL && *label != '\0')
 991         entryData.password = label;
 992 #endif
 993
 994     dlinkDelete(&state->list, &state->def->queue);
 995
 996     ExternalACLEntryPointer entry;
 997     if (cbdataReferenceValid(state->def))
 998         entry = external_acl_cache_add(state->def, state->key, entryData);
 999
1000     do {
1001         void *cbdata;
1002         if (state->callback && cbdataReferenceValidDone(state->callback_data, &cbdata))
1003             state->callback(cbdata, entry);
1004
1005         next = state->queue;
1006         state->queue = NULL;
1007
1008         delete state;
1009
1010         state = next;
1011     } while (state);
1012 }
1013
1014 void
1015 ACLExternal::ExternalAclLookup(ACLChecklist *checklist, ACLExternal * me)
1016 {
1017     ExternalACLLookup::Start(checklist, me->data, false);
1018 }
1019
1020 void
1021 ExternalACLLookup::Start(ACLChecklist *checklist, external_acl_data *acl, bool inBackground)
1022 {
1023     external_acl *def = acl->def;
1024
1025     ACLFilledChecklist *ch = Filled(checklist);
1026     const char *key = makeExternalAclKey(ch, acl);
1027     assert(key); // XXX: will fail if EXT_ACL_IDENT case needs an async lookup
1028
1029     debugs(82, 2, HERE << (inBackground ? "bg" : "fg") << " lookup in '" <<
1030            def->name << "' for '" << key << "'");
1031
1032     /* Check for a pending lookup to hook into */
1033     // only possible if we are caching results.
1034     externalAclState *oldstate = NULL;
1035     if (def->cache_size > 0) {
1036         for (dlink_node *node = def->queue.head; node; node = node->next) {
1037             externalAclState *oldstatetmp = static_cast<externalAclState *>(node->data);
1038
1039             if (strcmp(key, oldstatetmp->key) == 0) {
1040                 oldstate = oldstatetmp;
1041                 break;
1042             }
1043         }
1044     }
1045
1046     // A background refresh has no need to piggiback on a pending request:
1047     // When the pending request completes, the cache will be refreshed anyway.
1048     if (oldstate && inBackground) {
1049         debugs(82, 7, HERE << "'" << def->name << "' queue is already being refreshed (ch=" << ch << ")");
1050         return;
1051     }
1052
1053     externalAclState *state = new externalAclState(def, key);
1054
1055     if (!inBackground) {
1056         state->callback = &ExternalACLLookup::LookupDone;
1057         state->callback_data = cbdataReference(checklist);
1058     }
1059
1060     if (oldstate) {
1061         /* Hook into pending lookup */
1062         state->queue = oldstate->queue;
1063         oldstate->queue = state;
1064     } else {
1065         /* No pending lookup found. Sumbit to helper */
1066
1067         MemBuf buf;
1068         buf.init();
1069         buf.appendf("%s\n", key);
1070         debugs(82, 4, "externalAclLookup: looking up for '" << key << "' in '" << def->name << "'.");
1071
1072         if (!def->theHelper->trySubmit(buf.buf, externalAclHandleReply, state)) {
1073             debugs(82, 7, HERE << "'" << def->name << "' submit to helper failed");
1074             assert(inBackground); // or the caller should have checked
1075             delete state;
1076             return;
1077         }
1078
1079         dlinkAdd(state, &state->list, &def->queue);
1080
1081         buf.clean();
1082     }
1083
1084     debugs(82, 4, "externalAclLookup: will wait for the result of '" << key <<
1085            "' in '" << def->name << "' (ch=" << ch << ").");
1086 }
1087
1088 static void
1089 externalAclStats(StoreEntry * sentry)
1090 {
1091     for (external_acl *p = Config.externalAclHelperList; p; p = p->next) {
1092         storeAppendPrintf(sentry, "External ACL Statistics: %s\n", p->name);
1093         storeAppendPrintf(sentry, "Cache size: %d\n", p->cache->count);
1094         assert(p->theHelper);
1095         p->theHelper->packStatsInto(sentry);
1096         storeAppendPrintf(sentry, "\n");
1097     }
1098 }
1099
1100 static void
1101 externalAclRegisterWithCacheManager(void)
1102 {
1103     Mgr::RegisterAction("external_acl",
1104                         "External ACL stats",
1105                         externalAclStats, 0, 1);
1106 }
1107
1108 void
1109 externalAclInit(void)
1110 {
1111     for (external_acl *p = Config.externalAclHelperList; p; p = p->next) {
1112         if (!p->cache)
1113             p->cache = hash_create((HASHCMP *) strcmp, hashPrime(1024), hash4);
1114
1115         if (!p->theHelper)
1116             p->theHelper = new helper(p->name);
1117
1118         p->theHelper->cmdline = p->cmdline;
1119
1120         p->theHelper->childs.updateLimits(p->children);
1121
1122         p->theHelper->ipc_type = IPC_TCP_SOCKET;
1123
1124         p->theHelper->addr = p->local_addr;
1125
1126         helperOpenServers(p->theHelper);
1127     }
1128
1129     externalAclRegisterWithCacheManager();
1130 }
1131
1132 void
1133 externalAclShutdown(void)
1134 {
1135     external_acl *p;
1136
1137     for (p = Config.externalAclHelperList; p; p = p->next) {
1138         helperShutdown(p->theHelper);
1139     }
1140 }
1141
1142 ExternalACLLookup ExternalACLLookup::instance_;
1143 ExternalACLLookup *
1144 ExternalACLLookup::Instance()
1145 {
1146     return &instance_;
1147 }
1148
1149 void
1150 ExternalACLLookup::checkForAsync(ACLChecklist *checklist)const
1151 {
1152     /* TODO: optimise this - we probably have a pointer to this
1153      * around somewhere */
1154     ACL *acl = ACL::FindByName(AclMatchedName);
1155     assert(acl);
1156     ACLExternal *me = dynamic_cast<ACLExternal *> (acl);
1157     assert (me);
1158     ACLExternal::ExternalAclLookup(checklist, me);
1159 }
1160
1161 /// Called when an async lookup returns
1162 void
1163 ExternalACLLookup::LookupDone(void *data, const ExternalACLEntryPointer &result)
1164 {
1165     ACLFilledChecklist *checklist = Filled(static_cast<ACLChecklist*>(data));
1166     checklist->extacl_entry = result;
1167     checklist->resumeNonBlockingCheck(ExternalACLLookup::Instance());
1168 }
1169
1170 ACL *
1171 ACLExternal::clone() const
1172 {
1173     return new ACLExternal(*this);
1174 }
1175
1176 ACLExternal::ACLExternal(char const *theClass) : data(NULL), class_(xstrdup(theClass))
1177 {}
1178
1179 ACLExternal::ACLExternal(ACLExternal const & old) : data(NULL), class_(old.class_ ? xstrdup(old.class_) : NULL)
1180 {
1181     /* we don't have copy constructors for the data yet */
1182     assert(!old.data);
1183 }
1184
1185 char const *
1186 ACLExternal::typeString() const
1187 {
1188     return class_;
1189 }
1190
1191 bool
1192 ACLExternal::isProxyAuth() const
1193 {
1194 #if USE_AUTH
1195     return data->def->require_auth;
1196 #else
1197     return false;
1198 #endif
1199 }
1200