src/external_acl.cc

   1 /*
   2  * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 82    External ACL */
  10
  11 #include "squid.h"
  12 #include "acl/Acl.h"
  13 #include "acl/FilledChecklist.h"
  14 #include "cache_cf.h"
  15 #include "client_side.h"
  16 #include "comm/Connection.h"
  17 #include "ConfigParser.h"
  18 #include "ExternalACL.h"
  19 #include "ExternalACLEntry.h"
  20 #include "fde.h"
  21 #include "format/ByteCode.h"
  22 #include "helper.h"
  23 #include "helper/Reply.h"
  24 #include "HttpHeaderTools.h"
  25 #include "HttpReply.h"
  26 #include "HttpRequest.h"
  27 #include "ip/tools.h"
  28 #include "MemBuf.h"
  29 #include "mgr/Registration.h"
  30 #include "rfc1738.h"
  31 #include "SquidConfig.h"
  32 #include "SquidString.h"
  33 #include "SquidTime.h"
  34 #include "Store.h"
  35 #include "tools.h"
  36 #include "URL.h"
  37 #include "wordlist.h"
  38 #if USE_OPENSSL
  39 #include "ssl/ServerBump.h"
  40 #include "ssl/support.h"
  41 #endif
  42 #if USE_AUTH
  43 #include "auth/Acl.h"
  44 #include "auth/Gadgets.h"
  45 #include "auth/UserRequest.h"
  46 #endif
  47 #if USE_IDENT
  48 #include "ident/AclIdent.h"
  49 #endif
  50
  51 #ifndef DEFAULT_EXTERNAL_ACL_TTL
  52 #define DEFAULT_EXTERNAL_ACL_TTL 1 * 60 * 60
  53 #endif
  54 #ifndef DEFAULT_EXTERNAL_ACL_CHILDREN
  55 #define DEFAULT_EXTERNAL_ACL_CHILDREN 5
  56 #endif
  57
  58 static char *makeExternalAclKey(ACLFilledChecklist * ch, external_acl_data * acl_data);
  59 static void external_acl_cache_delete(external_acl * def, const ExternalACLEntryPointer &entry);
  60 static int external_acl_entry_expired(external_acl * def, const ExternalACLEntryPointer &entry);
  61 static int external_acl_grace_expired(external_acl * def, const ExternalACLEntryPointer &entry);
  62 static void external_acl_cache_touch(external_acl * def, const ExternalACLEntryPointer &entry);
  63 static ExternalACLEntryPointer external_acl_cache_add(external_acl * def, const char *key, ExternalACLEntryData const &data);
  64
  65 /******************************************************************
  66  * external_acl directive
  67  */
  68
  69 class external_acl_format : public RefCountable
  70 {
  71     MEMPROXY_CLASS(external_acl_format);
  72
  73 public:
  74     typedef RefCount<external_acl_format> Pointer;
  75
  76     external_acl_format() : type(Format::LFT_NONE), header(NULL), member(NULL), separator(' '), header_id(HDR_BAD_HDR) {}
  77     ~external_acl_format() {
  78         xfree(header);
  79         xfree(member);
  80     }
  81
  82     Format::ByteCode_t type;
  83     external_acl_format::Pointer next;
  84     char *header;
  85     char *member;
  86     char separator;
  87     http_hdr_type header_id;
  88 };
  89
  90 class external_acl
  91 {
  92     /* FIXME: These are not really cbdata, but it is an easy way
  93      * to get them pooled, refcounted, accounted and freed properly...
  94      */
  95     CBDATA_CLASS(external_acl);
  96
  97 public:
  98     external_acl();
  99     ~external_acl();
 100
 101     external_acl *next;
 102
 103     void add(const ExternalACLEntryPointer &);
 104
 105     void trimCache();
 106
 107     int ttl;
 108
 109     int negative_ttl;
 110
 111     int grace;
 112
 113     char *name;
 114
 115     external_acl_format::Pointer format;
 116
 117     wordlist *cmdline;
 118
 119     Helper::ChildConfig children;
 120
 121     helper *theHelper;
 122
 123     hash_table *cache;
 124
 125     dlink_list lru_list;
 126
 127     int cache_size;
 128
 129     int cache_entries;
 130
 131     dlink_list queue;
 132
 133 #if USE_AUTH
 134     /**
 135      * Configuration flag. May only be altered by the configuration parser.
 136      *
 137      * Indicates that all uses of this external_acl_type helper require authentication
 138      * details to be processed. If none are available its a fail match.
 139      */
 140     bool require_auth;
 141 #endif
 142
 143     enum {
 144         QUOTE_METHOD_SHELL = 1,
 145         QUOTE_METHOD_URL
 146     } quote;
 147
 148     Ip::Address local_addr;
 149 };
 150
 151 CBDATA_CLASS_INIT(external_acl);
 152
 153 external_acl::external_acl() :
 154     next(NULL),
 155     ttl(DEFAULT_EXTERNAL_ACL_TTL),
 156     negative_ttl(-1),
 157     grace(1),
 158     name(NULL),
 159     cmdline(NULL),
 160     children(DEFAULT_EXTERNAL_ACL_CHILDREN),
 161     theHelper(NULL),
 162     cache(NULL),
 163     cache_size(256*1024),
 164     cache_entries(0),
 165 #if USE_AUTH
 166     require_auth(0),
 167 #endif
 168     quote(external_acl::QUOTE_METHOD_URL)
 169 {
 170     local_addr.setLocalhost();
 171 }
 172
 173 external_acl::~external_acl()
 174 {
 175     xfree(name);
 176     format = NULL;
 177     wordlistDestroy(&cmdline);
 178
 179     if (theHelper) {
 180         helperShutdown(theHelper);
 181         delete theHelper;
 182         theHelper = NULL;
 183     }
 184
 185     while (lru_list.tail) {
 186         ExternalACLEntryPointer e(static_cast<ExternalACLEntry *>(lru_list.tail->data));
 187         external_acl_cache_delete(this, e);
 188     }
 189     if (cache)
 190         hashFreeMemory(cache);
 191
 192     while (next) {
 193         external_acl *node = next;
 194         next = node->next;
 195         node->next = NULL; // prevent recursion
 196         delete node;
 197     }
 198 }
 199
 200 /**
 201  * Parse the External ACL format %<{.*} and %>{.*} token(s) to pass a specific
 202  * request or reply header to external helper.
 203  *
 204  \param header   - the token being parsed (without the identifying prefix)
 205  \param type     - format enum identifier for this element, pulled from identifying prefix
 206  \param format   - structure to contain all the info about this format element.
 207  */
 208 void
 209 parse_header_token(external_acl_format::Pointer format, char *header, const Format::ByteCode_t type)
 210 {
 211     /* header format */
 212     char *member, *end;
 213
 214     /** Cut away the closing brace */
 215     end = strchr(header, '}');
 216     if (end && strlen(end) == 1)
 217         *end = '\0';
 218     else
 219         self_destruct();
 220
 221     member = strchr(header, ':');
 222
 223     if (member) {
 224         /* Split in header and member */
 225         *member = '\0';
 226         ++member;
 227
 228         if (!xisalnum(*member)) {
 229             format->separator = *member;
 230             ++member;
 231         } else {
 232             format->separator = ',';
 233         }
 234
 235         format->member = xstrdup(member);
 236
 237         if (type == Format::LFT_ADAPTED_REQUEST_HEADER)
 238             format->type = Format::LFT_ADAPTED_REQUEST_HEADER_ELEM;
 239         else
 240             format->type = Format::LFT_REPLY_HEADER_ELEM;
 241
 242     } else {
 243         format->type = type;
 244     }
 245
 246     format->header = xstrdup(header);
 247     format->header_id = httpHeaderIdByNameDef(header, strlen(header));
 248 }
 249
 250 void
 251 parse_externalAclHelper(external_acl ** list)
 252 {
 253     external_acl *a = new external_acl;
 254     char *token = ConfigParser::NextToken();
 255
 256     if (!token)
 257         self_destruct();
 258
 259     a->name = xstrdup(token);
 260
 261     // Allow supported %macros inside quoted tokens
 262     ConfigParser::EnableMacros();
 263     token = ConfigParser::NextToken();
 264
 265     /* Parse options */
 266     while (token) {
 267         if (strncmp(token, "ttl=", 4) == 0) {
 268             a->ttl = atoi(token + 4);
 269         } else if (strncmp(token, "negative_ttl=", 13) == 0) {
 270             a->negative_ttl = atoi(token + 13);
 271         } else if (strncmp(token, "children=", 9) == 0) {
 272             a->children.n_max = atoi(token + 9);
 273             debugs(0, DBG_CRITICAL, "WARNING: external_acl_type option children=N has been deprecated in favor of children-max=N and children-startup=N");
 274         } else if (strncmp(token, "children-max=", 13) == 0) {
 275             a->children.n_max = atoi(token + 13);
 276         } else if (strncmp(token, "children-startup=", 17) == 0) {
 277             a->children.n_startup = atoi(token + 17);
 278         } else if (strncmp(token, "children-idle=", 14) == 0) {
 279             a->children.n_idle = atoi(token + 14);
 280         } else if (strncmp(token, "concurrency=", 12) == 0) {
 281             a->children.concurrency = atoi(token + 12);
 282         } else if (strncmp(token, "queue-size=", 11) == 0) {
 283             a->children.queue_size = atoi(token + 11);
 284             a->children.defaultQueueSize = false;
 285         } else if (strncmp(token, "cache=", 6) == 0) {
 286             a->cache_size = atoi(token + 6);
 287         } else if (strncmp(token, "grace=", 6) == 0) {
 288             a->grace = atoi(token + 6);
 289         } else if (strcmp(token, "protocol=2.5") == 0) {
 290             a->quote = external_acl::QUOTE_METHOD_SHELL;
 291         } else if (strcmp(token, "protocol=3.0") == 0) {
 292             debugs(3, DBG_PARSE_NOTE(2), "WARNING: external_acl_type option protocol=3.0 is deprecated. Remove this from your config.");
 293             a->quote = external_acl::QUOTE_METHOD_URL;
 294         } else if (strcmp(token, "quote=url") == 0) {
 295             debugs(3, DBG_PARSE_NOTE(2), "WARNING: external_acl_type option quote=url is deprecated. Remove this from your config.");
 296             a->quote = external_acl::QUOTE_METHOD_URL;
 297         } else if (strcmp(token, "quote=shell") == 0) {
 298             debugs(3, DBG_PARSE_NOTE(2), "WARNING: external_acl_type option quote=shell is deprecated. Use protocol=2.5 if still needed.");
 299             a->quote = external_acl::QUOTE_METHOD_SHELL;
 300
 301             /* INET6: allow admin to configure some helpers explicitly to
 302                       bind to IPv4/v6 localhost port. */
 303         } else if (strcmp(token, "ipv4") == 0) {
 304             if ( !a->local_addr.setIPv4() ) {
 305                 debugs(3, DBG_CRITICAL, "WARNING: Error converting " << a->local_addr << " to IPv4 in " << a->name );
 306             }
 307         } else if (strcmp(token, "ipv6") == 0) {
 308             if (!Ip::EnableIpv6)
 309                 debugs(3, DBG_CRITICAL, "WARNING: --enable-ipv6 required for external ACL helpers to use IPv6: " << a->name );
 310             // else nothing to do.
 311         } else {
 312             break;
 313         }
 314
 315         token = ConfigParser::NextToken();
 316     }
 317     ConfigParser::DisableMacros();
 318
 319     /* check that child startup value is sane. */
 320     if (a->children.n_startup > a->children.n_max)
 321         a->children.n_startup = a->children.n_max;
 322
 323     /* check that child idle value is sane. */
 324     if (a->children.n_idle > a->children.n_max)
 325         a->children.n_idle = a->children.n_max;
 326     if (a->children.n_idle < 1)
 327         a->children.n_idle = 1;
 328
 329     if (a->negative_ttl == -1)
 330         a->negative_ttl = a->ttl;
 331
 332     if (a->children.defaultQueueSize)
 333         a->children.queue_size = 2 * a->children.n_max;
 334
 335     /* Parse format */
 336     external_acl_format::Pointer *p = &a->format;
 337
 338     while (token) {
 339         /* stop on first non-format token found */
 340
 341         if (*token != '%')
 342             break;
 343
 344         external_acl_format::Pointer format = new external_acl_format;
 345
 346         if (strncmp(token, "%{", 2) == 0) {
 347             // deprecated. but assume the old configs all referred to request headers.
 348             debugs(82, DBG_PARSE_NOTE(DBG_IMPORTANT), "WARNING: external_acl_type format %{...} is being replaced by %>ha{...} for : " << token);
 349             parse_header_token(format, (token+2), Format::LFT_ADAPTED_REQUEST_HEADER);
 350         } else if (strncmp(token, "%>{", 3) == 0) {
 351             debugs(82, DBG_PARSE_NOTE(DBG_IMPORTANT), "WARNING: external_acl_type format %>{...} is being replaced by %>ha{...} for : " << token);
 352             parse_header_token(format, (token+3), Format::LFT_ADAPTED_REQUEST_HEADER);
 353         } else if (strncmp(token, "%>ha{", 5) == 0) {
 354             parse_header_token(format, (token+5), Format::LFT_ADAPTED_REQUEST_HEADER);
 355         } else if (strncmp(token, "%<{", 3) == 0) {
 356             debugs(82, DBG_PARSE_NOTE(DBG_IMPORTANT), "WARNING: external_acl_type format %<{...} is being replaced by %<h{...} for : " << token);
 357             parse_header_token(format, (token+3), Format::LFT_REPLY_HEADER);
 358         } else if (strncmp(token, "%<h{", 4) == 0) {
 359             parse_header_token(format, (token+4), Format::LFT_REPLY_HEADER);
 360 #if USE_AUTH
 361         } else if (strcmp(token, "%LOGIN") == 0 || strcmp(token, "%ul") == 0) {
 362             format->type = Format::LFT_USER_LOGIN;
 363             a->require_auth = true;
 364 #endif
 365         }
 366 #if USE_IDENT
 367         else if (strcmp(token, "%IDENT") == 0 || strcmp(token, "%ui") == 0)
 368             format->type = Format::LFT_USER_IDENT;
 369 #endif
 370         else if (strcmp(token, "%SRC") == 0 || strcmp(token, "%>a") == 0)
 371             format->type = Format::LFT_CLIENT_IP_ADDRESS;
 372         else if (strcmp(token, "%SRCPORT") == 0 || strcmp(token, "%>p") == 0)
 373             format->type = Format::LFT_CLIENT_PORT;
 374 #if USE_SQUID_EUI
 375         else if (strcmp(token, "%SRCEUI48") == 0)
 376             format->type = Format::LFT_EXT_ACL_CLIENT_EUI48;
 377         else if (strcmp(token, "%SRCEUI64") == 0)
 378             format->type = Format::LFT_EXT_ACL_CLIENT_EUI64;
 379 #endif
 380         else if (strcmp(token, "%MYADDR") == 0 || strcmp(token, "%la") == 0)
 381             format->type = Format::LFT_LOCAL_LISTENING_IP;
 382         else if (strcmp(token, "%MYPORT") == 0 || strcmp(token, "%lp") == 0)
 383             format->type = Format::LFT_LOCAL_LISTENING_PORT;
 384         else if (strcmp(token, "%URI") == 0 || strcmp(token, "%>ru") == 0)
 385             format->type = Format::LFT_CLIENT_REQ_URI;
 386         else if (strcmp(token, "%DST") == 0 || strcmp(token, "%>rd") == 0)
 387             format->type = Format::LFT_CLIENT_REQ_URLDOMAIN;
 388         else if (strcmp(token, "%PROTO") == 0 || strcmp(token, "%>rs") == 0)
 389             format->type = Format::LFT_CLIENT_REQ_URLSCHEME;
 390         else if (strcmp(token, "%PORT") == 0) // XXX: add a logformat token
 391             format->type = Format::LFT_CLIENT_REQ_URLPORT;
 392         else if (strcmp(token, "%PATH") == 0 || strcmp(token, "%>rp") == 0)
 393             format->type = Format::LFT_CLIENT_REQ_URLPATH;
 394         else if (strcmp(token, "%METHOD") == 0 || strcmp(token, "%>rm") == 0)
 395             format->type = Format::LFT_CLIENT_REQ_METHOD;
 396 #if USE_OPENSSL
 397         else if (strcmp(token, "%USER_CERT") == 0)
 398             format->type = Format::LFT_EXT_ACL_USER_CERT_RAW;
 399         else if (strcmp(token, "%USER_CERTCHAIN") == 0)
 400             format->type = Format::LFT_EXT_ACL_USER_CERTCHAIN_RAW;
 401         else if (strncmp(token, "%USER_CERT_", 11) == 0) {
 402             format->type = Format::LFT_EXT_ACL_USER_CERT;
 403             format->header = xstrdup(token + 11);
 404         } else if (strncmp(token, "%USER_CA_CERT_", 14) == 0) {
 405             format->type = Format::LFT_EXT_ACL_USER_CA_CERT;
 406             format->header = xstrdup(token + 14);
 407         } else if (strncmp(token, "%CA_CERT_", 9) == 0) {
 408             debugs(82, DBG_PARSE_NOTE(DBG_IMPORTANT), "WARNING: external_acl_type %CA_CERT_* code is obsolete. Use %USER_CA_CERT_* instead");
 409             format->type = Format::LFT_EXT_ACL_USER_CA_CERT;
 410             format->header = xstrdup(token + 9);
 411         } else if (strcmp(token, "%ssl::>sni") == 0)
 412             format->type = Format::LFT_SSL_CLIENT_SNI;
 413         else if (strcmp(token, "%ssl::<cert_subject") == 0)
 414             format->type = Format::LFT_SSL_SERVER_CERT_SUBJECT;
 415         else if (strcmp(token, "%ssl::<cert_issuer") == 0)
 416             format->type = Format::LFT_SSL_SERVER_CERT_ISSUER;
 417 #endif
 418 #if USE_AUTH
 419         else if (strcmp(token, "%EXT_USER") == 0 || strcmp(token, "%ue") == 0)
 420             format->type = Format::LFT_USER_EXTERNAL;
 421 #endif
 422         else if (strcmp(token, "%EXT_LOG") == 0 || strcmp(token, "%ea") == 0)
 423             format->type = Format::LFT_EXT_LOG;
 424         else if (strcmp(token, "%TAG") == 0  || strcmp(token, "%et") == 0)
 425             format->type = Format::LFT_TAG;
 426         else if (strcmp(token, "%ACL") == 0)
 427             format->type = Format::LFT_EXT_ACL_NAME;
 428         else if (strcmp(token, "%DATA") == 0)
 429             format->type = Format::LFT_EXT_ACL_DATA;
 430         else if (strcmp(token, "%%") == 0)
 431             format->type = Format::LFT_PERCENT;
 432         else {
 433             debugs(0, DBG_CRITICAL, "ERROR: Unknown Format token " << token);
 434             self_destruct();
 435         }
 436
 437         *p = format;
 438         p = &format->next;
 439         token = ConfigParser::NextToken();
 440     }
 441
 442     /* There must be at least one format token */
 443     if (!a->format)
 444         self_destruct();
 445
 446     /* helper */
 447     if (!token)
 448         self_destruct();
 449
 450     wordlistAdd(&a->cmdline, token);
 451
 452     /* arguments */
 453     parse_wordlist(&a->cmdline);
 454
 455     while (*list)
 456         list = &(*list)->next;
 457
 458     *list = a;
 459 }
 460
 461 void
 462 dump_externalAclHelper(StoreEntry * sentry, const char *name, const external_acl * list)
 463 {
 464     const external_acl *node;
 465     const wordlist *word;
 466
 467     for (node = list; node; node = node->next) {
 468         storeAppendPrintf(sentry, "%s %s", name, node->name);
 469
 470         if (!node->local_addr.isIPv6())
 471             storeAppendPrintf(sentry, " ipv4");
 472         else
 473             storeAppendPrintf(sentry, " ipv6");
 474
 475         if (node->ttl != DEFAULT_EXTERNAL_ACL_TTL)
 476             storeAppendPrintf(sentry, " ttl=%d", node->ttl);
 477
 478         if (node->negative_ttl != node->ttl)
 479             storeAppendPrintf(sentry, " negative_ttl=%d", node->negative_ttl);
 480
 481         if (node->grace)
 482             storeAppendPrintf(sentry, " grace=%d", node->grace);
 483
 484         if (node->children.n_max != DEFAULT_EXTERNAL_ACL_CHILDREN)
 485             storeAppendPrintf(sentry, " children-max=%d", node->children.n_max);
 486
 487         if (node->children.n_startup != 1)
 488             storeAppendPrintf(sentry, " children-startup=%d", node->children.n_startup);
 489
 490         if (node->children.n_idle != (node->children.n_max + node->children.n_startup) )
 491             storeAppendPrintf(sentry, " children-idle=%d", node->children.n_idle);
 492
 493         if (node->children.concurrency)
 494             storeAppendPrintf(sentry, " concurrency=%d", node->children.concurrency);
 495
 496         if (node->cache)
 497             storeAppendPrintf(sentry, " cache=%d", node->cache_size);
 498
 499         if (node->quote == external_acl::QUOTE_METHOD_SHELL)
 500             storeAppendPrintf(sentry, " protocol=2.5");
 501
 502         for (external_acl_format::Pointer format = node->format; format!= NULL; format = format->next) {
 503             switch (format->type) {
 504
 505             case Format::LFT_ADAPTED_REQUEST_HEADER:
 506                 storeAppendPrintf(sentry, " %%>ha{%s}", format->header);
 507                 break;
 508
 509             case Format::LFT_ADAPTED_REQUEST_HEADER_ELEM:
 510                 storeAppendPrintf(sentry, " %%>ha{%s:%s}", format->header, format->member);
 511                 break;
 512
 513             case Format::LFT_REPLY_HEADER:
 514                 storeAppendPrintf(sentry, " %%<h{%s}", format->header);
 515                 break;
 516
 517             case Format::LFT_REPLY_HEADER_ELEM:
 518                 storeAppendPrintf(sentry, " %%<h{%s:%s}", format->header, format->member);
 519                 break;
 520
 521 #define DUMP_EXT_ACL_TYPE_FMT(a, fmt, ...) \
 522             case Format::LFT_##a: \
 523                 storeAppendPrintf(sentry, fmt, ##__VA_ARGS__); \
 524                 break
 525 #if USE_AUTH
 526                 DUMP_EXT_ACL_TYPE_FMT(USER_LOGIN," %%ul");
 527 #endif
 528 #if USE_IDENT
 529
 530                 DUMP_EXT_ACL_TYPE_FMT(USER_IDENT," %%ui");
 531 #endif
 532                 DUMP_EXT_ACL_TYPE_FMT(CLIENT_IP_ADDRESS," %%>a");
 533                 DUMP_EXT_ACL_TYPE_FMT(CLIENT_PORT," %%>p");
 534 #if USE_SQUID_EUI
 535                 DUMP_EXT_ACL_TYPE_FMT(EXT_ACL_CLIENT_EUI48," %%SRCEUI48");
 536                 DUMP_EXT_ACL_TYPE_FMT(EXT_ACL_CLIENT_EUI64," %%SRCEUI64");
 537 #endif
 538                 DUMP_EXT_ACL_TYPE_FMT(LOCAL_LISTENING_IP," %%>la");
 539                 DUMP_EXT_ACL_TYPE_FMT(LOCAL_LISTENING_PORT," %%>lp");
 540                 DUMP_EXT_ACL_TYPE_FMT(CLIENT_REQ_URI," %%>ru");
 541                 DUMP_EXT_ACL_TYPE_FMT(CLIENT_REQ_URLDOMAIN," %%>rd");
 542                 DUMP_EXT_ACL_TYPE_FMT(CLIENT_REQ_URLSCHEME," %%>rs");
 543                 DUMP_EXT_ACL_TYPE_FMT(CLIENT_REQ_URLPORT," %%>rP");
 544                 DUMP_EXT_ACL_TYPE_FMT(CLIENT_REQ_URLPATH," %%>rp");
 545                 DUMP_EXT_ACL_TYPE_FMT(CLIENT_REQ_METHOD," %%>rm");
 546 #if USE_OPENSSL
 547                 DUMP_EXT_ACL_TYPE_FMT(EXT_ACL_USER_CERT_RAW, " %%USER_CERT_RAW");
 548                 DUMP_EXT_ACL_TYPE_FMT(EXT_ACL_USER_CERTCHAIN_RAW, " %%USER_CERTCHAIN_RAW");
 549                 DUMP_EXT_ACL_TYPE_FMT(EXT_ACL_USER_CERT, " %%USER_CERT_%s", format->header);
 550                 DUMP_EXT_ACL_TYPE_FMT(EXT_ACL_USER_CA_CERT, " %%USER_CA_CERT_%s", format->header);
 551                 DUMP_EXT_ACL_TYPE_FMT(SSL_CLIENT_SNI, "%%ssl::>sni");
 552                 DUMP_EXT_ACL_TYPE_FMT(SSL_SERVER_CERT_SUBJECT, "%%ssl::<cert_subject");
 553                 DUMP_EXT_ACL_TYPE_FMT(SSL_SERVER_CERT_ISSUER, "%%ssl::<cert_issuer");
 554 #endif
 555 #if USE_AUTH
 556                 DUMP_EXT_ACL_TYPE_FMT(USER_EXTERNAL," %%ue");
 557 #endif
 558                 DUMP_EXT_ACL_TYPE_FMT(EXT_LOG," %%ea");
 559                 DUMP_EXT_ACL_TYPE_FMT(TAG," %%et");
 560                 DUMP_EXT_ACL_TYPE_FMT(EXT_ACL_NAME," %%ACL");
 561                 DUMP_EXT_ACL_TYPE_FMT(EXT_ACL_DATA," %%DATA");
 562                 DUMP_EXT_ACL_TYPE_FMT(PERCENT, " %%%%");
 563             default:
 564                 fatal("unknown external_acl format error");
 565                 break;
 566             }
 567         }
 568
 569         for (word = node->cmdline; word; word = word->next)
 570             storeAppendPrintf(sentry, " %s", word->key);
 571
 572         storeAppendPrintf(sentry, "\n");
 573     }
 574 }
 575
 576 void
 577 free_externalAclHelper(external_acl ** list)
 578 {
 579     delete *list;
 580     *list = NULL;
 581 }
 582
 583 static external_acl *
 584 find_externalAclHelper(const char *name)
 585 {
 586     external_acl *node;
 587
 588     for (node = Config.externalAclHelperList; node; node = node->next) {
 589         if (strcmp(node->name, name) == 0)
 590             return node;
 591     }
 592
 593     return NULL;
 594 }
 595
 596 void
 597 external_acl::add(const ExternalACLEntryPointer &anEntry)
 598 {
 599     trimCache();
 600     assert(anEntry != NULL);
 601     assert (anEntry->def == NULL);
 602     anEntry->def = this;
 603     ExternalACLEntry *e = const_cast<ExternalACLEntry *>(anEntry.getRaw()); // XXX: make hash a std::map of Pointer.
 604     hash_join(cache, e);
 605     dlinkAdd(e, &e->lru, &lru_list);
 606     e->lock(); //cbdataReference(e); // lock it on behalf of the hash
 607     ++cache_entries;
 608 }
 609
 610 void
 611 external_acl::trimCache()
 612 {
 613     if (cache_size && cache_entries >= cache_size) {
 614         ExternalACLEntryPointer e(static_cast<ExternalACLEntry *>(lru_list.tail->data));
 615         external_acl_cache_delete(this, e);
 616     }
 617 }
 618
 619 /******************************************************************
 620  * external acl type
 621  */
 622
 623 class external_acl_data
 624 {
 625     CBDATA_CLASS(external_acl_data);
 626
 627 public:
 628     explicit external_acl_data(external_acl *aDef) : def(cbdataReference(aDef)), name(NULL), arguments(NULL) {}
 629     ~external_acl_data();
 630
 631     external_acl *def;
 632     const char *name;
 633     wordlist *arguments;
 634 };
 635
 636 CBDATA_CLASS_INIT(external_acl_data);
 637
 638 external_acl_data::~external_acl_data()
 639 {
 640     xfree(name);
 641     wordlistDestroy(&arguments);
 642     cbdataReferenceDone(def);
 643 }
 644
 645 void
 646 ACLExternal::parse()
 647 {
 648     if (data)
 649         self_destruct();
 650
 651     char *token = ConfigParser::strtokFile();
 652
 653     if (!token)
 654         self_destruct();
 655
 656     data = new external_acl_data(find_externalAclHelper(token));
 657
 658     if (!data->def)
 659         self_destruct();
 660
 661     // def->name is the name of the external_acl_type.
 662     // this is the name of the 'acl' directive being tested
 663     data->name = xstrdup(AclMatchedName);
 664
 665     while ((token = ConfigParser::strtokFile())) {
 666         wordlistAdd(&data->arguments, token);
 667     }
 668 }
 669
 670 bool
 671 ACLExternal::valid () const
 672 {
 673 #if USE_AUTH
 674     if (data->def->require_auth) {
 675         if (authenticateSchemeCount() == 0) {
 676             debugs(28, DBG_CRITICAL, "Can't use proxy auth because no authentication schemes were compiled.");
 677             return false;
 678         }
 679
 680         if (authenticateActiveSchemeCount() == 0) {
 681             debugs(28, DBG_CRITICAL, "Can't use proxy auth because no authentication schemes are fully configured.");
 682             return false;
 683         }
 684     }
 685 #endif
 686
 687     return true;
 688 }
 689
 690 bool
 691 ACLExternal::empty () const
 692 {
 693     return false;
 694 }
 695
 696 ACLExternal::~ACLExternal()
 697 {
 698     delete data;
 699     xfree(class_);
 700 }
 701
 702 static void
 703 copyResultsFromEntry(HttpRequest *req, const ExternalACLEntryPointer &entry)
 704 {
 705     if (req) {
 706 #if USE_AUTH
 707         if (entry->user.size())
 708             req->extacl_user = entry->user;
 709
 710         if (entry->password.size())
 711             req->extacl_passwd = entry->password;
 712 #endif
 713         if (!req->tag.size())
 714             req->tag = entry->tag;
 715
 716         if (entry->log.size())
 717             req->extacl_log = entry->log;
 718
 719         if (entry->message.size())
 720             req->extacl_message = entry->message;
 721
 722         // attach the helper kv-pair to the transaction
 723         UpdateRequestNotes(req->clientConnectionManager.get(), *req, entry->notes);
 724     }
 725 }
 726
 727 static allow_t
 728 aclMatchExternal(external_acl_data *acl, ACLFilledChecklist *ch)
 729 {
 730     debugs(82, 9, HERE << "acl=\"" << acl->def->name << "\"");
 731     ExternalACLEntryPointer entry = ch->extacl_entry;
 732
 733     external_acl_message = "MISSING REQUIRED INFORMATION";
 734
 735     if (entry != NULL) {
 736         if (entry->def == acl->def) {
 737             /* Ours, use it.. if the key matches */
 738             const char *key = makeExternalAclKey(ch, acl);
 739             if (!key)
 740                 return ACCESS_DUNNO; // insufficent data to continue
 741             if (strcmp(key, (char*)entry->key) != 0) {
 742                 debugs(82, 9, "entry key='" << (char *)entry->key << "', our key='" << key << "' dont match. Discarded.");
 743                 // too bad. need a new lookup.
 744                 entry = ch->extacl_entry = NULL;
 745             }
 746         } else {
 747             /* Not ours.. get rid of it */
 748             debugs(82, 9, "entry " << entry << " not valid or not ours. Discarded.");
 749             if (entry != NULL) {
 750                 debugs(82, 9, "entry def=" << entry->def << ", our def=" << acl->def);
 751                 const char *key = makeExternalAclKey(ch, acl); // may be nil
 752                 debugs(82, 9, "entry key='" << (char *)entry->key << "', our key='" << key << "'");
 753             }
 754             entry = ch->extacl_entry = NULL;
 755         }
 756     }
 757
 758     if (!entry) {
 759         debugs(82, 9, HERE << "No helper entry available");
 760 #if USE_AUTH
 761         if (acl->def->require_auth) {
 762             /* Make sure the user is authenticated */
 763             debugs(82, 3, HERE << acl->def->name << " check user authenticated.");
 764             const allow_t ti = AuthenticateAcl(ch);
 765             if (ti != ACCESS_ALLOWED) {
 766                 debugs(82, 2, HERE << acl->def->name << " user not authenticated (" << ti << ")");
 767                 return ti;
 768             }
 769             debugs(82, 3, HERE << acl->def->name << " user is authenticated.");
 770         }
 771 #endif
 772         const char *key = makeExternalAclKey(ch, acl);
 773
 774         if (!key) {
 775             /* Not sufficient data to process */
 776             return ACCESS_DUNNO;
 777         }
 778
 779         entry = static_cast<ExternalACLEntry *>(hash_lookup(acl->def->cache, key));
 780
 781         const ExternalACLEntryPointer staleEntry = entry;
 782         if (entry != NULL && external_acl_entry_expired(acl->def, entry))
 783             entry = NULL;
 784
 785         if (entry != NULL && external_acl_grace_expired(acl->def, entry)) {
 786             // refresh in the background
 787             ExternalACLLookup::Start(ch, acl, true);
 788             debugs(82, 4, HERE << "no need to wait for the refresh of '" <<
 789                    key << "' in '" << acl->def->name << "' (ch=" << ch << ").");
 790         }
 791
 792         if (!entry) {
 793             debugs(82, 2, HERE << acl->def->name << "(\"" << key << "\") = lookup needed");
 794
 795             if (!acl->def->theHelper->queueFull()) {
 796                 debugs(82, 2, HERE << "\"" << key << "\": queueing a call.");
 797                 if (!ch->goAsync(ExternalACLLookup::Instance()))
 798                     debugs(82, 2, "\"" << key << "\": no async support!");
 799                 debugs(82, 2, HERE << "\"" << key << "\": return -1.");
 800                 return ACCESS_DUNNO; // expired cached or simply absent entry
 801             } else {
 802                 if (!staleEntry) {
 803                     debugs(82, DBG_IMPORTANT, "WARNING: external ACL '" << acl->def->name <<
 804                            "' queue overload. Request rejected '" << key << "'.");
 805                     external_acl_message = "SYSTEM TOO BUSY, TRY AGAIN LATER";
 806                     return ACCESS_DUNNO;
 807                 } else {
 808                     debugs(82, DBG_IMPORTANT, "WARNING: external ACL '" << acl->def->name <<
 809                            "' queue overload. Using stale result. '" << key << "'.");
 810                     entry = staleEntry;
 811                     /* Fall thru to processing below */
 812                 }
 813             }
 814         }
 815     }
 816
 817     debugs(82, 4, HERE << "entry = { date=" <<
 818            (long unsigned int) entry->date <<
 819            ", result=" << entry->result <<
 820            " tag=" << entry->tag <<
 821            " log=" << entry->log << " }");
 822 #if USE_AUTH
 823     debugs(82, 4, HERE << "entry user=" << entry->user);
 824 #endif
 825
 826     external_acl_cache_touch(acl->def, entry);
 827     external_acl_message = entry->message.termedBuf();
 828
 829     debugs(82, 2, HERE << acl->def->name << " = " << entry->result);
 830     copyResultsFromEntry(ch->request, entry);
 831     return entry->result;
 832 }
 833
 834 int
 835 ACLExternal::match(ACLChecklist *checklist)
 836 {
 837     allow_t answer = aclMatchExternal(data, Filled(checklist));
 838
 839     // convert to tri-state ACL match 1,0,-1
 840     switch (answer) {
 841     case ACCESS_ALLOWED:
 842         return 1; // match
 843
 844     case ACCESS_DENIED:
 845         return 0; // non-match
 846
 847     case ACCESS_DUNNO:
 848     case ACCESS_AUTH_REQUIRED:
 849     default:
 850         // If the answer is not allowed or denied (matches/not matches) and
 851         // async authentication is not in progress, then we are done.
 852         if (checklist->keepMatching())
 853             checklist->markFinished(answer, "aclMatchExternal exception");
 854         return -1; // other
 855     }
 856 }
 857
 858 SBufList
 859 ACLExternal::dump() const
 860 {
 861     external_acl_data const *acl = data;
 862     SBufList rv;
 863     rv.push_back(SBuf(acl->def->name));
 864
 865     for (wordlist *arg = acl->arguments; arg; arg = arg->next) {
 866         SBuf s;
 867         s.Printf(" %s", arg->key);
 868         rv.push_back(s);
 869     }
 870
 871     return rv;
 872 }
 873
 874 /******************************************************************
 875  * external_acl cache
 876  */
 877
 878 static void
 879 external_acl_cache_touch(external_acl * def, const ExternalACLEntryPointer &entry)
 880 {
 881     // this must not be done when nothing is being cached.
 882     if (def->cache_size <= 0 || (def->ttl <= 0 && entry->result == 1) || (def->negative_ttl <= 0 && entry->result != 1))
 883         return;
 884
 885     dlinkDelete(&entry->lru, &def->lru_list);
 886     ExternalACLEntry *e = const_cast<ExternalACLEntry *>(entry.getRaw()); // XXX: make hash a std::map of Pointer.
 887     dlinkAdd(e, &entry->lru, &def->lru_list);
 888 }
 889
 890 static char *
 891 makeExternalAclKey(ACLFilledChecklist * ch, external_acl_data * acl_data)
 892 {
 893     static MemBuf mb;
 894     char buf[256];
 895     int first = 1;
 896     wordlist *arg;
 897     HttpRequest *request = ch->request;
 898     HttpReply *reply = ch->reply;
 899     mb.reset();
 900     bool data_used = false;
 901
 902     for (external_acl_format::Pointer format = acl_data->def->format; format != NULL; format = format->next) {
 903         const char *str = NULL;
 904         String sb;
 905
 906         switch (format->type) {
 907 #if USE_AUTH
 908         case Format::LFT_USER_LOGIN:
 909             // if this ACL line was the cause of credentials fetch
 910             // they may not already be in the checklist
 911             if (ch->auth_user_request == NULL && ch->request)
 912                 ch->auth_user_request = ch->request->auth_user_request;
 913
 914             if (ch->auth_user_request != NULL)
 915                 str = ch->auth_user_request->username();
 916             break;
 917 #endif
 918 #if USE_IDENT
 919         case Format::LFT_USER_IDENT:
 920             str = ch->rfc931;
 921
 922             if (!str || !*str) {
 923                 // if we fail to go async, we still return NULL and the caller
 924                 // will detect the failure in ACLExternal::match().
 925                 (void)ch->goAsync(IdentLookup::Instance());
 926                 return NULL;
 927             }
 928
 929             break;
 930 #endif
 931
 932         case Format::LFT_CLIENT_IP_ADDRESS:
 933             str = ch->src_addr.toStr(buf,sizeof(buf));
 934             break;
 935
 936         case Format::LFT_CLIENT_PORT:
 937             snprintf(buf, sizeof(buf), "%d", request->client_addr.port());
 938             str = buf;
 939             break;
 940
 941 #if USE_SQUID_EUI
 942         case Format::LFT_EXT_ACL_CLIENT_EUI48:
 943             if (request->clientConnectionManager.valid() && request->clientConnectionManager->clientConnection != NULL &&
 944                     request->clientConnectionManager->clientConnection->remoteEui48.encode(buf, sizeof(buf)))
 945                 str = buf;
 946             break;
 947
 948         case Format::LFT_EXT_ACL_CLIENT_EUI64:
 949             if (request->clientConnectionManager.valid() && request->clientConnectionManager->clientConnection != NULL &&
 950                     request->clientConnectionManager->clientConnection->remoteEui64.encode(buf, sizeof(buf)))
 951                 str = buf;
 952             break;
 953 #endif
 954
 955         case Format::LFT_LOCAL_LISTENING_IP:
 956             str = request->my_addr.toStr(buf, sizeof(buf));
 957             break;
 958
 959         case Format::LFT_LOCAL_LISTENING_PORT:
 960             snprintf(buf, sizeof(buf), "%d", request->my_addr.port());
 961             str = buf;
 962             break;
 963
 964         case Format::LFT_CLIENT_REQ_URI:
 965             str = urlCanonical(request);
 966             break;
 967
 968         case Format::LFT_CLIENT_REQ_URLDOMAIN:
 969             str = request->url.host();
 970             break;
 971
 972         case Format::LFT_CLIENT_REQ_URLSCHEME:
 973             str = request->url.getScheme().c_str();
 974             break;
 975
 976         case Format::LFT_CLIENT_REQ_URLPORT:
 977             snprintf(buf, sizeof(buf), "%u", request->url.port());
 978             str = buf;
 979             break;
 980
 981         case Format::LFT_CLIENT_REQ_URLPATH: {
 982             SBuf tmp = request->url.path();
 983             str = tmp.c_str();
 984         }
 985         break;
 986
 987         case Format::LFT_CLIENT_REQ_METHOD: {
 988             const SBuf &s = request->method.image();
 989             sb.append(s.rawContent(), s.length());
 990         }
 991         str = sb.termedBuf();
 992         break;
 993
 994         case Format::LFT_ADAPTED_REQUEST_HEADER:
 995             if (format->header_id == -1)
 996                 sb = request->header.getByName(format->header);
 997             else
 998                 sb = request->header.getStrOrList(format->header_id);
 999             str = sb.termedBuf();
1000             break;
1001
1002         case Format::LFT_ADAPTED_REQUEST_HEADER_ELEM:
1003             if (format->header_id == -1)
1004                 sb = request->header.getByNameListMember(format->header, format->member, format->separator);
1005             else
1006                 sb = request->header.getListMember(format->header_id, format->member, format->separator);
1007             str = sb.termedBuf();
1008             break;
1009
1010         case Format::LFT_REPLY_HEADER:
1011             if (reply) {
1012                 if (format->header_id == -1)
1013                     sb = reply->header.getByName(format->header);
1014                 else
1015                     sb = reply->header.getStrOrList(format->header_id);
1016                 str = sb.termedBuf();
1017             }
1018             break;
1019
1020         case Format::LFT_REPLY_HEADER_ELEM:
1021             if (reply) {
1022                 if (format->header_id == -1)
1023                     sb = reply->header.getByNameListMember(format->header, format->member, format->separator);
1024                 else
1025                     sb = reply->header.getListMember(format->header_id, format->member, format->separator);
1026                 str = sb.termedBuf();
1027             }
1028             break;
1029
1030 #if USE_OPENSSL
1031
1032         case Format::LFT_EXT_ACL_USER_CERT_RAW:
1033
1034             if (ch->conn() != NULL && Comm::IsConnOpen(ch->conn()->clientConnection)) {
1035                 SSL *ssl = fd_table[ch->conn()->clientConnection->fd].ssl;
1036
1037                 if (ssl)
1038                     str = sslGetUserCertificatePEM(ssl);
1039             }
1040
1041             break;
1042
1043         case Format::LFT_EXT_ACL_USER_CERTCHAIN_RAW:
1044
1045             if (ch->conn() != NULL && Comm::IsConnOpen(ch->conn()->clientConnection)) {
1046                 SSL *ssl = fd_table[ch->conn()->clientConnection->fd].ssl;
1047
1048                 if (ssl)
1049                     str = sslGetUserCertificateChainPEM(ssl);
1050             }
1051
1052             break;
1053
1054         case Format::LFT_EXT_ACL_USER_CERT:
1055
1056             if (ch->conn() != NULL && Comm::IsConnOpen(ch->conn()->clientConnection)) {
1057                 SSL *ssl = fd_table[ch->conn()->clientConnection->fd].ssl;
1058
1059                 if (ssl)
1060                     str = sslGetUserAttribute(ssl, format->header);
1061             }
1062
1063             break;
1064
1065         case Format::LFT_EXT_ACL_USER_CA_CERT:
1066
1067             if (ch->conn() != NULL && Comm::IsConnOpen(ch->conn()->clientConnection)) {
1068                 SSL *ssl = fd_table[ch->conn()->clientConnection->fd].ssl;
1069
1070                 if (ssl)
1071                     str = sslGetCAAttribute(ssl, format->header);
1072             }
1073
1074             break;
1075
1076         case Format::LFT_SSL_CLIENT_SNI:
1077             if (ch->conn() != NULL) {
1078                 if (Ssl::ServerBump * srvBump = ch->conn()->serverBump()) {
1079                     if (!srvBump->clientSni.isEmpty())
1080                         str = srvBump->clientSni.c_str();
1081                 }
1082             }
1083             break;
1084
1085         case Format::LFT_SSL_SERVER_CERT_SUBJECT:
1086         case Format::LFT_SSL_SERVER_CERT_ISSUER: {
1087             X509 *serverCert = NULL;
1088             if (ch->serverCert.get())
1089                 serverCert = ch->serverCert.get();
1090             else if (ch->conn() && ch->conn()->serverBump())
1091                 serverCert = ch->conn()->serverBump()->serverCert.get();
1092
1093             if (serverCert) {
1094                 if (format->type == Format::LFT_SSL_SERVER_CERT_SUBJECT)
1095                     str = Ssl::GetX509UserAttribute(serverCert, "DN");
1096                 else
1097                     str = Ssl::GetX509CAAttribute(serverCert, "DN");
1098             }
1099             break;
1100         }
1101
1102 #endif
1103 #if USE_AUTH
1104         case Format::LFT_USER_EXTERNAL:
1105             str = request->extacl_user.termedBuf();
1106             break;
1107 #endif
1108         case Format::LFT_EXT_LOG:
1109             str = request->extacl_log.termedBuf();
1110             break;
1111         case Format::LFT_TAG:
1112             str = request->tag.termedBuf();
1113             break;
1114         case Format::LFT_EXT_ACL_NAME:
1115             str = acl_data->name;
1116             break;
1117         case Format::LFT_EXT_ACL_DATA:
1118             data_used = true;
1119             for (arg = acl_data->arguments; arg; arg = arg->next) {
1120                 if (!first)
1121                     sb.append(" ", 1);
1122
1123                 if (acl_data->def->quote == external_acl::QUOTE_METHOD_URL) {
1124                     const char *quoted = rfc1738_escape(arg->key);
1125                     sb.append(quoted, strlen(quoted));
1126                 } else {
1127                     static MemBuf mb2;
1128                     mb2.init();
1129                     strwordquote(&mb2, arg->key);
1130                     sb.append(mb2.buf, mb2.size);
1131                     mb2.clean();
1132                 }
1133
1134                 first = 0;
1135             }
1136             break;
1137         case Format::LFT_PERCENT:
1138             str = "%";
1139             break;
1140
1141         default:
1142             // TODO: replace this function with Format::assemble()
1143             // For now die on unsupported logformat codes.
1144             fatalf("ERROR: unknown external_acl_type format %u", (uint8_t)format->type);
1145             break;
1146         }
1147
1148         if (str)
1149             if (!*str)
1150                 str = NULL;
1151
1152         if (!str)
1153             str = "-";
1154
1155         if (!first)
1156             mb.append(" ", 1);
1157
1158         if (acl_data->def->quote == external_acl::QUOTE_METHOD_URL) {
1159             const char *quoted = rfc1738_escape(str);
1160             mb.append(quoted, strlen(quoted));
1161         } else {
1162             strwordquote(&mb, str);
1163         }
1164
1165         sb.clean();
1166
1167         first = 0;
1168     }
1169
1170     if (!data_used) {
1171         for (arg = acl_data->arguments; arg; arg = arg->next) {
1172             if (!first)
1173                 mb.append(" ", 1);
1174
1175             if (acl_data->def->quote == external_acl::QUOTE_METHOD_URL) {
1176                 const char *quoted = rfc1738_escape(arg->key);
1177                 mb.append(quoted, strlen(quoted));
1178             } else {
1179                 strwordquote(&mb, arg->key);
1180             }
1181
1182             first = 0;
1183         }
1184     }
1185
1186     return mb.buf;
1187 }
1188
1189 static int
1190 external_acl_entry_expired(external_acl * def, const ExternalACLEntryPointer &entry)
1191 {
1192     if (def->cache_size <= 0)
1193         return 1;
1194
1195     if (entry->date + (entry->result == 1 ? def->ttl : def->negative_ttl) < squid_curtime)
1196         return 1;
1197     else
1198         return 0;
1199 }
1200
1201 static int
1202 external_acl_grace_expired(external_acl * def, const ExternalACLEntryPointer &entry)
1203 {
1204     if (def->cache_size <= 0)
1205         return 1;
1206
1207     int ttl;
1208     ttl = entry->result == 1 ? def->ttl : def->negative_ttl;
1209     ttl = (ttl * (100 - def->grace)) / 100;
1210
1211     if (entry->date + ttl <= squid_curtime)
1212         return 1;
1213     else
1214         return 0;
1215 }
1216
1217 static ExternalACLEntryPointer
1218 external_acl_cache_add(external_acl * def, const char *key, ExternalACLEntryData const & data)
1219 {
1220     ExternalACLEntryPointer entry;
1221
1222     // do not bother caching this result if TTL is going to expire it immediately
1223     if (def->cache_size <= 0 || (def->ttl <= 0 && data.result == 1) || (def->negative_ttl <= 0 && data.result != 1)) {
1224         debugs(82,6, HERE);
1225         entry = new ExternalACLEntry;
1226         entry->key = xstrdup(key);
1227         entry->update(data);
1228         entry->def = def;
1229         return entry;
1230     }
1231
1232     entry = static_cast<ExternalACLEntry *>(hash_lookup(def->cache, key));
1233     debugs(82, 2, "external_acl_cache_add: Adding '" << key << "' = " << data.result);
1234
1235     if (entry != NULL) {
1236         debugs(82, 3, "updating existing entry");
1237         entry->update(data);
1238         external_acl_cache_touch(def, entry);
1239         return entry;
1240     }
1241
1242     entry = new ExternalACLEntry;
1243     entry->key = xstrdup(key);
1244     entry->update(data);
1245
1246     def->add(entry);
1247
1248     return entry;
1249 }
1250
1251 static void
1252 external_acl_cache_delete(external_acl * def, const ExternalACLEntryPointer &entry)
1253 {
1254     assert(entry != NULL);
1255     assert(def->cache_size > 0 && entry->def == def);
1256     ExternalACLEntry *e = const_cast<ExternalACLEntry *>(entry.getRaw()); // XXX: make hash a std::map of Pointer.
1257     hash_remove_link(def->cache, e);
1258     dlinkDelete(&e->lru, &def->lru_list);
1259     e->unlock(); // unlock on behalf of the hash
1260     def->cache_entries -= 1;
1261 }
1262
1263 /******************************************************************
1264  * external_acl helpers
1265  */
1266
1267 class externalAclState
1268 {
1269     CBDATA_CLASS(externalAclState);
1270
1271 public:
1272     externalAclState(external_acl* aDef, const char *aKey) :
1273         callback(NULL),
1274         callback_data(NULL),
1275         key(xstrdup(aKey)),
1276         def(cbdataReference(aDef)),
1277         queue(NULL)
1278     {}
1279     ~externalAclState();
1280
1281     EAH *callback;
1282     void *callback_data;
1283     char *key;
1284     external_acl *def;
1285     dlink_node list;
1286     externalAclState *queue;
1287 };
1288
1289 CBDATA_CLASS_INIT(externalAclState);
1290
1291 externalAclState::~externalAclState()
1292 {
1293     xfree(key);
1294     cbdataReferenceDone(callback_data);
1295     cbdataReferenceDone(def);
1296 }
1297
1298 /*
1299  * The helper program receives queries on stdin, one
1300  * per line, and must return the result on on stdout
1301  *
1302  * General result syntax:
1303  *
1304  *   OK/ERR keyword=value ...
1305  *
1306  * Keywords:
1307  *
1308  *   user=      The users name (login)
1309  *   message=   Message describing the reason
1310  *   tag=   A string tag to be applied to the request that triggered the acl match.
1311  *          applies to both OK and ERR responses.
1312  *          Won't override existing request tags.
1313  *   log=   A string to be used in access logging
1314  *
1315  * Other keywords may be added to the protocol later
1316  *
1317  * value needs to be URL-encoded or enclosed in double quotes (")
1318  * with \-escaping on any whitespace, quotes, or slashes (\).
1319  */
1320 static void
1321 externalAclHandleReply(void *data, const Helper::Reply &reply)
1322 {
1323     externalAclState *state = static_cast<externalAclState *>(data);
1324     externalAclState *next;
1325     ExternalACLEntryData entryData;
1326     entryData.result = ACCESS_DENIED;
1327
1328     debugs(82, 2, HERE << "reply=" << reply);
1329
1330     if (reply.result == Helper::Okay)
1331         entryData.result = ACCESS_ALLOWED;
1332     // XXX: handle other non-DENIED results better
1333
1334     // XXX: make entryData store a proper Helper::Reply object instead of copying.
1335
1336     entryData.notes.append(&reply.notes);
1337
1338     const char *label = reply.notes.findFirst("tag");
1339     if (label != NULL && *label != '\0')
1340         entryData.tag = label;
1341
1342     label = reply.notes.findFirst("message");
1343     if (label != NULL && *label != '\0')
1344         entryData.message = label;
1345
1346     label = reply.notes.findFirst("log");
1347     if (label != NULL && *label != '\0')
1348         entryData.log = label;
1349
1350 #if USE_AUTH
1351     label = reply.notes.findFirst("user");
1352     if (label != NULL && *label != '\0')
1353         entryData.user = label;
1354
1355     label = reply.notes.findFirst("password");
1356     if (label != NULL && *label != '\0')
1357         entryData.password = label;
1358 #endif
1359
1360     dlinkDelete(&state->list, &state->def->queue);
1361
1362     ExternalACLEntryPointer entry;
1363     if (cbdataReferenceValid(state->def)) {
1364         // only cache OK and ERR results.
1365         if (reply.result == Helper::Okay || reply.result == Helper::Error)
1366             entry = external_acl_cache_add(state->def, state->key, entryData);
1367         else {
1368             const ExternalACLEntryPointer oldentry = static_cast<ExternalACLEntry *>(hash_lookup(state->def->cache, state->key));
1369
1370             if (oldentry != NULL)
1371                 external_acl_cache_delete(state->def, oldentry);
1372         }
1373     }
1374
1375     do {
1376         void *cbdata;
1377         if (state->callback && cbdataReferenceValidDone(state->callback_data, &cbdata))
1378             state->callback(cbdata, entry);
1379
1380         next = state->queue;
1381         state->queue = NULL;
1382
1383         delete state;
1384
1385         state = next;
1386     } while (state);
1387 }
1388
1389 void
1390 ACLExternal::ExternalAclLookup(ACLChecklist *checklist, ACLExternal * me)
1391 {
1392     ExternalACLLookup::Start(checklist, me->data, false);
1393 }
1394
1395 void
1396 ExternalACLLookup::Start(ACLChecklist *checklist, external_acl_data *acl, bool inBackground)
1397 {
1398     external_acl *def = acl->def;
1399
1400     ACLFilledChecklist *ch = Filled(checklist);
1401     const char *key = makeExternalAclKey(ch, acl);
1402     assert(key); // XXX: will fail if EXT_ACL_IDENT case needs an async lookup
1403
1404     debugs(82, 2, HERE << (inBackground ? "bg" : "fg") << " lookup in '" <<
1405            def->name << "' for '" << key << "'");
1406
1407     /* Check for a pending lookup to hook into */
1408     // only possible if we are caching results.
1409     externalAclState *oldstate = NULL;
1410     if (def->cache_size > 0) {
1411         for (dlink_node *node = def->queue.head; node; node = node->next) {
1412             externalAclState *oldstatetmp = static_cast<externalAclState *>(node->data);
1413
1414             if (strcmp(key, oldstatetmp->key) == 0) {
1415                 oldstate = oldstatetmp;
1416                 break;
1417             }
1418         }
1419     }
1420
1421     // A background refresh has no need to piggiback on a pending request:
1422     // When the pending request completes, the cache will be refreshed anyway.
1423     if (oldstate && inBackground) {
1424         debugs(82, 7, HERE << "'" << def->name << "' queue is already being refreshed (ch=" << ch << ")");
1425         return;
1426     }
1427
1428     externalAclState *state = new externalAclState(def, key);
1429
1430     if (!inBackground) {
1431         state->callback = &ExternalACLLookup::LookupDone;
1432         state->callback_data = cbdataReference(checklist);
1433     }
1434
1435     if (oldstate) {
1436         /* Hook into pending lookup */
1437         state->queue = oldstate->queue;
1438         oldstate->queue = state;
1439     } else {
1440         /* No pending lookup found. Sumbit to helper */
1441
1442         MemBuf buf;
1443         buf.init();
1444         buf.appendf("%s\n", key);
1445         debugs(82, 4, "externalAclLookup: looking up for '" << key << "' in '" << def->name << "'.");
1446
1447         if (!def->theHelper->trySubmit(buf.buf, externalAclHandleReply, state)) {
1448             debugs(82, 7, HERE << "'" << def->name << "' submit to helper failed");
1449             assert(inBackground); // or the caller should have checked
1450             delete state;
1451             return;
1452         }
1453
1454         dlinkAdd(state, &state->list, &def->queue);
1455
1456         buf.clean();
1457     }
1458
1459     debugs(82, 4, "externalAclLookup: will wait for the result of '" << key <<
1460            "' in '" << def->name << "' (ch=" << ch << ").");
1461 }
1462
1463 static void
1464 externalAclStats(StoreEntry * sentry)
1465 {
1466     for (external_acl *p = Config.externalAclHelperList; p; p = p->next) {
1467         storeAppendPrintf(sentry, "External ACL Statistics: %s\n", p->name);
1468         storeAppendPrintf(sentry, "Cache size: %d\n", p->cache->count);
1469         assert(p->theHelper);
1470         p->theHelper->packStatsInto(sentry);
1471         storeAppendPrintf(sentry, "\n");
1472     }
1473 }
1474
1475 static void
1476 externalAclRegisterWithCacheManager(void)
1477 {
1478     Mgr::RegisterAction("external_acl",
1479                         "External ACL stats",
1480                         externalAclStats, 0, 1);
1481 }
1482
1483 void
1484 externalAclInit(void)
1485 {
1486     for (external_acl *p = Config.externalAclHelperList; p; p = p->next) {
1487         if (!p->cache)
1488             p->cache = hash_create((HASHCMP *) strcmp, hashPrime(1024), hash4);
1489
1490         if (!p->theHelper)
1491             p->theHelper = new helper(p->name);
1492
1493         p->theHelper->cmdline = p->cmdline;
1494
1495         p->theHelper->childs.updateLimits(p->children);
1496
1497         p->theHelper->ipc_type = IPC_TCP_SOCKET;
1498
1499         p->theHelper->addr = p->local_addr;
1500
1501         helperOpenServers(p->theHelper);
1502     }
1503
1504     externalAclRegisterWithCacheManager();
1505 }
1506
1507 void
1508 externalAclShutdown(void)
1509 {
1510     external_acl *p;
1511
1512     for (p = Config.externalAclHelperList; p; p = p->next) {
1513         helperShutdown(p->theHelper);
1514     }
1515 }
1516
1517 ExternalACLLookup ExternalACLLookup::instance_;
1518 ExternalACLLookup *
1519 ExternalACLLookup::Instance()
1520 {
1521     return &instance_;
1522 }
1523
1524 void
1525 ExternalACLLookup::checkForAsync(ACLChecklist *checklist)const
1526 {
1527     /* TODO: optimise this - we probably have a pointer to this
1528      * around somewhere */
1529     ACL *acl = ACL::FindByName(AclMatchedName);
1530     assert(acl);
1531     ACLExternal *me = dynamic_cast<ACLExternal *> (acl);
1532     assert (me);
1533     ACLExternal::ExternalAclLookup(checklist, me);
1534 }
1535
1536 /// Called when an async lookup returns
1537 void
1538 ExternalACLLookup::LookupDone(void *data, const ExternalACLEntryPointer &result)
1539 {
1540     ACLFilledChecklist *checklist = Filled(static_cast<ACLChecklist*>(data));
1541     checklist->extacl_entry = result;
1542     checklist->resumeNonBlockingCheck(ExternalACLLookup::Instance());
1543 }
1544
1545 /* This registers "external" in the registry. To do dynamic definitions
1546  * of external ACL's, rather than a static prototype, have a Prototype instance
1547  * prototype in the class that defines each external acl 'class'.
1548  * Then, then the external acl instance is created, it self registers under
1549  * it's name.
1550  * Be sure that clone is fully functional for that acl class though!
1551  */
1552 ACL::Prototype ACLExternal::RegistryProtoype(&ACLExternal::RegistryEntry_, "external");
1553
1554 ACLExternal ACLExternal::RegistryEntry_("external");
1555
1556 ACL *
1557 ACLExternal::clone() const
1558 {
1559     return new ACLExternal(*this);
1560 }
1561
1562 ACLExternal::ACLExternal(char const *theClass) : data(NULL), class_(xstrdup(theClass))
1563 {}
1564
1565 ACLExternal::ACLExternal(ACLExternal const & old) : data(NULL), class_(old.class_ ? xstrdup(old.class_) : NULL)
1566 {
1567     /* we don't have copy constructors for the data yet */
1568     assert(!old.data);
1569 }
1570
1571 char const *
1572 ACLExternal::typeString() const
1573 {
1574     return class_;
1575 }
1576
1577 bool
1578 ACLExternal::isProxyAuth() const
1579 {
1580 #if USE_AUTH
1581     return data->def->require_auth;
1582 #else
1583     return false;
1584 #endif
1585 }
1586