]> git.ipfire.org Git - thirdparty/squid.git/blobdiff - src/external_acl.cc
SourceFormat Enforcement
[thirdparty/squid.git] / src / external_acl.cc
index bb5099d41043a898bcfef865efa27731f47e2ed6..d7b01e597a6518afc6dddc6211e686282e876494 100644 (file)
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
+ * Copyright (C) 1996-2017 The Squid Software Foundation and contributors
  *
  * Squid software is distributed under GPLv2+ license and includes
  * contributions from numerous individuals and organizations.
 #include "acl/FilledChecklist.h"
 #include "cache_cf.h"
 #include "client_side.h"
+#include "client_side_request.h"
 #include "comm/Connection.h"
 #include "ConfigParser.h"
 #include "ExternalACL.h"
 #include "ExternalACLEntry.h"
 #include "fde.h"
-#include "format/ByteCode.h"
+#include "format/Token.h"
 #include "helper.h"
 #include "helper/Reply.h"
+#include "http/Stream.h"
 #include "HttpHeaderTools.h"
 #include "HttpReply.h"
 #include "HttpRequest.h"
@@ -66,27 +68,6 @@ static ExternalACLEntryPointer external_acl_cache_add(external_acl * def, const
  * external_acl directive
  */
 
-class external_acl_format : public RefCountable
-{
-    MEMPROXY_CLASS(external_acl_format);
-
-public:
-    typedef RefCount<external_acl_format> Pointer;
-
-    external_acl_format() : type(Format::LFT_NONE), header(NULL), member(NULL), separator(' '), header_id(HDR_BAD_HDR) {}
-    ~external_acl_format() {
-        xfree(header);
-        xfree(member);
-    }
-
-    Format::ByteCode_t type;
-    external_acl_format::Pointer next;
-    char *header;
-    char *member;
-    char separator;
-    http_hdr_type header_id;
-};
-
 class external_acl
 {
     /* FIXME: These are not really cbdata, but it is an easy way
@@ -112,7 +93,7 @@ public:
 
     char *name;
 
-    external_acl_format::Pointer format;
+    Format::Format format;
 
     wordlist *cmdline;
 
@@ -140,10 +121,7 @@ public:
     bool require_auth;
 #endif
 
-    enum {
-        QUOTE_METHOD_SHELL = 1,
-        QUOTE_METHOD_URL
-    } quote;
+    Format::Quoting quote; // default quoting to use, set by protocol= parameter
 
     Ip::Address local_addr;
 };
@@ -151,10 +129,12 @@ public:
 CBDATA_CLASS_INIT(external_acl);
 
 external_acl::external_acl() :
+    next(NULL),
     ttl(DEFAULT_EXTERNAL_ACL_TTL),
     negative_ttl(-1),
     grace(1),
     name(NULL),
+    format("external_acl_type"),
     cmdline(NULL),
     children(DEFAULT_EXTERNAL_ACL_CHILDREN),
     theHelper(NULL),
@@ -164,7 +144,7 @@ external_acl::external_acl() :
 #if USE_AUTH
     require_auth(0),
 #endif
-    quote(external_acl::QUOTE_METHOD_URL)
+    quote(Format::LOG_QUOTE_URL)
 {
     local_addr.setLocalhost();
 }
@@ -172,7 +152,6 @@ external_acl::external_acl() :
 external_acl::~external_acl()
 {
     xfree(name);
-    format = NULL;
     wordlistDestroy(&cmdline);
 
     if (theHelper) {
@@ -196,65 +175,17 @@ external_acl::~external_acl()
     }
 }
 
-/**
- * Parse the External ACL format %<{.*} and %>{.*} token(s) to pass a specific
- * request or reply header to external helper.
- *
- \param header   - the token being parsed (without the identifying prefix)
- \param type     - format enum identifier for this element, pulled from identifying prefix
- \param format   - structure to contain all the info about this format element.
- */
-void
-parse_header_token(external_acl_format::Pointer format, char *header, const Format::ByteCode_t type)
-{
-    /* header format */
-    char *member, *end;
-
-    /** Cut away the closing brace */
-    end = strchr(header, '}');
-    if (end && strlen(end) == 1)
-        *end = '\0';
-    else
-        self_destruct();
-
-    member = strchr(header, ':');
-
-    if (member) {
-        /* Split in header and member */
-        *member = '\0';
-        ++member;
-
-        if (!xisalnum(*member)) {
-            format->separator = *member;
-            ++member;
-        } else {
-            format->separator = ',';
-        }
-
-        format->member = xstrdup(member);
-
-        if (type == Format::LFT_ADAPTED_REQUEST_HEADER)
-            format->type = Format::LFT_ADAPTED_REQUEST_HEADER_ELEM;
-        else
-            format->type = Format::LFT_REPLY_HEADER_ELEM;
-
-    } else {
-        format->type = type;
-    }
-
-    format->header = xstrdup(header);
-    format->header_id = httpHeaderIdByNameDef(header, strlen(header));
-}
-
 void
 parse_externalAclHelper(external_acl ** list)
 {
-    external_acl *a = new external_acl;
     char *token = ConfigParser::NextToken();
 
-    if (!token)
+    if (!token) {
         self_destruct();
+        return;
+    }
 
+    external_acl *a = new external_acl;
     a->name = xstrdup(token);
 
     // Allow supported %macros inside quoted tokens
@@ -286,16 +217,16 @@ parse_externalAclHelper(external_acl ** list)
         } else if (strncmp(token, "grace=", 6) == 0) {
             a->grace = atoi(token + 6);
         } else if (strcmp(token, "protocol=2.5") == 0) {
-            a->quote = external_acl::QUOTE_METHOD_SHELL;
+            a->quote = Format::LOG_QUOTE_SHELL;
         } else if (strcmp(token, "protocol=3.0") == 0) {
             debugs(3, DBG_PARSE_NOTE(2), "WARNING: external_acl_type option protocol=3.0 is deprecated. Remove this from your config.");
-            a->quote = external_acl::QUOTE_METHOD_URL;
+            a->quote = Format::LOG_QUOTE_URL;
         } else if (strcmp(token, "quote=url") == 0) {
             debugs(3, DBG_PARSE_NOTE(2), "WARNING: external_acl_type option quote=url is deprecated. Remove this from your config.");
-            a->quote = external_acl::QUOTE_METHOD_URL;
+            a->quote = Format::LOG_QUOTE_URL;
         } else if (strcmp(token, "quote=shell") == 0) {
             debugs(3, DBG_PARSE_NOTE(2), "WARNING: external_acl_type option quote=shell is deprecated. Use protocol=2.5 if still needed.");
-            a->quote = external_acl::QUOTE_METHOD_SHELL;
+            a->quote = Format::LOG_QUOTE_SHELL;
 
             /* INET6: allow admin to configure some helpers explicitly to
                       bind to IPv4/v6 localhost port. */
@@ -331,120 +262,84 @@ parse_externalAclHelper(external_acl ** list)
     if (a->children.defaultQueueSize)
         a->children.queue_size = 2 * a->children.n_max;
 
-    /* Parse format */
-    external_acl_format::Pointer *p = &a->format;
-
+    /* Legacy external_acl_type format parser.
+     * Handles a series of %... tokens where any non-% means
+     * the start of another parameter field (ie the path to binary).
+     */
+    enum Format::Quoting quote = Format::LOG_QUOTE_NONE;
+    Format::Token **fmt = &a->format.format;
+    bool data_used = false;
     while (token) {
-        /* stop on first non-format token found */
-
+        /* stop on first non-% token found */
         if (*token != '%')
             break;
 
-        external_acl_format::Pointer format = new external_acl_format;
-
-        if (strncmp(token, "%{", 2) == 0) {
-            // deprecated. but assume the old configs all referred to request headers.
-            debugs(82, DBG_PARSE_NOTE(DBG_IMPORTANT), "WARNING: external_acl_type format %{...} is being replaced by %>ha{...} for : " << token);
-            parse_header_token(format, (token+2), Format::LFT_ADAPTED_REQUEST_HEADER);
-        } else if (strncmp(token, "%>{", 3) == 0) {
-            debugs(82, DBG_PARSE_NOTE(DBG_IMPORTANT), "WARNING: external_acl_type format %>{...} is being replaced by %>ha{...} for : " << token);
-            parse_header_token(format, (token+3), Format::LFT_ADAPTED_REQUEST_HEADER);
-        } else if (strncmp(token, "%>ha{", 5) == 0) {
-            parse_header_token(format, (token+5), Format::LFT_ADAPTED_REQUEST_HEADER);
-        } else if (strncmp(token, "%<{", 3) == 0) {
-            debugs(82, DBG_PARSE_NOTE(DBG_IMPORTANT), "WARNING: external_acl_type format %<{...} is being replaced by %<h{...} for : " << token);
-            parse_header_token(format, (token+3), Format::LFT_REPLY_HEADER);
-        } else if (strncmp(token, "%<h{", 4) == 0) {
-            parse_header_token(format, (token+4), Format::LFT_REPLY_HEADER);
-#if USE_AUTH
-        } else if (strcmp(token, "%LOGIN") == 0 || strcmp(token, "%ul") == 0) {
-            format->type = Format::LFT_USER_LOGIN;
-            a->require_auth = true;
-#endif
-        }
-#if USE_IDENT
-        else if (strcmp(token, "%IDENT") == 0 || strcmp(token, "%ui") == 0)
-            format->type = Format::LFT_USER_IDENT;
-#endif
-        else if (strcmp(token, "%SRC") == 0 || strcmp(token, "%>a") == 0)
-            format->type = Format::LFT_CLIENT_IP_ADDRESS;
-        else if (strcmp(token, "%SRCPORT") == 0 || strcmp(token, "%>p") == 0)
-            format->type = Format::LFT_CLIENT_PORT;
-#if USE_SQUID_EUI
-        else if (strcmp(token, "%SRCEUI48") == 0)
-            format->type = Format::LFT_EXT_ACL_CLIENT_EUI48;
-        else if (strcmp(token, "%SRCEUI64") == 0)
-            format->type = Format::LFT_EXT_ACL_CLIENT_EUI64;
-#endif
-        else if (strcmp(token, "%MYADDR") == 0 || strcmp(token, "%la") == 0)
-            format->type = Format::LFT_LOCAL_LISTENING_IP;
-        else if (strcmp(token, "%MYPORT") == 0 || strcmp(token, "%lp") == 0)
-            format->type = Format::LFT_LOCAL_LISTENING_PORT;
-        else if (strcmp(token, "%URI") == 0 || strcmp(token, "%>ru") == 0)
-            format->type = Format::LFT_CLIENT_REQ_URI;
-        else if (strcmp(token, "%DST") == 0 || strcmp(token, "%>rd") == 0)
-            format->type = Format::LFT_CLIENT_REQ_URLDOMAIN;
-        else if (strcmp(token, "%PROTO") == 0 || strcmp(token, "%>rs") == 0)
-            format->type = Format::LFT_CLIENT_REQ_URLSCHEME;
-        else if (strcmp(token, "%PORT") == 0) // XXX: add a logformat token
-            format->type = Format::LFT_CLIENT_REQ_URLPORT;
-        else if (strcmp(token, "%PATH") == 0 || strcmp(token, "%>rp") == 0)
-            format->type = Format::LFT_CLIENT_REQ_URLPATH;
-        else if (strcmp(token, "%METHOD") == 0 || strcmp(token, "%>rm") == 0)
-            format->type = Format::LFT_CLIENT_REQ_METHOD;
-#if USE_OPENSSL
-        else if (strcmp(token, "%USER_CERT") == 0)
-            format->type = Format::LFT_EXT_ACL_USER_CERT_RAW;
-        else if (strcmp(token, "%USER_CERTCHAIN") == 0)
-            format->type = Format::LFT_EXT_ACL_USER_CERTCHAIN_RAW;
-        else if (strncmp(token, "%USER_CERT_", 11) == 0) {
-            format->type = Format::LFT_EXT_ACL_USER_CERT;
-            format->header = xstrdup(token + 11);
+        *fmt = new Format::Token;
+        // these tokens are whitespace delimited
+        (*fmt)->space = true;
+
+        // set the default encoding to match the protocol= config
+        // this will be overridden by explicit %macro attributes
+        (*fmt)->quote = a->quote;
+
+        // compatibility for old tokens incompatible with Format::Token syntax
+#if USE_OPENSSL // dont bother if we dont have to.
+        if (strncmp(token, "%USER_CERT_", 11) == 0) {
+            (*fmt)->type = Format::LFT_EXT_ACL_USER_CERT;
+            (*fmt)->data.string = xstrdup(token + 11);
+            (*fmt)->data.header.header = (*fmt)->data.string;
         } else if (strncmp(token, "%USER_CA_CERT_", 14) == 0) {
-            format->type = Format::LFT_EXT_ACL_USER_CA_CERT;
-            format->header = xstrdup(token + 14);
+            (*fmt)->type = Format::LFT_EXT_ACL_USER_CA_CERT;
+            (*fmt)->data.string = xstrdup(token + 14);
+            (*fmt)->data.header.header = (*fmt)->data.string;
         } else if (strncmp(token, "%CA_CERT_", 9) == 0) {
             debugs(82, DBG_PARSE_NOTE(DBG_IMPORTANT), "WARNING: external_acl_type %CA_CERT_* code is obsolete. Use %USER_CA_CERT_* instead");
-            format->type = Format::LFT_EXT_ACL_USER_CA_CERT;
-            format->header = xstrdup(token + 9);
-        } else if (strcmp(token, "%ssl::>sni") == 0)
-            format->type = Format::LFT_SSL_CLIENT_SNI;
-        else if (strcmp(token, "%ssl::<cert_subject") == 0)
-            format->type = Format::LFT_SSL_SERVER_CERT_SUBJECT;
-        else if (strcmp(token, "%ssl::<cert_issuer") == 0)
-            format->type = Format::LFT_SSL_SERVER_CERT_ISSUER;
+            (*fmt)->type = Format::LFT_EXT_ACL_USER_CA_CERT;
+            (*fmt)->data.string = xstrdup(token + 9);
+            (*fmt)->data.header.header = (*fmt)->data.string;
+        } else
 #endif
+        {
+            // we can use the Format::Token::parse() method since it
+            // only pulls off one token. Since we already checked
+            // for '%' prefix above this is guaranteed to be a token.
+            const size_t len = (*fmt)->parse(token, &quote);
+            assert(len == strlen(token));
+        }
+
+        // process special token-specific actions (only if necessary)
 #if USE_AUTH
-        else if (strcmp(token, "%EXT_USER") == 0 || strcmp(token, "%ue") == 0)
-            format->type = Format::LFT_USER_EXTERNAL;
+        if ((*fmt)->type == Format::LFT_USER_LOGIN)
+            a->require_auth = true;
 #endif
-        else if (strcmp(token, "%EXT_LOG") == 0 || strcmp(token, "%ea") == 0)
-            format->type = Format::LFT_EXT_LOG;
-        else if (strcmp(token, "%TAG") == 0  || strcmp(token, "%et") == 0)
-            format->type = Format::LFT_TAG;
-        else if (strcmp(token, "%ACL") == 0)
-            format->type = Format::LFT_EXT_ACL_NAME;
-        else if (strcmp(token, "%DATA") == 0)
-            format->type = Format::LFT_EXT_ACL_DATA;
-        else if (strcmp(token, "%%") == 0)
-            format->type = Format::LFT_PERCENT;
-        else {
-            debugs(0, DBG_CRITICAL, "ERROR: Unknown Format token " << token);
-            self_destruct();
-        }
 
-        *p = format;
-        p = &format->next;
+        if ((*fmt)->type == Format::LFT_EXT_ACL_DATA)
+            data_used = true;
+
+        fmt = &((*fmt)->next);
         token = ConfigParser::NextToken();
     }
 
     /* There must be at least one format token */
-    if (!a->format)
+    if (!a->format.format) {
+        delete a;
         self_destruct();
+        return;
+    }
+
+    // format has implicit %DATA on the end if not used explicitly
+    if (!data_used) {
+        *fmt = new Format::Token;
+        (*fmt)->type = Format::LFT_EXT_ACL_DATA;
+        (*fmt)->quote = Format::LOG_QUOTE_NONE;
+    }
 
     /* helper */
-    if (!token)
+    if (!token) {
+        delete a;
         self_destruct();
+        return;
+    }
 
     wordlistAdd(&a->cmdline, token);
 
@@ -483,87 +378,22 @@ dump_externalAclHelper(StoreEntry * sentry, const char *name, const external_acl
         if (node->children.n_max != DEFAULT_EXTERNAL_ACL_CHILDREN)
             storeAppendPrintf(sentry, " children-max=%d", node->children.n_max);
 
-        if (node->children.n_startup != 1)
+        if (node->children.n_startup != 0) // sync with helper/ChildConfig.cc default
             storeAppendPrintf(sentry, " children-startup=%d", node->children.n_startup);
 
-        if (node->children.n_idle != (node->children.n_max + node->children.n_startup) )
+        if (node->children.n_idle != 1) // sync with helper/ChildConfig.cc default
             storeAppendPrintf(sentry, " children-idle=%d", node->children.n_idle);
 
-        if (node->children.concurrency)
+        if (node->children.concurrency != 0)
             storeAppendPrintf(sentry, " concurrency=%d", node->children.concurrency);
 
         if (node->cache)
             storeAppendPrintf(sentry, " cache=%d", node->cache_size);
 
-        if (node->quote == external_acl::QUOTE_METHOD_SHELL)
+        if (node->quote == Format::LOG_QUOTE_SHELL)
             storeAppendPrintf(sentry, " protocol=2.5");
 
-        for (external_acl_format::Pointer format = node->format; format!= NULL; format = format->next) {
-            switch (format->type) {
-
-            case Format::LFT_ADAPTED_REQUEST_HEADER:
-                storeAppendPrintf(sentry, " %%>ha{%s}", format->header);
-                break;
-
-            case Format::LFT_ADAPTED_REQUEST_HEADER_ELEM:
-                storeAppendPrintf(sentry, " %%>ha{%s:%s}", format->header, format->member);
-                break;
-
-            case Format::LFT_REPLY_HEADER:
-                storeAppendPrintf(sentry, " %%<h{%s}", format->header);
-                break;
-
-            case Format::LFT_REPLY_HEADER_ELEM:
-                storeAppendPrintf(sentry, " %%<h{%s:%s}", format->header, format->member);
-                break;
-
-#define DUMP_EXT_ACL_TYPE_FMT(a, fmt, ...) \
-            case Format::LFT_##a: \
-                storeAppendPrintf(sentry, fmt, ##__VA_ARGS__); \
-                break
-#if USE_AUTH
-                DUMP_EXT_ACL_TYPE_FMT(USER_LOGIN," %%ul");
-#endif
-#if USE_IDENT
-
-                DUMP_EXT_ACL_TYPE_FMT(USER_IDENT," %%ui");
-#endif
-                DUMP_EXT_ACL_TYPE_FMT(CLIENT_IP_ADDRESS," %%>a");
-                DUMP_EXT_ACL_TYPE_FMT(CLIENT_PORT," %%>p");
-#if USE_SQUID_EUI
-                DUMP_EXT_ACL_TYPE_FMT(EXT_ACL_CLIENT_EUI48," %%SRCEUI48");
-                DUMP_EXT_ACL_TYPE_FMT(EXT_ACL_CLIENT_EUI64," %%SRCEUI64");
-#endif
-                DUMP_EXT_ACL_TYPE_FMT(LOCAL_LISTENING_IP," %%>la");
-                DUMP_EXT_ACL_TYPE_FMT(LOCAL_LISTENING_PORT," %%>lp");
-                DUMP_EXT_ACL_TYPE_FMT(CLIENT_REQ_URI," %%>ru");
-                DUMP_EXT_ACL_TYPE_FMT(CLIENT_REQ_URLDOMAIN," %%>rd");
-                DUMP_EXT_ACL_TYPE_FMT(CLIENT_REQ_URLSCHEME," %%>rs");
-                DUMP_EXT_ACL_TYPE_FMT(CLIENT_REQ_URLPORT," %%>rP");
-                DUMP_EXT_ACL_TYPE_FMT(CLIENT_REQ_URLPATH," %%>rp");
-                DUMP_EXT_ACL_TYPE_FMT(CLIENT_REQ_METHOD," %%>rm");
-#if USE_OPENSSL
-                DUMP_EXT_ACL_TYPE_FMT(EXT_ACL_USER_CERT_RAW, " %%USER_CERT_RAW");
-                DUMP_EXT_ACL_TYPE_FMT(EXT_ACL_USER_CERTCHAIN_RAW, " %%USER_CERTCHAIN_RAW");
-                DUMP_EXT_ACL_TYPE_FMT(EXT_ACL_USER_CERT, " %%USER_CERT_%s", format->header);
-                DUMP_EXT_ACL_TYPE_FMT(EXT_ACL_USER_CA_CERT, " %%USER_CA_CERT_%s", format->header);
-                DUMP_EXT_ACL_TYPE_FMT(SSL_CLIENT_SNI, "%%ssl::>sni");
-                DUMP_EXT_ACL_TYPE_FMT(SSL_SERVER_CERT_SUBJECT, "%%ssl::<cert_subject");
-                DUMP_EXT_ACL_TYPE_FMT(SSL_SERVER_CERT_ISSUER, "%%ssl::<cert_issuer");
-#endif
-#if USE_AUTH
-                DUMP_EXT_ACL_TYPE_FMT(USER_EXTERNAL," %%ue");
-#endif
-                DUMP_EXT_ACL_TYPE_FMT(EXT_LOG," %%ea");
-                DUMP_EXT_ACL_TYPE_FMT(TAG," %%et");
-                DUMP_EXT_ACL_TYPE_FMT(EXT_ACL_NAME," %%ACL");
-                DUMP_EXT_ACL_TYPE_FMT(EXT_ACL_DATA," %%DATA");
-                DUMP_EXT_ACL_TYPE_FMT(PERCENT, " %%%%");
-            default:
-                fatal("unknown external_acl format error");
-                break;
-            }
-        }
+        node->format.dump(sentry, NULL, false);
 
         for (word = node->cmdline; word; word = word->next)
             storeAppendPrintf(sentry, " %s", word->key);
@@ -644,18 +474,25 @@ external_acl_data::~external_acl_data()
 void
 ACLExternal::parse()
 {
-    if (data)
+    if (data) {
         self_destruct();
+        return;
+    }
 
     char *token = ConfigParser::strtokFile();
 
-    if (!token)
+    if (!token) {
         self_destruct();
+        return;
+    }
 
     data = new external_acl_data(find_externalAclHelper(token));
 
-    if (!data->def)
+    if (!data->def) {
+        delete data;
         self_destruct();
+        return;
+    }
 
     // def->name is the name of the external_acl_type.
     // this is the name of the 'acl' directive being tested
@@ -717,6 +554,9 @@ copyResultsFromEntry(HttpRequest *req, const ExternalACLEntryPointer &entry)
 
         if (entry->message.size())
             req->extacl_message = entry->message;
+
+        // attach the helper kv-pair to the transaction
+        UpdateRequestNotes(req->clientConnectionManager.get(), *req, entry->notes);
     }
 }
 
@@ -788,7 +628,8 @@ aclMatchExternal(external_acl_data *acl, ACLFilledChecklist *ch)
         if (!entry) {
             debugs(82, 2, HERE << acl->def->name << "(\"" << key << "\") = lookup needed");
 
-            if (!acl->def->theHelper->queueFull()) {
+            // TODO: All other helpers allow temporary overload. Should not we?
+            if (!acl->def->theHelper->willOverload()) {
                 debugs(82, 2, HERE << "\"" << key << "\": queueing a call.");
                 if (!ch->goAsync(ExternalACLLookup::Instance()))
                     debugs(82, 2, "\"" << key << "\": no async support!");
@@ -797,12 +638,12 @@ aclMatchExternal(external_acl_data *acl, ACLFilledChecklist *ch)
             } else {
                 if (!staleEntry) {
                     debugs(82, DBG_IMPORTANT, "WARNING: external ACL '" << acl->def->name <<
-                           "' queue overload. Request rejected '" << key << "'.");
+                           "' queue full. Request rejected '" << key << "'.");
                     external_acl_message = "SYSTEM TOO BUSY, TRY AGAIN LATER";
                     return ACCESS_DUNNO;
                 } else {
                     debugs(82, DBG_IMPORTANT, "WARNING: external ACL '" << acl->def->name <<
-                           "' queue overload. Using stale result. '" << key << "'.");
+                           "' queue full. Using stale result. '" << key << "'.");
                     entry = staleEntry;
                     /* Fall thru to processing below */
                 }
@@ -887,234 +728,25 @@ static char *
 makeExternalAclKey(ACLFilledChecklist * ch, external_acl_data * acl_data)
 {
     static MemBuf mb;
-    char buf[256];
-    int first = 1;
-    wordlist *arg;
-    HttpRequest *request = ch->request;
-    HttpReply *reply = ch->reply;
     mb.reset();
-    bool data_used = false;
-
-    for (external_acl_format::Pointer format = acl_data->def->format; format != NULL; format = format->next) {
-        const char *str = NULL;
-        String sb;
-
-        switch (format->type) {
-#if USE_AUTH
-        case Format::LFT_USER_LOGIN:
-            // if this ACL line was the cause of credentials fetch
-            // they may not already be in the checklist
-            if (ch->auth_user_request == NULL && ch->request)
-                ch->auth_user_request = ch->request->auth_user_request;
-
-            if (ch->auth_user_request != NULL)
-                str = ch->auth_user_request->username();
-            break;
-#endif
-#if USE_IDENT
-        case Format::LFT_USER_IDENT:
-            str = ch->rfc931;
-
-            if (!str || !*str) {
-                // if we fail to go async, we still return NULL and the caller
-                // will detect the failure in ACLExternal::match().
-                (void)ch->goAsync(IdentLookup::Instance());
-                return NULL;
-            }
-
-            break;
-#endif
-
-        case Format::LFT_CLIENT_IP_ADDRESS:
-            str = ch->src_addr.toStr(buf,sizeof(buf));
-            break;
-
-        case Format::LFT_CLIENT_PORT:
-            snprintf(buf, sizeof(buf), "%d", request->client_addr.port());
-            str = buf;
-            break;
-
-#if USE_SQUID_EUI
-        case Format::LFT_EXT_ACL_CLIENT_EUI48:
-            if (request->clientConnectionManager.valid() && request->clientConnectionManager->clientConnection != NULL &&
-                    request->clientConnectionManager->clientConnection->remoteEui48.encode(buf, sizeof(buf)))
-                str = buf;
-            break;
-
-        case Format::LFT_EXT_ACL_CLIENT_EUI64:
-            if (request->clientConnectionManager.valid() && request->clientConnectionManager->clientConnection != NULL &&
-                    request->clientConnectionManager->clientConnection->remoteEui64.encode(buf, sizeof(buf)))
-                str = buf;
-            break;
-#endif
-
-        case Format::LFT_LOCAL_LISTENING_IP:
-            str = request->my_addr.toStr(buf, sizeof(buf));
-            break;
-
-        case Format::LFT_LOCAL_LISTENING_PORT:
-            snprintf(buf, sizeof(buf), "%d", request->my_addr.port());
-            str = buf;
-            break;
-
-        case Format::LFT_CLIENT_REQ_URI:
-            str = urlCanonical(request);
-            break;
-
-        case Format::LFT_CLIENT_REQ_URLDOMAIN:
-            str = request->GetHost();
-            break;
-
-        case Format::LFT_CLIENT_REQ_URLSCHEME:
-            str = request->url.getScheme().c_str();
-            break;
-
-        case Format::LFT_CLIENT_REQ_URLPORT:
-            snprintf(buf, sizeof(buf), "%d", request->port);
-            str = buf;
-            break;
-
-        case Format::LFT_CLIENT_REQ_URLPATH:
-            str = request->urlpath.termedBuf();
-            break;
-
-        case Format::LFT_CLIENT_REQ_METHOD: {
-            const SBuf &s = request->method.image();
-            sb.append(s.rawContent(), s.length());
-        }
-        str = sb.termedBuf();
-        break;
-
-        case Format::LFT_ADAPTED_REQUEST_HEADER:
-            if (format->header_id == -1)
-                sb = request->header.getByName(format->header);
-            else
-                sb = request->header.getStrOrList(format->header_id);
-            str = sb.termedBuf();
-            break;
-
-        case Format::LFT_ADAPTED_REQUEST_HEADER_ELEM:
-            if (format->header_id == -1)
-                sb = request->header.getByNameListMember(format->header, format->member, format->separator);
-            else
-                sb = request->header.getListMember(format->header_id, format->member, format->separator);
-            str = sb.termedBuf();
-            break;
-
-        case Format::LFT_REPLY_HEADER:
-            if (reply) {
-                if (format->header_id == -1)
-                    sb = reply->header.getByName(format->header);
-                else
-                    sb = reply->header.getStrOrList(format->header_id);
-                str = sb.termedBuf();
-            }
-            break;
-
-        case Format::LFT_REPLY_HEADER_ELEM:
-            if (reply) {
-                if (format->header_id == -1)
-                    sb = reply->header.getByNameListMember(format->header, format->member, format->separator);
-                else
-                    sb = reply->header.getListMember(format->header_id, format->member, format->separator);
-                str = sb.termedBuf();
-            }
-            break;
-
-#if USE_OPENSSL
-
-        case Format::LFT_EXT_ACL_USER_CERT_RAW:
-
-            if (ch->conn() != NULL && Comm::IsConnOpen(ch->conn()->clientConnection)) {
-                SSL *ssl = fd_table[ch->conn()->clientConnection->fd].ssl;
-
-                if (ssl)
-                    str = sslGetUserCertificatePEM(ssl);
-            }
-
-            break;
-
-        case Format::LFT_EXT_ACL_USER_CERTCHAIN_RAW:
-
-            if (ch->conn() != NULL && Comm::IsConnOpen(ch->conn()->clientConnection)) {
-                SSL *ssl = fd_table[ch->conn()->clientConnection->fd].ssl;
-
-                if (ssl)
-                    str = sslGetUserCertificateChainPEM(ssl);
-            }
-
-            break;
-
-        case Format::LFT_EXT_ACL_USER_CERT:
-
-            if (ch->conn() != NULL && Comm::IsConnOpen(ch->conn()->clientConnection)) {
-                SSL *ssl = fd_table[ch->conn()->clientConnection->fd].ssl;
 
-                if (ssl)
-                    str = sslGetUserAttribute(ssl, format->header);
-            }
-
-            break;
-
-        case Format::LFT_EXT_ACL_USER_CA_CERT:
-
-            if (ch->conn() != NULL && Comm::IsConnOpen(ch->conn()->clientConnection)) {
-                SSL *ssl = fd_table[ch->conn()->clientConnection->fd].ssl;
-
-                if (ssl)
-                    str = sslGetCAAttribute(ssl, format->header);
-            }
-
-            break;
-
-        case Format::LFT_SSL_CLIENT_SNI:
-            if (ch->conn() != NULL) {
-                if (Ssl::ServerBump * srvBump = ch->conn()->serverBump()) {
-                    if (!srvBump->clientSni.isEmpty())
-                        str = srvBump->clientSni.c_str();
-                }
-            }
-            break;
+    // check for special case tokens in the format
+    for (Format::Token *t = acl_data->def->format.format; t ; t = t->next) {
 
-        case Format::LFT_SSL_SERVER_CERT_SUBJECT:
-        case Format::LFT_SSL_SERVER_CERT_ISSUER: {
-            X509 *serverCert = NULL;
-            if (ch->serverCert.get())
-                serverCert = ch->serverCert.get();
-            else if (ch->conn() && ch->conn()->serverBump())
-                serverCert = ch->conn()->serverBump()->serverCert.get();
-
-            if (serverCert) {
-                if (format->type == Format::LFT_SSL_SERVER_CERT_SUBJECT)
-                    str = Ssl::GetX509UserAttribute(serverCert, "DN");
-                else
-                    str = Ssl::GetX509CAAttribute(serverCert, "DN");
-            }
-            break;
+        if (t->type == Format::LFT_EXT_ACL_NAME) {
+            // setup for %ACL
+            safe_free(ch->al->lastAclName);
+            ch->al->lastAclName = xstrdup(acl_data->name);
         }
 
-#endif
-#if USE_AUTH
-        case Format::LFT_USER_EXTERNAL:
-            str = request->extacl_user.termedBuf();
-            break;
-#endif
-        case Format::LFT_EXT_LOG:
-            str = request->extacl_log.termedBuf();
-            break;
-        case Format::LFT_TAG:
-            str = request->tag.termedBuf();
-            break;
-        case Format::LFT_EXT_ACL_NAME:
-            str = acl_data->name;
-            break;
-        case Format::LFT_EXT_ACL_DATA:
-            data_used = true;
-            for (arg = acl_data->arguments; arg; arg = arg->next) {
-                if (!first)
+        if (t->type == Format::LFT_EXT_ACL_DATA) {
+            // setup string for %DATA
+            SBuf sb;
+            for (auto arg = acl_data->arguments; arg; arg = arg->next) {
+                if (sb.length())
                     sb.append(" ", 1);
 
-                if (acl_data->def->quote == external_acl::QUOTE_METHOD_URL) {
+                if (acl_data->def->quote == Format::LOG_QUOTE_URL) {
                     const char *quoted = rfc1738_escape(arg->key);
                     sb.append(quoted, strlen(quoted));
                 } else {
@@ -1124,59 +756,26 @@ makeExternalAclKey(ACLFilledChecklist * ch, external_acl_data * acl_data)
                     sb.append(mb2.buf, mb2.size);
                     mb2.clean();
                 }
-
-                first = 0;
             }
-            break;
-        case Format::LFT_PERCENT:
-            str = "%";
-            break;
 
-        default:
-            // TODO: replace this function with Format::assemble()
-            // For now die on unsupported logformat codes.
-            fatalf("ERROR: unknown external_acl_type format %u", (uint8_t)format->type);
-            break;
-        }
-
-        if (str)
-            if (!*str)
-                str = NULL;
-
-        if (!str)
-            str = "-";
-
-        if (!first)
-            mb.append(" ", 1);
-
-        if (acl_data->def->quote == external_acl::QUOTE_METHOD_URL) {
-            const char *quoted = rfc1738_escape(str);
-            mb.append(quoted, strlen(quoted));
-        } else {
-            strwordquote(&mb, str);
+            ch->al->lastAclData = sb;
         }
 
-        sb.clean();
-
-        first = 0;
-    }
-
-    if (!data_used) {
-        for (arg = acl_data->arguments; arg; arg = arg->next) {
-            if (!first)
-                mb.append(" ", 1);
-
-            if (acl_data->def->quote == external_acl::QUOTE_METHOD_URL) {
-                const char *quoted = rfc1738_escape(arg->key);
-                mb.append(quoted, strlen(quoted));
-            } else {
-                strwordquote(&mb, arg->key);
+#if USE_IDENT
+        if (t->type == Format::LFT_USER_IDENT) {
+            if (!*ch->rfc931) {
+                // if we fail to go async, we still return NULL and the caller
+                // will detect the failure in ACLExternal::match().
+                (void)ch->goAsync(IdentLookup::Instance());
+                return NULL;
             }
-
-            first = 0;
         }
+#endif
     }
 
+    // assemble the full helper lookup string
+    acl_data->def->format.assemble(mb, ch->al, 0);
+
     return mb.buf;
 }
 
@@ -1460,7 +1059,8 @@ externalAclStats(StoreEntry * sentry)
     for (external_acl *p = Config.externalAclHelperList; p; p = p->next) {
         storeAppendPrintf(sentry, "External ACL Statistics: %s\n", p->name);
         storeAppendPrintf(sentry, "Cache size: %d\n", p->cache->count);
-        helperStats(sentry, p->theHelper);
+        assert(p->theHelper);
+        p->theHelper->packStatsInto(sentry);
         storeAppendPrintf(sentry, "\n");
     }
 }
@@ -1532,18 +1132,6 @@ ExternalACLLookup::LookupDone(void *data, const ExternalACLEntryPointer &result)
 {
     ACLFilledChecklist *checklist = Filled(static_cast<ACLChecklist*>(data));
     checklist->extacl_entry = result;
-
-    // attach the helper kv-pair to the transaction
-    if (checklist->extacl_entry != NULL) {
-        if (HttpRequest * req = checklist->request) {
-            // XXX: we have no access to the transaction / AccessLogEntry so cant SyncNotes().
-            // workaround by using anything already set in HttpRequest
-            // OR use new and rely on a later Sync copying these to AccessLogEntry
-
-            UpdateRequestNotes(checklist->conn(), *req, checklist->extacl_entry->notes);
-        }
-    }
-
     checklist->resumeNonBlockingCheck(ExternalACLLookup::Instance());
 }