MINOR: cache: Prepare helper functions for Vary support

The Vary functionality is based on a secondary key that needs to be
calculated for every request to which a server answers with a Vary
header. The Vary header, which can only be found in server responses,
determines which headers of the request need to be taken into account in
the secondary key. Since we do not want to have to store all the headers
of the request until we have the response, we will pre-calculate as many
sub-hashes as there are headers that we want to manage in a Vary
context. We will only focus on a subset of headers which are likely to
be mentioned in a Vary response (accept-encoding and referer for now).
Every managed header will have its own normalization function which is
in charge of transforming the header value into a core representation,
more robust to insignificant changes that could exist between multiple
clients. For instance, two accept-encoding values mentioning the same
encodings but in different orders should give the same hash.
This patch adds a function that parses a Vary header value and checks if
all the values belong to our supported subset. It also adds the
normalization functions for our two headers, as well as utility
functions that can prebuild a secondary key for a given request and
transform it into an actual secondary key after the vary signature is
determined from the response.

diff --git a/include/haproxy/http_ana-t.h b/include/haproxy/http_ana-t.h
index 9449b7955d1212749c7c0050b61d6acede54467b..5fb8298deb9d6ab04789567e449b1931d40c7054 100644 (file)
--- a/include/haproxy/http_ana-t.h
+++ b/include/haproxy/http_ana-t.h
@@ -90,6 +90,13 @@
 
 #define HTTP_MSGF_BODYLESS    0x00000040  /* The message has no body (content-length = 0) */
 
+/* Maximum length of the cache secondary key (sum of all the possible parts of
+ * the secondary key). The actual keys might be smaller for some
+ * request/response pairs, because they depend on the responses' optional Vary
+ * header. The differents sizes can be found in the vary_information object (see
+ * cache.c).*/
+#define HTTP_CACHE_SEC_KEY_LEN (sizeof(int)+sizeof(int))
+
 
 /* Redirect flags */
 enum {
@@ -163,6 +170,7 @@ struct http_msg {
        struct channel *chn;                   /* pointer to the channel transporting the message */
 };
 
+
 /* This is an HTTP transaction. It contains both a request message and a
  * response message (which can be empty).
  */
@@ -176,6 +184,7 @@ struct http_txn {
        struct http_reply *http_reply;  /* The HTTP reply to use as reply */
 
        char cache_hash[20];               /* Store the cache hash  */
+       char cache_secondary_hash[HTTP_CACHE_SEC_KEY_LEN]; /* Optional cache secondary key. */
        char *uri;                      /* first line if log needed, NULL otherwise */
        char *cli_cookie;               /* cookie presented by the client, in capture mode */
        char *srv_cookie;               /* cookie presented by the server, in capture mode */

diff --git a/src/cache.c b/src/cache.c
index bd77a6ce3cadb5fb9ca18bc643d40d35c0c3890b..5a27624dec5bc34d35511a773fb236a707f2eef4 100644 (file)
--- a/src/cache.c
+++ b/src/cache.c
@@ -61,6 +61,41 @@ struct cache_flt_conf {
        unsigned int flags;   /* CACHE_FLT_F_* */
 };
 
+
+/*
+ * Vary-related structures and functions
+ */
+enum vary_header_bit {
+       VARY_ACCEPT_ENCODING = (1 << 0),
+       VARY_REFERER =         (1 << 1),
+       VARY_LAST  /* should always be last */
+};
+
+typedef int(*http_header_normalizer)(struct ist value, char *buf, unsigned int *buf_len);
+
+struct vary_hashing_information {
+       struct ist hdr_name;                 /* Header name */
+       enum vary_header_bit value;          /* Bit repesenting the header in a vary signature */
+       unsigned int hash_length;            /* Size of the sub hash for this header's value */
+       http_header_normalizer norm_fn;      /* Normalization function */
+};
+
+static int accept_encoding_normalizer(struct ist value, char *buf, unsigned int *buf_len);
+static int default_normalizer(struct ist value, char *buf, unsigned int *buf_len);
+
+/* Warning : do not forget to update HTTP_CACHE_SEC_KEY_LEN when new items are
+ * added to this array. */
+const struct vary_hashing_information vary_information[] = {
+       { IST("accept-encoding"), VARY_ACCEPT_ENCODING, sizeof(int), &accept_encoding_normalizer },
+       { IST("referer"), VARY_REFERER, sizeof(int), &default_normalizer },
+};
+
+static int http_request_prebuild_full_secondary_key(struct stream *s);
+static int http_request_build_secondary_key(struct stream *s, int vary_signature);
+static int http_request_reduce_secondary_key(unsigned int vary_signature,
+                                            char prebuilt_key[HTTP_CACHE_SEC_KEY_LEN]);
+
+
 /*
  * cache ctx for filters
  */
@@ -86,6 +121,10 @@ struct cache_entry {
                               * be used in case of an "If-Modified-Since"-based
                               * conditional request. */
 
+       unsigned int secondary_key_signature;  /* Bitfield of the HTTP headers that should be used
+                                               * to build secondary keys for this cache entry. */
+       char secondary_key[HTTP_CACHE_SEC_KEY_LEN];  /* Optional secondary key. */
+
        unsigned char data[0];
 };
 
@@ -123,6 +162,29 @@ struct cache_entry *entry_exist(struct cache *cache, char *hash)
 
 }
 
+struct cache_entry *secondary_entry_exist(struct cache *cache, struct cache_entry *entry,
+                                         char *secondary_key)
+{
+       struct eb32_node *node = &entry->eb;
+
+       if (!entry->secondary_key_signature)
+               return NULL;
+
+       while (entry && memcmp(entry->secondary_key, secondary_key, HTTP_CACHE_SEC_KEY_LEN) != 0) {
+               node = eb32_next_dup(node);
+               entry = node ? eb32_entry(node, struct cache_entry, eb) : NULL;
+       }
+
+       /* Expired entry */
+       if (entry && entry->expire <= now.tv_sec) {
+               eb32_delete(&entry->eb);
+               entry->eb.key = 0;
+               entry = NULL;
+       }
+
+       return entry;
+}
+
 static inline struct shared_context *shctx_ptr(struct cache *cache)
 {
        return (struct shared_context *)((unsigned char *)cache - ((struct shared_context *)NULL)->data);
@@ -589,6 +651,40 @@ static time_t get_last_modified_time(struct htx *htx)
        return last_modified;
 }
 
+/*
+ * Checks the vary header's value. The headers on which vary should be applied
+ * must be explicitely supported in the vary_information array (see cache.c). If
+ * any other header is mentioned, we won't store the response.
+ * Returns 1 if Vary-based storage can work, 0 otherwise.
+ */
+static int http_check_vary_header(struct htx *htx, unsigned int *vary_signature)
+{
+       unsigned int vary_idx;
+       unsigned int vary_info_count;
+       const struct vary_hashing_information *vary_info;
+       struct http_hdr_ctx ctx = { .blk = NULL };
+
+       int retval = 1;
+
+       *vary_signature = 0;
+
+       vary_info_count = sizeof(vary_information)/sizeof(*vary_information);
+       while (retval && http_find_header(htx, ist("Vary"), &ctx, 0)) {
+               for (vary_idx = 0; vary_idx < vary_info_count; ++vary_idx) {
+                       vary_info = &vary_information[vary_idx];
+                       if (isteqi(ctx.value, vary_info->hdr_name)) {
+                               *vary_signature |= vary_info->value;
+                               break;
+                       }
+               }
+               retval = (vary_idx < vary_info_count);
+       }
+
+       return retval;
+}
+
+
+
 /*
  * This function will store the headers of the response in a buffer and then
  * register a filter to store the data
@@ -1606,6 +1702,182 @@ struct flt_ops cache_ops = {
 };
 
 
+int accept_encoding_cmp(const void *a, const void *b)
+{
+       const struct ist ist_a = *(const struct ist*)a;
+       const struct ist ist_b = *(const struct ist*)b;
+
+       return istdiff(ist_a, ist_b);
+}
+
+/*
+ * Build a hash of the accept-encoding header. The different parts of the
+ * header value are first sorted, appended and then a crc is calculated
+ * for the newly constructed buffer.
+ * Returns 0 in case of success.
+ */
+static int accept_encoding_normalizer(struct ist value, char *buf, unsigned int *buf_len)
+{
+       int retval = 0;
+       struct ist values[16] = {{}};
+       unsigned int count = 0;
+       char *comma = NULL;
+       struct buffer *trash = get_trash_chunk();
+       int hash_value = 0;
+
+       /* The hash will be built out of a sorted list of accepted encodings. */
+       while((comma = istchr(value, ',')) != NULL) {
+               values[count++] = ist2(istptr(value), comma-istptr(value));
+               value = ist2(comma+1, istlen(value) - (comma-istptr(value)) - 1);
+       }
+       values[count++] = value;
+
+       /* Sort the values alphabetically. */
+       qsort(values, count, sizeof(struct ist), &accept_encoding_cmp);
+
+       while (count)
+               chunk_istcat(trash, values[--count]);
+
+       hash_value = hash_crc32(b_orig(trash), b_data(trash));
+
+       memcpy(buf, &hash_value, sizeof(hash_value));
+       *buf_len = sizeof(hash_value);
+
+       return retval;
+}
+
+/*
+ * Normalizer used by default for User-Agent and Referer headers. It only
+ * calculates a simple crc of the whole value.
+ * Returns 0 in case of success.
+ */
+static int default_normalizer(struct ist value, char *buf, unsigned int *buf_len)
+{
+       int hash_value = 0;
+
+       hash_value = hash_crc32(istptr(value), istlen(value));
+
+       memcpy(buf, &hash_value, sizeof(hash_value));
+       *buf_len = sizeof(hash_value);
+
+       return 0;
+}
+
+
+/*
+ * Pre-calculate the hashes of all the supported headers (in our Vary
+ * implementation) of a given request. We have to calculate all the hashes
+ * in advance because the actual Vary signature won't be known until the first
+ * response.
+ * Only the first occurrence of every header will be taken into account in the
+ * hash.
+ * If the header is not present, the hash portion of the given header will be
+ * filled with zeros.
+ * Returns 0 in case of success.
+ */
+static int http_request_prebuild_full_secondary_key(struct stream *s)
+{
+       struct http_txn *txn = s->txn;
+       struct htx *htx = htxbuf(&s->req.buf);
+       struct http_hdr_ctx ctx = { .blk = NULL };
+
+       unsigned int idx;
+       const struct vary_hashing_information *info = NULL;
+       unsigned int hash_length = 0;
+       int retval = 0;
+       int offset = 0;
+
+       for (idx = 0; idx < sizeof(vary_information)/sizeof(*vary_information) && !retval; ++idx) {
+               info = &vary_information[idx];
+
+               ctx.blk = NULL;
+               if (info->norm_fn != NULL && http_find_header(htx, info->hdr_name, &ctx, 1)) {
+                       retval = info->norm_fn(ctx.value, &txn->cache_secondary_hash[offset], &hash_length);
+                       offset += hash_length;
+               }
+               else {
+                       /* Fill hash with 0s. */
+                       hash_length = info->hash_length;
+                       memset(&txn->cache_secondary_hash[offset], 0, hash_length);
+                       offset += hash_length;
+               }
+       }
+
+       return retval;
+}
+
+
+/*
+ * Calculate the secondary key for a request for which we already have a known
+ * vary signature. The key is made by aggregating hashes calculated for every
+ * header mentioned in the vary signature.
+ * Only the first occurrence of every header will be taken into account in the
+ * hash.
+ * If the header is not present, the hash portion of the given header will be
+ * filled with zeros.
+ * Returns 0 in case of success.
+ */
+static int http_request_build_secondary_key(struct stream *s, int vary_signature)
+{
+       struct http_txn *txn = s->txn;
+       struct htx *htx = htxbuf(&s->req.buf);
+       struct http_hdr_ctx ctx = { .blk = NULL };
+
+       unsigned int idx;
+       const struct vary_hashing_information *info = NULL;
+       unsigned int hash_length = 0;
+       int retval = 0;
+       int offset = 0;
+
+       for (idx = 0; idx < sizeof(vary_information)/sizeof(*vary_information) && !retval; ++idx) {
+               info = &vary_information[idx];
+
+               ctx.blk = NULL;
+               if ((vary_signature & info->value) && info->norm_fn != NULL &&
+                   http_find_header(htx, info->hdr_name, &ctx, 1)) {
+                       retval = info->norm_fn(ctx.value, &txn->cache_secondary_hash[offset], &hash_length);
+                       offset += hash_length;
+               }
+               else {
+                       /* Fill hash with 0s. */
+                       hash_length = info->hash_length;
+                       memset(&txn->cache_secondary_hash[offset], 0, hash_length);
+                       offset += hash_length;
+               }
+       }
+
+       return retval;
+}
+
+/*
+ * Build the actual secondary key of a given request out of the prebuilt key and
+ * the actual vary signature (extracted from the response).
+ * Returns 0 in case of success.
+ */
+static int http_request_reduce_secondary_key(unsigned int vary_signature,
+                                            char prebuilt_key[HTTP_CACHE_SEC_KEY_LEN])
+{
+       int offset = 0;
+       int global_offset = 0;
+       int vary_info_count = 0;
+       int keep = 0;
+       unsigned int vary_idx;
+       const struct vary_hashing_information *vary_info;
+
+       vary_info_count = sizeof(vary_information)/sizeof(*vary_information);
+       for (vary_idx = 0; vary_idx < vary_info_count; ++vary_idx) {
+               vary_info = &vary_information[vary_idx];
+               keep = (vary_signature & vary_info->value) ? 0xff : 0;
+
+               for (offset = 0; offset < vary_info->hash_length; ++offset,++global_offset) {
+                       prebuilt_key[global_offset] &= keep;
+               }
+       }
+
+       return 0;
+}
+
+
 
 static int
 parse_cache_flt(char **args, int *cur_arg, struct proxy *px,