From: Emeric Brun Date: Wed, 28 Nov 2012 17:47:52 +0000 (+0100) Subject: MEDIUM: ssl: manage shared cache by blocks for huge sessions. X-Git-Tag: v1.5-dev15~27 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=af9619da3ee527671bb69250bbd6eabb2cff8c57;p=thirdparty%2Fhaproxy.git MEDIUM: ssl: manage shared cache by blocks for huge sessions. Sessions using client certs are huge (more than 1 kB) and do not fit in session cache, or require a huge cache. In this new implementation sshcachesize set a number of available blocks instead a number of available sessions. Each block is large enough (128 bytes) to store a simple session (without client certs). Huge sessions will take multiple blocks depending on client certificate size. Note: some unused code for session sync with remote peers was temporarily removed. --- diff --git a/doc/configuration.txt b/doc/configuration.txt index ed1ddc2648..81cd231bc9 100644 --- a/doc/configuration.txt +++ b/doc/configuration.txt @@ -878,14 +878,16 @@ tune.sndbuf.server notifying haproxy again. tune.ssl.cachesize - Sets the size of the global SSL session cache, in number of sessions. Each - entry uses approximately 600 bytes of memory. The default value may be forced - at build time, otherwise defaults to 20000. When the cache is full, the most - idle entries are purged and reassigned. Higher values reduce the occurrence - of such a purge, hence the number of CPU-intensive SSL handshakes by ensuring - that all users keep their session as long as possible. All entries are pre- - allocated upon startup and are shared between all processes if "nbproc" is - greater than 1. + Sets the size of the global SSL session cache, in a number of blocks. A block + is large enough to contain an encoded session without peer certificate. + An encoded session with peer certificate is stored in multiple blocks + depending on the size of the peer certificate. A block use approximatively + 200 bytes of memory. The default value may be forced at build time, otherwise + defaults to 20000. When the cache is full, the most idle entries are purged + and reassigned. Higher values reduce the occurrence of such a purge, hence + the number of CPU-intensive SSL handshakes by ensuring that all users keep + their session as long as possible. All entries are pre-allocated upon startup + and are shared between all processes if "nbproc" is greater than 1. tune.ssl.lifetime Sets how long a cached SSL session may remain valid. This time is expressed diff --git a/include/proto/shctx.h b/include/proto/shctx.h index 379be355f7..a09c38c4b7 100644 --- a/include/proto/shctx.h +++ b/include/proto/shctx.h @@ -16,13 +16,12 @@ #include #include -#ifndef SHSESS_MAX_FOOTER_LEN -#define SHSESS_MAX_FOOTER_LEN sizeof(uint32_t) \ - + EVP_MAX_MD_SIZE +#ifndef SHSESS_BLOCK_MIN_SIZE +#define SHSESS_BLOCK_MIN_SIZE 128 #endif #ifndef SHSESS_MAX_DATA_LEN -#define SHSESS_MAX_DATA_LEN 512 +#define SHSESS_MAX_DATA_LEN 4096 #endif #ifndef SHCTX_DEFAULT_SIZE @@ -33,37 +32,15 @@ #define SHCTX_APPNAME "haproxy" #endif -#define SHSESS_MAX_ENCODED_LEN SSL_MAX_SSL_SESSION_ID_LENGTH \ - + SHSESS_MAX_DATA_LEN \ - + SHSESS_MAX_FOOTER_LEN - - - -/* Callback called on a new session event: - * session contains the sessionid zeros padded to SSL_MAX_SSL_SESSION_ID_LENGTH - * followed by ASN1 session encoding. - * len is set to SSL_MAX_SSL_SESSION_ID_LENGTH + ASN1 session length - * len is always less than SSL_MAX_SSL_SESSION_ID_LENGTH + SHSESS_MAX_DATA_LEN. - * Remaining Bytes from len to SHSESS_MAX_ENCODED_LEN can be used to add a footer. - * cdate is the creation date timestamp. - */ -void shsess_set_new_cbk(void (*func)(unsigned char *session, unsigned int len, long cdate)); - -/* Add a session into the cache, - * session contains the sessionid zeros padded to SSL_MAX_SSL_SESSION_ID_LENGTH - * followed by ASN1 session encoding. - * len is set to SSL_MAX_SSL_SESSION_ID_LENGTH + ASN1 data length. - * if len greater than SHSESS_MAX_ENCODED_LEN, session is not added. - * if cdate not 0, on get events session creation date will be reset to cdate */ -void shctx_sess_add(const unsigned char *session, unsigned int session_len, long cdate); - /* Allocate shared memory context. - * size is maximum cached sessions. - * if set less or equal to 0, SHCTX_DEFAULT_SIZE is used. - * set use_shared_memory to 1 to use a mapped shared memory insteed - * of private. (ignored if compiled whith USE_PRIVATE_CACHE=1) - * Returns: -1 on alloc failure, size if it performs context alloc, - * and 0 if cache is already allocated */ + * is the number of allocated blocks into cache (default 128 bytes) + * A block is large enough to contain a classic session (without client cert) + * If is set less or equal to 0, SHCTX_DEFAULT_SIZE is used. + * Set to 1 to use a mapped shared memory instead + * of private. (ignored if compiled with USE_PRIVATE_CACHE=1). + * Returns: -1 on alloc failure, if it performs context alloc, + * and 0 if cache is already allocated. + */ int shared_context_init(int size, int use_shared_memory); /* Set shared cache callbacks on an ssl context. diff --git a/src/shctx.c b/src/shctx.c index 03961b6797..457aedbd27 100644 --- a/src/shctx.c +++ b/src/shctx.c @@ -24,20 +24,39 @@ #include #endif /* USE_SYSCALL_FUTEX */ #endif - +#include #include "ebmbtree.h" #include "proto/shctx.h" +struct shsess_packet_hdr { + unsigned int eol; + unsigned char final:1; + unsigned char seq:7; + unsigned char id[SSL_MAX_SSL_SESSION_ID_LENGTH]; +}; + +struct shsess_packet { + unsigned char version; + unsigned char sig[SHA_DIGEST_LENGTH]; + struct shsess_packet_hdr hdr; + unsigned char data[0]; +}; + struct shared_session { struct ebmb_node key; unsigned char key_data[SSL_MAX_SSL_SESSION_ID_LENGTH]; - long c_date; - int data_len; - unsigned char data[SHSESS_MAX_DATA_LEN]; - struct shared_session *p; - struct shared_session *n; + unsigned char data[SHSESS_BLOCK_MIN_SIZE]; }; +struct shared_block { + union { + struct shared_session session; + unsigned char data[sizeof(struct shared_session)]; + } data; + short int data_len; + struct shared_block *p; + struct shared_block *n; +}; struct shared_context { #ifndef USE_PRIVATE_CACHE @@ -47,8 +66,11 @@ struct shared_context { pthread_mutex_t mutex; #endif #endif - struct shared_session active; - struct shared_session free; + struct shsess_packet_hdr upd; + unsigned char data[SHSESS_MAX_DATA_LEN]; + short int data_len; + struct shared_block active; + struct shared_block free; }; /* Static shared context */ @@ -57,9 +79,6 @@ static struct shared_context *shctx = NULL; static int use_shared_mem = 0; #endif -/* Callbacks */ -static void (*shared_session_new_cbk)(unsigned char *session, unsigned int session_len, long cdate); - /* Lock functions */ #ifdef USE_PRIVATE_CACHE #define shared_context_lock() @@ -156,93 +175,215 @@ static inline void _shared_context_unlock(void) /* List Macros */ -#define shsess_unset(s) (s)->n->p = (s)->p; \ +#define shblock_unset(s) (s)->n->p = (s)->p; \ (s)->p->n = (s)->n; -#define shsess_set_free(s) shsess_unset(s) \ - (s)->p = &shctx->free; \ - (s)->n = shctx->free.n; \ - shctx->free.n->p = s; \ - shctx->free.n = s; +#define shblock_set_free(s) shblock_unset(s) \ + (s)->n = &shctx->free; \ + (s)->p = shctx->free.p; \ + shctx->free.p->n = s; \ + shctx->free.p = s; -#define shsess_set_active(s) shsess_unset(s) \ - (s)->p = &shctx->active; \ - (s)->n = shctx->active.n; \ - shctx->active.n->p = s; \ - shctx->active.n = s; +#define shblock_set_active(s) shblock_unset(s) \ + (s)->n = &shctx->active; \ + (s)->p = shctx->active.p; \ + shctx->active.p->n = s; \ + shctx->active.p = s; -#define shsess_get_next() (shctx->free.p == &shctx->free) ? \ - shctx->active.p : shctx->free.p; - /* Tree Macros */ #define shsess_tree_delete(s) ebmb_delete(&(s)->key); -#define shsess_tree_insert(s) (struct shared_session *)ebmb_insert(&shctx->active.key.node.branches, \ +#define shsess_tree_insert(s) (struct shared_session *)ebmb_insert(&shctx->active.data.session.key.node.branches, \ &(s)->key, SSL_MAX_SSL_SESSION_ID_LENGTH); -#define shsess_tree_lookup(k) (struct shared_session *)ebmb_lookup(&shctx->active.key.node.branches, \ +#define shsess_tree_lookup(k) (struct shared_session *)ebmb_lookup(&shctx->active.data.session.key.node.branches, \ (k), SSL_MAX_SSL_SESSION_ID_LENGTH); -/* Other Macros */ +/* shared session functions */ -#define shsess_set_key(s,k,l) { memcpy((s)->key_data, (k), (l)); \ - if ((l) < SSL_MAX_SSL_SESSION_ID_LENGTH) \ - memset((s)->key_data+(l), 0, SSL_MAX_SSL_SESSION_ID_LENGTH-(l)); }; +/* Free session blocks, returns number of freed blocks */ +static int shsess_free(struct shared_session *shsess) +{ + struct shared_block *block; + int ret = 1; + if (((struct shared_block *)shsess)->data_len <= sizeof(shsess->data)) { + shblock_set_free((struct shared_block *)shsess); + return ret; + } + block = ((struct shared_block *)shsess)->n; + shblock_set_free((struct shared_block *)shsess); + while (1) { + struct shared_block *next; + + if (block->data_len <= sizeof(block->data)) { + /* last block */ + shblock_set_free(block); + ret++; + break; + } + next = block->n; + shblock_set_free(block); + ret++; + block = next; + } + return ret; +} -/* SSL context callbacks */ +/* This function frees enough blocks to store a new session of data_len. + * Returns a ptr on a free block if it succeeds, or NULL if there are not + * enough blocks to store that session. + */ +static struct shared_session *shsess_get_next(int data_len) +{ + int head = 0; + struct shared_block *b; + + b = shctx->free.n; + while (b != &shctx->free) { + if (!head) { + data_len -= sizeof(b->data.session.data); + head = 1; + } + else + data_len -= sizeof(b->data.data); + if (data_len <= 0) + return &shctx->free.n->data.session; + b = b->n; + } + b = shctx->active.n; + while (b != &shctx->active) { + int freed; + + shsess_tree_delete(&b->data.session); + freed = shsess_free(&b->data.session); + if (!head) + data_len -= sizeof(b->data.session.data) + (freed-1)*sizeof(b->data.data); + else + data_len -= freed*sizeof(b->data.data); + if (data_len <= 0) + return &shctx->free.n->data.session; + b = shctx->active.n; + } + return NULL; +} -/* SSL callback used on new session creation */ -int shctx_new_cb(SSL *ssl, SSL_SESSION *sess) +/* store a session into the cache + * s_id : session id padded with zero to SSL_MAX_SSL_SESSION_ID_LENGTH + * data: asn1 encoded session + * data_len: asn1 encoded session length + * Returns 1 id session was stored (else 0) + */ +static int shsess_store(unsigned char *s_id, unsigned char *data, int data_len) { - struct shared_session *shsess; - unsigned char *data,*p; - unsigned int data_len; - unsigned char encsess[SHSESS_MAX_ENCODED_LEN]; - (void)ssl; - - /* check if session reserved size in aligned buffer is large enougth for the ASN1 encode session */ - data_len=i2d_SSL_SESSION(sess, NULL); - if(data_len > SHSESS_MAX_DATA_LEN) + struct shared_session *shsess, *oldshsess; + + shsess = shsess_get_next(data_len); + if (!shsess) { + /* Could not retrieve enough free blocks to store that session */ return 0; + } - /* process ASN1 session encoding before the lock: lower cost */ - p = data = encsess+SSL_MAX_SSL_SESSION_ID_LENGTH; - i2d_SSL_SESSION(sess, &p); + /* prepare key */ + memcpy(shsess->key_data, s_id, SSL_MAX_SSL_SESSION_ID_LENGTH); - shared_context_lock(); + /* it returns the already existing node + or current node if none, never returns null */ + oldshsess = shsess_tree_insert(shsess); + if (oldshsess != shsess) { + /* free all blocks used by old node */ + shsess_free(oldshsess); + shsess = oldshsess; + } - shsess = shsess_get_next(); + ((struct shared_block *)shsess)->data_len = data_len; + if (data_len <= sizeof(shsess->data)) { + /* Store on a single block */ + memcpy(shsess->data, data, data_len); + shblock_set_active((struct shared_block *)shsess); + } + else { + unsigned char *p; + /* Store on multiple blocks */ + int cur_len; + + memcpy(shsess->data, data, sizeof(shsess->data)); + p = data + sizeof(shsess->data); + cur_len = data_len - sizeof(shsess->data); + shblock_set_active((struct shared_block *)shsess); + while (1) { + /* Store next data on free block. + * shsess_get_next guarantees that there are enough + * free blocks in queue. + */ + struct shared_block *block; + + block = shctx->free.n; + if (cur_len <= sizeof(block->data)) { + /* This is the last block */ + block->data_len = cur_len; + memcpy(block->data.data, p, cur_len); + shblock_set_active(block); + break; + } + /* Intermediate block */ + block->data_len = cur_len; + memcpy(block->data.data, p, sizeof(block->data)); + p += sizeof(block->data.data); + cur_len -= sizeof(block->data.data); + shblock_set_active(block); + } + } - shsess_tree_delete(shsess); + return 1; +} - shsess_set_key(shsess, sess->session_id, sess->session_id_length); - /* it returns the already existing node or current node if none, never returns null */ - shsess = shsess_tree_insert(shsess); +/* SSL context callbacks */ - /* store ASN1 encoded session into cache */ - shsess->data_len = data_len; - memcpy(shsess->data, data, data_len); +/* SSL callback used on new session creation */ +int shctx_new_cb(SSL *ssl, SSL_SESSION *sess) +{ + unsigned char encsess[sizeof(struct shsess_packet)+SHSESS_MAX_DATA_LEN]; + struct shsess_packet *packet = (struct shsess_packet *)encsess; + unsigned char *p; + int data_len, sid_length; + + + /* Session id is already stored in to key and session id is known + * so we dont store it to keep size. + */ + sid_length = sess->session_id_length; + sess->session_id_length = 0; + sess->sid_ctx_length = 0; + + /* check if buffer is large enough for the ASN1 encoded session */ + data_len = i2d_SSL_SESSION(sess, NULL); + if (data_len > SHSESS_MAX_DATA_LEN) + goto err; + + /* process ASN1 session encoding before the lock */ + p = packet->data; + i2d_SSL_SESSION(sess, &p); + + memcpy(packet->hdr.id, sess->session_id, sid_length); + if (sid_length < SSL_MAX_SSL_SESSION_ID_LENGTH) + memset(&packet->hdr.id[sid_length], 0, SSL_MAX_SSL_SESSION_ID_LENGTH-sid_length); - /* store creation date */ - shsess->c_date = SSL_SESSION_get_time(sess); + shared_context_lock(); - shsess_set_active(shsess); + /* store to cache */ + shsess_store(packet->hdr.id, packet->data, data_len); shared_context_unlock(); - if (shared_session_new_cbk) { /* if user level callback is set */ - /* copy sessionid padded with 0 into the sessionid + data aligned buffer */ - memcpy(encsess, sess->session_id, sess->session_id_length); - if (sess->session_id_length < SSL_MAX_SSL_SESSION_ID_LENGTH) - memset(encsess+sess->session_id_length, 0, SSL_MAX_SSL_SESSION_ID_LENGTH-sess->session_id_length); - - shared_session_new_cbk(encsess, SSL_MAX_SSL_SESSION_ID_LENGTH+data_len, SSL_SESSION_get_time(sess)); - } +err: + /* reset original length values */ + sess->sid_ctx_length = ssl->sid_ctx_length; + sess->session_id_length = sid_length; return 0; /* do not increment session reference count */ } @@ -253,10 +394,8 @@ SSL_SESSION *shctx_get_cb(SSL *ssl, unsigned char *key, int key_len, int *do_cop struct shared_session *shsess; unsigned char data[SHSESS_MAX_DATA_LEN], *p; unsigned char tmpkey[SSL_MAX_SSL_SESSION_ID_LENGTH]; - unsigned int data_len; - long cdate; + int data_len; SSL_SESSION *sess; - (void)ssl; /* allow the session to be freed automatically by openssl */ *do_copy = 0; @@ -279,24 +418,52 @@ SSL_SESSION *shctx_get_cb(SSL *ssl, unsigned char *key, int key_len, int *do_cop return NULL; } - /* backup creation date to reset in session after ASN1 decode */ - cdate = shsess->c_date; - - /* copy ASN1 session data to decode outside the lock */ - data_len = shsess->data_len; - memcpy(data, shsess->data, shsess->data_len); - - shsess_set_active(shsess); + data_len = ((struct shared_block *)shsess)->data_len; + if (data_len <= sizeof(shsess->data)) { + /* Session stored on single block */ + memcpy(data, shsess->data, data_len); + shblock_set_active((struct shared_block *)shsess); + } + else { + /* Session stored on multiple blocks */ + struct shared_block *block; + + memcpy(data, shsess->data, sizeof(shsess->data)); + p = data + sizeof(shsess->data); + block = ((struct shared_block *)shsess)->n; + shblock_set_active((struct shared_block *)shsess); + while (1) { + /* Retrieve data from next block */ + struct shared_block *next; + + if (block->data_len <= sizeof(block->data.data)) { + /* This is the last block */ + memcpy(p, block->data.data, block->data_len); + p += block->data_len; + shblock_set_active(block); + break; + } + /* Intermediate block */ + memcpy(p, block->data.data, sizeof(block->data.data)); + p += sizeof(block->data.data); + next = block->n; + shblock_set_active(block); + block = next; + } + } shared_context_unlock(); /* decode ASN1 session */ p = data; sess = d2i_SSL_SESSION(NULL, (const unsigned char **)&p, data_len); - - /* reset creation date */ - if (sess) - SSL_SESSION_set_time(sess, cdate); + /* Reset session id and session id contenxt */ + if (sess) { + memcpy(sess->session_id, key, key_len); + sess->session_id_length = key_len; + memcpy(sess->sid_ctx, ssl->sid_ctx, ssl->sid_ctx_length); + sess->sid_ctx_length = ssl->sid_ctx_length; + } return sess; } @@ -321,59 +488,21 @@ void shctx_remove_cb(SSL_CTX *ctx, SSL_SESSION *sess) /* lookup for session */ shsess = shsess_tree_lookup(key); if (shsess) { - shsess_set_free(shsess); + /* free session */ + shsess_tree_delete(shsess); + shsess_free(shsess); } /* unlock cache */ shared_context_unlock(); } -/* User level function called to add a session to the cache (remote updates) */ -void shctx_sess_add(const unsigned char *encsess, unsigned int len, long cdate) -{ - struct shared_session *shsess; - - /* check buffer is at least padded key long + 1 byte - and data_len not too long */ - if ((len <= SSL_MAX_SSL_SESSION_ID_LENGTH) - || (len > SHSESS_MAX_DATA_LEN+SSL_MAX_SSL_SESSION_ID_LENGTH)) - return; - - shared_context_lock(); - - shsess = shsess_get_next(); - - shsess_tree_delete(shsess); - - shsess_set_key(shsess, encsess, SSL_MAX_SSL_SESSION_ID_LENGTH); - - /* it returns the already existing node or current node if none, never returns null */ - shsess = shsess_tree_insert(shsess); - - /* store into cache and update earlier on session get events */ - if (cdate) - shsess->c_date = (long)cdate; - - /* copy ASN1 session data into cache */ - shsess->data_len = len-SSL_MAX_SSL_SESSION_ID_LENGTH; - memcpy(shsess->data, encsess+SSL_MAX_SSL_SESSION_ID_LENGTH, shsess->data_len); - - shsess_set_active(shsess); - - shared_context_unlock(); -} - -/* Function used to set a callback on new session creation */ -void shsess_set_new_cbk(void (*func)(unsigned char *, unsigned int, long)) -{ - shared_session_new_cbk = func; -} - /* Allocate shared memory context. - * size is maximum cached sessions. - * if set less or equal to 0, SHCTX_DEFAULT_SIZE is used. - * Returns: -1 on alloc failure, size if it performs context alloc, - * and 0 if cache is already allocated */ + * is maximum cached sessions. + * If is set to less or equal to 0, SHCTX_DEFAULT_SIZE is used. + * Returns: -1 on alloc failure, if it performs context alloc, + * and 0 if cache is already allocated. + */ int shared_context_init(int size, int shared) { int i; @@ -382,7 +511,7 @@ int shared_context_init(int size, int shared) pthread_mutexattr_t attr; #endif /* USE_SYSCALL_FUTEX */ #endif - struct shared_session *prev,*cur; + struct shared_block *prev,*cur; int maptype = MAP_PRIVATE; if (shctx) @@ -391,12 +520,14 @@ int shared_context_init(int size, int shared) if (size<=0) size = SHCTX_DEFAULT_SIZE; + /* Increate size by one to reserve one node for lookup */ + size++; #ifndef USE_PRIVATE_CACHE if (shared) maptype = MAP_SHARED; #endif - shctx = (struct shared_context *)mmap(NULL, sizeof(struct shared_context)+(size*sizeof(struct shared_session)), + shctx = (struct shared_context *)mmap(NULL, sizeof(struct shared_context)+(size*sizeof(struct shared_block)), PROT_READ | PROT_WRITE, maptype | MAP_ANON, -1, 0); if (!shctx || shctx == MAP_FAILED) { shctx = NULL; @@ -415,12 +546,16 @@ int shared_context_init(int size, int shared) use_shared_mem = 1; #endif - memset(&shctx->active.key, 0, sizeof(struct ebmb_node)); - memset(&shctx->free.key, 0, sizeof(struct ebmb_node)); + memset(&shctx->active.data.session.key, 0, sizeof(struct ebmb_node)); + memset(&shctx->free.data.session.key, 0, sizeof(struct ebmb_node)); /* No duplicate authorized in tree: */ - //shctx->active.key.node.branches.b[1] = (void *)1; - shctx->active.key.node.branches = EB_ROOT_UNIQUE; + shctx->active.data.session.key.node.branches = EB_ROOT_UNIQUE; + + /* Init remote update cache */ + shctx->upd.eol = 0; + shctx->upd.seq = 0; + shctx->data_len = 0; cur = &shctx->active; cur->n = cur->p = cur; @@ -428,7 +563,7 @@ int shared_context_init(int size, int shared) cur = &shctx->free; for (i = 0 ; i < size ; i++) { prev = cur; - cur = (struct shared_session *)((char *)prev + sizeof(struct shared_session)); + cur = (struct shared_block *)((char *)prev + sizeof(struct shared_block)); prev->n = cur; cur->p = prev; }